/* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ using System; using System.Collections.Generic; using Newtonsoft.Json; using QuantConnect.Algorithm.Framework.Alphas.Serialization; using QuantConnect.Interfaces; using QuantConnect.Securities; namespace QuantConnect.Algorithm.Framework.Alphas { /// /// Defines a alpha prediction for a single symbol generated by the algorithm /// /// /// Serialization of this type is delegated to the which uses the as a model. /// [JsonConverter(typeof(InsightJsonConverter))] public class Insight { private readonly IPeriodSpecification _periodSpecification; /// /// Gets the unique identifier for this insight /// public Guid Id { get; protected set; } /// /// Gets the group id this insight belongs to, null if not in a group /// public Guid? GroupId { get; protected set; } /// /// Gets an identifier for the source model that generated this insight. /// public string SourceModel { get; set; } /// /// Gets the utc time this insight was generated /// /// /// The algorithm framework handles setting this value appropriately. /// If providing custom implementation, be sure /// to set this value to algorithm.UtcTime when the insight is generated. /// public DateTime GeneratedTimeUtc { get; set; } /// /// Gets the insight's prediction end time. This is the time when this /// insight prediction is expected to be fulfilled. This time takes into /// account market hours, weekends, as well as the symbol's data resolution /// public DateTime CloseTimeUtc { get; set; } /// /// Gets the symbol this insight is for /// public Symbol Symbol { get; private set; } /// /// Gets the type of insight, for example, price insight or volatility insight /// public InsightType Type { get; private set; } /// /// Gets the initial reference value this insight is predicting against. The value is dependent on the specified /// public decimal ReferenceValue { get; set; } /// /// Gets the final reference value, used for scoring, this insight is predicting against. The value is dependent on the specified /// public decimal ReferenceValueFinal { get; set; } /// /// Gets the predicted direction, down, flat or up /// public InsightDirection Direction { get; private set; } /// /// Gets the period over which this insight is expected to come to fruition /// public TimeSpan Period { get; internal set; } /// /// Gets the predicted percent change in the insight type (price/volatility) /// public double? Magnitude { get; private set; } /// /// Gets the confidence in this insight /// public double? Confidence { get; private set; } /// /// Gets the portfolio weight of this insight /// public double? Weight { get; private set; } /// /// Gets the most recent scores for this insight /// public InsightScore Score { get; protected set; } /// /// Gets the estimated value of this insight in the account currency /// public decimal EstimatedValue { get; set; } /// /// The insight's tag containing additional information /// public string Tag { get; protected set; } /// /// Determines whether or not this insight is considered expired at the specified /// /// The algorithm's current time in UTC. See /// True if this insight is expired, false otherwise public bool IsExpired(DateTime utcTime) { return CloseTimeUtc < utcTime; } /// /// Determines whether or not this insight is considered active at the specified /// /// The algorithm's current time in UTC. See /// True if this insight is active, false otherwise public bool IsActive(DateTime utcTime) { return !IsExpired(utcTime); } /// /// Expire this insight /// /// The algorithm's current time in UTC. See public void Expire(DateTime utcTime) { if (IsActive(utcTime)) { CloseTimeUtc = utcTime.Add(-Time.OneSecond); Period = CloseTimeUtc - GeneratedTimeUtc; } } /// /// Cancel this insight /// /// The algorithm's current time in UTC. See public void Cancel(DateTime utcTime) { Expire(utcTime); } /// /// Initializes a new instance of the class /// /// The symbol this insight is for /// The period over which the prediction will come true /// The type of insight, price/volatility /// The predicted direction /// The insight's tag containing additional information public Insight(Symbol symbol, TimeSpan period, InsightType type, InsightDirection direction, string tag = "") : this(symbol, period, type, direction, null, null, null, null, tag) { } /// /// Initializes a new instance of the class /// /// The symbol this insight is for /// The period over which the prediction will come true /// The type of insight, price/volatility /// The predicted direction /// The predicted magnitude as a percentage change /// The confidence in this insight /// An identifier defining the model that generated this insight /// The portfolio weight of this insight /// The insight's tag containing additional information public Insight(Symbol symbol, TimeSpan period, InsightType type, InsightDirection direction, double? magnitude, double? confidence, string sourceModel = null, double? weight = null, string tag = "") { Id = Guid.NewGuid(); Score = new InsightScore(); SourceModel = sourceModel; Symbol = symbol; Type = type; Direction = direction; Period = period; // Optional Magnitude = magnitude; Confidence = confidence; Weight = weight; Tag = tag; _periodSpecification = new TimeSpanPeriodSpecification(period); } /// /// Initializes a new instance of the class /// /// The symbol this insight is for /// Func that defines the expiry time /// The type of insight, price/volatility /// The predicted direction /// The insight's tag containing additional information public Insight(Symbol symbol, Func expiryFunc, InsightType type, InsightDirection direction, string tag = "") : this(symbol, expiryFunc, type, direction, null, null, null, null, tag) { } /// /// Initializes a new instance of the class /// /// The symbol this insight is for /// Func that defines the expiry time /// The type of insight, price/volatility /// The predicted direction /// The predicted magnitude as a percentage change /// The confidence in this insight /// An identifier defining the model that generated this insight /// The portfolio weight of this insight /// The insight's tag containing additional information public Insight(Symbol symbol, Func expiryFunc, InsightType type, InsightDirection direction, double? magnitude, double? confidence, string sourceModel = null, double? weight = null, string tag = "") : this(symbol, new FuncPeriodSpecification(expiryFunc), type, direction, magnitude, confidence, sourceModel, weight, tag) { } /// /// Initializes a new instance of the class. /// This constructor is provided mostly for testing purposes. When running inside an algorithm, /// the generated and close times are set based on the algorithm's time. /// /// The time this insight was generated in utc /// The symbol this insight is for /// The period over which the prediction will come true /// The type of insight, price/volatility /// The predicted direction /// The predicted magnitude as a percentage change /// The confidence in this insight /// An identifier defining the model that generated this insight /// The portfolio weight of this insight /// The insight's tag containing additional information public Insight(DateTime generatedTimeUtc, Symbol symbol, TimeSpan period, InsightType type, InsightDirection direction, double? magnitude, double? confidence, string sourceModel = null, double? weight = null, string tag = "") : this(symbol, period, type, direction, magnitude, confidence, sourceModel, weight, tag) { GeneratedTimeUtc = generatedTimeUtc; CloseTimeUtc = generatedTimeUtc + period; } /// /// Private constructor used to keep track of how a user defined the insight period. /// /// The symbol this insight is for /// A specification defining how the insight's period was defined, via time span, via resolution/barcount, via close time /// The type of insight, price/volatility /// The predicted direction /// The predicted magnitude as a percentage change /// The confidence in this insight /// An identifier defining the model that generated this insight /// The portfolio weight of this insight /// The insight's tag containing additional information private Insight(Symbol symbol, IPeriodSpecification periodSpec, InsightType type, InsightDirection direction, double? magnitude, double? confidence, string sourceModel = null, double? weight = null, string tag = "") { Id = Guid.NewGuid(); Score = new InsightScore(); SourceModel = sourceModel; Symbol = symbol; Type = type; Direction = direction; // Optional Magnitude = magnitude; Confidence = confidence; Weight = weight; Tag = tag; _periodSpecification = periodSpec; // keep existing behavior of Insight.Price such that we set the period immediately var period = (periodSpec as TimeSpanPeriodSpecification)?.Period; if (period != null) { Period = period.Value; } } /// /// Sets the insight period and close times if they have not already been set. /// /// The insight's security exchange hours public void SetPeriodAndCloseTime(SecurityExchangeHours exchangeHours) { if (GeneratedTimeUtc == default(DateTime)) { throw new InvalidOperationException(Messages.Insight.GeneratedTimeUtcNotSet(this)); } _periodSpecification.SetPeriodAndCloseTime(this, exchangeHours); } /// /// Creates a deep clone of this insight instance /// /// A new insight with identical values, but new instances public virtual Insight Clone() { return new Insight(Symbol, Period, Type, Direction, Magnitude, Confidence, weight: Weight, tag: Tag) { GeneratedTimeUtc = GeneratedTimeUtc, CloseTimeUtc = CloseTimeUtc, Score = Score, Id = Id, EstimatedValue = EstimatedValue, ReferenceValue = ReferenceValue, ReferenceValueFinal = ReferenceValueFinal, SourceModel = SourceModel, GroupId = GroupId }; } /// /// Creates a new insight for predicting the percent change in price over the specified period /// /// The symbol this insight is for /// The resolution used to define the insight's period and also used to determine the insight's close time /// The number of resolution time steps to make in market hours to compute the insight's closing time /// The predicted direction /// The predicted magnitude as a percent change /// The confidence in this insight /// The model generating this insight /// The portfolio weight of this insight /// The insight's tag containing additional information /// A new insight object for the specified parameters public static Insight Price(Symbol symbol, Resolution resolution, int barCount, InsightDirection direction, double? magnitude = null, double? confidence = null, string sourceModel = null, double? weight = null, string tag = "") { if (barCount < 1) { throw new ArgumentOutOfRangeException(nameof(barCount), Messages.Insight.InvalidBarCount); } var spec = new ResolutionBarCountPeriodSpecification(resolution, barCount); return new Insight(symbol, spec, InsightType.Price, direction, magnitude, confidence, sourceModel, weight, tag); } /// /// Creates a new insight for predicting the percent change in price over the specified period /// /// The symbol this insight is for /// The insight's closing time in the security's exchange time zone /// The predicted direction /// The predicted magnitude as a percent change /// The confidence in this insight /// The model generating this insight /// The portfolio weight of this insight /// The insight's tag containing additional information /// A new insight object for the specified parameters public static Insight Price(Symbol symbol, DateTime closeTimeLocal, InsightDirection direction, double? magnitude = null, double? confidence = null, string sourceModel = null, double? weight = null, string tag = "") { var spec = closeTimeLocal == Time.EndOfTime ? (IPeriodSpecification) new EndOfTimeCloseTimePeriodSpecification() : new CloseTimePeriodSpecification(closeTimeLocal); return new Insight(symbol, spec, InsightType.Price, direction, magnitude, confidence, sourceModel, weight, tag); } /// /// Creates a new insight for predicting the percent change in price over the specified period /// /// The symbol this insight is for /// The period over which the prediction will come true /// The predicted direction /// The predicted magnitude as a percent change /// The confidence in this insight /// The model generating this insight /// The portfolio weight of this insight /// The insight's tag containing additional information /// A new insight object for the specified parameters public static Insight Price(Symbol symbol, TimeSpan period, InsightDirection direction, double? magnitude = null, double? confidence = null, string sourceModel = null, double? weight = null, string tag = "") { if (period < Time.OneSecond) { throw new ArgumentOutOfRangeException(nameof(period), Messages.Insight.InvalidPeriod); } var spec = period == Time.EndOfTimeTimeSpan ? (IPeriodSpecification) new EndOfTimeCloseTimePeriodSpecification() : new TimeSpanPeriodSpecification(period); return new Insight(symbol, spec, InsightType.Price, direction, magnitude, confidence, sourceModel, weight, tag); } /// /// Creates a new insight for predicting the percent change in price over the specified period /// /// The symbol this insight is for /// Func that defines the expiry time /// The predicted direction /// The predicted magnitude as a percent change /// The confidence in this insight /// The model generating this insight /// The portfolio weight of this insight /// The insight's tag containing additional information /// A new insight object for the specified parameters public static Insight Price(Symbol symbol, Func expiryFunc, InsightDirection direction, double? magnitude = null, double? confidence = null, string sourceModel = null, double? weight = null, string tag = "") { return new Insight(symbol, expiryFunc, InsightType.Price, direction, magnitude, confidence, sourceModel, weight, tag); } /// /// Creates a new, unique group id and sets it on each insight /// /// The insights to be grouped public static IEnumerable Group(params Insight[] insights) { if (insights == null) { throw new ArgumentNullException(nameof(insights)); } var groupId = Guid.NewGuid(); foreach (var insight in insights) { if (insight.GroupId.HasValue) { throw new InvalidOperationException(Messages.Insight.InsightAlreadyAssignedToAGroup(insight)); } insight.GroupId = groupId; } return insights; } /// /// Creates a new, unique group id and sets it on each insight /// /// The insight to be grouped public static IEnumerable Group(Insight insight) => Group(new[] {insight}); /// /// Creates a new object from the specified serialized form /// /// The insight DTO /// A new insight containing the information specified public static Insight FromSerializedInsight(SerializedInsight serializedInsight) { var sid = SecurityIdentifier.Parse(serializedInsight.Symbol); var insight = new Insight( Time.UnixTimeStampToDateTime(serializedInsight.CreatedTime), new Symbol(sid, serializedInsight.Ticker ?? sid.Symbol), TimeSpan.FromSeconds(serializedInsight.Period), serializedInsight.Type, serializedInsight.Direction, serializedInsight.Magnitude, serializedInsight.Confidence, serializedInsight.SourceModel, serializedInsight.Weight, serializedInsight.Tag ) { Id = Guid.Parse(serializedInsight.Id), CloseTimeUtc = Time.UnixTimeStampToDateTime(serializedInsight.CloseTime), EstimatedValue = serializedInsight.EstimatedValue, ReferenceValue = serializedInsight.ReferenceValue, ReferenceValueFinal = serializedInsight.ReferenceValueFinal, GroupId = string.IsNullOrEmpty(serializedInsight.GroupId) ? (Guid?) null : Guid.Parse(serializedInsight.GroupId) }; // only set score values if non-zero or if they're the final scores if (serializedInsight.ScoreIsFinal) { insight.Score.SetScore(InsightScoreType.Magnitude, serializedInsight.ScoreMagnitude, insight.CloseTimeUtc); insight.Score.SetScore(InsightScoreType.Direction, serializedInsight.ScoreDirection, insight.CloseTimeUtc); insight.Score.Finalize(insight.CloseTimeUtc); } else { if (serializedInsight.ScoreMagnitude != 0) { insight.Score.SetScore(InsightScoreType.Magnitude, serializedInsight.ScoreMagnitude, insight.CloseTimeUtc); } if (serializedInsight.ScoreDirection != 0) { insight.Score.SetScore(InsightScoreType.Direction, serializedInsight.ScoreDirection, insight.CloseTimeUtc); } } return insight; } /// /// Computes the insight closing time from the given generated time, resolution and bar count. /// This will step through market hours using the given resolution, respecting holidays, early closes, weekends, etc.. /// /// The exchange hours of the insight's security /// The insight's generated time in utc /// The resolution used to 'step-through' market hours to compute a reasonable close time /// The number of resolution steps to take /// The insight's closing time in utc public static DateTime ComputeCloseTime(SecurityExchangeHours exchangeHours, DateTime generatedTimeUtc, Resolution resolution, int barCount) { if (barCount < 1) { throw new ArgumentOutOfRangeException(nameof(barCount), Messages.Insight.InvalidBarCount); } // remap ticks to seconds resolution = resolution == Resolution.Tick ? Resolution.Second : resolution; if (resolution == Resolution.Hour) { // remap hours to minutes to avoid complications w/ stepping through // for example 9->10 is an hour step but market opens at 9:30 barCount *= 60; resolution = Resolution.Minute; } var barSize = resolution.ToTimeSpan(); var startTimeLocal = generatedTimeUtc.ConvertFromUtc(exchangeHours.TimeZone); var closeTimeLocal = Time.GetEndTimeForTradeBars(exchangeHours, startTimeLocal, barSize, barCount, false); return closeTimeLocal.ConvertToUtc(exchangeHours.TimeZone); } /// /// computs the insight closing time from the given generated time and period /// /// The exchange hours of the insight's security /// The insight's generated time in utc /// The insight's period /// The insight's closing time in utc public static DateTime ComputeCloseTime(SecurityExchangeHours exchangeHours, DateTime generatedTimeUtc, TimeSpan period) { if (period < Time.OneSecond) { throw new ArgumentOutOfRangeException(nameof(period), Messages.Insight.InvalidPeriod); } var barSize = period.ToHigherResolutionEquivalent(false); // remap ticks to seconds barSize = barSize == Resolution.Tick ? Resolution.Second : barSize; // remap hours to minutes to avoid complications w/ stepping through, for example 9->10 is an hour step but market opens at 9:30 barSize = barSize == Resolution.Hour ? Resolution.Minute : barSize; var barCount = (int)(period.Ticks / barSize.ToTimeSpan().Ticks); var closeTimeUtc = ComputeCloseTime(exchangeHours, generatedTimeUtc, barSize, barCount); if (closeTimeUtc == generatedTimeUtc) { return ComputeCloseTime(exchangeHours, generatedTimeUtc, Resolution.Second, 1); } var totalPeriodUsed = barSize.ToTimeSpan().Multiply(barCount); if (totalPeriodUsed != period) { var delta = period - totalPeriodUsed; // interpret the remainder as fractional trading days if (barSize == Resolution.Daily) { var percentOfDay = delta.Ticks / (double) Time.OneDay.Ticks; delta = exchangeHours.RegularMarketDuration.Multiply(percentOfDay); } if (delta != TimeSpan.Zero) { // continue stepping forward using minute resolution for the remainder barCount = (int) (delta.Ticks / Time.OneMinute.Ticks); if (barCount > 0) { closeTimeUtc = ComputeCloseTime(exchangeHours, closeTimeUtc, Resolution.Minute, barCount); } } } return closeTimeUtc; } /// Returns a string that represents the current object. /// A string that represents the current object. /// 2 public override string ToString() { return Messages.Insight.ToString(this); } /// /// Returns a short string that represents the current object. /// /// A string that represents the current object. public string ShortToString() { return Messages.Insight.ShortToString(this); } /// /// Distinguishes between the different ways an insight's period/close times can be specified /// This was really only required since we can't properly acces certain data from within a static /// context (such as Insight.Price) or from within a constructor w/out requiring the users to properly /// fetch the required data and supply it as an argument. /// private interface IPeriodSpecification { void SetPeriodAndCloseTime(Insight insight, SecurityExchangeHours exchangeHours); } /// /// User defined the insight's period using a time span /// private class TimeSpanPeriodSpecification : IPeriodSpecification { public readonly TimeSpan Period; public TimeSpanPeriodSpecification(TimeSpan period) { if (period == TimeSpan.Zero) { period = Time.OneSecond; } Period = period; } public void SetPeriodAndCloseTime(Insight insight, SecurityExchangeHours exchangeHours) { insight.Period = Period; insight.CloseTimeUtc = ComputeCloseTime(exchangeHours, insight.GeneratedTimeUtc, Period); } } /// /// User defined insight's period using a resolution and bar count /// private class ResolutionBarCountPeriodSpecification : IPeriodSpecification { public readonly Resolution Resolution; public readonly int BarCount; public ResolutionBarCountPeriodSpecification(Resolution resolution, int barCount) { if (resolution == Resolution.Tick) { resolution = Resolution.Second; } if (resolution == Resolution.Hour) { // remap hours to minutes to avoid errors w/ half hours, for example, 9:30 open barCount *= 60; resolution = Resolution.Minute; } Resolution = resolution; BarCount = barCount; } public void SetPeriodAndCloseTime(Insight insight, SecurityExchangeHours exchangeHours) { insight.CloseTimeUtc = ComputeCloseTime(exchangeHours, insight.GeneratedTimeUtc, Resolution, BarCount); insight.Period = insight.CloseTimeUtc - insight.GeneratedTimeUtc; } } /// /// User defined the insight's local closing time /// private class CloseTimePeriodSpecification : IPeriodSpecification { public readonly DateTime CloseTimeLocal; public CloseTimePeriodSpecification(DateTime closeTimeLocal) { CloseTimeLocal = closeTimeLocal; } public void SetPeriodAndCloseTime(Insight insight, SecurityExchangeHours exchangeHours) { // Prevent close time to be defined to a date/time in closed market var closeTimeLocal = exchangeHours.IsOpen(CloseTimeLocal, false) ? CloseTimeLocal : exchangeHours.GetNextMarketOpen(CloseTimeLocal, false); insight.CloseTimeUtc = closeTimeLocal.ConvertToUtc(exchangeHours.TimeZone); if (insight.GeneratedTimeUtc > insight.CloseTimeUtc) { throw new ArgumentOutOfRangeException(nameof(closeTimeLocal), $"Insight closeTimeLocal must not be in the past."); } insight.Period = insight.CloseTimeUtc - insight.GeneratedTimeUtc; } } /// /// Special case for insights which close time is defined by a function /// and want insights to expiry with calendar rules /// private class FuncPeriodSpecification : IPeriodSpecification { public readonly Func _expiryFunc; public FuncPeriodSpecification(Func expiryFunc) { _expiryFunc = expiryFunc; } public void SetPeriodAndCloseTime(Insight insight, SecurityExchangeHours exchangeHours) { var closeTimeLocal = insight.GeneratedTimeUtc.ConvertFromUtc(exchangeHours.TimeZone); closeTimeLocal = _expiryFunc(closeTimeLocal); // Prevent close time to be defined to a date/time in closed market if (!exchangeHours.IsOpen(closeTimeLocal, false)) { closeTimeLocal = exchangeHours.GetNextMarketOpen(closeTimeLocal, false); } insight.CloseTimeUtc = closeTimeLocal.ConvertToUtc(exchangeHours.TimeZone); insight.Period = insight.CloseTimeUtc - insight.GeneratedTimeUtc; } } /// /// Special case for insights where we do not know whats the /// or . /// private class EndOfTimeCloseTimePeriodSpecification : IPeriodSpecification { public void SetPeriodAndCloseTime(Insight insight, SecurityExchangeHours exchangeHours) { insight.Period = Time.EndOfTimeTimeSpan; insight.CloseTimeUtc = Time.EndOfTime; } } } }