/*
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
using System;
using System.Collections.Generic;
using Newtonsoft.Json;
using QuantConnect.Algorithm.Framework.Alphas.Serialization;
using QuantConnect.Interfaces;
using QuantConnect.Securities;
namespace QuantConnect.Algorithm.Framework.Alphas
{
///
/// Defines a alpha prediction for a single symbol generated by the algorithm
///
///
/// Serialization of this type is delegated to the which uses the as a model.
///
[JsonConverter(typeof(InsightJsonConverter))]
public class Insight
{
private readonly IPeriodSpecification _periodSpecification;
///
/// Gets the unique identifier for this insight
///
public Guid Id { get; protected set; }
///
/// Gets the group id this insight belongs to, null if not in a group
///
public Guid? GroupId { get; protected set; }
///
/// Gets an identifier for the source model that generated this insight.
///
public string SourceModel { get; set; }
///
/// Gets the utc time this insight was generated
///
///
/// The algorithm framework handles setting this value appropriately.
/// If providing custom implementation, be sure
/// to set this value to algorithm.UtcTime when the insight is generated.
///
public DateTime GeneratedTimeUtc { get; set; }
///
/// Gets the insight's prediction end time. This is the time when this
/// insight prediction is expected to be fulfilled. This time takes into
/// account market hours, weekends, as well as the symbol's data resolution
///
public DateTime CloseTimeUtc { get; set; }
///
/// Gets the symbol this insight is for
///
public Symbol Symbol { get; private set; }
///
/// Gets the type of insight, for example, price insight or volatility insight
///
public InsightType Type { get; private set; }
///
/// Gets the initial reference value this insight is predicting against. The value is dependent on the specified
///
public decimal ReferenceValue { get; set; }
///
/// Gets the final reference value, used for scoring, this insight is predicting against. The value is dependent on the specified
///
public decimal ReferenceValueFinal { get; set; }
///
/// Gets the predicted direction, down, flat or up
///
public InsightDirection Direction { get; private set; }
///
/// Gets the period over which this insight is expected to come to fruition
///
public TimeSpan Period { get; internal set; }
///
/// Gets the predicted percent change in the insight type (price/volatility)
///
public double? Magnitude { get; private set; }
///
/// Gets the confidence in this insight
///
public double? Confidence { get; private set; }
///
/// Gets the portfolio weight of this insight
///
public double? Weight { get; private set; }
///
/// Gets the most recent scores for this insight
///
public InsightScore Score { get; protected set; }
///
/// Gets the estimated value of this insight in the account currency
///
public decimal EstimatedValue { get; set; }
///
/// The insight's tag containing additional information
///
public string Tag { get; protected set; }
///
/// Determines whether or not this insight is considered expired at the specified
///
/// The algorithm's current time in UTC. See
/// True if this insight is expired, false otherwise
public bool IsExpired(DateTime utcTime)
{
return CloseTimeUtc < utcTime;
}
///
/// Determines whether or not this insight is considered active at the specified
///
/// The algorithm's current time in UTC. See
/// True if this insight is active, false otherwise
public bool IsActive(DateTime utcTime)
{
return !IsExpired(utcTime);
}
///
/// Expire this insight
///
/// The algorithm's current time in UTC. See
public void Expire(DateTime utcTime)
{
if (IsActive(utcTime))
{
CloseTimeUtc = utcTime.Add(-Time.OneSecond);
Period = CloseTimeUtc - GeneratedTimeUtc;
}
}
///
/// Cancel this insight
///
/// The algorithm's current time in UTC. See
public void Cancel(DateTime utcTime)
{
Expire(utcTime);
}
///
/// Initializes a new instance of the class
///
/// The symbol this insight is for
/// The period over which the prediction will come true
/// The type of insight, price/volatility
/// The predicted direction
/// The insight's tag containing additional information
public Insight(Symbol symbol, TimeSpan period, InsightType type, InsightDirection direction, string tag = "")
: this(symbol, period, type, direction, null, null, null, null, tag)
{
}
///
/// Initializes a new instance of the class
///
/// The symbol this insight is for
/// The period over which the prediction will come true
/// The type of insight, price/volatility
/// The predicted direction
/// The predicted magnitude as a percentage change
/// The confidence in this insight
/// An identifier defining the model that generated this insight
/// The portfolio weight of this insight
/// The insight's tag containing additional information
public Insight(Symbol symbol, TimeSpan period, InsightType type, InsightDirection direction, double? magnitude, double? confidence, string sourceModel = null, double? weight = null, string tag = "")
{
Id = Guid.NewGuid();
Score = new InsightScore();
SourceModel = sourceModel;
Symbol = symbol;
Type = type;
Direction = direction;
Period = period;
// Optional
Magnitude = magnitude;
Confidence = confidence;
Weight = weight;
Tag = tag;
_periodSpecification = new TimeSpanPeriodSpecification(period);
}
///
/// Initializes a new instance of the class
///
/// The symbol this insight is for
/// Func that defines the expiry time
/// The type of insight, price/volatility
/// The predicted direction
/// The insight's tag containing additional information
public Insight(Symbol symbol, Func expiryFunc, InsightType type, InsightDirection direction, string tag = "")
: this(symbol, expiryFunc, type, direction, null, null, null, null, tag)
{
}
///
/// Initializes a new instance of the class
///
/// The symbol this insight is for
/// Func that defines the expiry time
/// The type of insight, price/volatility
/// The predicted direction
/// The predicted magnitude as a percentage change
/// The confidence in this insight
/// An identifier defining the model that generated this insight
/// The portfolio weight of this insight
/// The insight's tag containing additional information
public Insight(Symbol symbol, Func expiryFunc, InsightType type, InsightDirection direction, double? magnitude, double? confidence, string sourceModel = null, double? weight = null, string tag = "")
: this(symbol, new FuncPeriodSpecification(expiryFunc), type, direction, magnitude, confidence, sourceModel, weight, tag)
{
}
///
/// Initializes a new instance of the class.
/// This constructor is provided mostly for testing purposes. When running inside an algorithm,
/// the generated and close times are set based on the algorithm's time.
///
/// The time this insight was generated in utc
/// The symbol this insight is for
/// The period over which the prediction will come true
/// The type of insight, price/volatility
/// The predicted direction
/// The predicted magnitude as a percentage change
/// The confidence in this insight
/// An identifier defining the model that generated this insight
/// The portfolio weight of this insight
/// The insight's tag containing additional information
public Insight(DateTime generatedTimeUtc, Symbol symbol, TimeSpan period, InsightType type, InsightDirection direction, double? magnitude, double? confidence, string sourceModel = null, double? weight = null, string tag = "")
: this(symbol, period, type, direction, magnitude, confidence, sourceModel, weight, tag)
{
GeneratedTimeUtc = generatedTimeUtc;
CloseTimeUtc = generatedTimeUtc + period;
}
///
/// Private constructor used to keep track of how a user defined the insight period.
///
/// The symbol this insight is for
/// A specification defining how the insight's period was defined, via time span, via resolution/barcount, via close time
/// The type of insight, price/volatility
/// The predicted direction
/// The predicted magnitude as a percentage change
/// The confidence in this insight
/// An identifier defining the model that generated this insight
/// The portfolio weight of this insight
/// The insight's tag containing additional information
private Insight(Symbol symbol, IPeriodSpecification periodSpec, InsightType type, InsightDirection direction, double? magnitude, double? confidence, string sourceModel = null, double? weight = null, string tag = "")
{
Id = Guid.NewGuid();
Score = new InsightScore();
SourceModel = sourceModel;
Symbol = symbol;
Type = type;
Direction = direction;
// Optional
Magnitude = magnitude;
Confidence = confidence;
Weight = weight;
Tag = tag;
_periodSpecification = periodSpec;
// keep existing behavior of Insight.Price such that we set the period immediately
var period = (periodSpec as TimeSpanPeriodSpecification)?.Period;
if (period != null)
{
Period = period.Value;
}
}
///
/// Sets the insight period and close times if they have not already been set.
///
/// The insight's security exchange hours
public void SetPeriodAndCloseTime(SecurityExchangeHours exchangeHours)
{
if (GeneratedTimeUtc == default(DateTime))
{
throw new InvalidOperationException(Messages.Insight.GeneratedTimeUtcNotSet(this));
}
_periodSpecification.SetPeriodAndCloseTime(this, exchangeHours);
}
///
/// Creates a deep clone of this insight instance
///
/// A new insight with identical values, but new instances
public virtual Insight Clone()
{
return new Insight(Symbol, Period, Type, Direction, Magnitude, Confidence, weight: Weight, tag: Tag)
{
GeneratedTimeUtc = GeneratedTimeUtc,
CloseTimeUtc = CloseTimeUtc,
Score = Score,
Id = Id,
EstimatedValue = EstimatedValue,
ReferenceValue = ReferenceValue,
ReferenceValueFinal = ReferenceValueFinal,
SourceModel = SourceModel,
GroupId = GroupId
};
}
///
/// Creates a new insight for predicting the percent change in price over the specified period
///
/// The symbol this insight is for
/// The resolution used to define the insight's period and also used to determine the insight's close time
/// The number of resolution time steps to make in market hours to compute the insight's closing time
/// The predicted direction
/// The predicted magnitude as a percent change
/// The confidence in this insight
/// The model generating this insight
/// The portfolio weight of this insight
/// The insight's tag containing additional information
/// A new insight object for the specified parameters
public static Insight Price(Symbol symbol, Resolution resolution, int barCount, InsightDirection direction, double? magnitude = null, double? confidence = null, string sourceModel = null, double? weight = null, string tag = "")
{
if (barCount < 1)
{
throw new ArgumentOutOfRangeException(nameof(barCount), Messages.Insight.InvalidBarCount);
}
var spec = new ResolutionBarCountPeriodSpecification(resolution, barCount);
return new Insight(symbol, spec, InsightType.Price, direction, magnitude, confidence, sourceModel, weight, tag);
}
///
/// Creates a new insight for predicting the percent change in price over the specified period
///
/// The symbol this insight is for
/// The insight's closing time in the security's exchange time zone
/// The predicted direction
/// The predicted magnitude as a percent change
/// The confidence in this insight
/// The model generating this insight
/// The portfolio weight of this insight
/// The insight's tag containing additional information
/// A new insight object for the specified parameters
public static Insight Price(Symbol symbol, DateTime closeTimeLocal, InsightDirection direction, double? magnitude = null, double? confidence = null, string sourceModel = null, double? weight = null, string tag = "")
{
var spec = closeTimeLocal == Time.EndOfTime ? (IPeriodSpecification)
new EndOfTimeCloseTimePeriodSpecification() : new CloseTimePeriodSpecification(closeTimeLocal);
return new Insight(symbol, spec, InsightType.Price, direction, magnitude, confidence, sourceModel, weight, tag);
}
///
/// Creates a new insight for predicting the percent change in price over the specified period
///
/// The symbol this insight is for
/// The period over which the prediction will come true
/// The predicted direction
/// The predicted magnitude as a percent change
/// The confidence in this insight
/// The model generating this insight
/// The portfolio weight of this insight
/// The insight's tag containing additional information
/// A new insight object for the specified parameters
public static Insight Price(Symbol symbol, TimeSpan period, InsightDirection direction, double? magnitude = null, double? confidence = null, string sourceModel = null, double? weight = null, string tag = "")
{
if (period < Time.OneSecond)
{
throw new ArgumentOutOfRangeException(nameof(period), Messages.Insight.InvalidPeriod);
}
var spec = period == Time.EndOfTimeTimeSpan ? (IPeriodSpecification)
new EndOfTimeCloseTimePeriodSpecification() : new TimeSpanPeriodSpecification(period);
return new Insight(symbol, spec, InsightType.Price, direction, magnitude, confidence, sourceModel, weight, tag);
}
///
/// Creates a new insight for predicting the percent change in price over the specified period
///
/// The symbol this insight is for
/// Func that defines the expiry time
/// The predicted direction
/// The predicted magnitude as a percent change
/// The confidence in this insight
/// The model generating this insight
/// The portfolio weight of this insight
/// The insight's tag containing additional information
/// A new insight object for the specified parameters
public static Insight Price(Symbol symbol, Func expiryFunc, InsightDirection direction, double? magnitude = null, double? confidence = null, string sourceModel = null, double? weight = null, string tag = "")
{
return new Insight(symbol, expiryFunc, InsightType.Price, direction, magnitude, confidence, sourceModel, weight, tag);
}
///
/// Creates a new, unique group id and sets it on each insight
///
/// The insights to be grouped
public static IEnumerable Group(params Insight[] insights)
{
if (insights == null)
{
throw new ArgumentNullException(nameof(insights));
}
var groupId = Guid.NewGuid();
foreach (var insight in insights)
{
if (insight.GroupId.HasValue)
{
throw new InvalidOperationException(Messages.Insight.InsightAlreadyAssignedToAGroup(insight));
}
insight.GroupId = groupId;
}
return insights;
}
///
/// Creates a new, unique group id and sets it on each insight
///
/// The insight to be grouped
public static IEnumerable Group(Insight insight) => Group(new[] {insight});
///
/// Creates a new object from the specified serialized form
///
/// The insight DTO
/// A new insight containing the information specified
public static Insight FromSerializedInsight(SerializedInsight serializedInsight)
{
var sid = SecurityIdentifier.Parse(serializedInsight.Symbol);
var insight = new Insight(
Time.UnixTimeStampToDateTime(serializedInsight.CreatedTime),
new Symbol(sid, serializedInsight.Ticker ?? sid.Symbol),
TimeSpan.FromSeconds(serializedInsight.Period),
serializedInsight.Type,
serializedInsight.Direction,
serializedInsight.Magnitude,
serializedInsight.Confidence,
serializedInsight.SourceModel,
serializedInsight.Weight,
serializedInsight.Tag
)
{
Id = Guid.Parse(serializedInsight.Id),
CloseTimeUtc = Time.UnixTimeStampToDateTime(serializedInsight.CloseTime),
EstimatedValue = serializedInsight.EstimatedValue,
ReferenceValue = serializedInsight.ReferenceValue,
ReferenceValueFinal = serializedInsight.ReferenceValueFinal,
GroupId = string.IsNullOrEmpty(serializedInsight.GroupId) ? (Guid?) null : Guid.Parse(serializedInsight.GroupId)
};
// only set score values if non-zero or if they're the final scores
if (serializedInsight.ScoreIsFinal)
{
insight.Score.SetScore(InsightScoreType.Magnitude, serializedInsight.ScoreMagnitude, insight.CloseTimeUtc);
insight.Score.SetScore(InsightScoreType.Direction, serializedInsight.ScoreDirection, insight.CloseTimeUtc);
insight.Score.Finalize(insight.CloseTimeUtc);
}
else
{
if (serializedInsight.ScoreMagnitude != 0)
{
insight.Score.SetScore(InsightScoreType.Magnitude, serializedInsight.ScoreMagnitude, insight.CloseTimeUtc);
}
if (serializedInsight.ScoreDirection != 0)
{
insight.Score.SetScore(InsightScoreType.Direction, serializedInsight.ScoreDirection, insight.CloseTimeUtc);
}
}
return insight;
}
///
/// Computes the insight closing time from the given generated time, resolution and bar count.
/// This will step through market hours using the given resolution, respecting holidays, early closes, weekends, etc..
///
/// The exchange hours of the insight's security
/// The insight's generated time in utc
/// The resolution used to 'step-through' market hours to compute a reasonable close time
/// The number of resolution steps to take
/// The insight's closing time in utc
public static DateTime ComputeCloseTime(SecurityExchangeHours exchangeHours, DateTime generatedTimeUtc, Resolution resolution, int barCount)
{
if (barCount < 1)
{
throw new ArgumentOutOfRangeException(nameof(barCount), Messages.Insight.InvalidBarCount);
}
// remap ticks to seconds
resolution = resolution == Resolution.Tick ? Resolution.Second : resolution;
if (resolution == Resolution.Hour)
{
// remap hours to minutes to avoid complications w/ stepping through
// for example 9->10 is an hour step but market opens at 9:30
barCount *= 60;
resolution = Resolution.Minute;
}
var barSize = resolution.ToTimeSpan();
var startTimeLocal = generatedTimeUtc.ConvertFromUtc(exchangeHours.TimeZone);
var closeTimeLocal = Time.GetEndTimeForTradeBars(exchangeHours, startTimeLocal, barSize, barCount, false);
return closeTimeLocal.ConvertToUtc(exchangeHours.TimeZone);
}
///
/// computs the insight closing time from the given generated time and period
///
/// The exchange hours of the insight's security
/// The insight's generated time in utc
/// The insight's period
/// The insight's closing time in utc
public static DateTime ComputeCloseTime(SecurityExchangeHours exchangeHours, DateTime generatedTimeUtc, TimeSpan period)
{
if (period < Time.OneSecond)
{
throw new ArgumentOutOfRangeException(nameof(period), Messages.Insight.InvalidPeriod);
}
var barSize = period.ToHigherResolutionEquivalent(false);
// remap ticks to seconds
barSize = barSize == Resolution.Tick ? Resolution.Second : barSize;
// remap hours to minutes to avoid complications w/ stepping through, for example 9->10 is an hour step but market opens at 9:30
barSize = barSize == Resolution.Hour ? Resolution.Minute : barSize;
var barCount = (int)(period.Ticks / barSize.ToTimeSpan().Ticks);
var closeTimeUtc = ComputeCloseTime(exchangeHours, generatedTimeUtc, barSize, barCount);
if (closeTimeUtc == generatedTimeUtc)
{
return ComputeCloseTime(exchangeHours, generatedTimeUtc, Resolution.Second, 1);
}
var totalPeriodUsed = barSize.ToTimeSpan().Multiply(barCount);
if (totalPeriodUsed != period)
{
var delta = period - totalPeriodUsed;
// interpret the remainder as fractional trading days
if (barSize == Resolution.Daily)
{
var percentOfDay = delta.Ticks / (double) Time.OneDay.Ticks;
delta = exchangeHours.RegularMarketDuration.Multiply(percentOfDay);
}
if (delta != TimeSpan.Zero)
{
// continue stepping forward using minute resolution for the remainder
barCount = (int) (delta.Ticks / Time.OneMinute.Ticks);
if (barCount > 0)
{
closeTimeUtc = ComputeCloseTime(exchangeHours, closeTimeUtc, Resolution.Minute, barCount);
}
}
}
return closeTimeUtc;
}
/// Returns a string that represents the current object.
/// A string that represents the current object.
/// 2
public override string ToString()
{
return Messages.Insight.ToString(this);
}
///
/// Returns a short string that represents the current object.
///
/// A string that represents the current object.
public string ShortToString()
{
return Messages.Insight.ShortToString(this);
}
///
/// Distinguishes between the different ways an insight's period/close times can be specified
/// This was really only required since we can't properly acces certain data from within a static
/// context (such as Insight.Price) or from within a constructor w/out requiring the users to properly
/// fetch the required data and supply it as an argument.
///
private interface IPeriodSpecification
{
void SetPeriodAndCloseTime(Insight insight, SecurityExchangeHours exchangeHours);
}
///
/// User defined the insight's period using a time span
///
private class TimeSpanPeriodSpecification : IPeriodSpecification
{
public readonly TimeSpan Period;
public TimeSpanPeriodSpecification(TimeSpan period)
{
if (period == TimeSpan.Zero)
{
period = Time.OneSecond;
}
Period = period;
}
public void SetPeriodAndCloseTime(Insight insight, SecurityExchangeHours exchangeHours)
{
insight.Period = Period;
insight.CloseTimeUtc = ComputeCloseTime(exchangeHours, insight.GeneratedTimeUtc, Period);
}
}
///
/// User defined insight's period using a resolution and bar count
///
private class ResolutionBarCountPeriodSpecification : IPeriodSpecification
{
public readonly Resolution Resolution;
public readonly int BarCount;
public ResolutionBarCountPeriodSpecification(Resolution resolution, int barCount)
{
if (resolution == Resolution.Tick)
{
resolution = Resolution.Second;
}
if (resolution == Resolution.Hour)
{
// remap hours to minutes to avoid errors w/ half hours, for example, 9:30 open
barCount *= 60;
resolution = Resolution.Minute;
}
Resolution = resolution;
BarCount = barCount;
}
public void SetPeriodAndCloseTime(Insight insight, SecurityExchangeHours exchangeHours)
{
insight.CloseTimeUtc = ComputeCloseTime(exchangeHours, insight.GeneratedTimeUtc, Resolution, BarCount);
insight.Period = insight.CloseTimeUtc - insight.GeneratedTimeUtc;
}
}
///
/// User defined the insight's local closing time
///
private class CloseTimePeriodSpecification : IPeriodSpecification
{
public readonly DateTime CloseTimeLocal;
public CloseTimePeriodSpecification(DateTime closeTimeLocal)
{
CloseTimeLocal = closeTimeLocal;
}
public void SetPeriodAndCloseTime(Insight insight, SecurityExchangeHours exchangeHours)
{
// Prevent close time to be defined to a date/time in closed market
var closeTimeLocal = exchangeHours.IsOpen(CloseTimeLocal, false)
? CloseTimeLocal
: exchangeHours.GetNextMarketOpen(CloseTimeLocal, false);
insight.CloseTimeUtc = closeTimeLocal.ConvertToUtc(exchangeHours.TimeZone);
if (insight.GeneratedTimeUtc > insight.CloseTimeUtc)
{
throw new ArgumentOutOfRangeException(nameof(closeTimeLocal), $"Insight closeTimeLocal must not be in the past.");
}
insight.Period = insight.CloseTimeUtc - insight.GeneratedTimeUtc;
}
}
///
/// Special case for insights which close time is defined by a function
/// and want insights to expiry with calendar rules
///
private class FuncPeriodSpecification : IPeriodSpecification
{
public readonly Func _expiryFunc;
public FuncPeriodSpecification(Func expiryFunc)
{
_expiryFunc = expiryFunc;
}
public void SetPeriodAndCloseTime(Insight insight, SecurityExchangeHours exchangeHours)
{
var closeTimeLocal = insight.GeneratedTimeUtc.ConvertFromUtc(exchangeHours.TimeZone);
closeTimeLocal = _expiryFunc(closeTimeLocal);
// Prevent close time to be defined to a date/time in closed market
if (!exchangeHours.IsOpen(closeTimeLocal, false))
{
closeTimeLocal = exchangeHours.GetNextMarketOpen(closeTimeLocal, false);
}
insight.CloseTimeUtc = closeTimeLocal.ConvertToUtc(exchangeHours.TimeZone);
insight.Period = insight.CloseTimeUtc - insight.GeneratedTimeUtc;
}
}
///
/// Special case for insights where we do not know whats the
/// or .
///
private class EndOfTimeCloseTimePeriodSpecification : IPeriodSpecification
{
public void SetPeriodAndCloseTime(Insight insight, SecurityExchangeHours exchangeHours)
{
insight.Period = Time.EndOfTimeTimeSpan;
insight.CloseTimeUtc = Time.EndOfTime;
}
}
}
}