/*
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
using System;
using System.Collections.Generic;
using System.Globalization;
using Newtonsoft.Json.Converters;
using NodaTime;
using QuantConnect.Logging;
using QuantConnect.Securities;
using static QuantConnect.StringExtensions;
namespace QuantConnect
{
///
/// Time helper class collection for working with trading dates
///
public static class Time
{
///
/// Allows specifying an offset to trigger the tradable date event
///
/// Useful for delaying the tradable date event until new auxiliary data is available to refresh map and factor files
public static TimeSpan LiveAuxiliaryDataOffset { get; set; } = TimeSpan.FromHours(8);
///
/// Provides a value far enough in the future the current computer hardware will have decayed :)
///
///
/// new DateTime(2050, 12, 31)
///
public static readonly DateTime EndOfTime = new DateTime(2050, 12, 31);
///
/// Provides a time span based on
///
public static TimeSpan EndOfTimeTimeSpan = new TimeSpan(EndOfTime.Ticks);
///
/// Provides a common and normalized start time for Lean data
///
public static readonly DateTime Start = new DateTime(1998, 1, 2);
///
/// Provides a value far enough in the past that can be used as a lower bound on dates, 12/30/1899
///
///
/// DateTime.FromOADate(0)
///
public static readonly DateTime BeginningOfTime = DateTime.FromOADate(0);
///
/// Provides a value large enough that we won't hit the limit, while small enough
/// we can still do math against it without checking everywhere for
///
public static readonly TimeSpan MaxTimeSpan = TimeSpan.FromDays(1000*365);
///
/// One Year TimeSpan Period Constant
///
/// 365 days
public static readonly TimeSpan OneYear = TimeSpan.FromDays(365);
///
/// One Day TimeSpan Period Constant
///
public static readonly TimeSpan OneDay = TimeSpan.FromDays(1);
///
/// One Hour TimeSpan Period Constant
///
public static readonly TimeSpan OneHour = TimeSpan.FromHours(1);
///
/// One Minute TimeSpan Period Constant
///
public static readonly TimeSpan OneMinute = TimeSpan.FromMinutes(1);
///
/// One Second TimeSpan Period Constant
///
public static readonly TimeSpan OneSecond = TimeSpan.FromSeconds(1);
///
/// One Millisecond TimeSpan Period Constant
///
public static readonly TimeSpan OneMillisecond = TimeSpan.FromMilliseconds(1);
///
/// Live charting is sensitive to timezone so need to convert the local system time to a UTC and display in browser as UTC.
///
public struct DateTimeWithZone
{
private readonly DateTime utcDateTime;
private readonly TimeZoneInfo timeZone;
///
/// Initializes a new instance of the struct.
///
/// Date time.
/// Time zone.
public DateTimeWithZone(DateTime dateTime, TimeZoneInfo timeZone)
{
utcDateTime = TimeZoneInfo.ConvertTimeToUtc(dateTime, timeZone);
this.timeZone = timeZone;
}
///
/// Gets the universal time.
///
/// The universal time.
public DateTime UniversalTime { get { return utcDateTime; } }
///
/// Gets the time zone.
///
/// The time zone.
public TimeZoneInfo TimeZone { get { return timeZone; } }
///
/// Gets the local time.
///
/// The local time.
public DateTime LocalTime
{
get
{
return TimeZoneInfo.ConvertTime(utcDateTime, timeZone);
}
}
}
private static readonly DateTime EpochTime = new DateTime(1970, 1, 1, 0, 0, 0, 0);
private const long SecondToMillisecond = 1000;
///
/// Helper method to get the new live auxiliary data due time
///
/// The due time for the new auxiliary data emission
public static TimeSpan GetNextLiveAuxiliaryDataDueTime()
{
return GetNextLiveAuxiliaryDataDueTime(DateTime.UtcNow);
}
///
/// Helper method to get the new live auxiliary data due time
///
/// The current utc time
/// The due time for the new auxiliary data emission
public static TimeSpan GetNextLiveAuxiliaryDataDueTime(DateTime utcNow)
{
var nowNewYork = utcNow.ConvertFromUtc(TimeZones.NewYork);
if (nowNewYork.TimeOfDay < LiveAuxiliaryDataOffset)
{
return LiveAuxiliaryDataOffset - nowNewYork.TimeOfDay;
}
return nowNewYork.Date.AddDays(1).Add(+LiveAuxiliaryDataOffset) - nowNewYork;
}
///
/// Helper method to adjust a waiting time, in milliseconds, so it's uneven with the second turn around
///
/// The desired wait time
/// This is useful for real time performance in live trading. We want to avoid adding unnecessary cpu usage,
/// during periods where we know there will be cpu time demand, like a second turn around where data is emitted.
/// The adjusted wait time
public static int GetSecondUnevenWait(int waitTimeMillis)
{
return DateTime.UtcNow.GetSecondUnevenWait(waitTimeMillis);
}
///
/// Helper method to adjust a waiting time, in milliseconds, so it's uneven with the second turn around
///
/// The current time
/// The desired wait time
/// This is useful for real time performance in live trading. We want to avoid adding unnecessary cpu usage,
/// during periods where we know there will be cpu time demand, like a second turn around where data is emitted.
/// The adjusted wait time
public static int GetSecondUnevenWait(this DateTime now, int waitTimeMillis)
{
var wakeUpTime = now.AddMilliseconds(waitTimeMillis);
if (wakeUpTime.Millisecond < 100 || wakeUpTime.Millisecond > 900)
{
// if we are going to wake before/after the next second we add an offset to avoid it
var offsetMillis = waitTimeMillis >= 1000 ? 500 : 100;
return waitTimeMillis + offsetMillis;
}
return waitTimeMillis;
}
///
/// Create a C# DateTime from a UnixTimestamp
///
/// Double unix timestamp (Time since Midnight Jan 1 1970)
/// C# date timeobject
public static DateTime UnixTimeStampToDateTime(double unixTimeStamp)
{
DateTime time;
try
{
var ticks = unixTimeStamp * TimeSpan.TicksPerSecond;
time = EpochTime.AddTicks((long)ticks);
}
catch (Exception err)
{
Log.Error(err, Invariant($"UnixTimeStamp: {unixTimeStamp}"));
time = DateTime.Now;
}
return time;
}
///
/// Create a C# DateTime from a UnixTimestamp
///
/// Decimal unix timestamp (Time since Midnight Jan 1 1970)
/// C# date time object
public static DateTime UnixTimeStampToDateTime(decimal unixTimeStamp)
{
return UnixMillisecondTimeStampToDateTime(unixTimeStamp * SecondToMillisecond);
}
///
/// Create a C# DateTime from a UnixTimestamp
///
/// Long unix timestamp (Time since Midnight Jan 1 1970)
/// C# date time object
public static DateTime UnixTimeStampToDateTime(long unixTimeStamp)
{
return UnixTimeStampToDateTime(Convert.ToDecimal(unixTimeStamp));
}
///
/// Create a C# DateTime from a UnixTimestamp
///
/// Decimal unix timestamp (Time since Midnight Jan 1 1970) in milliseconds
/// C# date time object
public static DateTime UnixMillisecondTimeStampToDateTime(decimal unixTimeStamp)
{
DateTime time;
try
{
// Any residual decimal numbers that remain are nanoseconds from [0, 100) nanoseconds.
// If we cast to (long), only the integer component of the decimal is taken, and can
// potentially result in look-ahead bias in increments of 100 nanoseconds, i.e. 1 DateTime tick.
var ticks = Math.Ceiling(unixTimeStamp * TimeSpan.TicksPerMillisecond);
time = EpochTime.AddTicks((long)ticks);
}
catch (Exception err)
{
Log.Error(err, Invariant($"UnixTimeStamp: {unixTimeStamp}"));
time = DateTime.Now;
}
return time;
}
///
/// Create a C# DateTime from a UnixTimestamp
///
/// Int64 unix timestamp (Time since Midnight Jan 1 1970) in nanoseconds
/// C# date time object
public static DateTime UnixNanosecondTimeStampToDateTime(long unixTimeStamp)
{
DateTime time;
try
{
var ticks = unixTimeStamp / 100;
time = EpochTime.AddTicks(ticks);
}
catch (Exception err)
{
Log.Error(err, Invariant($"UnixTimeStamp: {unixTimeStamp}"));
time = DateTime.Now;
}
return time;
}
///
/// Convert a Datetime to Unix Timestamp
///
/// C# datetime object
/// Double unix timestamp
public static double DateTimeToUnixTimeStamp(DateTime time)
{
double timestamp = 0;
try
{
timestamp = (time - new DateTime(1970, 1, 1, 0, 0, 0, 0)).TotalSeconds;
}
catch (Exception err)
{
Log.Error(err, Invariant($"{time:o}"));
}
return timestamp;
}
///
/// Convert a Datetime to Unix Timestamp
///
/// C# datetime object
/// Double unix timestamp
public static double DateTimeToUnixTimeStampMilliseconds(DateTime time)
{
double timestamp = 0;
try
{
timestamp = (time - new DateTime(1970, 1, 1, 0, 0, 0, 0)).TotalMilliseconds;
}
catch (Exception err)
{
Log.Error(err, Invariant($"{time:o}"));
}
return timestamp;
}
///
/// Convert a Datetime to Unix Timestamp
///
/// C# datetime object
/// Int64 unix timestamp
public static long DateTimeToUnixTimeStampNanoseconds(DateTime time)
{
long timestamp = 0;
try
{
timestamp = (time - new DateTime(1970, 1, 1, 0, 0, 0, 0)).Ticks * 100;
}
catch (Exception err)
{
Log.Error(err, Invariant($"{time:o}"));
}
return timestamp;
}
///
/// Get the current time as a unix timestamp
///
/// Double value of the unix as UTC timestamp
public static double TimeStamp()
{
return DateTimeToUnixTimeStamp(DateTime.UtcNow);
}
///
/// Returns the timespan with the larger value
///
public static TimeSpan Max(TimeSpan one, TimeSpan two)
{
return TimeSpan.FromTicks(Math.Max(one.Ticks, two.Ticks));
}
///
/// Returns the timespan with the smaller value
///
public static TimeSpan Min(TimeSpan one, TimeSpan two)
{
return TimeSpan.FromTicks(Math.Min(one.Ticks, two.Ticks));
}
///
/// Returns the larger of two date times
///
public static DateTime Max(DateTime one, DateTime two)
{
return one > two ? one : two;
}
///
/// Returns the smaller of two date times
///
public static DateTime Min(DateTime one, DateTime two)
{
return one < two ? one : two;
}
///
/// Multiplies the specified interval by the multiplier
///
/// The interval to be multiplied, such as TimeSpan.FromSeconds(1)
/// The number of times to multiply the interval
/// The multiplied interval, such as 1s*5 = 5s
public static TimeSpan Multiply(this TimeSpan interval, double multiplier)
{
return TimeSpan.FromTicks((long) (interval.Ticks * multiplier));
}
///
/// Parse a standard YY MM DD date into a DateTime. Attempt common date formats
///
/// String date time to parse
/// Date time
public static DateTime ParseDate(string dateToParse)
{
try
{
//First try the exact options:
DateTime date;
if (DateTime.TryParseExact(dateToParse, DateFormat.SixCharacter, CultureInfo.InvariantCulture, DateTimeStyles.None, out date))
{
return date;
}
if (DateTime.TryParseExact(dateToParse, DateFormat.EightCharacter, CultureInfo.InvariantCulture, DateTimeStyles.None, out date))
{
return date;
}
if (DateTime.TryParseExact(dateToParse, DateFormat.TwelveCharacter, CultureInfo.InvariantCulture, DateTimeStyles.None, out date))
{
return date;
}
if (DateTime.TryParseExact(dateToParse.SafeSubstring(0, 19), DateFormat.JsonFormat, CultureInfo.InvariantCulture, DateTimeStyles.None, out date))
{
return date;
}
if (DateTime.TryParseExact(dateToParse, DateFormat.USShort, CultureInfo.InvariantCulture, DateTimeStyles.None, out date))
{
return date;
}
if (DateTime.TryParseExact(dateToParse, DateFormat.USShortDateOnly, CultureInfo.InvariantCulture, DateTimeStyles.None, out date))
{
return date;
}
if (DateTime.TryParseExact(dateToParse, DateFormat.US, CultureInfo.InvariantCulture, DateTimeStyles.None, out date))
{
return date;
}
if (DateTime.TryParseExact(dateToParse, DateFormat.USDateOnly, CultureInfo.InvariantCulture, DateTimeStyles.None, out date))
{
return date;
}
if (DateTime.TryParse(dateToParse, out date))
{
return date;
}
}
catch (Exception err)
{
Log.Error(err);
}
return DateTime.Now;
}
///
/// Parse a standard YY MM DD date into a DateTime. Attempt common date formats
///
/// String date time to parse
/// Date time
public static DateTime ParseFIXUtcTimestamp(string dateToParse)
{
try
{
//First try the exact options:
DateTime date;
if (DateTime.TryParseExact(dateToParse, DateFormat.FIX, CultureInfo.InvariantCulture, DateTimeStyles.None, out date))
{
return date;
}
if (DateTime.TryParseExact(dateToParse, DateFormat.FIXWithMillisecond, CultureInfo.InvariantCulture, DateTimeStyles.None, out date))
{
return date;
}
}
catch (Exception err)
{
Log.Error(err);
}
return DateTime.UtcNow;
}
///
/// Define an enumerable date time range using the given time step
///
/// DateTime start date time
/// DateTime end date time
/// Enumerable date time range
public static IEnumerable DateTimeRange(DateTime from, DateTime thru, TimeSpan step)
{
for (var dateTime = from; dateTime <= thru; dateTime = dateTime.Add(step))
yield return dateTime;
}
///
/// Define an enumerable date range and return each date as a datetime object in the date range
///
/// DateTime start date
/// DateTime end date
/// Enumerable date range
public static IEnumerable EachDay(DateTime from, DateTime thru)
{
return DateTimeRange(from.Date, thru.Date, TimeSpan.FromDays(1));
}
///
/// Define an enumerable date range of tradeable dates - skip the holidays and weekends when securities in this algorithm don't trade.
///
/// Securities we have in portfolio
/// Start date
/// End date
/// Enumerable date range
public static IEnumerable EachTradeableDay(ICollection securities, DateTime from, DateTime thru)
{
for (var day = from.Date; day.Date <= thru.Date; day = day.AddDays(1))
{
if (TradableDate(securities, day))
{
yield return day;
}
}
}
///
/// Define an enumerable date range of tradeable dates - skip the holidays and weekends when securities in this algorithm don't trade.
///
/// The security to get tradeable dates for
/// Start date
/// End date
/// True to include days with extended market hours only, like sunday for futures
/// Enumerable date range
public static IEnumerable EachTradeableDay(Security security, DateTime from, DateTime thru, bool extendedMarketHours = false)
{
return EachTradeableDay(security.Exchange.Hours, from, thru, extendedMarketHours);
}
///
/// Define an enumerable date range of tradeable dates - skip the holidays and weekends when securities in this algorithm don't trade.
///
/// The security to get tradeable dates for
/// Start date
/// End date
/// True to include days with extended market hours only, like sunday for futures
/// Enumerable date range
public static IEnumerable EachTradeableDay(SecurityExchangeHours exchange, DateTime from, DateTime thru, bool extendedMarketHours = false)
{
for (var day = from.Date; day.Date <= thru.Date; day = day.AddDays(1))
{
if (exchange.IsDateOpen(day, extendedMarketHours))
{
yield return day;
}
}
}
///
/// Define an enumerable date range of tradeable dates but expressed in a different time zone.
///
///
/// This is mainly used to bridge the gap between exchange time zone and data time zone for file written to disk. The returned
/// enumerable of dates is guaranteed to be the same size or longer than those generated via
///
/// The exchange hours
/// The start time in the exchange time zone
/// The end time in the exchange time zone (inclusive of the final day)
/// The timezone to project the dates into (inclusive of the final day)
/// True to include extended market hours trading in the search, false otherwise
///
public static IEnumerable EachTradeableDayInTimeZone(SecurityExchangeHours exchange, DateTime from, DateTime thru, DateTimeZone timeZone, bool includeExtendedMarketHours = true)
{
var currentExchangeTime = from;
thru = thru.Date.AddDays(1); // we want to include the full thru date
while (currentExchangeTime < thru)
{
// take steps of max size of one day in the data time zone
var currentInTimeZone = currentExchangeTime.ConvertTo(exchange.TimeZone, timeZone);
var currentInTimeZoneEod = currentInTimeZone.Date.AddDays(1);
var currentExchangeTimeEod = currentInTimeZoneEod.ConvertTo(timeZone, exchange.TimeZone);
// don't pass the end
if (currentExchangeTimeEod > thru)
{
currentExchangeTimeEod = thru;
}
// perform market open checks in the exchange time zone
if (exchange.IsOpen(currentExchangeTime, currentExchangeTimeEod, includeExtendedMarketHours))
{
yield return currentInTimeZone.Date;
}
currentExchangeTime = currentExchangeTimeEod;
}
}
///
/// Make sure this date is not a holiday, or weekend for the securities in this algorithm.
///
/// Security manager from the algorithm
/// DateTime to check if trade-able.
/// True if tradeable date
public static bool TradableDate(IEnumerable securities, DateTime day)
{
try
{
foreach (var security in securities)
{
if (security.Exchange.DateIsOpen(day.Date)) return true;
}
}
catch (Exception err)
{
Log.Error(err);
}
return false;
}
///
/// Could of the number of tradeable dates within this period.
///
/// Securities we're trading
/// Start of Date Loop
/// End of Date Loop
/// Number of dates
public static int TradeableDates(ICollection securities, DateTime start, DateTime finish)
{
var count = 0;
Log.Trace(Invariant($"Time.TradeableDates(): {Messages.Time.SecurityCount(securities.Count)}"));
try
{
foreach (var day in EachDay(start, finish))
{
if (TradableDate(securities, day))
{
count++;
}
}
}
catch (Exception err)
{
Log.Error(err);
}
return count;
}
///
/// Determines the start time required to produce the requested number of bars and the given size
///
/// The exchange hours used to test for market open hours
/// The end time of the last bar over the requested period
/// The length of each bar
/// The number of bars requested
/// True to allow extended market hours bars, otherwise false for only normal market hours
/// Timezone for this data
/// True if daily strict end times are enabled
/// The start time that would provide the specified number of bars ending at the specified end time, rounded down by the requested bar size
public static DateTime GetStartTimeForTradeBars(SecurityExchangeHours exchangeHours, DateTime end, TimeSpan barSize, int barCount,
bool extendedMarketHours, DateTimeZone dataTimeZone, bool dailyPreciseEndTime = false)
{
if (barSize <= TimeSpan.Zero)
{
throw new ArgumentException(Messages.Time.InvalidBarSize, nameof(barSize));
}
var current = end;
if (dailyPreciseEndTime && barSize == OneDay)
{
if (exchangeHours.IsDateOpen(current) && exchangeHours.GetNextMarketClose(current.Date, extendedMarketHours) > current)
{
// we round down, because data for today isn't ready/wont pass through current time.
// for example, for equities, current time is 3pm, 1 bar in daily should be yesterdays, today does not count
current = end.RoundDownInTimeZone(barSize, exchangeHours.TimeZone, dataTimeZone);
}
}
else
{
// need to round down in data timezone because data is stored in this time zone but only if not doing daily resolution or
// dailyPreciseEndTime is disabled because if we round down we might include 2 bars when we want 1, for example: say
// current is monday 8pm NY, if we round down we get minight monday which will return false as open, so we will return
// friday and monday data for daily equity, when we want only monday.
current = end.RoundDownInTimeZone(barSize, exchangeHours.TimeZone, dataTimeZone);
}
for (int i = 0; i < barCount;)
{
var previous = current;
current = current - barSize;
if (exchangeHours.IsOpen(current, previous, extendedMarketHours))
{
i++;
}
}
return current;
}
///
/// Determines the end time at which the requested number of bars of the given will have elapsed.
/// NOTE: The start time is not discretized by barSize units like is done in
///
/// The exchange hours used to test for market open hours
/// The end time of the last bar over the requested period
/// The length of each bar
/// The number of bars requested
/// True to allow extended market hours bars, otherwise false for only normal market hours
/// The start time that would provide the specified number of bars ending at the specified end time, rounded down by the requested bar size
public static DateTime GetEndTimeForTradeBars(SecurityExchangeHours exchangeHours, DateTime start, TimeSpan barSize, int barCount, bool extendedMarketHours)
{
if (barSize <= TimeSpan.Zero)
{
throw new ArgumentException(Messages.Time.InvalidBarSize, nameof(barSize));
}
var current = start;
if (barSize == OneDay)
{
for (int i = 0; i < barCount;)
{
current = current + OneDay;
if (exchangeHours.IsDateOpen(current))
{
i++;
}
}
return current;
}
for (int i = 0; i < barCount;)
{
var previous = current;
current = current + barSize;
if (exchangeHours.IsOpen(previous, current, extendedMarketHours))
{
i++;
}
}
return current;
}
///
/// Gets the number of trade bars of the specified that fit between the and
///
/// The exchange used to test for market open hours
/// The start time of the interval in the exchange time zone
/// The end time of the interval in the exchange time zone
/// The step size used to count number of bars between start and end
/// The number of bars of the specified size between start and end times
public static int GetNumberOfTradeBarsInInterval(SecurityExchangeHours exchangeHours, DateTime start, DateTime end, TimeSpan barSize)
{
if (barSize <= TimeSpan.Zero)
{
throw new ArgumentException(Messages.Time.InvalidBarSize, nameof(barSize));
}
var count = 0;
var current = start;
if (barSize == OneDay)
{
while (current < end)
{
if (exchangeHours.IsDateOpen(current))
{
count++;
}
current = current + OneDay;
}
return count;
}
while (current < end)
{
var previous = current;
current = current + barSize;
if (exchangeHours.IsOpen(previous, current, false))
{
count++;
}
}
return count;
}
///
/// Normalizes the current time within the specified period
/// time = start => 0
/// time = start + period => 1
///
/// The start time of the range
/// The current time we seek to normalize
/// The time span of the range
/// The normalized time
public static double NormalizeInstantWithinRange(DateTime start, DateTime current, TimeSpan period)
{
// normalization of a point time only has a value at that specific point
if (period == TimeSpan.Zero)
{
return start == current ? 1 : 0;
}
var delta = (current - start).TotalSeconds;
return delta / period.TotalSeconds;
}
///
/// Normalizes the step size as a percentage of the period.
///
/// The period to normalize against
/// The step size to be normaized
/// The normalized step size as a percentage of the period
public static double NormalizeTimeStep(TimeSpan period, TimeSpan stepSize)
{
// normalization of a time step for an instantaneous period will always be zero
if (period == TimeSpan.Zero)
{
return 0;
}
return stepSize.TotalSeconds / period.TotalSeconds;
}
///
/// Gets the absolute value of the specified time span
///
/// Time span whose absolute value we seek
/// The absolute value of the specified time span
public static TimeSpan Abs(this TimeSpan timeSpan)
{
return TimeSpan.FromTicks(Math.Abs(timeSpan.Ticks));
}
///
/// Helper method to deserialize month/year
///
public class MonthYearJsonConverter : IsoDateTimeConverter
{
///
/// Creates a new instance
///
public MonthYearJsonConverter()
{
DateTimeFormat = @"MM/yy";
}
}
}
}