/*
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
using System;
using System.Threading;
using QuantConnect.Logging;
using QuantConnect.Util;
using QuantConnect.Util.RateLimit;
namespace QuantConnect.Lean.Engine
{
///
/// Provides an implementation of that tracks the algorithm
/// manager's time loops and enforces a maximum amount of time that each time loop may take to execute.
/// The isolator uses the result provided by to determine if it should
/// terminate the algorithm for violation of the imposed limits.
///
public class AlgorithmTimeLimitManager : IIsolatorLimitResultProvider
{
private volatile bool _failed;
private volatile bool _stopped;
private long _additionalMinutes;
private volatile ReferenceWrapper _currentTimeStepTime;
private readonly TimeSpan _timeLoopMaximum;
///
/// Gets the additional time bucket which is responsible for tracking additional time requested
/// for processing via long-running scheduled events. In LEAN, we use the
///
public ITokenBucket AdditionalTimeBucket { get; }
///
/// Initializes a new instance of to manage the
/// creation of instances as it pertains to the
/// algorithm manager's time loop
///
/// Provides a bucket of additional time that can be requested to be
/// spent to give execution time for things such as training scheduled events
/// Specifies the maximum amount of time the algorithm is permitted to
/// spend in a single time loop. This value can be overriden if certain actions are taken by the
/// algorithm, such as invoking the training methods.
public AlgorithmTimeLimitManager(ITokenBucket additionalTimeBucket, TimeSpan timeLoopMaximum)
{
_timeLoopMaximum = timeLoopMaximum;
AdditionalTimeBucket = additionalTimeBucket;
_currentTimeStepTime = new ReferenceWrapper(DateTime.MinValue);
}
///
/// Invoked by the algorithm at the start of each time loop. This resets the current time step
/// elapsed time.
///
///
/// This class is the result of a mechanical refactor with the intention of preserving all existing
/// behavior, including setting the _currentTimeStepTime
to
///
public void StartNewTimeStep()
{
if (_stopped)
{
throw new InvalidOperationException("The AlgorithmTimeLimitManager may not be stopped and restarted.");
}
// maintains existing implementation behavior to reset the time to min value and then
// when the isolator pings IsWithinLimit, invocation of CurrentTimeStepElapsed will cause
// it to update to the current time. This was done as a performance improvement and moved
// accessing DateTime.UtcNow from the algorithm manager thread to the isolator thread
_currentTimeStepTime = new ReferenceWrapper(DateTime.MinValue);
Interlocked.Exchange(ref _additionalMinutes, 0L);
}
///
/// Stops this instance from tracking the algorithm manager's time loop elapsed time.
/// This is invoked at the end of the algorithm to prevent the isolator from terminating
/// the algorithm during final clean up and shutdown.
///
internal void StopEnforcingTimeLimit()
{
_stopped = true;
}
///
/// Determines whether or not the algorithm time loop is considered within the limits
///
public IsolatorLimitResult IsWithinLimit()
{
TimeSpan currentTimeStepElapsed;
var message = IsOutOfTime(out currentTimeStepElapsed) ? GetErrorMessage(currentTimeStepElapsed) : string.Empty;
return new IsolatorLimitResult(currentTimeStepElapsed, message);
}
///
/// Requests additional time to continue executing the current time step.
/// At time of writing, this is intended to be used to provide training scheduled events
/// additional time to allow complex training models time to execute while also preventing
/// abuse by enforcing certain control parameters set via the job packet.
///
/// Each time this method is invoked, this time limit manager will increase the allowable
/// execution time by the specified number of whole minutes
///
public void RequestAdditionalTime(int minutes)
{
if (!TryRequestAdditionalTime(minutes))
{
_failed = true;
Log.Debug($"AlgorithmTimeLimitManager.RequestAdditionalTime({minutes}): Failed to acquire additional time. Marking failed.");
}
}
///
/// Attempts to requests additional time to continue executing the current time step.
/// At time of writing, this is intended to be used to provide training scheduled events
/// additional time to allow complex training models time to execute while also preventing
/// abuse by enforcing certain control parameters set via the job packet.
///
/// Each time this method is invoked, this time limit manager will increase the allowable
/// execution time by the specified number of whole minutes
///
public bool TryRequestAdditionalTime(int minutes)
{
Log.Debug($"AlgorithmTimeLimitManager.TryRequestAdditionalTime({minutes}): Requesting additional time. Available: {AdditionalTimeBucket.AvailableTokens}");
// safely attempts to consume from the bucket, returning false if insufficient resources available
if (AdditionalTimeBucket.TryConsume(minutes))
{
var newValue = Interlocked.Add(ref _additionalMinutes, minutes);
Log.Debug($"AlgorithmTimeLimitManager.TryRequestAdditionalTime({minutes}): Success: AdditionalMinutes: {newValue}");
return true;
}
return false;
}
///
/// Determines whether or not the algorithm should be terminated due to exceeding the time limits
///
private bool IsOutOfTime(out TimeSpan currentTimeStepElapsed)
{
if (_stopped)
{
currentTimeStepElapsed = TimeSpan.Zero;
return false;
}
currentTimeStepElapsed = GetCurrentTimeStepElapsed();
if (_failed)
{
return true;
}
var additionalMinutes = TimeSpan.FromMinutes(Interlocked.Read(ref _additionalMinutes));
return currentTimeStepElapsed > _timeLoopMaximum.Add(additionalMinutes);
}
///
/// Gets the current amount of time that has elapsed since the beginning of the
/// most recent algorithm manager time loop
///
private TimeSpan GetCurrentTimeStepElapsed()
{
var currentValue = _currentTimeStepTime.Value;
if (currentValue == DateTime.MinValue)
{
_currentTimeStepTime = new ReferenceWrapper(DateTime.UtcNow);
return TimeSpan.Zero;
}
// here we use currentValue on purpose since '_currentTimeStepTime' could have been overwritten to 'DateTime.MinValue'
return DateTime.UtcNow - currentValue;
}
private string GetErrorMessage(TimeSpan currentTimeStepElapsed)
{
var message = $"Algorithm took longer than {_timeLoopMaximum.TotalMinutes} minutes on a single time loop.";
var minutesAboveStandardLimit = _additionalMinutes - (int) _timeLoopMaximum.TotalMinutes;
if (minutesAboveStandardLimit > 0)
{
message = $"{message} An additional {minutesAboveStandardLimit} minutes were also allocated and consumed.";
}
message = $"{message} CurrentTimeStepElapsed: {currentTimeStepElapsed.TotalMinutes:0.0} minutes";
return message;
}
}
}