/* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ using System; using System.Threading; using QuantConnect.Logging; using QuantConnect.Util; using QuantConnect.Util.RateLimit; namespace QuantConnect.Lean.Engine { /// /// Provides an implementation of that tracks the algorithm /// manager's time loops and enforces a maximum amount of time that each time loop may take to execute. /// The isolator uses the result provided by to determine if it should /// terminate the algorithm for violation of the imposed limits. /// public class AlgorithmTimeLimitManager : IIsolatorLimitResultProvider { private volatile bool _failed; private volatile bool _stopped; private long _additionalMinutes; private volatile ReferenceWrapper _currentTimeStepTime; private readonly TimeSpan _timeLoopMaximum; /// /// Gets the additional time bucket which is responsible for tracking additional time requested /// for processing via long-running scheduled events. In LEAN, we use the /// public ITokenBucket AdditionalTimeBucket { get; } /// /// Initializes a new instance of to manage the /// creation of instances as it pertains to the /// algorithm manager's time loop /// /// Provides a bucket of additional time that can be requested to be /// spent to give execution time for things such as training scheduled events /// Specifies the maximum amount of time the algorithm is permitted to /// spend in a single time loop. This value can be overriden if certain actions are taken by the /// algorithm, such as invoking the training methods. public AlgorithmTimeLimitManager(ITokenBucket additionalTimeBucket, TimeSpan timeLoopMaximum) { _timeLoopMaximum = timeLoopMaximum; AdditionalTimeBucket = additionalTimeBucket; _currentTimeStepTime = new ReferenceWrapper(DateTime.MinValue); } /// /// Invoked by the algorithm at the start of each time loop. This resets the current time step /// elapsed time. /// /// /// This class is the result of a mechanical refactor with the intention of preserving all existing /// behavior, including setting the _currentTimeStepTime to /// public void StartNewTimeStep() { if (_stopped) { throw new InvalidOperationException("The AlgorithmTimeLimitManager may not be stopped and restarted."); } // maintains existing implementation behavior to reset the time to min value and then // when the isolator pings IsWithinLimit, invocation of CurrentTimeStepElapsed will cause // it to update to the current time. This was done as a performance improvement and moved // accessing DateTime.UtcNow from the algorithm manager thread to the isolator thread _currentTimeStepTime = new ReferenceWrapper(DateTime.MinValue); Interlocked.Exchange(ref _additionalMinutes, 0L); } /// /// Stops this instance from tracking the algorithm manager's time loop elapsed time. /// This is invoked at the end of the algorithm to prevent the isolator from terminating /// the algorithm during final clean up and shutdown. /// internal void StopEnforcingTimeLimit() { _stopped = true; } /// /// Determines whether or not the algorithm time loop is considered within the limits /// public IsolatorLimitResult IsWithinLimit() { TimeSpan currentTimeStepElapsed; var message = IsOutOfTime(out currentTimeStepElapsed) ? GetErrorMessage(currentTimeStepElapsed) : string.Empty; return new IsolatorLimitResult(currentTimeStepElapsed, message); } /// /// Requests additional time to continue executing the current time step. /// At time of writing, this is intended to be used to provide training scheduled events /// additional time to allow complex training models time to execute while also preventing /// abuse by enforcing certain control parameters set via the job packet. /// /// Each time this method is invoked, this time limit manager will increase the allowable /// execution time by the specified number of whole minutes /// public void RequestAdditionalTime(int minutes) { if (!TryRequestAdditionalTime(minutes)) { _failed = true; Log.Debug($"AlgorithmTimeLimitManager.RequestAdditionalTime({minutes}): Failed to acquire additional time. Marking failed."); } } /// /// Attempts to requests additional time to continue executing the current time step. /// At time of writing, this is intended to be used to provide training scheduled events /// additional time to allow complex training models time to execute while also preventing /// abuse by enforcing certain control parameters set via the job packet. /// /// Each time this method is invoked, this time limit manager will increase the allowable /// execution time by the specified number of whole minutes /// public bool TryRequestAdditionalTime(int minutes) { Log.Debug($"AlgorithmTimeLimitManager.TryRequestAdditionalTime({minutes}): Requesting additional time. Available: {AdditionalTimeBucket.AvailableTokens}"); // safely attempts to consume from the bucket, returning false if insufficient resources available if (AdditionalTimeBucket.TryConsume(minutes)) { var newValue = Interlocked.Add(ref _additionalMinutes, minutes); Log.Debug($"AlgorithmTimeLimitManager.TryRequestAdditionalTime({minutes}): Success: AdditionalMinutes: {newValue}"); return true; } return false; } /// /// Determines whether or not the algorithm should be terminated due to exceeding the time limits /// private bool IsOutOfTime(out TimeSpan currentTimeStepElapsed) { if (_stopped) { currentTimeStepElapsed = TimeSpan.Zero; return false; } currentTimeStepElapsed = GetCurrentTimeStepElapsed(); if (_failed) { return true; } var additionalMinutes = TimeSpan.FromMinutes(Interlocked.Read(ref _additionalMinutes)); return currentTimeStepElapsed > _timeLoopMaximum.Add(additionalMinutes); } /// /// Gets the current amount of time that has elapsed since the beginning of the /// most recent algorithm manager time loop /// private TimeSpan GetCurrentTimeStepElapsed() { var currentValue = _currentTimeStepTime.Value; if (currentValue == DateTime.MinValue) { _currentTimeStepTime = new ReferenceWrapper(DateTime.UtcNow); return TimeSpan.Zero; } // here we use currentValue on purpose since '_currentTimeStepTime' could have been overwritten to 'DateTime.MinValue' return DateTime.UtcNow - currentValue; } private string GetErrorMessage(TimeSpan currentTimeStepElapsed) { var message = $"Algorithm took longer than {_timeLoopMaximum.TotalMinutes} minutes on a single time loop."; var minutesAboveStandardLimit = _additionalMinutes - (int) _timeLoopMaximum.TotalMinutes; if (minutesAboveStandardLimit > 0) { message = $"{message} An additional {minutesAboveStandardLimit} minutes were also allocated and consumed."; } message = $"{message} CurrentTimeStepElapsed: {currentTimeStepElapsed.TotalMinutes:0.0} minutes"; return message; } } }