/*
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
using QuantConnect.Util;
using QuantConnect.Data;
using QuantConnect.Packets;
using QuantConnect.Interfaces;
using QuantConnect.Securities;
using QuantConnect.Configuration;
namespace QuantConnect.DownloaderDataProvider.Launcher.Models
{
///
/// Class for downloading data from a brokerage.
///
public class BrokerageDataDownloader : IDataDownloader, IDisposable
{
///
/// Represents the Brokerage implementation.
///
private IBrokerage _brokerage;
///
/// Provides access to exchange hours and raw data times zones in various markets
///
private readonly MarketHoursDatabase _marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
///
/// Initializes a new instance of the class.
///
public BrokerageDataDownloader()
{
var liveNodeConfiguration = new LiveNodePacket()
{
Brokerage = Config.Get("data-downloader-brokerage"),
UserToken = Globals.UserToken,
UserId = Globals.UserId,
ProjectId = Globals.ProjectId,
OrganizationId = Globals.OrganizationID,
Version = Globals.Version,
DeploymentTarget = DeploymentTarget.LocalPlatform
};
try
{
// import the brokerage data for the configured brokerage
var brokerageFactory = Composer.Instance.Single(factory => factory.BrokerageType.MatchesTypeName(liveNodeConfiguration.Brokerage));
liveNodeConfiguration.BrokerageData = brokerageFactory.BrokerageData;
}
catch (InvalidOperationException error)
{
throw new InvalidOperationException($"{nameof(BrokerageDataDownloader)}.An error occurred while resolving brokerage data for a live job. Brokerage: {liveNodeConfiguration.Brokerage}.", error);
}
_brokerage = Composer.Instance.GetExportedValueByTypeName(liveNodeConfiguration.Brokerage);
_brokerage.Message += (object _, Brokerages.BrokerageMessageEvent e) =>
{
if (e.Type == Brokerages.BrokerageMessageType.Error)
{
Logging.Log.Error(e.Message);
}
else
{
Logging.Log.Trace(e.Message);
}
};
((IDataQueueHandler)_brokerage).SetJob(liveNodeConfiguration);
}
///
/// Get historical data enumerable for a single symbol, type and resolution given this start and end time (in UTC).
///
/// model class for passing in parameters for historical data
/// Enumerable of base data for this symbol
public IEnumerable? Get(DataDownloaderGetParameters dataDownloaderGetParameters)
{
var symbol = dataDownloaderGetParameters.Symbol;
var resolution = dataDownloaderGetParameters.Resolution;
var startUtc = dataDownloaderGetParameters.StartUtc;
var endUtc = dataDownloaderGetParameters.EndUtc;
var tickType = dataDownloaderGetParameters.TickType;
var dataType = LeanData.GetDataType(resolution, tickType);
var exchangeHours = _marketHoursDatabase.GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType);
var dataTimeZone = _marketHoursDatabase.GetDataTimeZone(symbol.ID.Market, symbol, symbol.SecurityType);
var symbols = new List { symbol };
if (symbol.IsCanonical())
{
symbols = GetChainSymbols(symbol, true).ToList();
}
return symbols
.Select(symbol =>
{
var request = new Data.HistoryRequest(startUtc, endUtc, dataType, symbol, resolution, exchangeHours: exchangeHours, dataTimeZone: dataTimeZone, resolution,
// let's not ask for extended market hours for hour and daily resolutions to match lean
includeExtendedMarketHours: resolution != Resolution.Hour && resolution != Resolution.Daily, false, DataNormalizationMode.Raw, tickType);
var history = _brokerage.GetHistory(request);
if (history == null)
{
Logging.Log.Trace($"{nameof(BrokerageDataDownloader)}.{nameof(Get)}: Ignoring history request for unsupported symbol {symbol}");
}
return history;
})
.Where(history => history != null)
.SelectMany(history => history);
}
///
/// Returns an IEnumerable of Future/Option contract symbols for the given root ticker
///
/// The Symbol to get futures/options chain for
/// Include expired contracts
private IEnumerable GetChainSymbols(Symbol symbol, bool includeExpired)
{
if (_brokerage is IDataQueueUniverseProvider universeProvider)
{
return universeProvider.LookupSymbols(symbol, includeExpired);
}
else
{
throw new InvalidOperationException($"{nameof(BrokerageDataDownloader)}.{nameof(GetChainSymbols)}: The current brokerage does not support fetching canonical symbols. Please ensure your brokerage instance supports this feature.");
}
}
public void Dispose()
{
_brokerage.DisposeSafely();
}
}
}