/* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ using System; using System.Globalization; using System.IO; using System.Runtime.CompilerServices; using ProtoBuf; using QuantConnect.Logging; using QuantConnect.Python; using QuantConnect.Util; using static QuantConnect.StringExtensions; namespace QuantConnect.Data.Market { /// /// QuoteBar class for second and minute resolution data: /// An OHLC implementation of the QuantConnect BaseData class with parameters for candles. /// [ProtoContract(SkipConstructor = true)] public class QuoteBar : BaseData, IBaseDataBar { // scale factor used in QC equity/option/indexOption data files private const decimal _scaleFactor = 1 / 10000m; /// /// Average bid size /// [ProtoMember(201)] [PandasColumn("bidsize")] public decimal LastBidSize { get; set; } /// /// Average ask size /// [ProtoMember(202)] [PandasColumn("asksize")] public decimal LastAskSize { get; set; } /// /// Bid OHLC /// [ProtoMember(203)] public Bar Bid { get; set; } /// /// Ask OHLC /// [ProtoMember(204)] public Bar Ask { get; set; } /// /// Opening price of the bar: Defined as the price at the start of the time period. /// public decimal Open { get { if (Bid != null && Ask != null) { if (Bid.Open != 0m && Ask.Open != 0m) return (Bid.Open + Ask.Open) / 2m; if (Bid.Open != 0) return Bid.Open; if (Ask.Open != 0) return Ask.Open; return 0m; } if (Bid != null) { return Bid.Open; } if (Ask != null) { return Ask.Open; } return 0m; } } /// /// High price of the QuoteBar during the time period. /// public decimal High { get { if (Bid != null && Ask != null) { if (Bid.High != 0m && Ask.High != 0m) return (Bid.High + Ask.High) / 2m; if (Bid.High != 0) return Bid.High; if (Ask.High != 0) return Ask.High; return 0m; } if (Bid != null) { return Bid.High; } if (Ask != null) { return Ask.High; } return 0m; } } /// /// Low price of the QuoteBar during the time period. /// public decimal Low { get { if (Bid != null && Ask != null) { if (Bid.Low != 0m && Ask.Low != 0m) return (Bid.Low + Ask.Low) / 2m; if (Bid.Low != 0) return Bid.Low; if (Ask.Low != 0) return Ask.Low; return 0m; } if (Bid != null) { return Bid.Low; } if (Ask != null) { return Ask.Low; } return 0m; } } /// /// Closing price of the QuoteBar. Defined as the price at Start Time + TimeSpan. /// public decimal Close { get { if (Bid != null && Ask != null) { if (Bid.Close != 0m && Ask.Close != 0m) return (Bid.Close + Ask.Close) / 2m; if (Bid.Close != 0) return Bid.Close; if (Ask.Close != 0) return Ask.Close; return 0m; } if (Bid != null) { return Bid.Close; } if (Ask != null) { return Ask.Close; } return Value; } } /// /// The closing time of this bar, computed via the Time and Period /// [PandasIgnore] public override DateTime EndTime { get { return Time + Period; } set { Period = value - Time; } } /// /// The period of this quote bar, (second, minute, daily, ect...) /// [ProtoMember(205)] [PandasIgnore] public TimeSpan Period { get; set; } /// /// Default initializer to setup an empty quotebar. /// public QuoteBar() { Symbol = Symbol.Empty; Time = new DateTime(); Bid = new Bar(); Ask = new Bar(); Value = 0; Period = QuantConnect.Time.OneMinute; DataType = MarketDataType.QuoteBar; } /// /// Initialize Quote Bar with Bid(OHLC) and Ask(OHLC) Values: /// /// DateTime Timestamp of the bar /// Market MarketType Symbol /// Bid OLHC bar /// Average bid size over period /// Ask OLHC bar /// Average ask size over period /// The period of this bar, specify null for default of 1 minute public QuoteBar(DateTime time, Symbol symbol, IBar bid, decimal lastBidSize, IBar ask, decimal lastAskSize, TimeSpan? period = null) { Symbol = symbol; Time = time; Bid = bid == null ? null : new Bar(bid.Open, bid.High, bid.Low, bid.Close); Ask = ask == null ? null : new Bar(ask.Open, ask.High, ask.Low, ask.Close); if (Bid != null) LastBidSize = lastBidSize; if (Ask != null) LastAskSize = lastAskSize; Value = Close; Period = period ?? QuantConnect.Time.OneMinute; DataType = MarketDataType.QuoteBar; } /// /// Update the quotebar - build the bar from this pricing information: /// /// The last trade price /// Current bid price /// Current asking price /// Volume of this trade /// The size of the current bid, if available, if not, pass 0 /// The size of the current ask, if available, if not, pass 0 [MethodImpl(MethodImplOptions.AggressiveInlining)] public override void Update(decimal lastTrade, decimal bidPrice, decimal askPrice, decimal volume, decimal bidSize, decimal askSize) { // update our bid and ask bars - handle null values, this is to give good values for midpoint OHLC if (Bid == null && bidPrice != 0) Bid = new Bar(bidPrice, bidPrice, bidPrice, bidPrice); else if (Bid != null) Bid.Update(ref bidPrice); if (Ask == null && askPrice != 0) Ask = new Bar(askPrice, askPrice, askPrice, askPrice); else if (Ask != null) Ask.Update(ref askPrice); if (bidSize > 0) { LastBidSize = bidSize; } if (askSize > 0) { LastAskSize = askSize; } // be prepared for updates without trades if (lastTrade != 0) Value = lastTrade; else if (askPrice != 0) Value = askPrice; else if (bidPrice != 0) Value = bidPrice; } /// /// QuoteBar Reader: Fetch the data from the QC storage and feed it line by line into the engine. /// /// Symbols, Resolution, DataType, /// The file data stream /// Date of this reader request /// true if we're in live mode, false for backtesting mode /// Enumerable iterator for returning each line of the required data. [StubsIgnore] public override BaseData Reader(SubscriptionDataConfig config, StreamReader stream, DateTime date, bool isLiveMode) { try { switch (config.SecurityType) { case SecurityType.Equity: return ParseEquity(config, stream, date); case SecurityType.Forex: case SecurityType.Crypto: case SecurityType.CryptoFuture: return ParseForex(config, stream, date); case SecurityType.Cfd: return ParseCfd(config, stream, date); case SecurityType.Option: case SecurityType.FutureOption: case SecurityType.IndexOption: return ParseOption(config, stream, date); case SecurityType.Future: return ParseFuture(config, stream, date); } } catch (Exception err) { Log.Error(Invariant($"QuoteBar.Reader(): Error parsing stream, Symbol: {config.Symbol.Value}, SecurityType: {config.SecurityType}, ") + Invariant($"Resolution: {config.Resolution}, Date: {date.ToStringInvariant("yyyy-MM-dd")}, Message: {err}") ); } // we need to consume a line anyway, to advance the stream stream.ReadLine(); // if we couldn't parse it above return a default instance return new QuoteBar { Symbol = config.Symbol, Period = config.Increment }; } /// /// QuoteBar Reader: Fetch the data from the QC storage and feed it line by line into the engine. /// /// Symbols, Resolution, DataType, /// Line from the data file requested /// Date of this reader request /// true if we're in live mode, false for backtesting mode /// Enumerable iterator for returning each line of the required data. public override BaseData Reader(SubscriptionDataConfig config, string line, DateTime date, bool isLiveMode) { try { switch (config.SecurityType) { case SecurityType.Equity: return ParseEquity(config, line, date); case SecurityType.Forex: case SecurityType.Crypto: case SecurityType.CryptoFuture: return ParseForex(config, line, date); case SecurityType.Cfd: return ParseCfd(config, line, date); case SecurityType.Option: case SecurityType.FutureOption: case SecurityType.IndexOption: return ParseOption(config, line, date); case SecurityType.Future: return ParseFuture(config, line, date); } } catch (Exception err) { Log.Error(Invariant($"QuoteBar.Reader(): Error parsing line: '{line}', Symbol: {config.Symbol.Value}, SecurityType: {config.SecurityType}, ") + Invariant($"Resolution: {config.Resolution}, Date: {date.ToStringInvariant("yyyy-MM-dd")}, Message: {err}") ); } // if we couldn't parse it above return a default instance return new QuoteBar { Symbol = config.Symbol, Period = config.Increment }; } /// /// Parse a quotebar representing a future with a scaling factor /// /// Symbols, Resolution, DataType /// Line from the data file requested /// Date of this reader request /// with the bid/ask set to same values public QuoteBar ParseFuture(SubscriptionDataConfig config, string line, DateTime date) { return ParseQuote(config, date, line, false); } /// /// Parse a quotebar representing a future with a scaling factor /// /// Symbols, Resolution, DataType /// The data stream of the requested file /// Date of this reader request /// with the bid/ask set to same values public QuoteBar ParseFuture(SubscriptionDataConfig config, StreamReader streamReader, DateTime date) { return ParseQuote(config, date, streamReader, false); } /// /// Parse a quotebar representing an option with a scaling factor /// /// Symbols, Resolution, DataType /// Line from the data file requested /// Date of this reader request /// with the bid/ask set to same values public QuoteBar ParseOption(SubscriptionDataConfig config, string line, DateTime date) { return ParseQuote(config, date, line, LeanData.OptionUseScaleFactor(config.Symbol)); } /// /// Parse a quotebar representing an option with a scaling factor /// /// Symbols, Resolution, DataType /// The data stream of the requested file /// Date of this reader request /// with the bid/ask set to same values public QuoteBar ParseOption(SubscriptionDataConfig config, StreamReader streamReader, DateTime date) { // scale factor only applies for equity and index options return ParseQuote(config, date, streamReader, useScaleFactor: LeanData.OptionUseScaleFactor(config.Symbol)); } /// /// Parse a quotebar representing a cfd without a scaling factor /// /// Symbols, Resolution, DataType /// Line from the data file requested /// Date of this reader request /// with the bid/ask set to same values public QuoteBar ParseCfd(SubscriptionDataConfig config, string line, DateTime date) { return ParseQuote(config, date, line, false); } /// /// Parse a quotebar representing a cfd without a scaling factor /// /// Symbols, Resolution, DataType /// The data stream of the requested file /// Date of this reader request /// with the bid/ask set to same values public QuoteBar ParseCfd(SubscriptionDataConfig config, StreamReader streamReader, DateTime date) { return ParseQuote(config, date, streamReader, false); } /// /// Parse a quotebar representing a forex without a scaling factor /// /// Symbols, Resolution, DataType /// Line from the data file requested /// Date of this reader request /// with the bid/ask set to same values public QuoteBar ParseForex(SubscriptionDataConfig config, string line, DateTime date) { return ParseQuote(config, date, line, false); } /// /// Parse a quotebar representing a forex without a scaling factor /// /// Symbols, Resolution, DataType /// The data stream of the requested file /// Date of this reader request /// with the bid/ask set to same values public QuoteBar ParseForex(SubscriptionDataConfig config, StreamReader streamReader, DateTime date) { return ParseQuote(config, date, streamReader, false); } /// /// Parse a quotebar representing an equity with a scaling factor /// /// Symbols, Resolution, DataType /// Line from the data file requested /// Date of this reader request /// with the bid/ask set to same values public QuoteBar ParseEquity(SubscriptionDataConfig config, string line, DateTime date) { return ParseQuote(config, date, line, true); } /// /// Parse a quotebar representing an equity with a scaling factor /// /// Symbols, Resolution, DataType /// The data stream of the requested file /// Date of this reader request /// with the bid/ask set to same values public QuoteBar ParseEquity(SubscriptionDataConfig config, StreamReader streamReader, DateTime date) { return ParseQuote(config, date, streamReader, true); } /// /// "Scaffold" code - If the data being read is formatted as a QuoteBar, use this method to deserialize it /// /// Symbols, Resolution, DataType, /// The data stream of the requested file /// Date of this reader request /// Whether the data has a scaling factor applied /// with the bid/ask prices set appropriately private QuoteBar ParseQuote(SubscriptionDataConfig config, DateTime date, StreamReader streamReader, bool useScaleFactor) { // Non-equity asset classes will not use scaling, including options that have a non-equity underlying asset class. var scaleFactor = useScaleFactor ? _scaleFactor : 1; var quoteBar = new QuoteBar { Period = config.Increment, Symbol = config.Symbol }; if (config.Resolution == Resolution.Daily || config.Resolution == Resolution.Hour) { // hourly and daily have different time format, and can use slow, robust c# parser. quoteBar.Time = streamReader.GetDateTime().ConvertTo(config.DataTimeZone, config.ExchangeTimeZone); } else { // Using custom int conversion for speed on high resolution data. quoteBar.Time = date.Date.AddMilliseconds(streamReader.GetInt32()).ConvertTo(config.DataTimeZone, config.ExchangeTimeZone); } var open = streamReader.GetDecimal(); var high = streamReader.GetDecimal(); var low = streamReader.GetDecimal(); var close = streamReader.GetDecimal(); var lastSize = streamReader.GetDecimal(); // only create the bid if it exists in the file if (open != 0 || high != 0 || low != 0 || close != 0) { // the Bid/Ask bars were already create above, we don't need to recreate them but just set their values quoteBar.Bid.Open = open * scaleFactor; quoteBar.Bid.High = high * scaleFactor; quoteBar.Bid.Low = low * scaleFactor; quoteBar.Bid.Close = close * scaleFactor; quoteBar.LastBidSize = lastSize; } else { quoteBar.Bid = null; } open = streamReader.GetDecimal(); high = streamReader.GetDecimal(); low = streamReader.GetDecimal(); close = streamReader.GetDecimal(); lastSize = streamReader.GetDecimal(); // only create the ask if it exists in the file if (open != 0 || high != 0 || low != 0 || close != 0) { // the Bid/Ask bars were already create above, we don't need to recreate them but just set their values quoteBar.Ask.Open = open * scaleFactor; quoteBar.Ask.High = high * scaleFactor; quoteBar.Ask.Low = low * scaleFactor; quoteBar.Ask.Close = close * scaleFactor; quoteBar.LastAskSize = lastSize; } else { quoteBar.Ask = null; } quoteBar.Value = quoteBar.Close; return quoteBar; } /// /// "Scaffold" code - If the data being read is formatted as a QuoteBar, use this method to deserialize it /// TODO: Once all Forex data refactored to use QuoteBar formatted data, use only this method /// /// Symbols, Resolution, DataType, /// Line from the data file requested /// Date of this reader request /// Whether the data has a scaling factor applied /// with the bid/ask prices set appropriately private QuoteBar ParseQuote(SubscriptionDataConfig config, DateTime date, string line, bool useScaleFactor) { var scaleFactor = useScaleFactor ? _scaleFactor : 1; var quoteBar = new QuoteBar { Period = config.Increment, Symbol = config.Symbol }; var csv = line.ToCsv(11); if (config.Resolution == Resolution.Daily || config.Resolution == Resolution.Hour) { // hourly and daily have different time format, and can use slow, robust c# parser. quoteBar.Time = DateTime.ParseExact(csv[0], DateFormat.TwelveCharacter, CultureInfo.InvariantCulture).ConvertTo(config.DataTimeZone, config.ExchangeTimeZone); } else { // Using custom "ToDecimal" conversion for speed on high resolution data. quoteBar.Time = date.Date.AddMilliseconds((double)csv[0].ToDecimal()).ConvertTo(config.DataTimeZone, config.ExchangeTimeZone); } // only create the bid if it exists in the file if (csv[1].Length != 0 || csv[2].Length != 0 || csv[3].Length != 0 || csv[4].Length != 0) { // the Bid/Ask bars were already create above, we don't need to recreate them but just set their values quoteBar.Bid.Open = csv[1].ToDecimal() * scaleFactor; quoteBar.Bid.High = csv[2].ToDecimal() * scaleFactor; quoteBar.Bid.Low = csv[3].ToDecimal() * scaleFactor; quoteBar.Bid.Close = csv[4].ToDecimal() * scaleFactor; quoteBar.LastBidSize = csv[5].ToDecimal(); } else { quoteBar.Bid = null; } // only create the ask if it exists in the file if (csv[6].Length != 0 || csv[7].Length != 0 || csv[8].Length != 0 || csv[9].Length != 0) { // the Bid/Ask bars were already create above, we don't need to recreate them but just set their values quoteBar.Ask.Open = csv[6].ToDecimal() * scaleFactor; quoteBar.Ask.High = csv[7].ToDecimal() * scaleFactor; quoteBar.Ask.Low = csv[8].ToDecimal() * scaleFactor; quoteBar.Ask.Close = csv[9].ToDecimal() * scaleFactor; quoteBar.LastAskSize = csv[10].ToDecimal(); } else { quoteBar.Ask = null; } quoteBar.Value = quoteBar.Close; return quoteBar; } /// /// Get Source for Custom Data File /// >> What source file location would you prefer for each type of usage: /// /// Configuration object /// Date of this source request if source spread across multiple files /// true if we're in live mode, false for backtesting mode /// String source location of the file public override SubscriptionDataSource GetSource(SubscriptionDataConfig config, DateTime date, bool isLiveMode) { if (isLiveMode) { // this data type is streamed in live mode return new SubscriptionDataSource(string.Empty, SubscriptionTransportMedium.Streaming); } var source = LeanData.GenerateZipFilePath(Globals.DataFolder, config.Symbol, date, config.Resolution, config.TickType); if (config.SecurityType == SecurityType.Future || config.SecurityType.IsOption()) { source += "#" + LeanData.GenerateZipEntryName(config.Symbol, date, config.Resolution, config.TickType); } return new SubscriptionDataSource(source, SubscriptionTransportMedium.LocalFile, FileFormat.Csv); } /// /// Return a new instance clone of this quote bar, used in fill forward /// /// A clone of the current quote bar public override BaseData Clone() { return new QuoteBar { Ask = Ask == null ? null : Ask.Clone(), Bid = Bid == null ? null : Bid.Clone(), LastAskSize = LastAskSize, LastBidSize = LastBidSize, Symbol = Symbol, Time = Time, Period = Period, Value = Value, DataType = DataType }; } /// /// Collapses QuoteBars into TradeBars object when /// algorithm requires FX data, but calls OnData() /// TODO: (2017) Remove this method in favor of using OnData() /// /// public TradeBar Collapse() { return new TradeBar(Time, Symbol, Open, High, Low, Close, 0, Period); } /// /// Convert this to string form. /// /// String representation of the public override string ToString() { return $"{Symbol}: " + $"Bid: O: {Bid?.Open.SmartRounding()} " + $"Bid: H: {Bid?.High.SmartRounding()} " + $"Bid: L: {Bid?.Low.SmartRounding()} " + $"Bid: C: {Bid?.Close.SmartRounding()} " + $"Ask: O: {Ask?.Open.SmartRounding()} " + $"Ask: H: {Ask?.High.SmartRounding()} " + $"Ask: L: {Ask?.Low.SmartRounding()} " + $"Ask: C: {Ask?.Close.SmartRounding()} "; } } }