/*
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
using System;
using ProtoBuf;
using System.IO;
using Newtonsoft.Json;
using QuantConnect.Util;
using QuantConnect.Logging;
using System.Globalization;
using System.Runtime.CompilerServices;
using QuantConnect.Python;
namespace QuantConnect.Data.Market
{
///
/// Tick class is the base representation for tick data. It is grouped into a Ticks object
/// which implements IDictionary and passed into an OnData event handler.
///
[ProtoContract(SkipConstructor = true)]
[ProtoInclude(1000, typeof(OpenInterest))]
public class Tick : BaseData
{
private Exchange _exchange = QuantConnect.Exchange.UNKNOWN;
private string _exchangeValue;
private uint? _parsedSaleCondition;
///
/// Type of the Tick: Trade or Quote.
///
[ProtoMember(10)]
[PandasIgnore]
public TickType TickType { get; set; } = TickType.Trade;
///
/// Quantity exchanged in a trade.
///
[ProtoMember(11)]
public decimal Quantity { get; set; }
///
/// Exchange code this tick came from
///
[PandasIgnore]
public string ExchangeCode
{
get
{
if (_exchange == null)
{
_exchange = Symbol != null
? _exchangeValue.GetPrimaryExchange(Symbol.SecurityType, Symbol.ID.Market) : _exchangeValue.GetPrimaryExchange();
}
return _exchange.Code;
}
set
{
_exchangeValue = value;
_exchange = null;
}
}
///
/// Exchange name this tick came from
///
[ProtoMember(12)]
public string Exchange
{
get
{
if (_exchange == null)
{
_exchange = Symbol != null
? _exchangeValue.GetPrimaryExchange(Symbol.SecurityType, Symbol.ID.Market) : _exchangeValue.GetPrimaryExchange();
}
return _exchange;
}
set
{
_exchangeValue = value;
_exchange = null;
}
}
///
/// Sale condition for the tick.
///
[PandasIgnore]
[ProtoMember(13)]
public string SaleCondition { get; set; } = string.Empty;
///
/// For performance parsed sale condition for the tick.
///
[JsonIgnore]
[PandasIgnore]
public uint ParsedSaleCondition
{
get
{
if (string.IsNullOrEmpty(SaleCondition))
{
return 0;
}
if (!_parsedSaleCondition.HasValue)
{
_parsedSaleCondition = uint.Parse(SaleCondition, NumberStyles.HexNumber, CultureInfo.InvariantCulture);
}
return _parsedSaleCondition.Value;
}
set
{
_parsedSaleCondition = value;
}
}
///
/// Bool whether this is a suspicious tick
///
[ProtoMember(14)]
public bool Suspicious { get; set; }
///
/// Bid Price for Tick
///
[ProtoMember(15)]
public decimal BidPrice { get; set; }
///
/// Asking price for the Tick quote.
///
[ProtoMember(16)]
public decimal AskPrice { get; set; }
///
/// Alias for "Value" - the last sale for this asset.
///
public decimal LastPrice
{
get
{
return Value;
}
}
///
/// Size of bid quote.
///
[ProtoMember(17)]
public decimal BidSize { get; set; }
///
/// Size of ask quote.
///
[ProtoMember(18)]
public decimal AskSize { get; set; }
//In Base Class: Alias of Closing:
//public decimal Price;
//Symbol of Asset.
//In Base Class: public Symbol Symbol;
//In Base Class: DateTime Of this TradeBar
//public DateTime Time;
///
/// Initialize tick class with a default constructor.
///
public Tick()
{
Value = 0;
Time = new DateTime();
DataType = MarketDataType.Tick;
Symbol = Symbol.Empty;
TickType = TickType.Trade;
Quantity = 0;
_exchange = QuantConnect.Exchange.UNKNOWN;
SaleCondition = string.Empty;
Suspicious = false;
BidSize = 0;
AskSize = 0;
}
///
/// Cloner constructor for fill forward engine implementation. Clone the original tick into this new tick:
///
/// Original tick we're cloning
public Tick(Tick original)
{
Symbol = original.Symbol;
Time = new DateTime(original.Time.Ticks);
Value = original.Value;
BidPrice = original.BidPrice;
AskPrice = original.AskPrice;
// directly set privates so we don't parse the exchange
_exchange = original._exchange;
_exchangeValue = original._exchangeValue;
SaleCondition = original.SaleCondition;
_parsedSaleCondition = original._parsedSaleCondition;
Quantity = original.Quantity;
Suspicious = original.Suspicious;
DataType = MarketDataType.Tick;
TickType = original.TickType;
BidSize = original.BidSize;
AskSize = original.AskSize;
}
///
/// Constructor for a FOREX tick where there is no last sale price. The volume in FX is so high its rare to find FX trade data.
/// To fake this the tick contains bid-ask prices and the last price is the midpoint.
///
/// Full date and time
/// Underlying currency pair we're trading
/// FX tick bid value
/// FX tick ask value
public Tick(DateTime time, Symbol symbol, decimal bid, decimal ask)
{
DataType = MarketDataType.Tick;
Time = time;
Symbol = symbol;
Value = (bid + ask) / 2;
TickType = TickType.Quote;
BidPrice = bid;
AskPrice = ask;
}
///
/// Initializes a new instance of the class to .
///
/// The time at which the open interest tick occurred.
/// The symbol associated with the open interest tick.
/// The value of the open interest for the specified symbol.
public Tick(DateTime time, Symbol symbol, decimal openInterest)
{
Time = time;
Symbol = symbol;
Value = openInterest;
DataType = MarketDataType.Tick;
TickType = TickType.OpenInterest;
}
///
/// Initializer for a last-trade equity tick with bid or ask prices.
///
/// Full date and time
/// Underlying equity security symbol
/// Bid value
/// Ask value
/// Last trade price
public Tick(DateTime time, Symbol symbol, decimal last, decimal bid, decimal ask)
{
DataType = MarketDataType.Tick;
Time = time;
Symbol = symbol;
Value = last;
TickType = TickType.Quote;
BidPrice = bid;
AskPrice = ask;
}
///
/// Trade tick type constructor
///
/// Full date and time
/// Underlying equity security symbol
/// The ticks sale condition
/// The ticks exchange
/// The quantity traded
/// The price of the trade
public Tick(DateTime time, Symbol symbol, string saleCondition, string exchange, decimal quantity, decimal price)
{
Value = price;
Time = time;
DataType = MarketDataType.Tick;
Symbol = symbol;
TickType = TickType.Trade;
Quantity = quantity;
Exchange = exchange;
SaleCondition = saleCondition;
Suspicious = false;
}
///
/// Trade tick type constructor
///
/// Full date and time
/// Underlying equity security symbol
/// The ticks sale condition
/// The ticks exchange
/// The quantity traded
/// The price of the trade
public Tick(DateTime time, Symbol symbol, string saleCondition, Exchange exchange, decimal quantity, decimal price)
: this(time, symbol, saleCondition, string.Empty, quantity, price)
{
// we were giving the exchange, set it directly
_exchange = exchange;
}
///
/// Quote tick type constructor
///
/// Full date and time
/// Underlying equity security symbol
/// The ticks sale condition
/// The ticks exchange
/// The bid size
/// The bid price
/// The ask size
/// The ask price
public Tick(DateTime time, Symbol symbol, string saleCondition, string exchange, decimal bidSize, decimal bidPrice, decimal askSize, decimal askPrice)
{
Time = time;
DataType = MarketDataType.Tick;
Symbol = symbol;
TickType = TickType.Quote;
Exchange = exchange;
SaleCondition = saleCondition;
Suspicious = false;
AskPrice = askPrice;
AskSize = askSize;
BidPrice = bidPrice;
BidSize = bidSize;
SetValue();
}
///
/// Quote tick type constructor
///
/// Full date and time
/// Underlying equity security symbol
/// The bid size
/// The bid price
/// The ask size
/// The ask price
public Tick(DateTime time, Symbol symbol, decimal bidSize, decimal bidPrice, decimal askSize, decimal askPrice)
: this(time, symbol, string.Empty, string.Empty, bidSize, bidPrice, askSize, askPrice)
{
}
///
/// Quote tick type constructor
///
/// Full date and time
/// Underlying equity security symbol
/// The ticks sale condition
/// The ticks exchange
/// The bid size
/// The bid price
/// The ask size
/// The ask price
public Tick(DateTime time, Symbol symbol, string saleCondition, Exchange exchange, decimal bidSize, decimal bidPrice, decimal askSize, decimal askPrice)
: this(time, symbol, saleCondition, string.Empty, bidSize, bidPrice, askSize, askPrice)
{
// we were giving the exchange, set it directly
_exchange = exchange;
}
///
/// Constructor for QuantConnect FXCM Data source:
///
/// Symbol for underlying asset
/// CSV line of data from FXCM
public Tick(Symbol symbol, string line)
{
var csv = line.Split(',');
DataType = MarketDataType.Tick;
Symbol = symbol;
Time = DateTime.ParseExact(csv[0], DateFormat.Forex, CultureInfo.InvariantCulture);
Value = (BidPrice + AskPrice) / 2;
TickType = TickType.Quote;
BidPrice = Convert.ToDecimal(csv[1], CultureInfo.InvariantCulture);
AskPrice = Convert.ToDecimal(csv[2], CultureInfo.InvariantCulture);
}
///
/// Constructor for QuantConnect tick data
///
/// Symbol for underlying asset
/// CSV line of data from QC tick csv
/// The base date of the tick
public Tick(Symbol symbol, string line, DateTime baseDate)
{
var csv = line.Split(',');
DataType = MarketDataType.Tick;
Symbol = symbol;
Time = baseDate.Date.AddTicks(Convert.ToInt64(10000 * csv[0].ToDecimal()));
Value = csv[1].ToDecimal() / GetScaleFactor(symbol);
TickType = TickType.Trade;
Quantity = csv[2].ToDecimal();
Exchange = csv[3].Trim();
SaleCondition = csv[4];
Suspicious = csv[5].ToInt32() == 1;
}
///
/// Parse a tick data line from quantconnect zip source files.
///
/// The source stream reader
/// Base date for the tick (ticks date is stored as int milliseconds since midnight)
/// Subscription configuration object
public Tick(SubscriptionDataConfig config, StreamReader reader, DateTime date)
{
try
{
DataType = MarketDataType.Tick;
Symbol = config.Symbol;
// Which security type is this data feed:
var scaleFactor = GetScaleFactor(config.Symbol);
switch (config.SecurityType)
{
case SecurityType.Equity:
{
TickType = config.TickType;
Time = date.Date.AddTicks(Convert.ToInt64(10000 * reader.GetDecimal())).ConvertTo(config.DataTimeZone, config.ExchangeTimeZone);
bool pastLineEnd;
if (TickType == TickType.Trade)
{
Value = reader.GetDecimal() / scaleFactor;
Quantity = reader.GetDecimal(out pastLineEnd);
if (!pastLineEnd)
{
Exchange = reader.GetString();
SaleCondition = reader.GetString();
Suspicious = reader.GetInt32() == 1;
}
}
else if (TickType == TickType.Quote)
{
BidPrice = reader.GetDecimal() / scaleFactor;
BidSize = reader.GetDecimal();
AskPrice = reader.GetDecimal() / scaleFactor;
AskSize = reader.GetDecimal(out pastLineEnd);
SetValue();
if (!pastLineEnd)
{
Exchange = reader.GetString();
SaleCondition = reader.GetString();
Suspicious = reader.GetInt32() == 1;
}
}
else
{
throw new InvalidOperationException($"Tick(): Unexpected tick type {TickType}");
}
break;
}
case SecurityType.Forex:
case SecurityType.Cfd:
{
TickType = TickType.Quote;
Time = date.Date.AddTicks(Convert.ToInt64(10000 * reader.GetDecimal()))
.ConvertTo(config.DataTimeZone, config.ExchangeTimeZone);
BidPrice = reader.GetDecimal();
AskPrice = reader.GetDecimal();
SetValue();
break;
}
case SecurityType.CryptoFuture:
case SecurityType.Crypto:
{
TickType = config.TickType;
Exchange = config.Market;
Time = date.Date.AddTicks(Convert.ToInt64(10000 * reader.GetDecimal()))
.ConvertTo(config.DataTimeZone, config.ExchangeTimeZone);
if (TickType == TickType.Trade)
{
Value = reader.GetDecimal();
Quantity = reader.GetDecimal(out var endOfLine);
Suspicious = !endOfLine && reader.GetInt32() == 1;
}
else if(TickType == TickType.Quote)
{
BidPrice = reader.GetDecimal();
BidSize = reader.GetDecimal();
AskPrice = reader.GetDecimal();
AskSize = reader.GetDecimal(out var endOfLine);
Suspicious = !endOfLine && reader.GetInt32() == 1;
SetValue();
}
break;
}
case SecurityType.Future:
case SecurityType.Option:
case SecurityType.FutureOption:
case SecurityType.IndexOption:
{
TickType = config.TickType;
Time = date.Date.AddTicks(Convert.ToInt64(10000 * reader.GetDecimal()))
.ConvertTo(config.DataTimeZone, config.ExchangeTimeZone);
if (TickType == TickType.Trade)
{
Value = reader.GetDecimal() / scaleFactor;
Quantity = reader.GetDecimal();
Exchange = reader.GetString();
SaleCondition = reader.GetString();
Suspicious = reader.GetInt32() == 1;
}
else if (TickType == TickType.OpenInterest)
{
Value = reader.GetDecimal();
}
else
{
BidPrice = reader.GetDecimal() / scaleFactor;
BidSize = reader.GetDecimal();
AskPrice = reader.GetDecimal() / scaleFactor;
AskSize = reader.GetDecimal();
Exchange = reader.GetString();
Suspicious = reader.GetInt32() == 1;
SetValue();
}
break;
}
}
}
catch (Exception err)
{
Log.Error(err);
}
}
///
/// Parse a tick data line from quantconnect zip source files.
///
/// CSV source line of the compressed source
/// Base date for the tick (ticks date is stored as int milliseconds since midnight)
/// Subscription configuration object
public Tick(SubscriptionDataConfig config, string line, DateTime date)
{
try
{
DataType = MarketDataType.Tick;
Symbol = config.Symbol;
// Which security type is this data feed:
var scaleFactor = GetScaleFactor(config.Symbol);
switch (config.SecurityType)
{
case SecurityType.Equity:
{
var index = 0;
TickType = config.TickType;
var csv = line.ToCsv(TickType == TickType.Trade ? 6 : 8);
Time = date.Date.AddTicks(Convert.ToInt64(10000 * csv[index++].ToDecimal())).ConvertTo(config.DataTimeZone, config.ExchangeTimeZone);
if (TickType == TickType.Trade)
{
Value = csv[index++].ToDecimal() / scaleFactor;
Quantity = csv[index++].ToDecimal();
if (csv.Count > index)
{
Exchange = csv[index++];
SaleCondition = csv[index++];
Suspicious = (csv[index++] == "1");
}
}
else if (TickType == TickType.Quote)
{
BidPrice = csv[index++].ToDecimal() / scaleFactor;
BidSize = csv[index++].ToDecimal();
AskPrice = csv[index++].ToDecimal() / scaleFactor;
AskSize = csv[index++].ToDecimal();
SetValue();
if (csv.Count > index)
{
Exchange = csv[index++];
SaleCondition = csv[index++];
Suspicious = (csv[index++] == "1");
}
}
else
{
throw new InvalidOperationException($"Tick(): Unexpected tick type {TickType}");
}
break;
}
case SecurityType.Forex:
case SecurityType.Cfd:
{
var csv = line.ToCsv(3);
TickType = TickType.Quote;
var ticks = (long)(csv[0].ToDecimal() * TimeSpan.TicksPerMillisecond);
Time = date.Date.AddTicks(ticks)
.ConvertTo(config.DataTimeZone, config.ExchangeTimeZone);
BidPrice = csv[1].ToDecimal();
AskPrice = csv[2].ToDecimal();
SetValue();
break;
}
case SecurityType.Crypto:
case SecurityType.CryptoFuture:
{
TickType = config.TickType;
Exchange = config.Market;
if (TickType == TickType.Trade)
{
var csv = line.ToCsv(3);
Time = date.Date.AddTicks(Convert.ToInt64(10000 * csv[0].ToDecimal()))
.ConvertTo(config.DataTimeZone, config.ExchangeTimeZone);
Value = csv[1].ToDecimal();
Quantity = csv[2].ToDecimal();
Suspicious = csv.Count >= 4 && csv[3] == "1";
}
if (TickType == TickType.Quote)
{
var csv = line.ToCsv(6);
Time = date.Date.AddTicks(Convert.ToInt64(10000 * csv[0].ToDecimal()))
.ConvertTo(config.DataTimeZone, config.ExchangeTimeZone);
BidPrice = csv[1].ToDecimal();
BidSize = csv[2].ToDecimal();
AskPrice = csv[3].ToDecimal();
AskSize = csv[4].ToDecimal();
Suspicious = csv.Count >= 6 && csv[5] == "1";
SetValue();
}
break;
}
case SecurityType.Future:
case SecurityType.Option:
case SecurityType.FutureOption:
case SecurityType.IndexOption:
{
var csv = line.ToCsv(7);
TickType = config.TickType;
Time = date.Date.AddTicks(Convert.ToInt64(10000 * csv[0].ToDecimal()))
.ConvertTo(config.DataTimeZone, config.ExchangeTimeZone);
if (TickType == TickType.Trade)
{
Value = csv[1].ToDecimal()/scaleFactor;
Quantity = csv[2].ToDecimal();
Exchange = csv[3];
SaleCondition = csv[4];
Suspicious = csv[5] == "1";
}
else if (TickType == TickType.OpenInterest)
{
Value = csv[1].ToDecimal();
}
else
{
if (csv[1].Length != 0)
{
BidPrice = csv[1].ToDecimal()/scaleFactor;
BidSize = csv[2].ToDecimal();
}
if (csv[3].Length != 0)
{
AskPrice = csv[3].ToDecimal()/scaleFactor;
AskSize = csv[4].ToDecimal();
}
Exchange = csv[5];
Suspicious = csv[6] == "1";
SetValue();
}
break;
}
}
}
catch (Exception err)
{
Log.Error(err);
}
}
///
/// Tick implementation of reader method: read a line of data from the source and convert it to a tick object.
///
/// Subscription configuration object for algorithm
/// Line from the datafeed source
/// Date of this reader request
/// true if we're in live mode, false for backtesting mode
/// New Initialized tick
public override BaseData Reader(SubscriptionDataConfig config, string line, DateTime date, bool isLiveMode)
{
if (isLiveMode)
{
// currently ticks don't come through the reader function
return new Tick();
}
return new Tick(config, line, date);
}
///
/// Tick implementation of reader method: read a line of data from the source and convert it to a tick object.
///
/// Subscription configuration object for algorithm
/// The source stream reader
/// Date of this reader request
/// true if we're in live mode, false for backtesting mode
/// New Initialized tick
[StubsIgnore]
public override BaseData Reader(SubscriptionDataConfig config, StreamReader stream, DateTime date, bool isLiveMode)
{
if (isLiveMode)
{
// currently ticks don't come through the reader function
return new Tick();
}
return new Tick(config, stream, date);
}
///
/// Get source for tick data feed - not used with QuantConnect data sources implementation.
///
/// Configuration object
/// Date of this source request if source spread across multiple files
/// true if we're in live mode, false for backtesting mode
/// String source location of the file to be opened with a stream
public override SubscriptionDataSource GetSource(SubscriptionDataConfig config, DateTime date, bool isLiveMode)
{
if (isLiveMode)
{
// this data type is streamed in live mode
return new SubscriptionDataSource(string.Empty, SubscriptionTransportMedium.Streaming);
}
var source = LeanData.GenerateZipFilePath(Globals.DataFolder, config.Symbol, date, config.Resolution, config.TickType);
if (config.SecurityType == SecurityType.Future || config.SecurityType.IsOption())
{
source += "#" + LeanData.GenerateZipEntryName(config.Symbol, date, config.Resolution, config.TickType);
}
return new SubscriptionDataSource(source, SubscriptionTransportMedium.LocalFile, FileFormat.Csv);
}
///
/// Update the tick price information - not used.
///
/// This trade price
/// Current bid price
/// Current asking price
/// Volume of this trade
/// The size of the current bid, if available
/// The size of the current ask, if available
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public override void Update(decimal lastTrade, decimal bidPrice, decimal askPrice, decimal volume, decimal bidSize, decimal askSize)
{
Value = lastTrade;
BidPrice = bidPrice;
AskPrice = askPrice;
BidSize = bidSize;
AskSize = askSize;
Quantity = Convert.ToDecimal(volume);
}
///
/// Check if tick contains valid data (either a trade, or a bid or ask)
///
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsValid()
{
// Indexes have zero volume in live trading, but is still a valid tick.
return (TickType == TickType.Trade && (LastPrice > 0.0m && (Quantity > 0 || Symbol.SecurityType == SecurityType.Index))) ||
(TickType == TickType.Quote && AskPrice > 0.0m && AskSize > 0) ||
(TickType == TickType.Quote && BidPrice > 0.0m && BidSize > 0) ||
(TickType == TickType.OpenInterest && Value > 0);
}
///
/// Clone implementation for tick class:
///
/// New tick object clone of the current class values.
public override BaseData Clone()
{
return new Tick(this);
}
///
/// Formats a string with the symbol and value.
///
/// string - a string formatted as SPY: 167.753
public override string ToString()
{
switch (TickType)
{
case TickType.Trade:
return $"{Symbol}: Price: {Price} Quantity: {Quantity}";
case TickType.Quote:
return $"{Symbol}: Bid: {BidSize}@{BidPrice} Ask: {AskSize}@{AskPrice}";
case TickType.OpenInterest:
return $"{Symbol}: OpenInterest: {Value}";
default:
throw new ArgumentOutOfRangeException();
}
}
///
/// Sets the tick Value based on ask and bid price
///
public void SetValue()
{
Value = BidPrice + AskPrice;
if (BidPrice * AskPrice != 0)
{
Value /= 2m;
}
}
///
/// Gets the scaling factor according to the of the provided.
/// Non-equity data will not be scaled, including options with an underlying non-equity asset class.
///
/// Symbol to get scaling factor for
/// Scaling factor
private static decimal GetScaleFactor(Symbol symbol)
{
return symbol.SecurityType == SecurityType.Equity || symbol.SecurityType == SecurityType.Option ? 10000m : 1;
}
}
}