/// <summary>
/// Gets the list of future contracts for a given underlying symbol
/// </summary>
/// <param name="symbol">The underlying symbol</param>
/// <param name="date">The date for which to request the future chain (only used in backtesting)</param>
/// <returns>The list of future contracts</returns>
public IEnumerable <Symbol> GetFutureContractList(Symbol symbol, DateTime date)
{
if (symbol.SecurityType != SecurityType.Future)
{
throw new NotSupportedException($"BacktestingFutureChainProvider.GetFutureContractList(): SecurityType.Future is expected but was {symbol.SecurityType}");
}
// build the future contract list from the open interest zip file entry names
// build the zip file name for open interest data
var zipFileName = LeanData.GenerateZipFilePath(Globals.DataFolder, symbol, date, Resolution.Minute, TickType.OpenInterest);
if (!File.Exists(zipFileName))
{
// lets give quote a chance - some futures do not have an open interest file
var zipFileNameQuote = LeanData.GenerateZipFilePath(Globals.DataFolder, symbol, date, Resolution.Minute, TickType.Quote);
if (!File.Exists(zipFileNameQuote))
{
Log.Error($"BacktestingFutureChainProvider.GetFutureContractList(): Failed, files not found: {zipFileName} {zipFileNameQuote}");
yield break;
}
zipFileName = zipFileNameQuote;
}
// generate and return the contract symbol for each zip entry
var zipEntryNames = Compression.GetZipEntryFileNames(zipFileName);
foreach (var zipEntryName in zipEntryNames)
{
yield return(LeanData.ReadSymbolFromZipEntry(symbol, Resolution.Minute, zipEntryName));
}
}
/// <summary>
/// Gets the list of option contracts for a given underlying symbol
/// </summary>
/// <param name="symbol">The underlying symbol</param>
/// <param name="date">The date for which to request the option chain (only used in backtesting)</param>
/// <returns>The list of option contracts</returns>
public IEnumerable <Symbol> GetOptionContractList(Symbol symbol, DateTime date)
{
if (symbol.SecurityType != SecurityType.Equity)
{
throw new NotSupportedException($"BacktestingOptionChainProvider.GetOptionContractList(): SecurityType.Equity is expected but was {symbol.SecurityType}");
}
// build the option contract list from the open interest zip file entry names
// create a canonical option symbol for the given underlying
var canonicalSymbol = Symbol.CreateOption(symbol.Value, symbol.ID.Market, default(OptionStyle), default(OptionRight), 0, SecurityIdentifier.DefaultDate);
// build the zip file name for open interest data
var zipFileName = LeanData.GenerateZipFilePath(Globals.DataFolder, canonicalSymbol, date, Resolution.Minute, TickType.OpenInterest);
if (!File.Exists(zipFileName))
{
Log.Trace($"BacktestingOptionChainProvider.GetOptionContractList(): File not found: {zipFileName}");
yield break;
}
// generate and return the contract symbol for each zip entry
var zipEntryNames = Compression.GetZipEntryFileNames(zipFileName);
foreach (var zipEntryName in zipEntryNames)
{
yield return(LeanData.ReadSymbolFromZipEntry(canonicalSymbol, Resolution.Minute, zipEntryName));
}
}
/// <summary>
/// Reader converts each line of the data source into BaseData objects. Each data type creates its own factory method, and returns a new instance of the object
/// each time it is called. The returned object is assumed to be time stamped in the config.ExchangeTimeZone.
/// </summary>
/// <param name="config">Subscription data config setup object</param>
/// <param name="line">Line of the source document</param>
/// <param name="date">Date of the requested data</param>
/// <param name="isLiveMode">true if we're in live mode, false for backtesting mode</param>
/// <returns>Instance of the T:BaseData object generated by this line of the CSV</returns>
public override BaseData Reader(SubscriptionDataConfig config, string line, DateTime date, bool isLiveMode)
{
var symbol = LeanData.ReadSymbolFromZipEntry(config.SecurityType, config.Resolution, line);
return(new ZipEntryName {
Time = date, Symbol = symbol
});
}
/// <summary>
/// Get the contract symbols associated with the given canonical symbol and date
/// </summary>
/// <param name="canonicalSymbol">The canonical symbol</param>
/// <param name="date">The date to search for</param>
protected IEnumerable <Symbol> GetSymbols(Symbol canonicalSymbol, DateTime date)
{
IEnumerable <string> entries = null;
foreach (var tickType in DataTypes)
{
// build the zip file name and fetch it with our provider
var zipFileName = LeanData.GenerateZipFilePath(Globals.DataFolder, canonicalSymbol, date, Resolution.Minute, tickType);
try
{
entries = DataCacheProvider.GetZipEntries(zipFileName);
}
catch
{
// the cache provider will throw if the file isn't available TODO: it's api should be more like TryGetZipEntries
}
if (entries != null)
{
break;
}
}
if (entries == null)
{
var mhdb = MarketHoursDatabase.FromDataFolder();
if (mhdb.TryGetEntry(canonicalSymbol.ID.Market, canonicalSymbol, canonicalSymbol.SecurityType, out var entry) && !entry.ExchangeHours.IsDateOpen(date))
{
if (!_loggedPreviousTradableDate)
{
_loggedPreviousTradableDate = true;
Log.Trace($"BacktestingCacheProvider.GetSymbols(): {date} is not a tradable date for {canonicalSymbol}. When requesting contracts " +
$" for non tradable dates, will return contracts of previous tradable date.");
}
// be user friendly, will return contracts from the previous tradable date
foreach (var symbols in GetSymbols(canonicalSymbol, Time.GetStartTimeForTradeBars(entry.ExchangeHours, date, Time.OneDay, 1, false, entry.DataTimeZone)))
{
yield return(symbols);
}
yield break;
}
if (Log.DebuggingEnabled)
{
Log.Debug($"BacktestingCacheProvider.GetSymbols(): found no source of contracts for {canonicalSymbol} for date {date.ToString(DateFormat.EightCharacter)} for any tick type");
}
yield break;
}
// generate and return the contract symbol for each zip entry
foreach (var zipEntryName in entries)
{
yield return(LeanData.ReadSymbolFromZipEntry(canonicalSymbol, Resolution.Minute, zipEntryName));
}
}
/// <summary>
/// Gets the list of option contracts for a given underlying symbol
/// </summary>
/// <param name="symbol">The underlying symbol</param>
/// <param name="date">The date for which to request the option chain (only used in backtesting)</param>
/// <returns>The list of option contracts</returns>
public IEnumerable <Symbol> GetOptionContractList(Symbol underlyingSymbol, DateTime date)
{
if (!underlyingSymbol.SecurityType.HasOptions())
{
throw new NotSupportedException($"BacktestingOptionChainProvider.GetOptionContractList(): SecurityType.Equity, SecurityType.Future, or SecurityType.Index is expected but was {underlyingSymbol.SecurityType}");
}
// build the option contract list from the open interest zip file entry names
// create a canonical option symbol for the given underlying
var canonicalSymbol = Symbol.CreateOption(
underlyingSymbol,
underlyingSymbol.ID.Market,
underlyingSymbol.SecurityType.DefaultOptionStyle(),
default(OptionRight),
0,
SecurityIdentifier.DefaultDate);
var fileExists = false;
var zipFileName = string.Empty;
// In order of trust-worthiness of containing the complete option chain, OpenInterest is guaranteed
// to have the complete option chain. Quotes come after open-interest
// because it's also likely to contain the option chain. Trades may be
// missing portions of the option chain, so we resort to it last.
foreach (var tickType in new[] { TickType.OpenInterest, TickType.Quote, TickType.Trade })
{
// build the zip file name for open interest data
zipFileName = LeanData.GenerateZipFilePath(Globals.DataFolder, canonicalSymbol, date, Resolution.Minute, tickType);
if (File.Exists(zipFileName))
{
fileExists = true;
break;
}
}
if (!fileExists)
{
Log.Trace($"BacktestingOptionChainProvider.GetOptionContractList(): File not found: {zipFileName}");
yield break;
}
// generate and return the contract symbol for each zip entry
var zipEntryNames = Compression.GetZipEntryFileNames(zipFileName);
foreach (var zipEntryName in zipEntryNames)
{
yield return(LeanData.ReadSymbolFromZipEntry(canonicalSymbol, Resolution.Minute, zipEntryName));
}
}
/// <summary>
/// Initialize a instance of LeanDataReader from a path to a zipped data file.
/// It also supports declaring the zip entry CSV file for options and futures.
/// </summary>
/// <param name="filepath">Absolute or relative path to a zipped data file, optionally the zip entry file can be declared by using '#' as separator.</param>
/// <example>
/// var dataReader = LeanDataReader("../relative/path/to/file.zip")
/// var dataReader = LeanDataReader("absolute/path/to/file.zip#zipEntry.csv")
/// </example>
public LeanDataReader(string filepath)
{
Symbol symbol;
DateTime date;
Resolution resolution;
string zipEntry = null;
var isFutureOrOption = filepath.Contains("#");
if (isFutureOrOption)
{
zipEntry = filepath.Split('#')[1];
filepath = filepath.Split('#')[0];
}
var fileInfo = new FileInfo(filepath);
if (!LeanData.TryParsePath(fileInfo.FullName, out symbol, out date, out resolution))
{
throw new ArgumentException($"File {filepath} cannot be parsed.");
}
if (isFutureOrOption)
{
symbol = LeanData.ReadSymbolFromZipEntry(symbol, resolution, zipEntry);
}
var marketHoursDataBase = MarketHoursDatabase.FromDataFolder();
var dataTimeZone = marketHoursDataBase.GetDataTimeZone(symbol.ID.Market, symbol, symbol.SecurityType);
var exchangeTimeZone = marketHoursDataBase.GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType).TimeZone;
var tickType = LeanData.GetCommonTickType(symbol.SecurityType);
var fileName = Path.GetFileNameWithoutExtension(fileInfo.Name);
if (fileName.Contains("_"))
{
tickType = (TickType)Enum.Parse(typeof(TickType), fileName.Split('_')[1], true);
}
var dataType = LeanData.GetDataType(resolution, tickType);
var config = new SubscriptionDataConfig(dataType, symbol, resolution,
dataTimeZone, exchangeTimeZone, tickType: tickType,
fillForward: false, extendedHours: true, isInternalFeed: true);
_date = date;
_zipPath = fileInfo.FullName;
_zipentry = zipEntry;
_config = config;
}
/// <summary>
/// Enumerate over the tick zip file and return a list of BaseData.
/// </summary>
/// <returns>IEnumerable of ticks</returns>
public IEnumerable <BaseData> Parse()
{
var factory = (BaseData)ObjectActivator.GetActivator(_config.Type).Invoke(new object[0]);
// for futures and options if no entry was provided we just read all
if (_zipentry == null && (_config.SecurityType == SecurityType.Future || _config.SecurityType == SecurityType.Option || _config.SecurityType == SecurityType.FutureOption))
{
foreach (var entries in Compression.Unzip(_zipPath))
{
// we get the contract symbol from the zip entry
var symbol = LeanData.ReadSymbolFromZipEntry(_config.Symbol, _config.Resolution, entries.Key);
foreach (var line in entries.Value)
{
var dataPoint = factory.Reader(_config, line, _date, false);
dataPoint.Symbol = symbol;
yield return(dataPoint);
}
}
}
else
{
ZipFile zipFile;
using (var unzipped = Compression.Unzip(_zipPath, _zipentry, out zipFile))
{
if (unzipped == null)
{
yield break;
}
string line;
while ((line = unzipped.ReadLine()) != null)
{
yield return(factory.Reader(_config, line, _date, false));
}
}
zipFile.Dispose();
}
}
/// <summary>
/// Gets the list of option contracts for a given underlying symbol
/// </summary>
/// <param name="underlyingSymbol">The underlying symbol</param>
/// <param name="date">The date for which to request the option chain (only used in backtesting)</param>
/// <returns>The list of option contracts</returns>
public IEnumerable <Symbol> GetOptionContractList(Symbol underlyingSymbol, DateTime date)
{
if (!underlyingSymbol.SecurityType.HasOptions())
{
throw new NotSupportedException($"BacktestingOptionChainProvider.GetOptionContractList(): SecurityType.Equity, SecurityType.Future, or SecurityType.Index is expected but was {underlyingSymbol.SecurityType}");
}
// Resolve any mapping before requesting option contract list for equities
// Needs to be done in order for the data file key to be accurate
Symbol mappedSymbol;
if (underlyingSymbol.SecurityType.RequiresMapping())
{
var mapFileResolver = _mapFileProvider.Get(underlyingSymbol.ID.Market);
var mapFile = mapFileResolver.ResolveMapFile(underlyingSymbol.ID.Symbol, date);
var ticker = mapFile.GetMappedSymbol(date, underlyingSymbol.Value);
mappedSymbol = underlyingSymbol.UpdateMappedSymbol(ticker);
}
else
{
mappedSymbol = underlyingSymbol;
}
// build the option contract list from the open interest zip file entry names
// create a canonical option symbol for the given underlying
var canonicalSymbol = Symbol.CreateOption(
mappedSymbol,
mappedSymbol.ID.Market,
mappedSymbol.SecurityType.DefaultOptionStyle(),
default(OptionRight),
0,
SecurityIdentifier.DefaultDate);
var zipFileName = string.Empty;
Stream stream = null;
// In order of trust-worthiness of containing the complete option chain, OpenInterest is guaranteed
// to have the complete option chain. Quotes come after open-interest
// because it's also likely to contain the option chain. Trades may be
// missing portions of the option chain, so we resort to it last.
foreach (var tickType in new[] { TickType.OpenInterest, TickType.Quote, TickType.Trade })
{
// build the zip file name and fetch it with our provider
zipFileName = LeanData.GenerateZipFilePath(Globals.DataFolder, canonicalSymbol, date, Resolution.Minute, tickType);
stream = _dataProvider.Fetch(zipFileName);
if (stream != null)
{
break;
}
}
if (stream == null)
{
Log.Trace($"BacktestingOptionChainProvider.GetOptionContractList(): File not found: {zipFileName}");
yield break;
}
// generate and return the contract symbol for each zip entry
var zipEntryNames = Compression.GetZipEntryFileNames(stream);
foreach (var zipEntryName in zipEntryNames)
{
yield return(LeanData.ReadSymbolFromZipEntry(canonicalSymbol, Resolution.Minute, zipEntryName));
}
stream.DisposeSafely();
}
/// <summary>
/// Enumerate over the tick zip file and return a list of BaseData.
/// </summary>
/// <returns>IEnumerable of ticks</returns>
public IEnumerable <BaseData> Parse()
{
if (!File.Exists(_zipPath))
{
Log.Error($"LeanDataReader.Parse(): File does not exist: {_zipPath}");
yield break;
}
var factory = (BaseData)ObjectActivator.GetActivator(_config.Type).Invoke(new object[0]);
if (_config.Type.ImplementsStreamReader())
{
using (var zip = new ZipFile(_zipPath))
{
foreach (var zipEntry in zip.Where(x => _zipentry == null || string.Equals(x.FileName, _zipentry, StringComparison.OrdinalIgnoreCase)))
{
// we get the contract symbol from the zip entry if not already provided with the zip entry
var symbol = _config.Symbol;
if (_zipentry == null && (_config.SecurityType == SecurityType.Future || _config.SecurityType.IsOption()))
{
symbol = LeanData.ReadSymbolFromZipEntry(_config.Symbol, _config.Resolution, zipEntry.FileName);
}
using (var entryReader = new StreamReader(zipEntry.OpenReader()))
{
while (!entryReader.EndOfStream)
{
var dataPoint = factory.Reader(_config, entryReader, _date, false);
dataPoint.Symbol = symbol;
yield return(dataPoint);
}
}
}
}
}
// for futures and options if no entry was provided we just read all
else if (_zipentry == null && (_config.SecurityType == SecurityType.Future || _config.SecurityType.IsOption()))
{
foreach (var entries in Compression.Unzip(_zipPath))
{
// we get the contract symbol from the zip entry
var symbol = LeanData.ReadSymbolFromZipEntry(_config.Symbol, _config.Resolution, entries.Key);
foreach (var line in entries.Value)
{
var dataPoint = factory.Reader(_config, line, _date, false);
dataPoint.Symbol = symbol;
yield return(dataPoint);
}
}
}
else
{
ZipFile zipFile;
using (var unzipped = Compression.Unzip(_zipPath, _zipentry, out zipFile))
{
if (unzipped == null)
{
yield break;
}
string line;
while ((line = unzipped.ReadLine()) != null)
{
yield return(factory.Reader(_config, line, _date, false));
}
}
zipFile.Dispose();
}
}