/// <summary> /// Gets the list of future contracts for a given underlying symbol /// </summary> /// <param name="symbol">The underlying symbol</param> /// <param name="date">The date for which to request the future chain (only used in backtesting)</param> /// <returns>The list of future contracts</returns> public IEnumerable <Symbol> GetFutureContractList(Symbol symbol, DateTime date) { if (symbol.SecurityType != SecurityType.Future) { throw new NotSupportedException($"BacktestingFutureChainProvider.GetFutureContractList(): SecurityType.Future is expected but was {symbol.SecurityType}"); } // build the future contract list from the open interest zip file entry names // build the zip file name for open interest data var zipFileName = LeanData.GenerateZipFilePath(Globals.DataFolder, symbol, date, Resolution.Minute, TickType.OpenInterest); if (!File.Exists(zipFileName)) { // lets give quote a chance - some futures do not have an open interest file var zipFileNameQuote = LeanData.GenerateZipFilePath(Globals.DataFolder, symbol, date, Resolution.Minute, TickType.Quote); if (!File.Exists(zipFileNameQuote)) { Log.Error($"BacktestingFutureChainProvider.GetFutureContractList(): Failed, files not found: {zipFileName} {zipFileNameQuote}"); yield break; } zipFileName = zipFileNameQuote; } // generate and return the contract symbol for each zip entry var zipEntryNames = Compression.GetZipEntryFileNames(zipFileName); foreach (var zipEntryName in zipEntryNames) { yield return(LeanData.ReadSymbolFromZipEntry(symbol, Resolution.Minute, zipEntryName)); } }
/// <summary> /// Gets the list of option contracts for a given underlying symbol /// </summary> /// <param name="symbol">The underlying symbol</param> /// <param name="date">The date for which to request the option chain (only used in backtesting)</param> /// <returns>The list of option contracts</returns> public IEnumerable <Symbol> GetOptionContractList(Symbol symbol, DateTime date) { if (symbol.SecurityType != SecurityType.Equity) { throw new NotSupportedException($"BacktestingOptionChainProvider.GetOptionContractList(): SecurityType.Equity is expected but was {symbol.SecurityType}"); } // build the option contract list from the open interest zip file entry names // create a canonical option symbol for the given underlying var canonicalSymbol = Symbol.CreateOption(symbol.Value, symbol.ID.Market, default(OptionStyle), default(OptionRight), 0, SecurityIdentifier.DefaultDate); // build the zip file name for open interest data var zipFileName = LeanData.GenerateZipFilePath(Globals.DataFolder, canonicalSymbol, date, Resolution.Minute, TickType.OpenInterest); if (!File.Exists(zipFileName)) { Log.Trace($"BacktestingOptionChainProvider.GetOptionContractList(): File not found: {zipFileName}"); yield break; } // generate and return the contract symbol for each zip entry var zipEntryNames = Compression.GetZipEntryFileNames(zipFileName); foreach (var zipEntryName in zipEntryNames) { yield return(LeanData.ReadSymbolFromZipEntry(canonicalSymbol, Resolution.Minute, zipEntryName)); } }
/// <summary> /// Reader converts each line of the data source into BaseData objects. Each data type creates its own factory method, and returns a new instance of the object /// each time it is called. The returned object is assumed to be time stamped in the config.ExchangeTimeZone. /// </summary> /// <param name="config">Subscription data config setup object</param> /// <param name="line">Line of the source document</param> /// <param name="date">Date of the requested data</param> /// <param name="isLiveMode">true if we're in live mode, false for backtesting mode</param> /// <returns>Instance of the T:BaseData object generated by this line of the CSV</returns> public override BaseData Reader(SubscriptionDataConfig config, string line, DateTime date, bool isLiveMode) { var symbol = LeanData.ReadSymbolFromZipEntry(config.SecurityType, config.Resolution, line); return(new ZipEntryName { Time = date, Symbol = symbol }); }
/// <summary> /// Get the contract symbols associated with the given canonical symbol and date /// </summary> /// <param name="canonicalSymbol">The canonical symbol</param> /// <param name="date">The date to search for</param> protected IEnumerable <Symbol> GetSymbols(Symbol canonicalSymbol, DateTime date) { IEnumerable <string> entries = null; foreach (var tickType in DataTypes) { // build the zip file name and fetch it with our provider var zipFileName = LeanData.GenerateZipFilePath(Globals.DataFolder, canonicalSymbol, date, Resolution.Minute, tickType); try { entries = DataCacheProvider.GetZipEntries(zipFileName); } catch { // the cache provider will throw if the file isn't available TODO: it's api should be more like TryGetZipEntries } if (entries != null) { break; } } if (entries == null) { var mhdb = MarketHoursDatabase.FromDataFolder(); if (mhdb.TryGetEntry(canonicalSymbol.ID.Market, canonicalSymbol, canonicalSymbol.SecurityType, out var entry) && !entry.ExchangeHours.IsDateOpen(date)) { if (!_loggedPreviousTradableDate) { _loggedPreviousTradableDate = true; Log.Trace($"BacktestingCacheProvider.GetSymbols(): {date} is not a tradable date for {canonicalSymbol}. When requesting contracts " + $" for non tradable dates, will return contracts of previous tradable date."); } // be user friendly, will return contracts from the previous tradable date foreach (var symbols in GetSymbols(canonicalSymbol, Time.GetStartTimeForTradeBars(entry.ExchangeHours, date, Time.OneDay, 1, false, entry.DataTimeZone))) { yield return(symbols); } yield break; } if (Log.DebuggingEnabled) { Log.Debug($"BacktestingCacheProvider.GetSymbols(): found no source of contracts for {canonicalSymbol} for date {date.ToString(DateFormat.EightCharacter)} for any tick type"); } yield break; } // generate and return the contract symbol for each zip entry foreach (var zipEntryName in entries) { yield return(LeanData.ReadSymbolFromZipEntry(canonicalSymbol, Resolution.Minute, zipEntryName)); } }
/// <summary> /// Gets the list of option contracts for a given underlying symbol /// </summary> /// <param name="symbol">The underlying symbol</param> /// <param name="date">The date for which to request the option chain (only used in backtesting)</param> /// <returns>The list of option contracts</returns> public IEnumerable <Symbol> GetOptionContractList(Symbol underlyingSymbol, DateTime date) { if (!underlyingSymbol.SecurityType.HasOptions()) { throw new NotSupportedException($"BacktestingOptionChainProvider.GetOptionContractList(): SecurityType.Equity, SecurityType.Future, or SecurityType.Index is expected but was {underlyingSymbol.SecurityType}"); } // build the option contract list from the open interest zip file entry names // create a canonical option symbol for the given underlying var canonicalSymbol = Symbol.CreateOption( underlyingSymbol, underlyingSymbol.ID.Market, underlyingSymbol.SecurityType.DefaultOptionStyle(), default(OptionRight), 0, SecurityIdentifier.DefaultDate); var fileExists = false; var zipFileName = string.Empty; // In order of trust-worthiness of containing the complete option chain, OpenInterest is guaranteed // to have the complete option chain. Quotes come after open-interest // because it's also likely to contain the option chain. Trades may be // missing portions of the option chain, so we resort to it last. foreach (var tickType in new[] { TickType.OpenInterest, TickType.Quote, TickType.Trade }) { // build the zip file name for open interest data zipFileName = LeanData.GenerateZipFilePath(Globals.DataFolder, canonicalSymbol, date, Resolution.Minute, tickType); if (File.Exists(zipFileName)) { fileExists = true; break; } } if (!fileExists) { Log.Trace($"BacktestingOptionChainProvider.GetOptionContractList(): File not found: {zipFileName}"); yield break; } // generate and return the contract symbol for each zip entry var zipEntryNames = Compression.GetZipEntryFileNames(zipFileName); foreach (var zipEntryName in zipEntryNames) { yield return(LeanData.ReadSymbolFromZipEntry(canonicalSymbol, Resolution.Minute, zipEntryName)); } }
/// <summary> /// Initialize a instance of LeanDataReader from a path to a zipped data file. /// It also supports declaring the zip entry CSV file for options and futures. /// </summary> /// <param name="filepath">Absolute or relative path to a zipped data file, optionally the zip entry file can be declared by using '#' as separator.</param> /// <example> /// var dataReader = LeanDataReader("../relative/path/to/file.zip") /// var dataReader = LeanDataReader("absolute/path/to/file.zip#zipEntry.csv") /// </example> public LeanDataReader(string filepath) { Symbol symbol; DateTime date; Resolution resolution; string zipEntry = null; var isFutureOrOption = filepath.Contains("#"); if (isFutureOrOption) { zipEntry = filepath.Split('#')[1]; filepath = filepath.Split('#')[0]; } var fileInfo = new FileInfo(filepath); if (!LeanData.TryParsePath(fileInfo.FullName, out symbol, out date, out resolution)) { throw new ArgumentException($"File {filepath} cannot be parsed."); } if (isFutureOrOption) { symbol = LeanData.ReadSymbolFromZipEntry(symbol, resolution, zipEntry); } var marketHoursDataBase = MarketHoursDatabase.FromDataFolder(); var dataTimeZone = marketHoursDataBase.GetDataTimeZone(symbol.ID.Market, symbol, symbol.SecurityType); var exchangeTimeZone = marketHoursDataBase.GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType).TimeZone; var tickType = LeanData.GetCommonTickType(symbol.SecurityType); var fileName = Path.GetFileNameWithoutExtension(fileInfo.Name); if (fileName.Contains("_")) { tickType = (TickType)Enum.Parse(typeof(TickType), fileName.Split('_')[1], true); } var dataType = LeanData.GetDataType(resolution, tickType); var config = new SubscriptionDataConfig(dataType, symbol, resolution, dataTimeZone, exchangeTimeZone, tickType: tickType, fillForward: false, extendedHours: true, isInternalFeed: true); _date = date; _zipPath = fileInfo.FullName; _zipentry = zipEntry; _config = config; }
/// <summary> /// Enumerate over the tick zip file and return a list of BaseData. /// </summary> /// <returns>IEnumerable of ticks</returns> public IEnumerable <BaseData> Parse() { var factory = (BaseData)ObjectActivator.GetActivator(_config.Type).Invoke(new object[0]); // for futures and options if no entry was provided we just read all if (_zipentry == null && (_config.SecurityType == SecurityType.Future || _config.SecurityType == SecurityType.Option || _config.SecurityType == SecurityType.FutureOption)) { foreach (var entries in Compression.Unzip(_zipPath)) { // we get the contract symbol from the zip entry var symbol = LeanData.ReadSymbolFromZipEntry(_config.Symbol, _config.Resolution, entries.Key); foreach (var line in entries.Value) { var dataPoint = factory.Reader(_config, line, _date, false); dataPoint.Symbol = symbol; yield return(dataPoint); } } } else { ZipFile zipFile; using (var unzipped = Compression.Unzip(_zipPath, _zipentry, out zipFile)) { if (unzipped == null) { yield break; } string line; while ((line = unzipped.ReadLine()) != null) { yield return(factory.Reader(_config, line, _date, false)); } } zipFile.Dispose(); } }
/// <summary> /// Gets the list of option contracts for a given underlying symbol /// </summary> /// <param name="underlyingSymbol">The underlying symbol</param> /// <param name="date">The date for which to request the option chain (only used in backtesting)</param> /// <returns>The list of option contracts</returns> public IEnumerable <Symbol> GetOptionContractList(Symbol underlyingSymbol, DateTime date) { if (!underlyingSymbol.SecurityType.HasOptions()) { throw new NotSupportedException($"BacktestingOptionChainProvider.GetOptionContractList(): SecurityType.Equity, SecurityType.Future, or SecurityType.Index is expected but was {underlyingSymbol.SecurityType}"); } // Resolve any mapping before requesting option contract list for equities // Needs to be done in order for the data file key to be accurate Symbol mappedSymbol; if (underlyingSymbol.SecurityType.RequiresMapping()) { var mapFileResolver = _mapFileProvider.Get(underlyingSymbol.ID.Market); var mapFile = mapFileResolver.ResolveMapFile(underlyingSymbol.ID.Symbol, date); var ticker = mapFile.GetMappedSymbol(date, underlyingSymbol.Value); mappedSymbol = underlyingSymbol.UpdateMappedSymbol(ticker); } else { mappedSymbol = underlyingSymbol; } // build the option contract list from the open interest zip file entry names // create a canonical option symbol for the given underlying var canonicalSymbol = Symbol.CreateOption( mappedSymbol, mappedSymbol.ID.Market, mappedSymbol.SecurityType.DefaultOptionStyle(), default(OptionRight), 0, SecurityIdentifier.DefaultDate); var zipFileName = string.Empty; Stream stream = null; // In order of trust-worthiness of containing the complete option chain, OpenInterest is guaranteed // to have the complete option chain. Quotes come after open-interest // because it's also likely to contain the option chain. Trades may be // missing portions of the option chain, so we resort to it last. foreach (var tickType in new[] { TickType.OpenInterest, TickType.Quote, TickType.Trade }) { // build the zip file name and fetch it with our provider zipFileName = LeanData.GenerateZipFilePath(Globals.DataFolder, canonicalSymbol, date, Resolution.Minute, tickType); stream = _dataProvider.Fetch(zipFileName); if (stream != null) { break; } } if (stream == null) { Log.Trace($"BacktestingOptionChainProvider.GetOptionContractList(): File not found: {zipFileName}"); yield break; } // generate and return the contract symbol for each zip entry var zipEntryNames = Compression.GetZipEntryFileNames(stream); foreach (var zipEntryName in zipEntryNames) { yield return(LeanData.ReadSymbolFromZipEntry(canonicalSymbol, Resolution.Minute, zipEntryName)); } stream.DisposeSafely(); }
/// <summary> /// Enumerate over the tick zip file and return a list of BaseData. /// </summary> /// <returns>IEnumerable of ticks</returns> public IEnumerable <BaseData> Parse() { if (!File.Exists(_zipPath)) { Log.Error($"LeanDataReader.Parse(): File does not exist: {_zipPath}"); yield break; } var factory = (BaseData)ObjectActivator.GetActivator(_config.Type).Invoke(new object[0]); if (_config.Type.ImplementsStreamReader()) { using (var zip = new ZipFile(_zipPath)) { foreach (var zipEntry in zip.Where(x => _zipentry == null || string.Equals(x.FileName, _zipentry, StringComparison.OrdinalIgnoreCase))) { // we get the contract symbol from the zip entry if not already provided with the zip entry var symbol = _config.Symbol; if (_zipentry == null && (_config.SecurityType == SecurityType.Future || _config.SecurityType.IsOption())) { symbol = LeanData.ReadSymbolFromZipEntry(_config.Symbol, _config.Resolution, zipEntry.FileName); } using (var entryReader = new StreamReader(zipEntry.OpenReader())) { while (!entryReader.EndOfStream) { var dataPoint = factory.Reader(_config, entryReader, _date, false); dataPoint.Symbol = symbol; yield return(dataPoint); } } } } } // for futures and options if no entry was provided we just read all else if (_zipentry == null && (_config.SecurityType == SecurityType.Future || _config.SecurityType.IsOption())) { foreach (var entries in Compression.Unzip(_zipPath)) { // we get the contract symbol from the zip entry var symbol = LeanData.ReadSymbolFromZipEntry(_config.Symbol, _config.Resolution, entries.Key); foreach (var line in entries.Value) { var dataPoint = factory.Reader(_config, line, _date, false); dataPoint.Symbol = symbol; yield return(dataPoint); } } } else { ZipFile zipFile; using (var unzipped = Compression.Unzip(_zipPath, _zipentry, out zipFile)) { if (unzipped == null) { yield break; } string line; while ((line = unzipped.ReadLine()) != null) { yield return(factory.Reader(_config, line, _date, false)); } } zipFile.Dispose(); } }