/// <summary> /// Creates a <see cref="SubscriptionDataReader"/> to read the specified request /// </summary> /// <param name="request">The subscription request to be read</param> /// <param name="dataProvider">Provider used to get data when it is not present on disk</param> /// <returns>An enumerator reading the subscription request</returns> public IEnumerator <BaseData> CreateEnumerator(SubscriptionRequest request, IDataProvider dataProvider) { var mapFileResolver = request.Configuration.TickerShouldBeMapped() ? _mapFileProvider.Get(request.Security.Symbol.ID.Market) : MapFileResolver.Empty; var dataReader = new SubscriptionDataReader(request.Configuration, request.StartTimeLocal, request.EndTimeLocal, mapFileResolver, _factorFileProvider, _tradableDaysProvider(request), _isLiveMode, _zipDataCacheProvider ); dataReader.InvalidConfigurationDetected += (sender, args) => { _resultHandler.ErrorMessage(args.Message); }; dataReader.NumericalPrecisionLimited += (sender, args) => { _resultHandler.DebugMessage(args.Message); }; dataReader.StartDateLimited += (sender, args) => { _resultHandler.DebugMessage(args.Message); }; dataReader.DownloadFailed += (sender, args) => { _resultHandler.ErrorMessage(args.Message, args.StackTrace); }; dataReader.ReaderErrorDetected += (sender, args) => { _resultHandler.RuntimeError(args.Message, args.StackTrace); }; var result = CorporateEventEnumeratorFactory.CreateEnumerators( dataReader, request.Configuration, _factorFileProvider, dataReader, mapFileResolver, _includeAuxiliaryData, request.StartTimeLocal, _enablePriceScaling); return(result); }
/// <summary> /// Gets a <see cref="FactorFile"/> instance for the specified symbol, or null if not found /// </summary> /// <param name="symbol">The security's symbol whose factor file we seek</param> /// <returns>The resolved factor file, or null if not found</returns> public FactorFile Get(Symbol symbol) { FactorFile factorFile; if (_cache.TryGetValue(symbol, out factorFile)) { return(factorFile); } var market = symbol.ID.Market; // we first need to resolve the map file to get a permtick, that's how the factor files are stored var mapFileResolver = _mapFileProvider.Get(market); if (mapFileResolver == null) { return(GetFactorFile(symbol, symbol.Value, market)); } var mapFile = mapFileResolver.ResolveMapFile(symbol.ID.Symbol, symbol.ID.Date); if (mapFile.IsNullOrEmpty()) { return(GetFactorFile(symbol, symbol.Value, market)); } return(GetFactorFile(symbol, mapFile.Permtick, market)); }
private Subscription CreateSubscription(Universe universe, IResultHandler resultHandler, Security security, DateTime startTimeUtc, DateTime endTimeUtc, IReadOnlyRef <TimeSpan> fillForwardResolution) { var config = security.SubscriptionDataConfig; var localStartTime = startTimeUtc.ConvertFromUtc(security.Exchange.TimeZone); var localEndTime = endTimeUtc.ConvertFromUtc(security.Exchange.TimeZone); var tradeableDates = Time.EachTradeableDay(security, localStartTime, localEndTime); // ReSharper disable once PossibleMultipleEnumeration if (!tradeableDates.Any()) { _algorithm.Error(string.Format("No data loaded for {0} because there were no tradeable dates for this security.", security.Symbol)); return(null); } // get the map file resolver for this market var mapFileResolver = _mapFileProvider.Get(config.Market); // ReSharper disable once PossibleMultipleEnumeration IEnumerator <BaseData> enumerator = new SubscriptionDataReader(config, localStartTime, localEndTime, resultHandler, mapFileResolver, tradeableDates, false); // optionally apply fill forward logic, but never for tick data if (config.FillDataForward && config.Resolution != Resolution.Tick) { enumerator = new FillForwardEnumerator(enumerator, security.Exchange, fillForwardResolution, security.IsExtendedMarketHours, localEndTime, config.Resolution.ToTimeSpan()); } // finally apply exchange/user filters enumerator = SubscriptionFilterEnumerator.WrapForDataFeed(resultHandler, enumerator, security, localEndTime); var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, startTimeUtc, endTimeUtc); var subscription = new Subscription(universe, security, enumerator, timeZoneOffsetProvider, startTimeUtc, endTimeUtc, false); return(subscription); }
public static MapFile GetMapFile(IMapFileProvider mapFileProvider, string market, string symbol) { var resolver = mapFileProvider.Get(market); var mapFile = resolver.ResolveMapFile(symbol, DateTime.Today); return(mapFile); }
/// <summary> /// Creates a <see cref="SubscriptionDataReader"/> to read the specified request /// </summary> /// <param name="request">The subscription request to be read</param> /// <param name="dataProvider">Provider used to get data when it is not present on disk</param> /// <returns>An enumerator reading the subscription request</returns> public IEnumerator <BaseData> CreateEnumerator(SubscriptionRequest request, IDataProvider dataProvider) { var mapFileResolver = request.Configuration.SecurityType == SecurityType.Equity || request.Configuration.SecurityType == SecurityType.Option ? _mapFileProvider.Get(request.Security.Symbol.ID.Market) : MapFileResolver.Empty; var dataReader = new SubscriptionDataReader(request.Configuration, request.StartTimeLocal, request.EndTimeLocal, mapFileResolver, _factorFileProvider, _tradableDaysProvider(request), _isLiveMode, _zipDataCacheProvider, _includeAuxiliaryData ); dataReader.InvalidConfigurationDetected += (sender, args) => { _resultHandler.ErrorMessage(args.Message); }; dataReader.NumericalPrecisionLimited += (sender, args) => { _resultHandler.DebugMessage(args.Message); }; dataReader.DownloadFailed += (sender, args) => { _resultHandler.ErrorMessage(args.Message, args.StackTrace); }; dataReader.ReaderErrorDetected += (sender, args) => { _resultHandler.RuntimeError(args.Message, args.StackTrace); }; dataReader.Initialize(); return(dataReader); }
/// <summary> /// Converts a SecurityDefinition to a <see cref="Symbol" /> /// </summary> /// <param name="securityDefinition">Security definition</param> /// <param name="tradingDate"> /// The date that the stock was being traded. This is used to resolve /// the ticker that the stock was trading under on this date. /// </param> /// <returns>Symbol if matching Lean Symbol was found on the trading date, null otherwise</returns> private Symbol SecurityDefinitionToSymbol(SecurityDefinition securityDefinition, DateTime tradingDate) { if (securityDefinition == null) { return(null); } var market = securityDefinition.SecurityIdentifier.Market; var mapFileResolver = _mapFileProvider.Get(market); // Get the first ticker the symbol traded under, and then lookup the // trading date to get the ticker on the trading date. var mapFile = mapFileResolver .ResolveMapFile(securityDefinition.SecurityIdentifier.Symbol, securityDefinition.SecurityIdentifier.Date); // The mapped ticker will be null if the map file is null or there's // no entry found for the given trading date. var mappedTicker = mapFile?.GetMappedSymbol(tradingDate, null); // If we're null, then try again; get the last entry of the map file and use // it as the Symbol we return to the caller. mappedTicker ??= mapFile? .LastOrDefault()? .MappedSymbol; return(string.IsNullOrWhiteSpace(mappedTicker) ? null : new Symbol(securityDefinition.SecurityIdentifier, mappedTicker)); }
/// <summary> /// Creates a <see cref="SubscriptionDataReader"/> to read the specified request /// </summary> /// <param name="request">The subscription request to be read</param> /// <param name="dataProvider">Provider used to get data when it is not present on disk</param> /// <returns>An enumerator reading the subscription request</returns> public IEnumerator <BaseData> CreateEnumerator(SubscriptionRequest request, IDataProvider dataProvider) { var mapFileResolver = request.Configuration.SecurityType == SecurityType.Equity || request.Configuration.SecurityType == SecurityType.Option ? _mapFileProvider.Get(request.Security.Symbol.ID.Market) : MapFileResolver.Empty; var dataReader = new SubscriptionDataReader(request.Configuration, request.StartTimeLocal, request.EndTimeLocal, mapFileResolver, _factorFileProvider, _tradableDaysProvider(request), _isLiveMode, _zipDataCacheProvider ); dataReader.InvalidConfigurationDetected += (sender, args) => { _resultHandler.ErrorMessage(args.Message); }; dataReader.NumericalPrecisionLimited += (sender, args) => { _resultHandler.DebugMessage(args.Message); }; dataReader.DownloadFailed += (sender, args) => { _resultHandler.ErrorMessage(args.Message, args.StackTrace); }; dataReader.ReaderErrorDetected += (sender, args) => { _resultHandler.RuntimeError(args.Message, args.StackTrace); }; var enumerator = CorporateEventEnumeratorFactory.CreateEnumerators( request.Configuration, _factorFileProvider, dataReader, mapFileResolver, _includeAuxiliaryData); // has to be initialized after adding all the enumerators since it will execute a MoveNext dataReader.Initialize(); return(new SynchronizingEnumerator(dataReader, enumerator)); }
/// <summary> /// Gets a <see cref="FactorFile"/> instance for the specified symbol, or null if not found /// </summary> /// <param name="symbol">The security's symbol whose factor file we seek</param> /// <returns>The resolved factor file, or null if not found</returns> public IFactorProvider Get(Symbol symbol) { symbol = symbol.GetFactorFileSymbol(); IFactorProvider factorFile; if (_cache.TryGetValue(symbol, out factorFile)) { return(factorFile); } // we first need to resolve the map file to get a permtick, that's how the factor files are stored var mapFileResolver = _mapFileProvider.Get(AuxiliaryDataKey.Create(symbol)); if (mapFileResolver == null) { return(GetFactorFile(symbol, symbol.Value)); } var mapFile = mapFileResolver.ResolveMapFile(symbol); if (mapFile.IsNullOrEmpty()) { return(GetFactorFile(symbol, symbol.Value)); } return(GetFactorFile(symbol, mapFile.Permtick)); }
private Subscription CreateSubscription(Universe universe, Security security, SubscriptionDataConfig config, DateTime startTimeUtc, DateTime endTimeUtc) { var localStartTime = startTimeUtc.ConvertFromUtc(security.Exchange.TimeZone); var localEndTime = endTimeUtc.ConvertFromUtc(security.Exchange.TimeZone); var tradeableDates = Time.EachTradeableDayInTimeZone(security.Exchange.Hours, localStartTime, localEndTime, config.DataTimeZone, config.ExtendedMarketHours); // ReSharper disable once PossibleMultipleEnumeration if (!tradeableDates.Any()) { _algorithm.Error(string.Format("No data loaded for {0} because there were no tradeable dates for this security.", security.Symbol)); return(null); } // get the map file resolver for this market var mapFileResolver = MapFileResolver.Empty; if (config.SecurityType == SecurityType.Equity) { mapFileResolver = _mapFileProvider.Get(config.Market); } // ReSharper disable once PossibleMultipleEnumeration var enumerator = CreateSubscriptionEnumerator(security, config, localStartTime, localEndTime, mapFileResolver, tradeableDates); var enqueueable = new EnqueueableEnumerator <BaseData>(true); // add this enumerator to our exchange ScheduleEnumerator(enumerator, enqueueable, GetLowerThreshold(config.Resolution), GetUpperThreshold(config.Resolution)); var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, startTimeUtc, endTimeUtc); var subscription = new Subscription(universe, security, enqueueable, timeZoneOffsetProvider, startTimeUtc, endTimeUtc, false); return(subscription); }
/// <summary> /// Helper method to resolve the mapping file to use. /// </summary> /// <remarks>This method is aware of the data type being added for <see cref="SecurityType.Base"/> /// to the <see cref="SecurityIdentifier.Symbol"/> value</remarks> /// <param name="mapFileProvider">The map file provider</param> /// <param name="dataConfig">The configuration to fetch the map file for</param> /// <returns>The mapping file to use</returns> public static MapFile ResolveMapFile(this IMapFileProvider mapFileProvider, SubscriptionDataConfig dataConfig) { var resolver = MapFileResolver.Empty; if (dataConfig.TickerShouldBeMapped()) { resolver = mapFileProvider.Get(AuxiliaryDataKey.Create(dataConfig.Symbol)); } return(resolver.ResolveMapFile(dataConfig.Symbol, dataConfig.Type.Name, dataConfig.DataMappingMode)); }
/// <summary> /// Resolves the first ticker/date of the security represented by <paramref name="tickerToday"/> /// </summary> /// <param name="mapFileProvider">The IMapFileProvider instance used for resolving map files</param> /// <param name="tickerToday">The security's ticker as it trades today</param> /// <param name="market">The market the security exists in</param> /// <param name="mappingResolveDate">The date to use to resolve the map file. Default value is <see cref="DateTime.Today"/></param> /// <returns>The security's first ticker/date if mapping data available, otherwise, the provided ticker and DefaultDate are returned</returns> private static Tuple <string, DateTime> GetFirstTickerAndDate(IMapFileProvider mapFileProvider, string tickerToday, string market, DateTime?mappingResolveDate = null) { var resolver = mapFileProvider.Get(market); var mapFile = resolver.ResolveMapFile(tickerToday, mappingResolveDate ?? DateTime.Today); // if we have mapping data, use the first ticker/date from there, otherwise use provided ticker and DefaultDate return(mapFile.Any() ? Tuple.Create(mapFile.FirstTicker, mapFile.FirstDate) : Tuple.Create(tickerToday, DefaultDate)); }
/// <summary> /// Creates the correct enumerator factory for the given request /// </summary> private ISubscriptionEnumeratorFactory GetEnumeratorFactory(SubscriptionRequest request) { if (request.IsUniverseSubscription) { if (request.Universe is UserDefinedUniverse) { // Trigger universe selection when security added/removed after Initialize var universe = (UserDefinedUniverse)request.Universe; universe.CollectionChanged += (sender, args) => { var items = args.Action == NotifyCollectionChangedAction.Add ? args.NewItems : args.Action == NotifyCollectionChangedAction.Remove ? args.OldItems : null; if (items == null || _frontierUtc == DateTime.MinValue) { return; } var symbol = items.OfType <Symbol>().FirstOrDefault(); if (symbol == null) { return; } var collection = new BaseDataCollection(_frontierUtc, symbol); var changes = _universeSelection.ApplyUniverseSelection(universe, _frontierUtc, collection); _algorithm.OnSecuritiesChanged(changes); }; return(new UserDefinedUniverseSubscriptionEnumeratorFactory(request.Universe as UserDefinedUniverse, MarketHoursDatabase.FromDataFolder())); } if (request.Configuration.Type == typeof(CoarseFundamental)) { return(new BaseDataCollectionSubscriptionEnumeratorFactory()); } if (request.Configuration.Type == typeof(FineFundamental)) { return(new FineFundamentalSubscriptionEnumeratorFactory()); } if (request.Universe is OptionChainUniverse) { return(new OptionChainUniverseSubscriptionEnumeratorFactory((req, e) => ConfigureEnumerator(req, true, e))); } } var mapFileResolver = request.Configuration.SecurityType == SecurityType.Equity ? _mapFileProvider.Get(request.Security.Symbol.ID.Market) : MapFileResolver.Empty; return(new PostCreateConfigureSubscriptionEnumeratorFactory( new SubscriptionDataReaderSubscriptionEnumeratorFactory(_resultHandler, mapFileResolver, _factorFileProvider, false, true), enumerator => ConfigureEnumerator(request, false, enumerator) )); }
private string GetMappedTicker(Symbol symbol) { var ticker = symbol.ID.Symbol; if (symbol.ID.SecurityType == SecurityType.Equity) { var mapFile = _mapFileProvider.Get(AuxiliaryDataKey.Create(symbol)).ResolveMapFile(symbol); ticker = mapFile.GetMappedSymbol(DateTime.UtcNow, symbol.Value); } return(ticker); }
private string GetMappedTicker(Symbol symbol) { var ticker = symbol.Value; if (symbol.ID.SecurityType == SecurityType.Equity) { var mapFile = _mapFileProvider.Get(symbol.ID.Market).ResolveMapFile(symbol.ID.Symbol, symbol.ID.Date); ticker = mapFile.GetMappedSymbol(DateTime.UtcNow, symbol.Value); } return(ticker); }
/// <summary> /// Creates the correct enumerator factory for the given request /// </summary> private ISubscriptionEnumeratorFactory GetEnumeratorFactory(SubscriptionRequest request) { if (request.IsUniverseSubscription) { if (request.Universe is ITimeTriggeredUniverse) { var universe = request.Universe as UserDefinedUniverse; if (universe != null) { // Trigger universe selection when security added/removed after Initialize universe.CollectionChanged += (sender, args) => { var items = args.Action == NotifyCollectionChangedAction.Add ? args.NewItems : args.Action == NotifyCollectionChangedAction.Remove ? args.OldItems : null; if (items == null) { return; } var symbol = items.OfType <Symbol>().FirstOrDefault(); if (symbol == null) { return; } var collection = new BaseDataCollection(_algorithm.UtcTime, symbol); var changes = _universeSelection.ApplyUniverseSelection(universe, _algorithm.UtcTime, collection); _algorithm.OnSecuritiesChanged(changes); }; } return(new TimeTriggeredUniverseSubscriptionEnumeratorFactory(request.Universe as ITimeTriggeredUniverse, MarketHoursDatabase.FromDataFolder())); } if (request.Configuration.Type == typeof(CoarseFundamental)) { return(new BaseDataCollectionSubscriptionEnumeratorFactory()); } if (request.Universe is OptionChainUniverse) { return(new OptionChainUniverseSubscriptionEnumeratorFactory((req, e) => ConfigureEnumerator(req, true, e), _mapFileProvider.Get(request.Security.Symbol.ID.Market), _factorFileProvider)); } if (request.Universe is FuturesChainUniverse) { return(new FuturesChainUniverseSubscriptionEnumeratorFactory((req, e) => ConfigureEnumerator(req, true, e))); } } return(_subscriptionfactory); }
/// <summary> /// Gets the primary exchange for a given security identifier /// </summary> /// <param name="securityIdentifier">The security identifier to get the primary exchange for</param> /// <returns>Returns the primary exchange or null if not found</returns> public string GetPrimaryExchange(SecurityIdentifier securityIdentifier) { PrimaryExchange primaryExchange; if (!_primaryExchangeBySid.TryGetValue(securityIdentifier, out primaryExchange)) { var mapFile = _mapFileProvider.Get(securityIdentifier.Market).ResolveMapFile(securityIdentifier.Symbol, securityIdentifier.Date); if (mapFile != null && mapFile.Any()) { primaryExchange = mapFile.Last().PrimaryExchange; } _primaryExchangeBySid[securityIdentifier] = primaryExchange; } return(primaryExchange.ToString()); }
/// <summary> /// Creates a <see cref="SubscriptionDataReader"/> to read the specified request /// </summary> /// <param name="request">The subscription request to be read</param> /// <param name="dataProvider">Provider used to get data when it is not present on disk</param> /// <returns>An enumerator reading the subscription request</returns> public IEnumerator <BaseData> CreateEnumerator(SubscriptionRequest request, IDataProvider dataProvider) { var mapFileResolver = request.Configuration.TickerShouldBeMapped() ? _mapFileProvider.Get(request.Security.Symbol.ID.Market) : MapFileResolver.Empty; var dataReader = new SubscriptionDataReader(request.Configuration, request.StartTimeLocal, request.EndTimeLocal, mapFileResolver, _factorFileProvider, _tradableDaysProvider(request), _isLiveMode, _zipDataCacheProvider, dataProvider ); dataReader.InvalidConfigurationDetected += (sender, args) => { _resultHandler.ErrorMessage(args.Message); }; dataReader.StartDateLimited += (sender, args) => { // Queue this warning into our dictionary to report on dispose if (_startDateLimitedWarnings.Count <= _startDateLimitedWarningsMaxCount) { _startDateLimitedWarnings.TryAdd(args.Symbol, args.Message); } }; dataReader.DownloadFailed += (sender, args) => { _resultHandler.ErrorMessage(args.Message, args.StackTrace); }; dataReader.ReaderErrorDetected += (sender, args) => { _resultHandler.RuntimeError(args.Message, args.StackTrace); }; dataReader.NumericalPrecisionLimited += (sender, args) => { // Set a hard limit to keep this warning list from getting unnecessarily large if (_numericalPrecisionLimitedWarnings.Count <= _numericalPrecisionLimitedWarningsMaxCount) { _numericalPrecisionLimitedWarnings.TryAdd(args.Symbol, args.Message); } }; var result = CorporateEventEnumeratorFactory.CreateEnumerators( dataReader, request.Configuration, _factorFileProvider, dataReader, mapFileResolver, request.StartTimeLocal, _enablePriceScaling); return(result); }
/// Hydrate the <see cref="_factorFiles"/> from the latest zipped factor file on disk private void HydrateFactorFileFromLatestZip(string market) { if (market != QuantConnect.Market.USA.ToLowerInvariant()) { // don't explode for other markets which request factor files and we don't have return; } // start the search with yesterday, today's file will be available tomorrow var todayNewYork = DateTime.UtcNow.ConvertFromUtc(TimeZones.NewYork).Date; var date = todayNewYork.AddDays(-1); var count = 0; do { var zipFileName = $"equity/{market}/factor_files/factor_files_{date:yyyyMMdd}.zip"; var factorFilePath = Path.Combine(Globals.DataFolder, zipFileName); // Fetch a stream for our zip from our data provider var stream = _dataProvider.Fetch(factorFilePath); // If the file was found we can read the file if (stream != null) { var mapFileResolver = _mapFileProvider.Get(market); foreach (var keyValuePair in FactorFileZipHelper.ReadFactorFileZip(stream, mapFileResolver, market)) { // we merge with existing, this will allow to hold multiple markets _factorFiles[keyValuePair.Key] = keyValuePair.Value; } stream.DisposeSafely(); Log.Trace($"LocalZipFactorFileProvider.Get({market}): Fetched factor files for: {date.ToShortDateString()} NY"); return; } // Otherwise we will search back another day Log.Debug($"LocalZipFactorFileProvider.Get(): No factor file found for date {date.ToShortDateString()}"); // prevent infinite recursion if something is wrong if (count++ > 7) { throw new InvalidOperationException($"LocalZipFactorFileProvider.Get(): Could not find any factor files going all the way back to {date}"); } date = date.AddDays(-1); }while (true); }
/// <summary> /// Converts a Lean symbol instance to an InteractiveBrokers symbol /// </summary> /// <param name="symbol">A Lean symbol instance</param> /// <returns>The InteractiveBrokers symbol</returns> public string GetBrokerageSymbol(Symbol symbol) { if (string.IsNullOrWhiteSpace(symbol?.Value)) { throw new ArgumentException("Invalid symbol: " + (symbol == null ? "null" : symbol.ToString())); } if (symbol.SecurityType != SecurityType.Equity) { throw new ArgumentException($"Invalid security type: {symbol.SecurityType}"); } var mapFile = _mapFileProvider.Get(symbol.ID.Market).ResolveMapFile(symbol.ID.Symbol, symbol.ID.Date); return(mapFile.GetMappedSymbol(DateTime.UtcNow, symbol.Value)); }
/// <summary> /// Gets the primary exchange for a given security identifier /// </summary> /// <param name="securityIdentifier">The security identifier to get the primary exchange for</param> /// <returns>Returns the primary exchange or null if not found</returns> public Exchange GetPrimaryExchange(SecurityIdentifier securityIdentifier) { Exchange primaryExchange; if (!_primaryExchangeBySid.TryGetValue(securityIdentifier, out primaryExchange)) { var mapFile = _mapFileProvider.Get(AuxiliaryDataKey.Create(securityIdentifier)) .ResolveMapFile(securityIdentifier.Symbol, securityIdentifier.Date); if (mapFile != null && mapFile.Any()) { primaryExchange = mapFile.Last().PrimaryExchange; } _primaryExchangeBySid[securityIdentifier] = primaryExchange; } return(primaryExchange); }
/// <summary> /// Gets the list of option contracts for a given underlying symbol /// </summary> /// <param name="symbol">The underlying symbol</param> /// <param name="date">The date for which to request the option chain (only used in backtesting)</param> /// <returns>The list of option contracts</returns> public virtual IEnumerable <Symbol> GetOptionContractList(Symbol symbol, DateTime date) { if (!symbol.SecurityType.HasOptions()) { if (symbol.SecurityType.IsOption() && symbol.Underlying != null) { // be user friendly and take the underlying symbol = symbol.Underlying; } else { throw new NotSupportedException($"BacktestingOptionChainProvider.GetOptionContractList(): " + $"{nameof(SecurityType.Equity)}, {nameof(SecurityType.Future)}, or {nameof(SecurityType.Index)} is expected but was {symbol.SecurityType}"); } } // Resolve any mapping before requesting option contract list for equities // Needs to be done in order for the data file key to be accurate Symbol mappedSymbol; if (symbol.RequiresMapping()) { var mapFileResolver = _mapFileProvider.Get(AuxiliaryDataKey.Create(symbol)); var mapFile = mapFileResolver.ResolveMapFile(symbol); var ticker = mapFile.GetMappedSymbol(date, symbol.Value); mappedSymbol = symbol.UpdateMappedSymbol(ticker); } else { mappedSymbol = symbol; } // create a canonical option symbol for the given underlying var canonicalSymbol = Symbol.CreateOption( mappedSymbol, mappedSymbol.ID.Market, mappedSymbol.SecurityType.DefaultOptionStyle(), default(OptionRight), 0, SecurityIdentifier.DefaultDate); return(GetSymbols(canonicalSymbol, date)); }
/// <summary> /// Helper overload that will search the mapfiles to resolve the first date. This implementation /// uses the configured <see cref="IMapFileProvider"/> via the <see cref="Composer.Instance"/> /// </summary> /// <param name="symbol">The symbol as it is known today</param> /// <param name="market">The market</param> /// <param name="mapSymbol">Specifies if symbol should be mapped using map file provider</param> /// <param name="mapFileProvider">Specifies the IMapFileProvider to use for resolving symbols, specify null to load from Composer</param> /// <returns>A new <see cref="SecurityIdentifier"/> representing the specified symbol today</returns> public static SecurityIdentifier GenerateEquity(string symbol, string market, bool mapSymbol = true, IMapFileProvider mapFileProvider = null) { if (mapSymbol) { mapFileProvider = mapFileProvider ?? Composer.Instance.GetExportedValueByTypeName <IMapFileProvider>(MapFileProviderTypeName); var resolver = mapFileProvider.Get(market); var mapFile = resolver.ResolveMapFile(symbol, DateTime.Today); var firstDate = mapFile.FirstDate; if (mapFile.Any()) { symbol = mapFile.OrderBy(x => x.Date).First().MappedSymbol; } return(GenerateEquity(firstDate, symbol, market)); } else { return(GenerateEquity(DefaultDate, symbol, market)); } }
/// <summary> /// Adds a new subscription for universe selection /// </summary> /// <param name="request">The subscription request</param> private Subscription CreateUniverseSubscription(SubscriptionRequest request) { ISubscriptionEnumeratorFactory factory = _subscriptionFactory; if (request.Universe is ITimeTriggeredUniverse) { factory = new TimeTriggeredUniverseSubscriptionEnumeratorFactory(request.Universe as ITimeTriggeredUniverse, MarketHoursDatabase.FromDataFolder(), _timeProvider); if (request.Universe is UserDefinedUniverse) { // for user defined universe we do not use a worker task, since calls to AddData can happen in any moment // and we have to be able to inject selection data points into the enumerator return(SubscriptionUtils.Create(request, factory.CreateEnumerator(request, _dataProvider))); } } if (request.Configuration.Type == typeof(CoarseFundamental)) { factory = new BaseDataCollectionSubscriptionEnumeratorFactory(); } if (request.Universe is OptionChainUniverse) { factory = new OptionChainUniverseSubscriptionEnumeratorFactory((req) => { var mapFileResolver = req.Security.Symbol.SecurityType == SecurityType.Option ? _mapFileProvider.Get(req.Configuration.Market) : null; var underlyingFactory = new BaseDataSubscriptionEnumeratorFactory(false, mapFileResolver, _factorFileProvider); return(ConfigureEnumerator(req, true, underlyingFactory.CreateEnumerator(req, _dataProvider))); }); } if (request.Universe is FuturesChainUniverse) { factory = new FuturesChainUniverseSubscriptionEnumeratorFactory((req, e) => ConfigureEnumerator(req, true, e)); } // define our data enumerator var enumerator = factory.CreateEnumerator(request, _dataProvider); return(SubscriptionUtils.CreateAndScheduleWorker(request, enumerator, _factorFileProvider, true)); }
/// Hydrate the <see cref="_factorFiles"/> from the latest zipped factor file on disk private void HydrateFactorFileFromLatestZip(AuxiliaryDataKey key) { var market = key.Market; // start the search with yesterday, today's file will be available tomorrow var todayNewYork = DateTime.UtcNow.ConvertFromUtc(TimeZones.NewYork).Date; var date = todayNewYork.AddDays(-1); var count = 0; do { var factorFilePath = FactorFileZipHelper.GetFactorFileZipFileName(market, date, key.SecurityType); // Fetch a stream for our zip from our data provider var stream = _dataProvider.Fetch(factorFilePath); // If the file was found we can read the file if (stream != null) { var mapFileResolver = _mapFileProvider.Get(key); foreach (var keyValuePair in FactorFileZipHelper.ReadFactorFileZip(stream, mapFileResolver, market, key.SecurityType)) { // we merge with existing, this will allow to hold multiple markets _factorFiles[keyValuePair.Key] = keyValuePair.Value; } stream.DisposeSafely(); Log.Trace($"LocalZipFactorFileProvider.Get({market}): Fetched factor files for: {date.ToShortDateString()} NY"); return; } // Otherwise we will search back another day Log.Debug($"LocalZipFactorFileProvider.Get(): No factor file found for date {date.ToShortDateString()}"); // prevent infinite recursion if something is wrong if (count++ > 7) { throw new InvalidOperationException($"LocalZipFactorFileProvider.Get(): Could not find any factor files going all the way back to {date}"); } date = date.AddDays(-1); }while (true); }
/// <summary> /// Helper method to create a new instance. /// Knows which security types should create one and determines the appropriate delisting event provider to use /// </summary> public static bool TryCreate(SubscriptionDataConfig dataConfig, ITimeProvider timeProvider, IDataQueueHandler dataQueueHandler, SecurityCache securityCache, IMapFileProvider mapFileProvider, out IEnumerator <BaseData> enumerator) { enumerator = null; var securityType = dataConfig.SecurityType; if (securityType == SecurityType.FutureOption || securityType == SecurityType.Future || securityType == SecurityType.Option || securityType == SecurityType.Equity) { var mapfile = mapFileProvider.Get(dataConfig.Symbol.ID.Market).ResolveMapFile(dataConfig.Symbol, dataConfig.Type); var delistingEventProvider = new DelistingEventProvider(); if (securityType == SecurityType.Equity) { delistingEventProvider = new LiveDataBasedDelistingEventProvider(dataConfig, dataQueueHandler); } enumerator = new LiveDelistingEventProviderEnumerator(timeProvider, dataConfig, securityCache, delistingEventProvider, mapfile); return(true); } return(false); }
/// <summary> /// Creates a <see cref="SubscriptionDataReader"/> to read the specified request /// </summary> /// <param name="request">The subscription request to be read</param> /// <param name="dataProvider">Provider used to get data when it is not present on disk</param> /// <returns>An enumerator reading the subscription request</returns> public IEnumerator <BaseData> CreateEnumerator(SubscriptionRequest request, IDataProvider dataProvider) { var mapFileResolver = request.Configuration.SecurityType == SecurityType.Equity || request.Configuration.SecurityType == SecurityType.Option ? _mapFileProvider.Get(request.Security.Symbol.ID.Market) : MapFileResolver.Empty; return(new SubscriptionDataReader(request.Configuration, request.StartTimeLocal, request.EndTimeLocal, _resultHandler, mapFileResolver, _factorFileProvider, _dataProvider, _tradableDaysProvider(request), _isLiveMode, _zipDataCacheProvider, _includeAuxiliaryData )); }
public FactorFile Get(Symbol symbol) { FactorFile factorFile; if (_cache.TryGetValue(symbol, out factorFile)) { return(factorFile); } var market = symbol.ID.Market; var mapFileResolver = _mapFileProvider.Get(market); if (mapFileResolver == null) { return(GetFactorFile(symbol, symbol.Value, market)); } var mapFile = mapFileResolver.ResolveMapFile(symbol.ID.Symbol, symbol.ID.Date); if (mapFile.IsNullOrEmpty()) { return(GetFactorFile(symbol, symbol.Value, market)); } return(GetFactorFile(symbol, mapFile.Permtick, market)); }
/// <summary> /// Runs this instance. /// </summary> /// <returns></returns> public bool Run() { var startTime = DateTime.UtcNow; var success = true; Log.Trace($"CoarseUniverseGeneratorProgram.ProcessDailyFolder(): Processing: {_dailyDataFolder.FullName}"); var symbolsProcessed = 0; var filesRead = 0; var dailyFilesNotFound = 0; var coarseFilesGenerated = 0; var mapFileResolver = _mapFileProvider.Get(_market); var blackListedTickers = new HashSet <string>(); if (_blackListedTickersFile.Exists) { blackListedTickers = File.ReadAllLines(_blackListedTickersFile.FullName).ToHashSet(); } var marketFolder = _dailyDataFolder.Parent; var fineFundamentalFolder = new DirectoryInfo(Path.Combine(marketFolder.FullName, "fundamental", "fine")); if (!fineFundamentalFolder.Exists) { Log.Error($"CoarseUniverseGenerator.Run(): FAIL, Fine Fundamental folder not found at {fineFundamentalFolder}! "); return(false); } var securityIdentifierContexts = PopulateSidContex(mapFileResolver, blackListedTickers); var dailyPricesByTicker = new ConcurrentDictionary <string, List <TradeBar> >(); var outputCoarseContent = new ConcurrentDictionary <DateTime, List <string> >(); var parallelOptions = new ParallelOptions { MaxDegreeOfParallelism = Environment.ProcessorCount / 2 }; try { Parallel.ForEach(securityIdentifierContexts, parallelOptions, sidContext => { var symbol = new Symbol(sidContext.SID, sidContext.LastTicker); var symbolCount = Interlocked.Increment(ref symbolsProcessed); Log.Debug($"CoarseUniverseGeneratorProgram.Run(): Processing {symbol}"); var factorFile = _factorFileProvider.Get(symbol); // Populate dailyPricesByTicker with all daily data by ticker for all tickers of this security. foreach (var ticker in sidContext.Tickers) { var dailyFile = new FileInfo(Path.Combine(_dailyDataFolder.FullName, $"{ticker}.zip")); if (!dailyFile.Exists) { Log.Error($"CoarseUniverseGeneratorProgram.Run(): {dailyFile} not found!"); Interlocked.Increment(ref dailyFilesNotFound); continue; } if (!dailyPricesByTicker.ContainsKey(ticker)) { dailyPricesByTicker.AddOrUpdate(ticker, ParseDailyFile(dailyFile)); Interlocked.Increment(ref filesRead); } } // Look for daily data for each ticker of the actual security for (int mapFileRowIndex = sidContext.MapFileRows.Length - 1; mapFileRowIndex >= 1; mapFileRowIndex--) { var ticker = sidContext.MapFileRows[mapFileRowIndex].Item2.ToLowerInvariant(); var endDate = sidContext.MapFileRows[mapFileRowIndex].Item1; var startDate = sidContext.MapFileRows[mapFileRowIndex - 1].Item1; List <TradeBar> tickerDailyData; if (!dailyPricesByTicker.TryGetValue(ticker, out tickerDailyData)) { Log.Error($"CoarseUniverseGeneratorProgram.Run(): Daily data for ticker {ticker.ToUpperInvariant()} not found!"); continue; } var tickerFineFundamentalFolder = Path.Combine(fineFundamentalFolder.FullName, ticker); var fineAvailableDates = Enumerable.Empty <DateTime>(); if (Directory.Exists(tickerFineFundamentalFolder)) { fineAvailableDates = Directory.GetFiles(tickerFineFundamentalFolder, "*.zip") .Select(f => DateTime.ParseExact(Path.GetFileNameWithoutExtension(f), DateFormat.EightCharacter, CultureInfo.InvariantCulture)) .ToList(); } // Get daily data only for the time the ticker was foreach (var tradeBar in tickerDailyData.Where(tb => tb.Time >= startDate && tb.Time <= endDate)) { var coarseRow = GenerateFactorFileRow(ticker, sidContext, factorFile, tradeBar, fineAvailableDates, fineFundamentalFolder); outputCoarseContent.AddOrUpdate(tradeBar.Time, new List <string> { coarseRow }, (time, list) => { lock (list) { list.Add(coarseRow); return(list); } }); } } if (symbolCount % 1000 == 0) { var elapsed = DateTime.UtcNow - startTime; Log.Trace($"CoarseUniverseGeneratorProgram.Run(): Processed {symbolCount} in {elapsed:g} at {symbolCount / elapsed.TotalMinutes:F2} symbols/minute "); } }); _destinationFolder.Create(); var startWriting = DateTime.UtcNow; Parallel.ForEach(outputCoarseContent, coarseByDate => { var filename = $"{coarseByDate.Key.ToString(DateFormat.EightCharacter, CultureInfo.InvariantCulture)}.csv"; var filePath = Path.Combine(_destinationFolder.FullName, filename); Log.Debug($"CoarseUniverseGeneratorProgram.Run(): Saving {filename} with {coarseByDate.Value.Count} entries."); File.WriteAllLines(filePath, coarseByDate.Value.OrderBy(cr => cr)); var filesCount = Interlocked.Increment(ref coarseFilesGenerated); if (filesCount % 1000 == 0) { var elapsed = DateTime.UtcNow - startWriting; Log.Trace($"CoarseUniverseGeneratorProgram.Run(): Processed {filesCount} in {elapsed:g} at {filesCount / elapsed.TotalSeconds:F2} files/second "); } }); Log.Trace($"\n\nTotal of {coarseFilesGenerated} coarse files generated in {DateTime.UtcNow - startTime:g}:\n" + $"\t => {filesRead} daily data files read.\n"); } catch (Exception e) { Log.Error(e, $"CoarseUniverseGeneratorProgram.Run(): FAILED!"); success = false; } return(success); }
/// <summary> /// Gets the list of option contracts for a given underlying symbol /// </summary> /// <param name="underlyingSymbol">The underlying symbol</param> /// <param name="date">The date for which to request the option chain (only used in backtesting)</param> /// <returns>The list of option contracts</returns> public IEnumerable <Symbol> GetOptionContractList(Symbol underlyingSymbol, DateTime date) { if (!underlyingSymbol.SecurityType.HasOptions()) { throw new NotSupportedException($"BacktestingOptionChainProvider.GetOptionContractList(): SecurityType.Equity, SecurityType.Future, or SecurityType.Index is expected but was {underlyingSymbol.SecurityType}"); } // Resolve any mapping before requesting option contract list for equities // Needs to be done in order for the data file key to be accurate Symbol mappedSymbol; if (underlyingSymbol.SecurityType.RequiresMapping()) { var mapFileResolver = _mapFileProvider.Get(underlyingSymbol.ID.Market); var mapFile = mapFileResolver.ResolveMapFile(underlyingSymbol.ID.Symbol, date); var ticker = mapFile.GetMappedSymbol(date, underlyingSymbol.Value); mappedSymbol = underlyingSymbol.UpdateMappedSymbol(ticker); } else { mappedSymbol = underlyingSymbol; } // build the option contract list from the open interest zip file entry names // create a canonical option symbol for the given underlying var canonicalSymbol = Symbol.CreateOption( mappedSymbol, mappedSymbol.ID.Market, mappedSymbol.SecurityType.DefaultOptionStyle(), default(OptionRight), 0, SecurityIdentifier.DefaultDate); var zipFileName = string.Empty; Stream stream = null; // In order of trust-worthiness of containing the complete option chain, OpenInterest is guaranteed // to have the complete option chain. Quotes come after open-interest // because it's also likely to contain the option chain. Trades may be // missing portions of the option chain, so we resort to it last. foreach (var tickType in new[] { TickType.OpenInterest, TickType.Quote, TickType.Trade }) { // build the zip file name and fetch it with our provider zipFileName = LeanData.GenerateZipFilePath(Globals.DataFolder, canonicalSymbol, date, Resolution.Minute, tickType); stream = _dataProvider.Fetch(zipFileName); if (stream != null) { break; } } if (stream == null) { Log.Trace($"BacktestingOptionChainProvider.GetOptionContractList(): File not found: {zipFileName}"); yield break; } // generate and return the contract symbol for each zip entry var zipEntryNames = Compression.GetZipEntryFileNames(stream); foreach (var zipEntryName in zipEntryNames) { yield return(LeanData.ReadSymbolFromZipEntry(canonicalSymbol, Resolution.Minute, zipEntryName)); } stream.DisposeSafely(); }