private static MapFile GetMapFileToUse( SubscriptionDataConfig config, MapFileResolver mapFileResolver) { var mapFileToUse = new MapFile(config.Symbol.Value, new List<MapFileRow>()); // load up the map and factor files for equities, options, and custom data if (config.TickerShouldBeMapped()) { try { var mapFile = mapFileResolver.ResolveMapFile(config.Symbol, config.Type); // only take the resolved map file if it has data, otherwise we'll use the empty one we defined above if (mapFile.Any()) { mapFileToUse = mapFile; } } catch (Exception err) { Log.Error(err, "CorporateEventEnumeratorFactory.GetMapFileToUse():" + " Map File: " + config.Symbol.ID + ": "); } } return mapFileToUse; }
/// <summary> /// Helper method to create a new instance. /// Knows which security types should create one and determines the appropriate delisting event provider to use /// </summary> public static bool TryCreate(SubscriptionDataConfig dataConfig, ITimeProvider timeProvider, IDataQueueHandler dataQueueHandler, SecurityCache securityCache, IMapFileProvider mapFileProvider, IFactorFileProvider fileProvider, DateTime startTime, out IEnumerator <BaseData> enumerator) { enumerator = null; var securityType = dataConfig.SecurityType; if (securityType.IsOption() || securityType == SecurityType.Future || securityType == SecurityType.Equity) { var providers = new List <ITradableDateEventProvider> { securityType == SecurityType.Equity ? new LiveDataBasedDelistingEventProvider(dataConfig, dataQueueHandler) : new DelistingEventProvider() }; if (dataConfig.TickerShouldBeMapped()) { providers.Add(new LiveMappingEventProvider()); } enumerator = new LiveAuxiliaryDataEnumerator(dataConfig, fileProvider, mapFileProvider, providers.ToArray(), startTime, timeProvider, securityCache); } return(enumerator != null); }
/// <summary> /// Helper method to resolve the mapping file to use. /// </summary> /// <remarks>This method is aware of the data type being added for <see cref="SecurityType.Base"/> /// to the <see cref="SecurityIdentifier.Symbol"/> value</remarks> /// <param name="mapFileProvider">The map file provider</param> /// <param name="dataConfig">The configuration to fetch the map file for</param> /// <returns>The mapping file to use</returns> public static MapFile ResolveMapFile(this IMapFileProvider mapFileProvider, SubscriptionDataConfig dataConfig) { var resolver = MapFileResolver.Empty; if (dataConfig.TickerShouldBeMapped()) { resolver = mapFileProvider.Get(AuxiliaryDataKey.Create(dataConfig.Symbol)); } return(resolver.ResolveMapFile(dataConfig.Symbol, dataConfig.Type.Name, dataConfig.DataMappingMode)); }
/// <summary> /// Creates a new <see cref="AuxiliaryDataEnumerator"/> that will hold the /// corporate event providers /// </summary> /// <param name="rawDataEnumerator">The underlying raw data enumerator</param> /// <param name="config">The <see cref="SubscriptionDataConfig"/></param> /// <param name="factorFileProvider">Used for getting factor files</param> /// <param name="tradableDayNotifier">Tradable dates provider</param> /// <param name="mapFileProvider">The <see cref="MapFile"/> provider to use</param> /// <param name="startTime">Start date for the data request</param> /// <param name="enablePriceScaling">Applies price factor</param> /// <returns>The new auxiliary data enumerator</returns> public static IEnumerator <BaseData> CreateEnumerators( IEnumerator <BaseData> rawDataEnumerator, SubscriptionDataConfig config, IFactorFileProvider factorFileProvider, ITradableDatesNotifier tradableDayNotifier, IMapFileProvider mapFileProvider, DateTime startTime, bool enablePriceScaling = true) { var tradableEventProviders = new List <ITradableDateEventProvider>(); if (config.EmitSplitsAndDividends()) { tradableEventProviders.Add(new SplitEventProvider()); tradableEventProviders.Add(new DividendEventProvider()); } if (config.TickerShouldBeMapped()) { tradableEventProviders.Add(new MappingEventProvider()); } tradableEventProviders.Add(new DelistingEventProvider()); var enumerator = new AuxiliaryDataEnumerator( config, factorFileProvider, mapFileProvider, tradableEventProviders.ToArray(), tradableDayNotifier, startTime); // avoid price scaling for backtesting; calculate it directly in worker // and allow subscription to extract the the data depending on config data mode var dataEnumerator = rawDataEnumerator; if (enablePriceScaling && config.PricesShouldBeScaled()) { dataEnumerator = new PriceScaleFactorEnumerator( rawDataEnumerator, config, factorFileProvider); } return(new SynchronizingBaseDataEnumerator(dataEnumerator, enumerator)); }
/// <summary> /// Creates a subscription to process the request /// </summary> private Subscription CreateSubscription(HistoryRequest request, DateTime startUtc, DateTime endUtc) { // data reader expects these values in local times var startTimeLocal = startUtc.ConvertFromUtc(request.ExchangeHours.TimeZone); var endTimeLocal = endUtc.ConvertFromUtc(request.ExchangeHours.TimeZone); var config = new SubscriptionDataConfig(request.DataType, request.Symbol, request.Resolution, request.DataTimeZone, request.ExchangeHours.TimeZone, request.FillForwardResolution.HasValue, request.IncludeExtendedMarketHours, false, request.IsCustomData, request.TickType, true, request.DataNormalizationMode ); _dataPermissionManager.AssertConfiguration(config); var security = new Security( request.ExchangeHours, config, new Cash(Currencies.NullCurrency, 0, 1m), SymbolProperties.GetDefault(Currencies.NullCurrency), ErrorCurrencyConverter.Instance, RegisteredSecurityDataTypesProvider.Null, new SecurityCache() ); var mapFileResolver = MapFileResolver.Empty; if (config.TickerShouldBeMapped()) { mapFileResolver = _mapFileProvider.Get(config.Market); var mapFile = mapFileResolver.ResolveMapFile(config.Symbol.ID.Symbol, config.Symbol.ID.Date); config.MappedSymbol = mapFile.GetMappedSymbol(startTimeLocal, config.MappedSymbol); } // Tradable dates are defined with the data time zone to access the right source var tradableDates = Time.EachTradeableDayInTimeZone(request.ExchangeHours, startTimeLocal, endTimeLocal, request.DataTimeZone, request.IncludeExtendedMarketHours); var dataReader = new SubscriptionDataReader(config, startTimeLocal, endTimeLocal, mapFileResolver, _factorFileProvider, tradableDates, false, _dataCacheProvider ); dataReader.InvalidConfigurationDetected += (sender, args) => { OnInvalidConfigurationDetected(args); }; dataReader.NumericalPrecisionLimited += (sender, args) => { OnNumericalPrecisionLimited(args); }; dataReader.StartDateLimited += (sender, args) => { OnStartDateLimited(args); }; dataReader.DownloadFailed += (sender, args) => { OnDownloadFailed(args); }; dataReader.ReaderErrorDetected += (sender, args) => { OnReaderErrorDetected(args); }; IEnumerator <BaseData> reader = dataReader; var intraday = GetIntradayDataEnumerator(dataReader, request); if (intraday != null) { // we optionally concatenate the intraday data enumerator reader = new ConcatEnumerator(true, reader, intraday); } reader = CorporateEventEnumeratorFactory.CreateEnumerators( reader, config, _factorFileProvider, dataReader, mapFileResolver, false, startTimeLocal); // optionally apply fill forward behavior if (request.FillForwardResolution.HasValue) { // copy forward Bid/Ask bars for QuoteBars if (request.DataType == typeof(QuoteBar)) { reader = new QuoteBarFillForwardEnumerator(reader); } var readOnlyRef = Ref.CreateReadOnly(() => request.FillForwardResolution.Value.ToTimeSpan()); reader = new FillForwardEnumerator(reader, security.Exchange, readOnlyRef, request.IncludeExtendedMarketHours, endTimeLocal, config.Increment, config.DataTimeZone, startTimeLocal); } // since the SubscriptionDataReader performs an any overlap condition on the trade bar's entire // range (time->end time) we can end up passing the incorrect data (too far past, possibly future), // so to combat this we deliberately filter the results from the data reader to fix these cases // which only apply to non-tick data reader = new SubscriptionFilterEnumerator(reader, security, endTimeLocal, config.ExtendedMarketHours, false); reader = new FilterEnumerator <BaseData>(reader, data => { // allow all ticks if (config.Resolution == Resolution.Tick) { return(true); } // filter out future data if (data.EndTime > endTimeLocal) { return(false); } // filter out data before the start return(data.EndTime > startTimeLocal); }); var subscriptionRequest = new SubscriptionRequest(false, null, security, config, request.StartTimeUtc, request.EndTimeUtc); if (_parallelHistoryRequestsEnabled) { return(SubscriptionUtils.CreateAndScheduleWorker(subscriptionRequest, reader, _factorFileProvider, false)); } return(SubscriptionUtils.Create(subscriptionRequest, reader)); }
/// <summary> /// Initializes the <see cref="SubscriptionDataReader"/> instance /// </summary> /// <remarks>Should be called after all consumers of <see cref="NewTradableDate"/> event are set, /// since it will produce events.</remarks> public void Initialize() { if (_initialized) { return; } //Save the type of data we'll be getting from the source. try { _dataFactory = _config.Type.GetBaseDataInstance(); } catch (ArgumentException exception) { OnInvalidConfigurationDetected(new InvalidConfigurationDetectedEventArgs(exception.Message)); _endOfStream = true; return; } //If its quandl set the access token in data factory: var quandl = _dataFactory as Quandl; if (quandl != null) { if (!Quandl.IsAuthCodeSet) { Quandl.SetAuthCode(Config.Get("quandl-auth-token")); } } // If Tiingo data, set the access token in data factory var tiingo = _dataFactory as TiingoDailyData; if (tiingo != null) { if (!Tiingo.IsAuthCodeSet) { Tiingo.SetAuthCode(Config.Get("tiingo-auth-token")); } } // If USEnergyInformation data, set the access token in data factory var energyInformation = _dataFactory as USEnergyInformation; if (energyInformation != null) { if (!USEnergyInformation.IsAuthCodeSet) { USEnergyInformation.SetAuthCode(Config.Get("us-energy-information-auth-token")); } } _factorFile = new FactorFile(_config.Symbol.Value, new List <FactorFileRow>()); _mapFile = new MapFile(_config.Symbol.Value, new List <MapFileRow>()); // load up the map files for equities, options, and custom data if it supports it. // Only load up factor files for equities if (_config.TickerShouldBeMapped()) { try { var mapFile = _mapFileResolver.ResolveMapFile(_config.Symbol, _config.Type); // only take the resolved map file if it has data, otherwise we'll use the empty one we defined above if (mapFile.Any()) { _mapFile = mapFile; } if (!_config.IsCustomData && _config.SecurityType != SecurityType.Option) { var factorFile = _factorFileProvider.Get(_config.Symbol); _hasScaleFactors = factorFile != null; if (_hasScaleFactors) { _factorFile = factorFile; // if factor file has minimum date, update start period if before minimum date if (!_isLiveMode && _factorFile != null && _factorFile.FactorFileMinimumDate.HasValue) { if (_periodStart < _factorFile.FactorFileMinimumDate.Value) { _periodStart = _factorFile.FactorFileMinimumDate.Value; OnNumericalPrecisionLimited( new NumericalPrecisionLimitedEventArgs( $"Data for symbol {_config.Symbol.Value} has been limited due to numerical precision issues in the factor file. " + $"The starting date has been set to {_factorFile.FactorFileMinimumDate.Value.ToShortDateString()}.")); } } } } } catch (Exception err) { Log.Error(err, "Fetching Price/Map Factors: " + _config.Symbol.ID + ": "); } } // Estimate delisting date. switch (_config.Symbol.ID.SecurityType) { case SecurityType.Future: _delistingDate = _config.Symbol.ID.Date; break; case SecurityType.Option: _delistingDate = OptionSymbol.GetLastDayOfTrading(_config.Symbol); break; default: _delistingDate = _mapFile.DelistingDate; break; } // adding a day so we stop at EOD _delistingDate = _delistingDate.AddDays(1); _subscriptionFactoryEnumerator = ResolveDataEnumerator(true); _initialized = true; }
/// <summary> /// Creates a subscription to process the request /// </summary> private Subscription CreateSubscription(HistoryRequest request, DateTime start, DateTime end) { // data reader expects these values in local times start = start.ConvertFromUtc(request.ExchangeHours.TimeZone); end = end.ConvertFromUtc(request.ExchangeHours.TimeZone); var config = new SubscriptionDataConfig(request.DataType, request.Symbol, request.Resolution, request.DataTimeZone, request.ExchangeHours.TimeZone, request.FillForwardResolution.HasValue, request.IncludeExtendedMarketHours, false, request.IsCustomData, request.TickType, true, request.DataNormalizationMode ); var security = new Security( request.ExchangeHours, config, new Cash(Currencies.NullCurrency, 0, 1m), SymbolProperties.GetDefault(Currencies.NullCurrency), ErrorCurrencyConverter.Instance, RegisteredSecurityDataTypesProvider.Null ); var mapFileResolver = MapFileResolver.Empty; if (config.TickerShouldBeMapped()) { mapFileResolver = _mapFileProvider.Get(config.Market); var mapFile = mapFileResolver.ResolveMapFile(config.Symbol.ID.Symbol, config.Symbol.ID.Date); config.MappedSymbol = mapFile.GetMappedSymbol(start, config.MappedSymbol); } var dataReader = new SubscriptionDataReader(config, start, end, mapFileResolver, _factorFileProvider, Time.EachTradeableDay(request.ExchangeHours, start, end), false, _dataCacheProvider ); dataReader.InvalidConfigurationDetected += (sender, args) => { OnInvalidConfigurationDetected(new InvalidConfigurationDetectedEventArgs(args.Message)); }; dataReader.NumericalPrecisionLimited += (sender, args) => { OnNumericalPrecisionLimited(new NumericalPrecisionLimitedEventArgs(args.Message)); }; dataReader.DownloadFailed += (sender, args) => { OnDownloadFailed(new DownloadFailedEventArgs(args.Message, args.StackTrace)); }; dataReader.ReaderErrorDetected += (sender, args) => { OnReaderErrorDetected(new ReaderErrorDetectedEventArgs(args.Message, args.StackTrace)); }; var reader = CorporateEventEnumeratorFactory.CreateEnumerators( dataReader, config, _factorFileProvider, dataReader, mapFileResolver, false); // has to be initialized after adding all the enumerators since it will execute a MoveNext dataReader.Initialize(); // optionally apply fill forward behavior if (request.FillForwardResolution.HasValue) { // copy forward Bid/Ask bars for QuoteBars if (request.DataType == typeof(QuoteBar)) { reader = new QuoteBarFillForwardEnumerator(reader); } var readOnlyRef = Ref.CreateReadOnly(() => request.FillForwardResolution.Value.ToTimeSpan()); reader = new FillForwardEnumerator(reader, security.Exchange, readOnlyRef, security.IsExtendedMarketHours, end, config.Increment, config.DataTimeZone, start); } // since the SubscriptionDataReader performs an any overlap condition on the trade bar's entire // range (time->end time) we can end up passing the incorrect data (too far past, possibly future), // so to combat this we deliberately filter the results from the data reader to fix these cases // which only apply to non-tick data reader = new SubscriptionFilterEnumerator(reader, security, end); reader = new FilterEnumerator <BaseData>(reader, data => { // allow all ticks if (config.Resolution == Resolution.Tick) { return(true); } // filter out future data if (data.EndTime > end) { return(false); } // filter out data before the start return(data.EndTime > start); }); var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, start, end); var subscriptionDataEnumerator = new SubscriptionDataEnumerator(config, security.Exchange.Hours, timeZoneOffsetProvider, reader); var subscriptionRequest = new SubscriptionRequest(false, null, security, config, start, end); return(new Subscription(subscriptionRequest, subscriptionDataEnumerator, timeZoneOffsetProvider)); }