/// <summary> /// Creates a subscription to process the request /// </summary> private Subscription CreateSubscription(HistoryRequest request, DateTime startUtc, DateTime endUtc) { // data reader expects these values in local times var startTimeLocal = startUtc.ConvertFromUtc(request.ExchangeHours.TimeZone); var endTimeLocal = endUtc.ConvertFromUtc(request.ExchangeHours.TimeZone); var config = new SubscriptionDataConfig(request.DataType, request.Symbol, request.Resolution, request.DataTimeZone, request.ExchangeHours.TimeZone, request.FillForwardResolution.HasValue, request.IncludeExtendedMarketHours, false, request.IsCustomData, request.TickType, true, request.DataNormalizationMode ); _dataPermissionManager.AssertConfiguration(config); var security = new Security( request.ExchangeHours, config, new Cash(Currencies.NullCurrency, 0, 1m), SymbolProperties.GetDefault(Currencies.NullCurrency), ErrorCurrencyConverter.Instance, RegisteredSecurityDataTypesProvider.Null, new SecurityCache() ); var mapFileResolver = MapFileResolver.Empty; if (config.TickerShouldBeMapped()) { mapFileResolver = _mapFileProvider.Get(config.Market); var mapFile = mapFileResolver.ResolveMapFile(config.Symbol.ID.Symbol, config.Symbol.ID.Date); config.MappedSymbol = mapFile.GetMappedSymbol(startTimeLocal, config.MappedSymbol); } // Tradable dates are defined with the data time zone to access the right source var tradableDates = Time.EachTradeableDayInTimeZone(request.ExchangeHours, startTimeLocal, endTimeLocal, request.DataTimeZone, request.IncludeExtendedMarketHours); var dataReader = new SubscriptionDataReader(config, startTimeLocal, endTimeLocal, mapFileResolver, _factorFileProvider, tradableDates, false, _dataCacheProvider ); dataReader.InvalidConfigurationDetected += (sender, args) => { OnInvalidConfigurationDetected(args); }; dataReader.NumericalPrecisionLimited += (sender, args) => { OnNumericalPrecisionLimited(args); }; dataReader.StartDateLimited += (sender, args) => { OnStartDateLimited(args); }; dataReader.DownloadFailed += (sender, args) => { OnDownloadFailed(args); }; dataReader.ReaderErrorDetected += (sender, args) => { OnReaderErrorDetected(args); }; IEnumerator <BaseData> reader = dataReader; var intraday = GetIntradayDataEnumerator(dataReader, request); if (intraday != null) { // we optionally concatenate the intraday data enumerator reader = new ConcatEnumerator(true, reader, intraday); } reader = CorporateEventEnumeratorFactory.CreateEnumerators( reader, config, _factorFileProvider, dataReader, mapFileResolver, false, startTimeLocal); // optionally apply fill forward behavior if (request.FillForwardResolution.HasValue) { // copy forward Bid/Ask bars for QuoteBars if (request.DataType == typeof(QuoteBar)) { reader = new QuoteBarFillForwardEnumerator(reader); } var readOnlyRef = Ref.CreateReadOnly(() => request.FillForwardResolution.Value.ToTimeSpan()); reader = new FillForwardEnumerator(reader, security.Exchange, readOnlyRef, request.IncludeExtendedMarketHours, endTimeLocal, config.Increment, config.DataTimeZone, startTimeLocal); } // since the SubscriptionDataReader performs an any overlap condition on the trade bar's entire // range (time->end time) we can end up passing the incorrect data (too far past, possibly future), // so to combat this we deliberately filter the results from the data reader to fix these cases // which only apply to non-tick data reader = new SubscriptionFilterEnumerator(reader, security, endTimeLocal, config.ExtendedMarketHours, false); reader = new FilterEnumerator <BaseData>(reader, data => { // allow all ticks if (config.Resolution == Resolution.Tick) { return(true); } // filter out future data if (data.EndTime > endTimeLocal) { return(false); } // filter out data before the start return(data.EndTime > startTimeLocal); }); var subscriptionRequest = new SubscriptionRequest(false, null, security, config, request.StartTimeUtc, request.EndTimeUtc); if (_parallelHistoryRequestsEnabled) { return(SubscriptionUtils.CreateAndScheduleWorker(subscriptionRequest, reader, _factorFileProvider, false)); } return(SubscriptionUtils.Create(subscriptionRequest, reader)); }
/// <summary> /// Creates a subscription to process the request /// </summary> private Subscription CreateSubscription(HistoryRequest request, DateTime start, DateTime end) { // data reader expects these values in local times start = start.ConvertFromUtc(request.ExchangeHours.TimeZone); end = end.ConvertFromUtc(request.ExchangeHours.TimeZone); var config = new SubscriptionDataConfig(request.DataType, request.Symbol, request.Resolution, request.DataTimeZone, request.ExchangeHours.TimeZone, request.FillForwardResolution.HasValue, request.IncludeExtendedMarketHours, false, request.IsCustomData, request.TickType, true, request.DataNormalizationMode ); var security = new Security( request.ExchangeHours, config, new Cash(Currencies.NullCurrency, 0, 1m), SymbolProperties.GetDefault(Currencies.NullCurrency), ErrorCurrencyConverter.Instance ); var mapFileResolver = config.SecurityType == SecurityType.Equity ? _mapFileProvider.Get(config.Market) : MapFileResolver.Empty; if (config.SecurityType == SecurityType.Equity) { var mapFile = mapFileResolver.ResolveMapFile(config.Symbol.ID.Symbol, config.Symbol.ID.Date); config.MappedSymbol = mapFile.GetMappedSymbol(start, config.MappedSymbol); } var dataReader = new SubscriptionDataReader(config, start, end, mapFileResolver, _factorFileProvider, Time.EachTradeableDay(request.ExchangeHours, start, end), false, _dataCacheProvider ); dataReader.InvalidConfigurationDetected += (sender, args) => { OnInvalidConfigurationDetected(new InvalidConfigurationDetectedEventArgs(args.Message)); }; dataReader.NumericalPrecisionLimited += (sender, args) => { OnNumericalPrecisionLimited(new NumericalPrecisionLimitedEventArgs(args.Message)); }; dataReader.DownloadFailed += (sender, args) => { OnDownloadFailed(new DownloadFailedEventArgs(args.Message, args.StackTrace)); }; dataReader.ReaderErrorDetected += (sender, args) => { OnReaderErrorDetected(new ReaderErrorDetectedEventArgs(args.Message, args.StackTrace)); }; var reader = CorporateEventEnumeratorFactory.CreateEnumerators( dataReader, config, _factorFileProvider, dataReader, mapFileResolver, false); // has to be initialized after adding all the enumerators since it will execute a MoveNext dataReader.Initialize(); // optionally apply fill forward behavior if (request.FillForwardResolution.HasValue) { // copy forward Bid/Ask bars for QuoteBars if (request.DataType == typeof(QuoteBar)) { reader = new QuoteBarFillForwardEnumerator(reader); } var readOnlyRef = Ref.CreateReadOnly(() => request.FillForwardResolution.Value.ToTimeSpan()); reader = new FillForwardEnumerator(reader, security.Exchange, readOnlyRef, security.IsExtendedMarketHours, end, config.Increment, config.DataTimeZone); } // since the SubscriptionDataReader performs an any overlap condition on the trade bar's entire // range (time->end time) we can end up passing the incorrect data (too far past, possibly future), // so to combat this we deliberately filter the results from the data reader to fix these cases // which only apply to non-tick data reader = new SubscriptionFilterEnumerator(reader, security, end); reader = new FilterEnumerator <BaseData>(reader, data => { // allow all ticks if (config.Resolution == Resolution.Tick) { return(true); } // filter out future data if (data.EndTime > end) { return(false); } // filter out data before the start return(data.EndTime > start); }); var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, start, end); var subscriptionDataEnumerator = SubscriptionData.Enumerator(config, security, timeZoneOffsetProvider, reader); var subscriptionRequest = new SubscriptionRequest(false, null, security, config, start, end); return(new Subscription(subscriptionRequest, subscriptionDataEnumerator, timeZoneOffsetProvider)); }
/// <summary> /// Creates a new subscription for the specified security /// </summary> /// <param name="request">The subscription request</param> /// <returns>A new subscription instance of the specified security</returns> protected Subscription CreateDataSubscription(SubscriptionRequest request) { Subscription subscription = null; try { var localEndTime = request.EndTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone); var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc); IEnumerator <BaseData> enumerator; if (!_channelProvider.ShouldStreamSubscription(_job, request.Configuration)) { if (!Quandl.IsAuthCodeSet) { // we're not using the SubscriptionDataReader, so be sure to set the auth token here Quandl.SetAuthCode(Config.Get("quandl-auth-token")); } if (!Tiingo.IsAuthCodeSet) { // we're not using the SubscriptionDataReader, so be sure to set the auth token here Tiingo.SetAuthCode(Config.Get("tiingo-auth-token")); } if (!USEnergyAPI.IsAuthCodeSet) { // we're not using the SubscriptionDataReader, so be sure to set the auth token here USEnergyAPI.SetAuthCode(Config.Get("us-energy-information-auth-token")); } if (!FredApi.IsAuthCodeSet) { // we're not using the SubscriptionDataReader, so be sure to set the auth token here FredApi.SetAuthCode(Config.Get("fred-auth-token")); } if (!TradingEconomicsCalendar.IsAuthCodeSet) { // we're not using the SubscriptionDataReader, so be sure to set the auth token here TradingEconomicsCalendar.SetAuthCode(Config.Get("trading-economics-auth-token")); } var factory = new LiveCustomDataSubscriptionEnumeratorFactory(_timeProvider); var enumeratorStack = factory.CreateEnumerator(request, _dataProvider); _customExchange.AddEnumerator(request.Configuration.Symbol, enumeratorStack); var enqueable = new EnqueueableEnumerator <BaseData>(); _customExchange.SetDataHandler(request.Configuration.Symbol, data => { enqueable.Enqueue(data); subscription.OnNewDataAvailable(); }); enumerator = enqueable; } else { EventHandler handler = (sender, args) => subscription?.OnNewDataAvailable(); enumerator = _dataQueueHandler.Subscribe(request.Configuration, handler); if (request.Configuration.SecurityType == SecurityType.Equity && CorporateEventEnumeratorFactory.ShouldEmitAuxiliaryBaseData(request.Configuration)) { var dividends = _dataQueueHandler.Subscribe(new SubscriptionDataConfig(request.Configuration, typeof(Dividend)), handler); var splits = _dataQueueHandler.Subscribe(new SubscriptionDataConfig(request.Configuration, typeof(Split)), handler); enumerator = new LiveEquityDataSynchronizingEnumerator(_timeProvider, request.Configuration.ExchangeTimeZone, enumerator, dividends, splits); } } if (request.Configuration.FillDataForward) { var fillForwardResolution = _subscriptions.UpdateAndGetFillForwardResolution(request.Configuration); enumerator = new LiveFillForwardEnumerator(_frontierTimeProvider, enumerator, request.Security.Exchange, fillForwardResolution, request.Configuration.ExtendedMarketHours, localEndTime, request.Configuration.Increment, request.Configuration.DataTimeZone); } // define market hours and user filters to incoming data if (request.Configuration.IsFilteredSubscription) { enumerator = new SubscriptionFilterEnumerator(enumerator, request.Security, localEndTime, request.Configuration.ExtendedMarketHours, true); } // finally, make our subscriptions aware of the frontier of the data feed, prevents future data from spewing into the feed enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, timeZoneOffsetProvider); var subscriptionDataEnumerator = new SubscriptionDataEnumerator(request.Configuration, request.Security.Exchange.Hours, timeZoneOffsetProvider, enumerator); subscription = new Subscription(request, subscriptionDataEnumerator, timeZoneOffsetProvider); } catch (Exception err) { Log.Error(err); } return(subscription); }
/// <summary> /// Creates a subscription to process the request /// </summary> private Subscription CreateSubscription(HistoryRequest request) { var config = request.ToSubscriptionDataConfig(); DataPermissionManager.AssertConfiguration(config, request.StartTimeLocal, request.EndTimeLocal); var security = new Security( request.ExchangeHours, config, new Cash(Currencies.NullCurrency, 0, 1m), SymbolProperties.GetDefault(Currencies.NullCurrency), ErrorCurrencyConverter.Instance, RegisteredSecurityDataTypesProvider.Null, new SecurityCache() ); var dataReader = new SubscriptionDataReader(config, request, _mapFileProvider, _factorFileProvider, _dataCacheProvider, _dataProvider ); dataReader.InvalidConfigurationDetected += (sender, args) => { OnInvalidConfigurationDetected(args); }; dataReader.NumericalPrecisionLimited += (sender, args) => { OnNumericalPrecisionLimited(args); }; dataReader.StartDateLimited += (sender, args) => { OnStartDateLimited(args); }; dataReader.DownloadFailed += (sender, args) => { OnDownloadFailed(args); }; dataReader.ReaderErrorDetected += (sender, args) => { OnReaderErrorDetected(args); }; IEnumerator <BaseData> reader = dataReader; var intraday = GetIntradayDataEnumerator(dataReader, request); if (intraday != null) { // we optionally concatenate the intraday data enumerator reader = new ConcatEnumerator(true, reader, intraday); } reader = CorporateEventEnumeratorFactory.CreateEnumerators( reader, config, _factorFileProvider, dataReader, _mapFileProvider, request.StartTimeLocal); // optionally apply fill forward behavior if (request.FillForwardResolution.HasValue) { // copy forward Bid/Ask bars for QuoteBars if (request.DataType == typeof(QuoteBar)) { reader = new QuoteBarFillForwardEnumerator(reader); } var readOnlyRef = Ref.CreateReadOnly(() => request.FillForwardResolution.Value.ToTimeSpan()); reader = new FillForwardEnumerator(reader, security.Exchange, readOnlyRef, request.IncludeExtendedMarketHours, request.EndTimeLocal, config.Increment, config.DataTimeZone); } // since the SubscriptionDataReader performs an any overlap condition on the trade bar's entire // range (time->end time) we can end up passing the incorrect data (too far past, possibly future), // so to combat this we deliberately filter the results from the data reader to fix these cases // which only apply to non-tick data reader = new SubscriptionFilterEnumerator(reader, security, request.EndTimeLocal, config.ExtendedMarketHours, false, request.ExchangeHours); reader = new FilterEnumerator <BaseData>(reader, data => { // allow all ticks if (config.Resolution == Resolution.Tick) { return(true); } // filter out all aux data if (data.DataType == MarketDataType.Auxiliary) { return(false); } // filter out future data if (data.EndTime > request.EndTimeLocal) { return(false); } // filter out data before the start return(data.EndTime > request.StartTimeLocal); }); var subscriptionRequest = new SubscriptionRequest(false, null, security, config, request.StartTimeUtc, request.EndTimeUtc); if (_parallelHistoryRequestsEnabled) { return(SubscriptionUtils.CreateAndScheduleWorker(subscriptionRequest, reader, _factorFileProvider, false)); } return(SubscriptionUtils.Create(subscriptionRequest, reader)); }