public void AggregatesUntilNull() { var time = new DateTime(2015, 10, 20); var underlying = Enumerable.Range(0, 5).Select(x => new Tick { Time = time }).ToList(); underlying.AddRange(new Tick[] { null, null, null }); var aggregator = new BaseDataCollectionAggregatorEnumerator(underlying.GetEnumerator(), Symbols.SPY); Assert.IsTrue(aggregator.MoveNext()); Assert.IsNotNull(aggregator.Current); Assert.AreEqual(5, aggregator.Current.Data.Count); }
public void AggregatesUntilTimeChange() { var time = new DateTime(2015, 10, 20); var underlying = Enumerable.Range(0, 5).Select(x => new Tick { Time = time }).ToList(); underlying.AddRange(Enumerable.Range(0, 5).Select(x => new Tick {Time = time.AddSeconds(1)})); var aggregator = new BaseDataCollectionAggregatorEnumerator(underlying.GetEnumerator(), "underlying"); Assert.IsTrue(aggregator.MoveNext()); Assert.IsNotNull(aggregator.Current); Assert.AreEqual(5, aggregator.Current.Data.Count); }
/// <summary> /// Creates an enumerator for the specified security/configuration /// </summary> private IEnumerator <BaseData> CreateSubscriptionEnumerator(Security security, SubscriptionDataConfig config, DateTime localStartTime, DateTime localEndTime, MapFileResolver mapFileResolver, IEnumerable <DateTime> tradeableDates, bool useSubscriptionDataReader, bool aggregate) { IEnumerator <BaseData> enumerator; if (useSubscriptionDataReader) { enumerator = new SubscriptionDataReader(config, localStartTime, localEndTime, _resultHandler, mapFileResolver, _factorFileProvider, tradeableDates, false); } else { var sourceFactory = (BaseData)Activator.CreateInstance(config.Type); enumerator = (from date in tradeableDates let source = sourceFactory.GetSource(config, date, false) let factory = SubscriptionDataSourceReader.ForSource(source, config, date, false) let entriesForDate = factory.Read(source) from entry in entriesForDate select entry).GetEnumerator(); } if (aggregate) { enumerator = new BaseDataCollectionAggregatorEnumerator(enumerator, config.Symbol); } // optionally apply fill forward logic, but never for tick data if (config.FillDataForward && config.Resolution != Resolution.Tick) { enumerator = new FillForwardEnumerator(enumerator, security.Exchange, _fillForwardResolution, security.IsExtendedMarketHours, localEndTime, config.Resolution.ToTimeSpan()); } // optionally apply exchange/user filters if (config.IsFilteredSubscription) { enumerator = SubscriptionFilterEnumerator.WrapForDataFeed(_resultHandler, enumerator, security, localEndTime); } return(enumerator); }
public void AggregatesUntilTimeChange() { var time = new DateTime(2015, 10, 20); var underlying = Enumerable.Range(0, 5).Select(x => new Tick { Time = time }).ToList(); underlying.AddRange(Enumerable.Range(0, 5).Select(x => new Tick { Time = time.AddSeconds(1) })); var aggregator = new BaseDataCollectionAggregatorEnumerator(underlying.GetEnumerator(), Symbols.SPY); Assert.IsTrue(aggregator.MoveNext()); Assert.IsNotNull(aggregator.Current); Assert.AreEqual(5, aggregator.Current.Data.Count); }
/// <summary> /// Configure the enumerator with aggregation/fill-forward/filter behaviors. Returns new instance if re-configured /// </summary> protected IEnumerator <BaseData> ConfigureEnumerator(SubscriptionRequest request, bool aggregate, IEnumerator <BaseData> enumerator) { if (aggregate) { enumerator = new BaseDataCollectionAggregatorEnumerator(enumerator, request.Configuration.Symbol); } enumerator = TryAddFillForwardEnumerator(request, enumerator, request.Configuration.FillDataForward); // optionally apply exchange/user filters if (request.Configuration.IsFilteredSubscription) { enumerator = SubscriptionFilterEnumerator.WrapForDataFeed(_resultHandler, enumerator, request.Security, request.EndTimeLocal, request.Configuration.ExtendedMarketHours, false, request.ExchangeHours); } return(enumerator); }
/// <summary> /// Configure the enumerator with aggregation/fill-forward/filter behaviors. Returns new instance if re-configured /// </summary> private IEnumerator <BaseData> ConfigureEnumerator(SubscriptionRequest request, bool aggregate, IEnumerator <BaseData> enumerator) { if (aggregate) { enumerator = new BaseDataCollectionAggregatorEnumerator(enumerator, request.Configuration.Symbol); } // optionally apply fill forward logic, but never for tick data if (request.Configuration.FillDataForward && request.Configuration.Resolution != Resolution.Tick) { enumerator = new FillForwardEnumerator(enumerator, request.Security.Exchange, _fillForwardResolution, request.Security.IsExtendedMarketHours, request.EndTimeLocal, request.Configuration.Resolution.ToTimeSpan()); } // optionally apply exchange/user filters if (request.Configuration.IsFilteredSubscription) { enumerator = SubscriptionFilterEnumerator.WrapForDataFeed(_resultHandler, enumerator, request.Security, request.EndTimeLocal); } return(enumerator); }
/// <summary> /// If required will add a new enumerator for the underlying symbol /// </summary> protected IEnumerator <BaseData> TryAppendUnderlyingEnumerator(SubscriptionRequest request, IEnumerator <BaseData> parent, Func <SubscriptionRequest, IEnumerator <BaseData> > createEnumerator) { if (request.Configuration.Symbol.SecurityType.IsOption() && request.Configuration.Symbol.HasUnderlying) { // TODO: creating this subscription request/config is bad var underlyingRequests = new SubscriptionRequest(request, isUniverseSubscription: false, configuration: new SubscriptionDataConfig(request.Configuration, symbol: request.Configuration.Symbol.Underlying, objectType: typeof(TradeBar), tickType: TickType.Trade)); var underlying = createEnumerator(underlyingRequests); underlying = new FilterEnumerator <BaseData>(underlying, data => data.DataType != MarketDataType.Auxiliary); parent = new SynchronizingBaseDataEnumerator(parent, underlying); // we aggregate both underlying and chain data parent = new BaseDataCollectionAggregatorEnumerator(parent, request.Configuration.Symbol); // only let through if underlying and chain data present parent = new FilterEnumerator <BaseData>(parent, data => (data as BaseDataCollection).Underlying != null); parent = ConfigureEnumerator(request, false, parent); } return(parent); }
/// <summary> /// Creates a new subscription for universe selection /// </summary> /// <param name="request">The subscription request</param> private Subscription CreateUniverseSubscription(SubscriptionRequest request) { Subscription subscription = null; // TODO : Consider moving the creating of universe subscriptions to a separate, testable class // grab the relevant exchange hours var config = request.Universe.Configuration; var localEndTime = request.EndTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone); var tzOffsetProvider = new TimeZoneOffsetProvider(request.Configuration.ExchangeTimeZone, request.StartTimeUtc, request.EndTimeUtc); IEnumerator <BaseData> enumerator = null; var timeTriggered = request.Universe as ITimeTriggeredUniverse; if (timeTriggered != null) { Log.Trace($"LiveTradingDataFeed.CreateUniverseSubscription(): Creating user defined universe: {config.Symbol.ID}"); // spoof a tick on the requested interval to trigger the universe selection function var enumeratorFactory = new TimeTriggeredUniverseSubscriptionEnumeratorFactory(timeTriggered, MarketHoursDatabase.FromDataFolder(), _frontierTimeProvider); enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider); enumerator = new FrontierAwareEnumerator(enumerator, _timeProvider, tzOffsetProvider); var enqueueable = new EnqueueableEnumerator <BaseData>(); _customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable)); enumerator = enqueueable; } else if (config.Type == typeof(CoarseFundamental) || config.Type == typeof(ETFConstituentData)) { Log.Trace($"LiveTradingDataFeed.CreateUniverseSubscription(): Creating {config.Type.Name} universe: {config.Symbol.ID}"); // Will try to pull data from the data folder every 10min, file with yesterdays date. // If lean is started today it will trigger initial coarse universe selection var factory = new LiveCustomDataSubscriptionEnumeratorFactory(_timeProvider, // we adjust time to the previous tradable date time => Time.GetStartTimeForTradeBars(request.Security.Exchange.Hours, time, Time.OneDay, 1, false, config.DataTimeZone), TimeSpan.FromMinutes(10) ); var enumeratorStack = factory.CreateEnumerator(request, _dataProvider); // aggregates each coarse data point into a single BaseDataCollection var aggregator = new BaseDataCollectionAggregatorEnumerator(enumeratorStack, config.Symbol, true); _customExchange.AddEnumerator(config.Symbol, aggregator); var enqueable = new EnqueueableEnumerator <BaseData>(); _customExchange.SetDataHandler(config.Symbol, data => { enqueable.Enqueue(data); subscription.OnNewDataAvailable(); }); enumerator = GetConfiguredFrontierAwareEnumerator(enqueable, tzOffsetProvider, // advance time if before 23pm or after 5am and not on Saturdays time => time.Hour < 23 && time.Hour > 5 && time.DayOfWeek != DayOfWeek.Saturday); } else if (request.Universe is OptionChainUniverse) { Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating option chain universe: " + config.Symbol.ID); Func <SubscriptionRequest, IEnumerator <BaseData> > configure = (subRequest) => { var fillForwardResolution = _subscriptions.UpdateAndGetFillForwardResolution(subRequest.Configuration); var input = Subscribe(subRequest.Configuration, (sender, args) => subscription.OnNewDataAvailable()); return(new LiveFillForwardEnumerator(_frontierTimeProvider, input, subRequest.Security.Exchange, fillForwardResolution, subRequest.Configuration.ExtendedMarketHours, localEndTime, subRequest.Configuration.Increment, subRequest.Configuration.DataTimeZone)); }; var symbolUniverse = _dataQueueHandler as IDataQueueUniverseProvider; if (symbolUniverse == null) { throw new NotSupportedException("The DataQueueHandler does not support Options."); } var enumeratorFactory = new OptionChainUniverseSubscriptionEnumeratorFactory(configure, symbolUniverse, _timeProvider); enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider); enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, tzOffsetProvider); } else if (request.Universe is FuturesChainUniverse) { Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating futures chain universe: " + config.Symbol.ID); var symbolUniverse = _dataQueueHandler as IDataQueueUniverseProvider; if (symbolUniverse == null) { throw new NotSupportedException("The DataQueueHandler does not support Futures."); } var enumeratorFactory = new FuturesChainUniverseSubscriptionEnumeratorFactory(symbolUniverse, _timeProvider); enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider); enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, tzOffsetProvider); } else { Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating custom universe: " + config.Symbol.ID); var factory = new LiveCustomDataSubscriptionEnumeratorFactory(_timeProvider); var enumeratorStack = factory.CreateEnumerator(request, _dataProvider); enumerator = new BaseDataCollectionAggregatorEnumerator(enumeratorStack, config.Symbol, liveMode: true); var enqueueable = new EnqueueableEnumerator <BaseData>(); _customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable)); enumerator = enqueueable; } // create the subscription var subscriptionDataEnumerator = new SubscriptionDataEnumerator(request.Configuration, request.Security.Exchange.Hours, tzOffsetProvider, enumerator, request.IsUniverseSubscription); subscription = new Subscription(request, subscriptionDataEnumerator, tzOffsetProvider); // send the subscription for the new symbol through to the data queuehandler if (_channelProvider.ShouldStreamSubscription(subscription.Configuration)) { Subscribe(request.Configuration, (sender, args) => subscription.OnNewDataAvailable()); } return(subscription); }
/// <summary> /// Creates a new subscription for universe selection /// </summary> /// <param name="request">The subscription request</param> private Subscription CreateUniverseSubscription(SubscriptionRequest request) { Subscription subscription = null; // TODO : Consider moving the creating of universe subscriptions to a separate, testable class // grab the relevant exchange hours var config = request.Universe.Configuration; var localEndTime = request.EndTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone); var tzOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc); IEnumerator <BaseData> enumerator; var timeTriggered = request.Universe as ITimeTriggeredUniverse; if (timeTriggered != null) { Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating user defined universe: " + config.Symbol.ToString()); // spoof a tick on the requested interval to trigger the universe selection function var enumeratorFactory = new TimeTriggeredUniverseSubscriptionEnumeratorFactory(timeTriggered, MarketHoursDatabase.FromDataFolder()); enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider); enumerator = new FrontierAwareEnumerator(enumerator, _timeProvider, tzOffsetProvider); var enqueueable = new EnqueueableEnumerator <BaseData>(); _customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable)); enumerator = enqueueable; // Trigger universe selection when security added/removed after Initialize if (timeTriggered is UserDefinedUniverse) { var userDefined = (UserDefinedUniverse)timeTriggered; userDefined.CollectionChanged += (sender, args) => { var items = args.Action == NotifyCollectionChangedAction.Add ? args.NewItems : args.Action == NotifyCollectionChangedAction.Remove ? args.OldItems : null; var currentFrontierUtcTime = _frontierTimeProvider.GetUtcNow(); if (items == null || currentFrontierUtcTime == DateTime.MinValue) { return; } var symbol = items.OfType <Symbol>().FirstOrDefault(); if (symbol == null) { return; } var collection = new BaseDataCollection(currentFrontierUtcTime, symbol); var changes = _universeSelection.ApplyUniverseSelection(userDefined, currentFrontierUtcTime, collection); _algorithm.OnSecuritiesChanged(changes); subscription.OnNewDataAvailable(); }; } } else if (config.Type == typeof(CoarseFundamental)) { Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating coarse universe: " + config.Symbol.ToString()); // we subscribe using a normalized symbol, without a random GUID, // since the ticker plant will send the coarse data using this symbol var normalizedSymbol = CoarseFundamental.CreateUniverseSymbol(config.Symbol.ID.Market, false); // since we're binding to the data queue exchange we'll need to let him // know that we expect this data _dataQueueHandler.Subscribe(_job, new[] { normalizedSymbol }); var enqueable = new EnqueueableEnumerator <BaseData>(); // We `AddDataHandler` not `Set` so we can have multiple handlers for the coarse data _exchange.AddDataHandler(normalizedSymbol, data => { enqueable.Enqueue(data); subscription.OnNewDataAvailable(); }); enumerator = GetConfiguredFrontierAwareEnumerator(enqueable, tzOffsetProvider, // advance time if before 23pm or after 5am and not on Saturdays time => time.Hour < 23 && time.Hour > 5 && time.DayOfWeek != DayOfWeek.Saturday); } else if (request.Universe is OptionChainUniverse) { Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating option chain universe: " + config.Symbol.ToString()); Func <SubscriptionRequest, IEnumerator <BaseData>, IEnumerator <BaseData> > configure = (subRequest, input) => { // we check if input enumerator is an underlying enumerator. If yes, we subscribe it to the data. var aggregator = input as TradeBarBuilderEnumerator; if (aggregator != null) { _exchange.SetDataHandler(request.Configuration.Symbol, data => { aggregator.ProcessData((Tick)data); }); } var fillForwardResolution = _subscriptions.UpdateAndGetFillForwardResolution(request.Configuration); return(new LiveFillForwardEnumerator(_frontierTimeProvider, input, request.Security.Exchange, fillForwardResolution, request.Configuration.ExtendedMarketHours, localEndTime, request.Configuration.Increment, request.Configuration.DataTimeZone, request.StartTimeLocal)); }; var symbolUniverse = _dataQueueHandler as IDataQueueUniverseProvider; if (symbolUniverse == null) { throw new NotSupportedException("The DataQueueHandler does not support Options."); } var enumeratorFactory = new OptionChainUniverseSubscriptionEnumeratorFactory(configure, symbolUniverse, _timeProvider); enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider); enumerator = GetConfiguredFrontierAwareEnumerator(enumerator, tzOffsetProvider, time => symbolUniverse.CanAdvanceTime(config.SecurityType)); } else if (request.Universe is FuturesChainUniverse) { Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating futures chain universe: " + config.Symbol.ToString()); var symbolUniverse = _dataQueueHandler as IDataQueueUniverseProvider; if (symbolUniverse == null) { throw new NotSupportedException("The DataQueueHandler does not support Futures."); } var enumeratorFactory = new FuturesChainUniverseSubscriptionEnumeratorFactory(symbolUniverse, _timeProvider); enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider); enumerator = GetConfiguredFrontierAwareEnumerator(enumerator, tzOffsetProvider, time => symbolUniverse.CanAdvanceTime(config.SecurityType)); } else { Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating custom universe: " + config.Symbol.ToString()); var factory = new LiveCustomDataSubscriptionEnumeratorFactory(_timeProvider); var enumeratorStack = factory.CreateEnumerator(request, _dataProvider); enumerator = new BaseDataCollectionAggregatorEnumerator(enumeratorStack, config.Symbol, liveMode: true); var enqueueable = new EnqueueableEnumerator <BaseData>(); _customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable)); enumerator = enqueueable; } // create the subscription var subscriptionDataEnumerator = new SubscriptionDataEnumerator(request.Configuration, request.Security.Exchange.Hours, tzOffsetProvider, enumerator); subscription = new Subscription(request, subscriptionDataEnumerator, tzOffsetProvider); return(subscription); }
/// <summary> /// Creates a new subscription for universe selection /// </summary> /// <param name="request">The subscription request</param> private Subscription CreateUniverseSubscription(SubscriptionRequest request) { // TODO : Consider moving the creating of universe subscriptions to a separate, testable class // grab the relevant exchange hours var config = request.Universe.Configuration; var localEndTime = request.EndTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone); var tzOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc); IEnumerator <BaseData> enumerator; var userDefined = request.Universe as UserDefinedUniverse; if (userDefined != null) { Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating user defined universe: " + config.Symbol.ToString()); // spoof a tick on the requested interval to trigger the universe selection function var enumeratorFactory = new UserDefinedUniverseSubscriptionEnumeratorFactory(userDefined, MarketHoursDatabase.FromDataFolder()); enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider); enumerator = new FrontierAwareEnumerator(enumerator, _timeProvider, tzOffsetProvider); var enqueueable = new EnqueueableEnumerator <BaseData>(); _customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable)); enumerator = enqueueable; // Trigger universe selection when security added/removed after Initialize userDefined.CollectionChanged += (sender, args) => { var items = args.Action == NotifyCollectionChangedAction.Add ? args.NewItems : args.Action == NotifyCollectionChangedAction.Remove ? args.OldItems : null; if (items == null || _frontierUtc == DateTime.MinValue) { return; } var symbol = items.OfType <Symbol>().FirstOrDefault(); if (symbol == null) { return; } var collection = new BaseDataCollection(_frontierUtc, symbol); var changes = _universeSelection.ApplyUniverseSelection(userDefined, _frontierUtc, collection); _algorithm.OnSecuritiesChanged(changes); }; } else if (config.Type == typeof(CoarseFundamental)) { Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating coarse universe: " + config.Symbol.ToString()); // since we're binding to the data queue exchange we'll need to let him // know that we expect this data _dataQueueHandler.Subscribe(_job, new[] { request.Security.Symbol }); var enqueable = new EnqueueableEnumerator <BaseData>(); _exchange.SetDataHandler(config.Symbol, data => { enqueable.Enqueue(data); }); enumerator = enqueable; } else if (request.Universe is OptionChainUniverse) { Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating option chain universe: " + config.Symbol.ToString()); Func <SubscriptionRequest, IEnumerator <BaseData>, IEnumerator <BaseData> > configure = (subRequest, input) => { // we check if input enumerator is an underlying enumerator. If yes, we subscribe it to the data. var aggregator = input as TradeBarBuilderEnumerator; if (aggregator != null) { _exchange.SetDataHandler(request.Configuration.Symbol, data => { aggregator.ProcessData((Tick)data); }); } var subscriptionConfigs = _subscriptions.Select(x => x.Configuration).Concat(new[] { request.Configuration }); UpdateFillForwardResolution(subscriptionConfigs); return(new LiveFillForwardEnumerator(_frontierTimeProvider, input, request.Security.Exchange, _fillForwardResolution, request.Configuration.ExtendedMarketHours, localEndTime, request.Configuration.Increment, request.Configuration.DataTimeZone)); }; var symbolUniverse = _dataQueueHandler as IDataQueueUniverseProvider; if (symbolUniverse == null) { throw new NotSupportedException("The DataQueueHandler does not support Options."); } var enumeratorFactory = new OptionChainUniverseSubscriptionEnumeratorFactory(configure, symbolUniverse, _timeProvider); enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider); enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, tzOffsetProvider); } else if (request.Universe is FuturesChainUniverse) { Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating futures chain universe: " + config.Symbol.ToString()); var symbolUniverse = _dataQueueHandler as IDataQueueUniverseProvider; if (symbolUniverse == null) { throw new NotSupportedException("The DataQueueHandler does not support Futures."); } var enumeratorFactory = new FuturesChainUniverseSubscriptionEnumeratorFactory(symbolUniverse, _timeProvider); enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider); enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, tzOffsetProvider); } else { Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating custom universe: " + config.Symbol.ToString()); var factory = new LiveCustomDataSubscriptionEnumeratorFactory(_timeProvider); var enumeratorStack = factory.CreateEnumerator(request, _dataProvider); enumerator = new BaseDataCollectionAggregatorEnumerator(enumeratorStack, config.Symbol); var enqueueable = new EnqueueableEnumerator <BaseData>(); _customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable)); enumerator = enqueueable; } // create the subscription var subscriptionDataEnumerator = SubscriptionData.Enumerator(request.Configuration, request.Security, tzOffsetProvider, enumerator); var subscription = new Subscription(request.Universe, request.Security, config, subscriptionDataEnumerator, tzOffsetProvider, request.StartTimeUtc, request.EndTimeUtc, true); return(subscription); }
/// <summary> /// Configure the enumerator with aggregation/fill-forward/filter behaviors. Returns new instance if re-configured /// </summary> private IEnumerator<BaseData> ConfigureEnumerator(SubscriptionRequest request, bool aggregate, IEnumerator<BaseData> enumerator) { if (aggregate) { enumerator = new BaseDataCollectionAggregatorEnumerator(enumerator, request.Configuration.Symbol); } // optionally apply fill forward logic, but never for tick data if (request.Configuration.FillDataForward && request.Configuration.Resolution != Resolution.Tick) { enumerator = new FillForwardEnumerator(enumerator, request.Security.Exchange, _fillForwardResolution, request.Security.IsExtendedMarketHours, request.EndTimeLocal, request.Configuration.Resolution.ToTimeSpan()); } // optionally apply exchange/user filters if (request.Configuration.IsFilteredSubscription) { enumerator = SubscriptionFilterEnumerator.WrapForDataFeed(_resultHandler, enumerator, request.Security, request.EndTimeLocal); } return enumerator; }
public void DoesNotEmitInvalidData() { var startTime = new DateTime(2014, 06, 06, 0, 0, 0); var endTime = new DateTime(2014, 06, 09, 20, 0, 0); var canonicalSymbol = Symbol.Create("AAPL", SecurityType.Option, Market.USA, "?AAPL"); var quoteCurrency = new Cash(Currencies.USD, 0, 1); var exchangeHours = MarketHoursDatabase.FromDataFolder().GetExchangeHours(Market.USA, canonicalSymbol, SecurityType.Option); var config = new SubscriptionDataConfig( typeof(ZipEntryName), canonicalSymbol, Resolution.Minute, TimeZones.Utc, TimeZones.NewYork, true, false, false, false, TickType.Quote, false, DataNormalizationMode.Raw ); var option = new Option( canonicalSymbol, exchangeHours, quoteCurrency, new OptionSymbolProperties(SymbolProperties.GetDefault(Currencies.USD)), ErrorCurrencyConverter.Instance, RegisteredSecurityDataTypesProvider.Null, new SecurityCache() ); var fillForwardResolution = Ref.CreateReadOnly(() => Resolution.Minute.ToTimeSpan()); Func <SubscriptionRequest, IEnumerator <BaseData>, IEnumerator <BaseData> > underlyingEnumeratorFunc = (req, input) => { input = new BaseDataCollectionAggregatorEnumerator(input, req.Configuration.Symbol); return(new FillForwardEnumerator( input, option.Exchange, fillForwardResolution, false, endTime, Resolution.Minute.ToTimeSpan(), TimeZones.Utc, startTime)); }; var factory = new OptionChainUniverseSubscriptionEnumeratorFactory(underlyingEnumeratorFunc, MapFileResolver.Create(Globals.DataFolder, Market.USA), new LocalDiskFactorFileProvider(new LocalDiskMapFileProvider())); var request = new SubscriptionRequest(true, null, option, config, startTime, endTime); var enumerator = factory.CreateEnumerator(request, new DefaultDataProvider()); var emittedCount = 0; foreach (var data in enumerator.AsEnumerable()) { emittedCount++; var optionData = data as OptionChainUniverseDataCollection; Assert.IsNotNull(optionData); Assert.IsNotNull(optionData.Underlying); Assert.AreNotEqual(0, optionData.Data.Count); } // 9:30 to 15:59 -> 6.5 hours * 60 => 390 minutes * 2 days = 780 Assert.AreEqual(780, emittedCount); }