/// <summary> /// Creates an enumerator to read the specified request. /// </summary> /// <param name="request">The subscription request to be read</param> /// <param name="dataProvider">Provider used to get data when it is not present on disk</param> /// <returns>An enumerator reading the subscription request</returns> public IEnumerator <BaseData> CreateEnumerator(SubscriptionRequest request, IDataProvider dataProvider) { var config = request.Configuration; // frontier value used to prevent emitting duplicate time stamps between refreshed enumerators // also provides some immediate fast-forward to handle spooling through remote files quickly var frontier = Ref.Create(DateTime.MinValue); var lastSourceRefreshTime = DateTime.MinValue; var sourceFactory = (BaseData)ObjectActivator.GetActivator(config.Type).Invoke(new object[] { config.Type }); // this is refreshing the enumerator stack for each new source var refresher = new RefreshEnumerator <BaseData>(() => { // rate limit the refresh of this enumerator stack var utcNow = _timeProvider.GetUtcNow(); var minimumTimeBetweenCalls = GetMinimumTimeBetweenCalls(config.Increment); if (utcNow - lastSourceRefreshTime < minimumTimeBetweenCalls) { return(Enumerable.Empty <BaseData>().GetEnumerator()); } lastSourceRefreshTime = utcNow; var localDate = utcNow.ConvertFromUtc(config.ExchangeTimeZone).Date; var source = sourceFactory.GetSource(config, localDate, true); // fetch the new source and enumerate the data source reader var enumerator = EnumerateDataSourceReader(config, dataProvider, frontier, source, localDate); if (SourceRequiresFastForward(source)) { // apply fast forward logic for file transport mediums var maximumDataAge = GetMaximumDataAge(config.Increment); enumerator = new FastForwardEnumerator(enumerator, _timeProvider, config.ExchangeTimeZone, maximumDataAge); } else { // rate limit calls to this enumerator stack enumerator = new RateLimitEnumerator <BaseData>(enumerator, _timeProvider, minimumTimeBetweenCalls); } if (source.Format == FileFormat.Collection) { // unroll collections into individual data points after fast forward/rate limiting applied enumerator = enumerator.SelectMany(data => { var collection = data as BaseDataCollection; return(collection?.Data.GetEnumerator() ?? new List <BaseData> { data }.GetEnumerator()); }); } return(enumerator); }); // prevent calls to the enumerator stack if current is in the future var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc); return(new FrontierAwareEnumerator(refresher, _timeProvider, timeZoneOffsetProvider)); }
public void LimitsBasedOnTimeBetweenCalls() { var currentTime = new DateTime(2015, 10, 10, 13, 6, 0); var timeProvider = new ManualTimeProvider(currentTime, TimeZones.Utc); var data = Enumerable.Range(0, 100).Select(x => new Tick {Symbol = CreateSymbol(x)}).GetEnumerator(); var rateLimit = new RateLimitEnumerator(data, timeProvider, Time.OneSecond); Assert.IsTrue(rateLimit.MoveNext()); while (rateLimit.MoveNext() && rateLimit.Current == null) { timeProvider.AdvanceSeconds(0.1); } var delta = (timeProvider.GetUtcNow() - currentTime).TotalSeconds; Assert.AreEqual(1, delta); Assert.AreEqual("1", data.Current.Symbol.Value); }
public void LimitsBasedOnTimeBetweenCalls() { var currentTime = new DateTime(2015, 10, 10, 13, 6, 0); var timeProvider = new ManualTimeProvider(currentTime, TimeZones.Utc); var data = Enumerable.Range(0, 100).Select(x => new Tick { Symbol = x.ToString() }).GetEnumerator(); var rateLimit = new RateLimitEnumerator(data, timeProvider, Time.OneSecond); Assert.IsTrue(rateLimit.MoveNext()); while (rateLimit.MoveNext() && rateLimit.Current == null) { timeProvider.AdvanceSeconds(0.1); } var delta = (timeProvider.GetUtcNow() - currentTime).TotalSeconds; Assert.AreEqual(1, delta); Assert.AreEqual("1", data.Current.Symbol.ToString()); }
/// <summary> /// Creates a new subscription for universe selection /// </summary> /// <param name="request">The subscription request</param> private Subscription CreateUniverseSubscription(SubscriptionRequest request) { // TODO : Consider moving the creating of universe subscriptions to a separate, testable class // grab the relevant exchange hours var config = request.Universe.Configuration; var localEndTime = request.EndTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone); var tzOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc); IEnumerator <BaseData> enumerator; var userDefined = request.Universe as UserDefinedUniverse; if (userDefined != null) { Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating user defined universe: " + config.Symbol.ToString()); // spoof a tick on the requested interval to trigger the universe selection function var enumeratorFactory = new UserDefinedUniverseSubscriptionEnumeratorFactory(userDefined, MarketHoursDatabase.FromDataFolder()); enumerator = enumeratorFactory.CreateEnumerator(request, _dataFileProvider); enumerator = new FrontierAwareEnumerator(enumerator, _timeProvider, tzOffsetProvider); var enqueueable = new EnqueueableEnumerator <BaseData>(); _customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable)); enumerator = enqueueable; // Trigger universe selection when security added/removed after Initialize userDefined.CollectionChanged += (sender, args) => { var items = args.Action == NotifyCollectionChangedAction.Add ? args.NewItems : args.Action == NotifyCollectionChangedAction.Remove ? args.OldItems : null; if (items == null || _frontierUtc == DateTime.MinValue) { return; } var symbol = items.OfType <Symbol>().FirstOrDefault(); if (symbol == null) { return; } var collection = new BaseDataCollection(_frontierUtc, symbol); var changes = _universeSelection.ApplyUniverseSelection(userDefined, _frontierUtc, collection); _algorithm.OnSecuritiesChanged(changes); }; } else if (config.Type == typeof(CoarseFundamental)) { Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating coarse universe: " + config.Symbol.ToString()); // since we're binding to the data queue exchange we'll need to let him // know that we expect this data _dataQueueHandler.Subscribe(_job, new[] { request.Security.Symbol }); var enqueable = new EnqueueableEnumerator <BaseData>(); _exchange.SetDataHandler(config.Symbol, data => { enqueable.Enqueue(data); }); enumerator = enqueable; } else if (request.Universe is OptionChainUniverse) { Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating option chain universe: " + config.Symbol.ToString()); Func <SubscriptionRequest, IEnumerator <BaseData>, IEnumerator <BaseData> > configure = (subRequest, input) => { // we check if input enumerator is an underlying enumerator. If yes, we subscribe it to the data. var aggregator = input as TradeBarBuilderEnumerator; if (aggregator != null) { _exchange.SetDataHandler(request.Configuration.Symbol, data => { aggregator.ProcessData((Tick)data); }); } var subscriptionConfigs = _subscriptions.Select(x => x.Configuration).Concat(new[] { request.Configuration }); UpdateFillForwardResolution(subscriptionConfigs); return(new LiveFillForwardEnumerator(_frontierTimeProvider, input, request.Security.Exchange, _fillForwardResolution, request.Configuration.ExtendedMarketHours, localEndTime, request.Configuration.Increment)); }; var symbolUniverse = _dataQueueHandler as IDataQueueUniverseProvider; var enumeratorFactory = new OptionChainUniverseSubscriptionEnumeratorFactory(configure, symbolUniverse, _timeProvider); enumerator = enumeratorFactory.CreateEnumerator(request, _dataFileProvider); enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, tzOffsetProvider); } else if (request.Universe is FuturesChainUniverse) { Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating futures chain universe: " + config.Symbol.ToString()); var symbolUniverse = _dataQueueHandler as IDataQueueUniverseProvider; var enumeratorFactory = new FuturesChainUniverseSubscriptionEnumeratorFactory(symbolUniverse, _timeProvider); enumerator = enumeratorFactory.CreateEnumerator(request, _dataFileProvider); enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, tzOffsetProvider); } else { Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating custom universe: " + config.Symbol.ToString()); // each time we exhaust we'll new up this enumerator stack var refresher = new RefreshEnumerator <BaseDataCollection>(() => { var sourceProvider = (BaseData)Activator.CreateInstance(config.Type); var dateInDataTimeZone = DateTime.UtcNow.ConvertFromUtc(config.DataTimeZone).Date; var source = sourceProvider.GetSource(config, dateInDataTimeZone, true); var factory = SubscriptionDataSourceReader.ForSource(source, _dataFileProvider, config, dateInDataTimeZone, false); var factorEnumerator = factory.Read(source).GetEnumerator(); var fastForward = new FastForwardEnumerator(factorEnumerator, _timeProvider, request.Security.Exchange.TimeZone, config.Increment); var frontierAware = new FrontierAwareEnumerator(fastForward, _frontierTimeProvider, tzOffsetProvider); return(new BaseDataCollectionAggregatorEnumerator(frontierAware, config.Symbol)); }); // rate limit the refreshing of the stack to the requested interval var minimumTimeBetweenCalls = Math.Min(config.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks); var rateLimit = new RateLimitEnumerator(refresher, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls)); var enqueueable = new EnqueueableEnumerator <BaseData>(); _customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, rateLimit, enqueueable)); enumerator = enqueueable; } // create the subscription var subscription = new Subscription(request.Universe, request.Security, config, enumerator, tzOffsetProvider, request.StartTimeUtc, request.EndTimeUtc, true); return(subscription); }
/// <summary> /// Creates a new subscription for the specified security /// </summary> /// <param name="request">The subscription request</param> /// <returns>A new subscription instance of the specified security</returns> protected Subscription CreateSubscription(SubscriptionRequest request) { Subscription subscription = null; try { var localEndTime = request.EndTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone); var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc); IEnumerator <BaseData> enumerator; if (request.Configuration.IsCustomData) { if (!Quandl.IsAuthCodeSet) { // we're not using the SubscriptionDataReader, so be sure to set the auth token here Quandl.SetAuthCode(Config.Get("quandl-auth-token")); } // each time we exhaust we'll new up this enumerator stack var refresher = new RefreshEnumerator <BaseData>(() => { var dateInDataTimeZone = DateTime.UtcNow.ConvertFromUtc(request.Configuration.DataTimeZone).Date; var enumeratorFactory = new BaseDataSubscriptionEnumeratorFactory(r => new[] { dateInDataTimeZone }); var factoryReadEnumerator = enumeratorFactory.CreateEnumerator(request, _dataFileProvider); var maximumDataAge = TimeSpan.FromTicks(Math.Max(request.Configuration.Increment.Ticks, TimeSpan.FromSeconds(5).Ticks)); var fastForward = new FastForwardEnumerator(factoryReadEnumerator, _timeProvider, request.Security.Exchange.TimeZone, maximumDataAge); return(new FrontierAwareEnumerator(fastForward, _frontierTimeProvider, timeZoneOffsetProvider)); }); // rate limit the refreshing of the stack to the requested interval // At Tick resolution, it will refresh at full speed // At Second and Minute resolution, it will refresh every second and minute respectively // At Hour and Daily resolutions, it will refresh every 30 minutes var minimumTimeBetweenCalls = Math.Min(request.Configuration.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks); var rateLimit = new RateLimitEnumerator(refresher, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls)); _customExchange.AddEnumerator(request.Configuration.Symbol, rateLimit); var enqueable = new EnqueueableEnumerator <BaseData>(); _customExchange.SetDataHandler(request.Configuration.Symbol, data => { enqueable.Enqueue(data); if (subscription != null) { subscription.RealtimePrice = data.Value; } }); enumerator = enqueable; } else if (request.Configuration.Resolution != Resolution.Tick) { // this enumerator allows the exchange to pump ticks into the 'back' of the enumerator, // and the time sync loop can pull aggregated trade bars off the front switch (request.Configuration.TickType) { case TickType.Quote: var quoteBarAggregator = new QuoteBarBuilderEnumerator(request.Configuration.Increment, request.Security.Exchange.TimeZone, _timeProvider); _exchange.AddDataHandler(request.Configuration.Symbol, data => { var tick = data as Tick; if (tick.TickType == TickType.Quote) { quoteBarAggregator.ProcessData(tick); if (subscription != null) { subscription.RealtimePrice = data.Value; } } }); enumerator = quoteBarAggregator; break; case TickType.Trade: default: var tradeBarAggregator = new TradeBarBuilderEnumerator(request.Configuration.Increment, request.Security.Exchange.TimeZone, _timeProvider); _exchange.AddDataHandler(request.Configuration.Symbol, data => { var tick = data as Tick; if (tick.TickType == TickType.Trade) { tradeBarAggregator.ProcessData(tick); if (subscription != null) { subscription.RealtimePrice = data.Value; } } }); enumerator = tradeBarAggregator; break; case TickType.OpenInterest: var oiAggregator = new OpenInterestEnumerator(request.Configuration.Increment, request.Security.Exchange.TimeZone, _timeProvider); _exchange.AddDataHandler(request.Configuration.Symbol, data => { var tick = data as Tick; if (tick.TickType == TickType.OpenInterest) { oiAggregator.ProcessData(tick); } }); enumerator = oiAggregator; break; } } else { // tick subscriptions can pass right through var tickEnumerator = new EnqueueableEnumerator <BaseData>(); _exchange.SetDataHandler(request.Configuration.Symbol, data => { tickEnumerator.Enqueue(data); if (subscription != null) { subscription.RealtimePrice = data.Value; } }); enumerator = tickEnumerator; } if (request.Configuration.FillDataForward) { var subscriptionConfigs = _subscriptions.Select(x => x.Configuration).Concat(new[] { request.Configuration }); UpdateFillForwardResolution(subscriptionConfigs); enumerator = new LiveFillForwardEnumerator(_frontierTimeProvider, enumerator, request.Security.Exchange, _fillForwardResolution, request.Configuration.ExtendedMarketHours, localEndTime, request.Configuration.Increment); } // define market hours and user filters to incoming data if (request.Configuration.IsFilteredSubscription) { enumerator = new SubscriptionFilterEnumerator(enumerator, request.Security, localEndTime); } // finally, make our subscriptions aware of the frontier of the data feed, prevents future data from spewing into the feed enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, timeZoneOffsetProvider); subscription = new Subscription(request.Universe, request.Security, request.Configuration, enumerator, timeZoneOffsetProvider, request.StartTimeUtc, request.EndTimeUtc, false); } catch (Exception err) { Log.Error(err); } return(subscription); }
/// <summary> /// Creates a new subscription for universe selection /// </summary> /// <param name="request">The subscription request</param> private Subscription CreateUniverseSubscription(SubscriptionRequest request) { // TODO : Consider moving the creating of universe subscriptions to a separate, testable class // grab the relevant exchange hours var config = request.Universe.Configuration; var tzOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc); IEnumerator<BaseData> enumerator; var userDefined = request.Universe as UserDefinedUniverse; if (userDefined != null) { Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating user defined universe: " + config.Symbol.ToString()); // spoof a tick on the requested interval to trigger the universe selection function var enumeratorFactory = new UserDefinedUniverseSubscriptionEnumeratorFactory(userDefined, MarketHoursDatabase.FromDataFolder()); enumerator = enumeratorFactory.CreateEnumerator(request); enumerator = new FrontierAwareEnumerator(enumerator, _timeProvider, tzOffsetProvider); var enqueueable = new EnqueueableEnumerator<BaseData>(); _customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable)); enumerator = enqueueable; // Trigger universe selection when security added/removed after Initialize userDefined.CollectionChanged += (sender, args) => { var items = args.Action == NotifyCollectionChangedAction.Add ? args.NewItems : args.Action == NotifyCollectionChangedAction.Remove ? args.OldItems : null; if (items == null || _frontierUtc == DateTime.MinValue) return; var symbol = items.OfType<Symbol>().FirstOrDefault(); if (symbol == null) return; var collection = new BaseDataCollection(_frontierUtc, symbol); var changes = _universeSelection.ApplyUniverseSelection(userDefined, _frontierUtc, collection); _algorithm.OnSecuritiesChanged(changes); }; } else if (config.Type == typeof (CoarseFundamental)) { Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating coarse universe: " + config.Symbol.ToString()); // since we're binding to the data queue exchange we'll need to let him // know that we expect this data _dataQueueHandler.Subscribe(_job, new[] {request.Security.Symbol}); var enqueable = new EnqueueableEnumerator<BaseData>(); _exchange.SetDataHandler(config.Symbol, data => { enqueable.Enqueue(data); }); enumerator = enqueable; } else { Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating custom universe: " + config.Symbol.ToString()); // each time we exhaust we'll new up this enumerator stack var refresher = new RefreshEnumerator<BaseDataCollection>(() => { var sourceProvider = (BaseData)Activator.CreateInstance(config.Type); var dateInDataTimeZone = DateTime.UtcNow.ConvertFromUtc(config.DataTimeZone).Date; var source = sourceProvider.GetSource(config, dateInDataTimeZone, true); var factory = SubscriptionDataSourceReader.ForSource(source, config, dateInDataTimeZone, false); var factorEnumerator = factory.Read(source).GetEnumerator(); var fastForward = new FastForwardEnumerator(factorEnumerator, _timeProvider, request.Security.Exchange.TimeZone, config.Increment); var frontierAware = new FrontierAwareEnumerator(fastForward, _frontierTimeProvider, tzOffsetProvider); return new BaseDataCollectionAggregatorEnumerator(frontierAware, config.Symbol); }); // rate limit the refreshing of the stack to the requested interval var minimumTimeBetweenCalls = Math.Min(config.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks); var rateLimit = new RateLimitEnumerator(refresher, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls)); var enqueueable = new EnqueueableEnumerator<BaseData>(); _customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, rateLimit, enqueueable)); enumerator = enqueueable; } // create the subscription var subscription = new Subscription(request.Universe, request.Security, config, enumerator, tzOffsetProvider, request.StartTimeUtc, request.EndTimeUtc, true); return subscription; }
/// <summary> /// Creates a new subscription for the specified security /// </summary> /// <param name="request">The subscription request</param> /// <returns>A new subscription instance of the specified security</returns> protected Subscription CreateSubscription(SubscriptionRequest request) { Subscription subscription = null; try { var localEndTime = request.EndTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone); var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc); IEnumerator<BaseData> enumerator; if (request.Configuration.IsCustomData) { if (!Quandl.IsAuthCodeSet) { // we're not using the SubscriptionDataReader, so be sure to set the auth token here Quandl.SetAuthCode(Config.Get("quandl-auth-token")); } // each time we exhaust we'll new up this enumerator stack var refresher = new RefreshEnumerator<BaseData>(() => { var dateInDataTimeZone = DateTime.UtcNow.ConvertFromUtc(request.Configuration.DataTimeZone).Date; var enumeratorFactory = new BaseDataCollectionSubscriptionEnumeratorFactory(r => new [] {dateInDataTimeZone}); var factoryReadEnumerator = enumeratorFactory.CreateEnumerator(request); var maximumDataAge = TimeSpan.FromTicks(Math.Max(request.Configuration.Increment.Ticks, TimeSpan.FromSeconds(5).Ticks)); return new FastForwardEnumerator(factoryReadEnumerator, _timeProvider, request.Security.Exchange.TimeZone, maximumDataAge); }); // rate limit the refreshing of the stack to the requested interval var minimumTimeBetweenCalls = Math.Min(request.Configuration.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks); var rateLimit = new RateLimitEnumerator(refresher, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls)); var frontierAware = new FrontierAwareEnumerator(rateLimit, _timeProvider, timeZoneOffsetProvider); _customExchange.AddEnumerator(request.Configuration.Symbol, frontierAware); var enqueable = new EnqueueableEnumerator<BaseData>(); _customExchange.SetDataHandler(request.Configuration.Symbol, data => { enqueable.Enqueue(data); if (subscription != null) subscription.RealtimePrice = data.Value; }); enumerator = enqueable; } else if (request.Configuration.Resolution != Resolution.Tick) { // this enumerator allows the exchange to pump ticks into the 'back' of the enumerator, // and the time sync loop can pull aggregated trade bars off the front var aggregator = new TradeBarBuilderEnumerator(request.Configuration.Increment, request.Security.Exchange.TimeZone, _timeProvider); _exchange.SetDataHandler(request.Configuration.Symbol, data => { aggregator.ProcessData((Tick) data); if (subscription != null) subscription.RealtimePrice = data.Value; }); enumerator = aggregator; } else { // tick subscriptions can pass right through var tickEnumerator = new EnqueueableEnumerator<BaseData>(); _exchange.SetDataHandler(request.Configuration.Symbol, data => { tickEnumerator.Enqueue(data); if (subscription != null) subscription.RealtimePrice = data.Value; }); enumerator = tickEnumerator; } if (request.Configuration.FillDataForward) { enumerator = new LiveFillForwardEnumerator(_frontierTimeProvider, enumerator, request.Security.Exchange, _fillForwardResolution, request.Configuration.ExtendedMarketHours, localEndTime, request.Configuration.Increment); } // define market hours and user filters to incoming data if (request.Configuration.IsFilteredSubscription) { enumerator = new SubscriptionFilterEnumerator(enumerator, request.Security, localEndTime); } // finally, make our subscriptions aware of the frontier of the data feed, prevents future data from spewing into the feed enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, timeZoneOffsetProvider); subscription = new Subscription(request.Universe, request.Security, request.Configuration, enumerator, timeZoneOffsetProvider, request.StartTimeUtc, request.EndTimeUtc, false); } catch (Exception err) { Log.Error(err); } return subscription; }
/// <summary> /// Creates a new subscription for universe selection /// </summary> /// <param name="universe">The universe to add a subscription for</param> /// <param name="startTimeUtc">The start time of the subscription in utc</param> /// <param name="endTimeUtc">The end time of the subscription in utc</param> protected virtual Subscription CreateUniverseSubscription(Universe universe, DateTime startTimeUtc, DateTime endTimeUtc) { // TODO : Consider moving the creating of universe subscriptions to a separate, testable class // grab the relevant exchange hours var config = universe.Configuration; var localStartTime = startTimeUtc.ConvertFromUtc(config.TimeZone); var localEndTime = endTimeUtc.ConvertFromUtc(config.TimeZone); var exchangeHours = SecurityExchangeHoursProvider.FromDataFolder().GetExchangeHours(config); // create a canonical security object var security = new Security(exchangeHours, config, universe.SubscriptionSettings.Leverage); IEnumerator <BaseData> enumerator; var userDefined = universe as UserDefinedUniverse; if (userDefined != null) { // spoof a tick on the requested interval to trigger the universe selection function enumerator = LinqExtensions.Range(localStartTime, localEndTime, dt => dt + userDefined.Interval) .Where(dt => security.Exchange.IsOpenDuringBar(dt, dt + userDefined.Interval, config.ExtendedMarketHours)) .Select(dt => new Tick { Time = dt }).GetEnumerator(); } else if (config.Type == typeof(CoarseFundamental)) { // since we're binding to the data queue exchange we'll need to let him // know that we expect this data _dataQueueHandler.Subscribe(_job, new Dictionary <SecurityType, List <string> > { { config.SecurityType, new List <string> { config.Symbol } } }); var enqueable = new EnqueableEnumerator <BaseData>(); _exchange.SetHandler(config.Symbol, data => { var universeData = data as BaseDataCollection; if (universeData != null) { enqueable.EnqueueRange(universeData.Data); } }); enumerator = enqueable; } else { // each time we exhaust we'll new up this enumerator stack var refresher = new RefreshEnumerator <BaseDataCollection>(() => { var sourceProvider = (BaseData)Activator.CreateInstance(config.Type); var currentLocalDate = DateTime.UtcNow.ConvertFromUtc(config.TimeZone).Date; var factory = new BaseDataSubscriptionFactory(config, currentLocalDate, true); var source = sourceProvider.GetSource(config, currentLocalDate, true); var factorEnumerator = factory.Read(source).GetEnumerator(); var fastForward = new FastForwardEnumerator(factorEnumerator, _timeProvider, config.TimeZone, config.Increment); var timeZoneOffsetProvider = new TimeZoneOffsetProvider(config.TimeZone, startTimeUtc, endTimeUtc); var frontierAware = new FrontierAwareEnumerator(fastForward, _frontierTimeProvider, timeZoneOffsetProvider); return(new BaseDataCollectionAggregatorEnumerator(frontierAware, config.Symbol)); }); // rate limit the refreshing of the stack to the requested interval var minimumTimeBetweenCalls = Math.Min(config.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks); var rateLimit = new RateLimitEnumerator(refresher, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls)); _customExchange.AddEnumerator(rateLimit); var enqueable = new EnqueableEnumerator <BaseData>(); _customExchange.SetHandler(config.Symbol, data => { var universeData = data as BaseDataCollection; if (universeData != null) { enqueable.EnqueueRange(universeData.Data); } else { enqueable.Enqueue(data); } }); enumerator = enqueable; } // create the subscription var subscription = new Subscription(universe, security, enumerator, new TimeZoneOffsetProvider(security.SubscriptionDataConfig.TimeZone, startTimeUtc, endTimeUtc), startTimeUtc, endTimeUtc, true); return(subscription); }
/// <summary> /// Creates a new subscription for the specified security /// </summary> /// <param name="universe"></param> /// <param name="security">The security to create a subscription for</param> /// <param name="utcStartTime">The start time of the subscription in UTC</param> /// <param name="utcEndTime">The end time of the subscription in UTC</param> /// <returns>A new subscription instance of the specified security</returns> protected Subscription CreateSubscription(Universe universe, Security security, DateTime utcStartTime, DateTime utcEndTime) { Subscription subscription = null; try { var config = security.SubscriptionDataConfig; var localStartTime = utcStartTime.ConvertFromUtc(config.TimeZone); var localEndTime = utcEndTime.ConvertFromUtc(config.TimeZone); var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.SubscriptionDataConfig.TimeZone, utcStartTime, utcEndTime); IEnumerator <BaseData> enumerator; if (config.IsCustomData) { // each time we exhaust we'll new up this enumerator stack var refresher = new RefreshEnumerator <BaseData>(() => { var sourceProvider = (BaseData)Activator.CreateInstance(config.Type); var currentLocalDate = DateTime.UtcNow.ConvertFromUtc(config.TimeZone).Date; var factory = new BaseDataSubscriptionFactory(config, currentLocalDate, true); var source = sourceProvider.GetSource(config, currentLocalDate, true); var factoryReadEnumerator = factory.Read(source).GetEnumerator(); var maximumDataAge = TimeSpan.FromTicks(Math.Max(config.Increment.Ticks, TimeSpan.FromSeconds(5).Ticks)); var fastForward = new FastForwardEnumerator(factoryReadEnumerator, _timeProvider, config.TimeZone, maximumDataAge); return(new FrontierAwareEnumerator(fastForward, _timeProvider, timeZoneOffsetProvider)); }); // rate limit the refreshing of the stack to the requested interval var minimumTimeBetweenCalls = Math.Min(config.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks); var rateLimit = new RateLimitEnumerator(refresher, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls)); _customExchange.AddEnumerator(rateLimit); var enqueable = new EnqueableEnumerator <BaseData>(); _customExchange.SetHandler(config.Symbol, data => { enqueable.Enqueue(data); if (subscription != null) { subscription.RealtimePrice = data.Value; } }); enumerator = enqueable; } else if (config.Resolution != Resolution.Tick) { // this enumerator allows the exchange to pump ticks into the 'back' of the enumerator, // and the time sync loop can pull aggregated trade bars off the front var aggregator = new TradeBarBuilderEnumerator(config.Increment, config.TimeZone, _timeProvider); _exchange.SetHandler(config.Symbol, data => { aggregator.ProcessData((Tick)data); if (subscription != null) { subscription.RealtimePrice = data.Value; } }); enumerator = aggregator; } else { // tick subscriptions can pass right through var tickEnumerator = new EnqueableEnumerator <BaseData>(); _exchange.SetHandler(config.Symbol, data => { tickEnumerator.Enqueue(data); if (subscription != null) { subscription.RealtimePrice = data.Value; } }); enumerator = tickEnumerator; } if (config.FillDataForward) { // TODO : Properly resolve fill forward resolution like in FileSystemDataFeed (make considerations for universe-only) enumerator = new LiveFillForwardEnumerator(_frontierTimeProvider, enumerator, security.Exchange, _fillForwardResolution, config.ExtendedMarketHours, localEndTime, config.Increment); } // define market hours and user filters to incoming data enumerator = new SubscriptionFilterEnumerator(enumerator, security, localEndTime); // finally, make our subscriptions aware of the frontier of the data feed, this will help enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, timeZoneOffsetProvider); subscription = new Subscription(universe, security, enumerator, timeZoneOffsetProvider, utcStartTime, utcEndTime, false); } catch (Exception err) { Log.Error(err); } return(subscription); }
/// <summary> /// Creates a new subscription for universe selection /// </summary> /// <param name="universe">The universe to add a subscription for</param> /// <param name="startTimeUtc">The start time of the subscription in utc</param> /// <param name="endTimeUtc">The end time of the subscription in utc</param> protected virtual Subscription CreateUniverseSubscription(Universe universe, DateTime startTimeUtc, DateTime endTimeUtc) { // TODO : Consider moving the creating of universe subscriptions to a separate, testable class // grab the relevant exchange hours var config = universe.Configuration; var marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); var exchangeHours = marketHoursDatabase.GetExchangeHours(config); Security security; if (!_algorithm.Securities.TryGetValue(config.Symbol, out security)) { // create a canonical security object security = new Security(exchangeHours, config, _algorithm.Portfolio.CashBook[CashBook.AccountCurrency], SymbolProperties.GetDefault(CashBook.AccountCurrency)); } var tzOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, startTimeUtc, endTimeUtc); IEnumerator <BaseData> enumerator; var userDefined = universe as UserDefinedUniverse; if (userDefined != null) { Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating user defined universe: " + config.Symbol.ToString()); // spoof a tick on the requested interval to trigger the universe selection function enumerator = userDefined.GetTriggerTimes(startTimeUtc, endTimeUtc, marketHoursDatabase) .Select(dt => new Tick { Time = dt }).GetEnumerator(); enumerator = new FrontierAwareEnumerator(enumerator, _timeProvider, tzOffsetProvider); var enqueueable = new EnqueueableEnumerator <BaseData>(); _customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable)); enumerator = enqueueable; } else if (config.Type == typeof(CoarseFundamental)) { Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating coarse universe: " + config.Symbol.ToString()); // since we're binding to the data queue exchange we'll need to let him // know that we expect this data _dataQueueHandler.Subscribe(_job, new[] { security.Symbol }); var enqueable = new EnqueueableEnumerator <BaseData>(); _exchange.SetDataHandler(config.Symbol, data => { enqueable.Enqueue(data); }); enumerator = enqueable; } else { Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating custom universe: " + config.Symbol.ToString()); // each time we exhaust we'll new up this enumerator stack var refresher = new RefreshEnumerator <BaseDataCollection>(() => { var sourceProvider = (BaseData)Activator.CreateInstance(config.Type); var dateInDataTimeZone = DateTime.UtcNow.ConvertFromUtc(config.DataTimeZone).Date; var source = sourceProvider.GetSource(config, dateInDataTimeZone, true); var factory = SubscriptionFactory.ForSource(source, config, dateInDataTimeZone, false); var factorEnumerator = factory.Read(source).GetEnumerator(); var fastForward = new FastForwardEnumerator(factorEnumerator, _timeProvider, security.Exchange.TimeZone, config.Increment); var frontierAware = new FrontierAwareEnumerator(fastForward, _frontierTimeProvider, tzOffsetProvider); return(new BaseDataCollectionAggregatorEnumerator(frontierAware, config.Symbol)); }); // rate limit the refreshing of the stack to the requested interval var minimumTimeBetweenCalls = Math.Min(config.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks); var rateLimit = new RateLimitEnumerator(refresher, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls)); var enqueueable = new EnqueueableEnumerator <BaseData>(); _customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, rateLimit, enqueueable)); enumerator = enqueueable; } // create the subscription var subscription = new Subscription(universe, security, config, enumerator, tzOffsetProvider, startTimeUtc, endTimeUtc, true); return(subscription); }
/// <summary> /// Creates an enumerator to read the specified request. /// </summary> /// <param name="request">The subscription request to be read</param> /// <param name="dataProvider">Provider used to get data when it is not present on disk</param> /// <returns>An enumerator reading the subscription request</returns> public IEnumerator <BaseData> CreateEnumerator(SubscriptionRequest request, IDataProvider dataProvider) { var config = request.Configuration; // frontier value used to prevent emitting duplicate time stamps between refreshed enumerators // also provides some immediate fast-forward to handle spooling through remote files quickly var frontier = Ref.Create(request.StartTimeLocal); var lastSourceRefreshTime = DateTime.MinValue; var sourceFactory = (BaseData)ObjectActivator.GetActivator(config.Type).Invoke(new object[] { config.Type }); // this is refreshing the enumerator stack for each new source var refresher = new RefreshEnumerator <BaseData>(() => { // rate limit the refresh of this enumerator stack var utcNow = _timeProvider.GetUtcNow(); var minimumTimeBetweenCalls = GetMinimumTimeBetweenCalls(config.Increment); if (utcNow - lastSourceRefreshTime < minimumTimeBetweenCalls) { return(Enumerable.Empty <BaseData>().GetEnumerator()); } lastSourceRefreshTime = utcNow; var localDate = utcNow.ConvertFromUtc(config.ExchangeTimeZone).Date; var source = sourceFactory.GetSource(config, localDate, true); // fetch the new source and enumerate the data source reader var enumerator = EnumerateDataSourceReader(config, dataProvider, frontier, source, localDate); if (SourceRequiresFastForward(source)) { // The FastForwardEnumerator implements these two features: // (1) make sure we never emit past data // (2) data filtering based on a maximum data age // For custom data we don't want feature (2) because we would reject data points emitted later // (e.g. Quandl daily data after a weekend), so we disable it using a huge maximum data age. // apply fast forward logic for file transport mediums var maximumDataAge = GetMaximumDataAge(Time.MaxTimeSpan); enumerator = new FastForwardEnumerator(enumerator, _timeProvider, config.ExchangeTimeZone, maximumDataAge); } else { // rate limit calls to this enumerator stack enumerator = new RateLimitEnumerator <BaseData>(enumerator, _timeProvider, minimumTimeBetweenCalls); } if (source.Format == FileFormat.Collection) { // unroll collections into individual data points after fast forward/rate limiting applied enumerator = enumerator.SelectMany(data => { var collection = data as BaseDataCollection; return(collection?.Data.GetEnumerator() ?? new List <BaseData> { data }.GetEnumerator()); }); } return(enumerator); }); return(refresher); }
/// <summary> /// Creates a new subscription for the specified security /// </summary> /// <param name="security">The security to create a subscription for</param> /// <param name="utcStartTime">The start time of the subscription in UTC</param> /// <param name="utcEndTime">The end time of the subscription in UTC</param> /// <param name="isUserDefinedSubscription">True for subscriptions manually added by user via AddSecurity</param> /// <returns>A new subscription instance of the specified security</returns> protected Subscription CreateSubscription(Security security, DateTime utcStartTime, DateTime utcEndTime, bool isUserDefinedSubscription) { Subscription subscription = null; try { var config = security.SubscriptionDataConfig; var localStartTime = utcStartTime.ConvertFromUtc(config.TimeZone); var localEndTime = utcEndTime.ConvertFromUtc(config.TimeZone); IEnumerator<BaseData> enumerator; if (config.IsCustomData) { // custom data uses backtest readers var tradeableDates = Time.EachTradeableDay(security, localStartTime, localEndTime); var reader = new SubscriptionDataReader(config, localStartTime, localEndTime, _resultHandler, tradeableDates, true, false); // apply fast forwarding, this is especially important for RemoteFile types that // can send in large chunks of old, irrelevant data var fastForward = new FastForwardEnumerator(reader, _timeProvider, config.TimeZone, config.Increment); // apply rate limits (1x per increment, max 30 minutes between calls) // TODO : Pull limits from config file? var minimumTimeBetweenCalls = Math.Min(config.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks); var rateLimit = new RateLimitEnumerator(fastForward, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls)); // add the enumerator to the exchange _customExchange.AddEnumerator(rateLimit); // this enumerator just allows the exchange to directly dump data into the 'back' of the enumerator var enqueable = new EnqueableEnumerator<BaseData>(); _customExchange.SetHandler(config.Symbol, data => { enqueable.Enqueue(data); if (subscription != null) subscription.RealtimePrice = data.Value; }); enumerator = enqueable; } else if (config.Resolution != Resolution.Tick) { // this enumerator allows the exchange to pump ticks into the 'back' of the enumerator, // and the time sync loop can pull aggregated trade bars off the front var aggregator = new TradeBarBuilderEnumerator(config.Increment, config.TimeZone, _timeProvider); _exchange.SetHandler(config.Symbol, data => { aggregator.ProcessData((Tick) data); if (subscription != null) subscription.RealtimePrice = data.Value; }); enumerator = aggregator; } else { // tick subscriptions can pass right through var tickEnumerator = new EnqueableEnumerator<BaseData>(); _exchange.SetHandler(config.Symbol, data => { tickEnumerator.Enqueue(data); if (subscription != null) subscription.RealtimePrice = data.Value; }); enumerator = tickEnumerator; } if (config.FillDataForward) { // TODO : Properly resolve fill forward resolution like in FileSystemDataFeed (make considerations for universe-only) enumerator = new LiveFillForwardEnumerator(_frontierTimeProvider, enumerator, security.Exchange, _fillForwardResolution.ToTimeSpan(), config.ExtendedMarketHours, localEndTime, config.Increment); } // define market hours and user filters to incoming data enumerator = new SubscriptionFilterEnumerator(enumerator, security, localEndTime); // finally, make our subscriptions aware of the frontier of the data feed, this will help var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.SubscriptionDataConfig.TimeZone, utcStartTime, utcEndTime); enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, timeZoneOffsetProvider); subscription = new Subscription(security, enumerator, timeZoneOffsetProvider, utcStartTime, utcEndTime, isUserDefinedSubscription); } catch (Exception err) { Log.Error(err); } return subscription; }
/// <summary> /// Creates a new subscription for the specified security /// </summary> /// <param name="universe"></param> /// <param name="security">The security to create a subscription for</param> /// <param name="config">The subscription config to be added</param> /// <param name="utcStartTime">The start time of the subscription in UTC</param> /// <param name="utcEndTime">The end time of the subscription in UTC</param> /// <returns>A new subscription instance of the specified security</returns> protected Subscription CreateSubscription(Universe universe, Security security, SubscriptionDataConfig config, DateTime utcStartTime, DateTime utcEndTime) { Subscription subscription = null; try { var localEndTime = utcEndTime.ConvertFromUtc(security.Exchange.TimeZone); var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, utcStartTime, utcEndTime); IEnumerator <BaseData> enumerator; if (config.IsCustomData) { if (!Quandl.IsAuthCodeSet) { // we're not using the SubscriptionDataReader, so be sure to set the auth token here Quandl.SetAuthCode(Config.Get("quandl-auth-token")); } // each time we exhaust we'll new up this enumerator stack var refresher = new RefreshEnumerator <BaseData>(() => { var sourceProvider = (BaseData)Activator.CreateInstance(config.Type); var dateInDataTimeZone = DateTime.UtcNow.ConvertFromUtc(config.DataTimeZone).Date; var source = sourceProvider.GetSource(config, dateInDataTimeZone, true); var factory = SubscriptionDataSourceReader.ForSource(source, config, dateInDataTimeZone, false); var factoryReadEnumerator = factory.Read(source).GetEnumerator(); var maximumDataAge = TimeSpan.FromTicks(Math.Max(config.Increment.Ticks, TimeSpan.FromSeconds(5).Ticks)); return(new FastForwardEnumerator(factoryReadEnumerator, _timeProvider, security.Exchange.TimeZone, maximumDataAge)); }); // rate limit the refreshing of the stack to the requested interval var minimumTimeBetweenCalls = Math.Min(config.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks); var rateLimit = new RateLimitEnumerator(refresher, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls)); var frontierAware = new FrontierAwareEnumerator(rateLimit, _timeProvider, timeZoneOffsetProvider); _customExchange.AddEnumerator(config.Symbol, frontierAware); var enqueable = new EnqueueableEnumerator <BaseData>(); _customExchange.SetDataHandler(config.Symbol, data => { enqueable.Enqueue(data); if (subscription != null) { subscription.RealtimePrice = data.Value; } }); enumerator = enqueable; } else if (config.Resolution != Resolution.Tick) { // this enumerator allows the exchange to pump ticks into the 'back' of the enumerator, // and the time sync loop can pull aggregated trade bars off the front var aggregator = new TradeBarBuilderEnumerator(config.Increment, security.Exchange.TimeZone, _timeProvider); _exchange.SetDataHandler(config.Symbol, data => { aggregator.ProcessData((Tick)data); if (subscription != null) { subscription.RealtimePrice = data.Value; } }); enumerator = aggregator; } else { // tick subscriptions can pass right through var tickEnumerator = new EnqueueableEnumerator <BaseData>(); _exchange.SetDataHandler(config.Symbol, data => { tickEnumerator.Enqueue(data); if (subscription != null) { subscription.RealtimePrice = data.Value; } }); enumerator = tickEnumerator; } if (config.FillDataForward) { enumerator = new LiveFillForwardEnumerator(_frontierTimeProvider, enumerator, security.Exchange, _fillForwardResolution, config.ExtendedMarketHours, localEndTime, config.Increment); } // define market hours and user filters to incoming data if (config.IsFilteredSubscription) { enumerator = new SubscriptionFilterEnumerator(enumerator, security, localEndTime); } // finally, make our subscriptions aware of the frontier of the data feed, prevents future data from spewing into the feed enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, timeZoneOffsetProvider); subscription = new Subscription(universe, security, config, enumerator, timeZoneOffsetProvider, utcStartTime, utcEndTime, false); } catch (Exception err) { Log.Error(err); } return(subscription); }
/// <summary> /// Creates a new subscription for universe selection /// </summary> /// <param name="universe">The universe to add a subscription for</param> /// <param name="startTimeUtc">The start time of the subscription in utc</param> /// <param name="endTimeUtc">The end time of the subscription in utc</param> protected virtual Subscription CreateUniverseSubscription(Universe universe, DateTime startTimeUtc, DateTime endTimeUtc) { // TODO : Consider moving the creating of universe subscriptions to a separate, testable class // grab the relevant exchange hours var config = universe.Configuration; var exchangeHours = MarketHoursDatabase.FromDataFolder().GetExchangeHours(universe.Market, config.Symbol, universe.SecurityType); var localStartTime = startTimeUtc.ConvertFromUtc(exchangeHours.TimeZone); var localEndTime = endTimeUtc.ConvertFromUtc(exchangeHours.TimeZone); // create a canonical security object var security = new Security(exchangeHours, config, universe.SubscriptionSettings.Leverage); IEnumerator<BaseData> enumerator; var userDefined = universe as UserDefinedUniverse; if (userDefined != null) { // spoof a tick on the requested interval to trigger the universe selection function enumerator = LinqExtensions.Range(localStartTime, localEndTime, dt => dt + userDefined.Interval) .Where(dt => security.Exchange.IsOpenDuringBar(dt, dt + userDefined.Interval, config.ExtendedMarketHours)) .Select(dt => new Tick { Time = dt }).GetEnumerator(); } else if (config.Type == typeof (CoarseFundamental)) { // since we're binding to the data queue exchange we'll need to let him // know that we expect this data _dataQueueHandler.Subscribe(_job, new Dictionary<SecurityType, List<Symbol>> { {config.SecurityType, new List<Symbol>{config.Symbol}} }); var enqueable = new EnqueableEnumerator<BaseData>(); _exchange.SetHandler(config.Symbol, data => { var universeData = data as BaseDataCollection; if (universeData != null) { enqueable.EnqueueRange(universeData.Data); } }); enumerator = enqueable; } else { // each time we exhaust we'll new up this enumerator stack var refresher = new RefreshEnumerator<BaseDataCollection>(() => { var sourceProvider = (BaseData)Activator.CreateInstance(config.Type); var currentLocalDate = DateTime.UtcNow.ConvertFromUtc(security.Exchange.TimeZone).Date; var factory = new BaseDataSubscriptionFactory(config, currentLocalDate, true); var source = sourceProvider.GetSource(config, currentLocalDate, true); var factorEnumerator = factory.Read(source).GetEnumerator(); var fastForward = new FastForwardEnumerator(factorEnumerator, _timeProvider, security.Exchange.TimeZone, config.Increment); var tzOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, startTimeUtc, endTimeUtc); var frontierAware = new FrontierAwareEnumerator(fastForward, _frontierTimeProvider, tzOffsetProvider); return new BaseDataCollectionAggregatorEnumerator(frontierAware, config.Symbol); }); // rate limit the refreshing of the stack to the requested interval var minimumTimeBetweenCalls = Math.Min(config.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks); var rateLimit = new RateLimitEnumerator(refresher, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls)); _customExchange.AddEnumerator(rateLimit); var enqueable = new EnqueableEnumerator<BaseData>(); _customExchange.SetHandler(config.Symbol, data => { var universeData = data as BaseDataCollection; if (universeData != null) { enqueable.EnqueueRange(universeData.Data); } else { enqueable.Enqueue(data); } }); enumerator = enqueable; } // create the subscription var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, startTimeUtc, endTimeUtc); var subscription = new Subscription(universe, security, enumerator, timeZoneOffsetProvider, startTimeUtc, endTimeUtc, true); return subscription; }
/// <summary> /// Creates a new subscription for the specified security /// </summary> /// <param name="universe"></param> /// <param name="security">The security to create a subscription for</param> /// <param name="utcStartTime">The start time of the subscription in UTC</param> /// <param name="utcEndTime">The end time of the subscription in UTC</param> /// <returns>A new subscription instance of the specified security</returns> protected Subscription CreateSubscription(Universe universe, Security security, DateTime utcStartTime, DateTime utcEndTime) { Subscription subscription = null; try { var config = security.SubscriptionDataConfig; var localEndTime = utcEndTime.ConvertFromUtc(security.Exchange.TimeZone); var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, utcStartTime, utcEndTime); IEnumerator<BaseData> enumerator; if (config.IsCustomData) { // each time we exhaust we'll new up this enumerator stack var refresher = new RefreshEnumerator<BaseData>(() => { var sourceProvider = (BaseData)Activator.CreateInstance(config.Type); var currentLocalDate = DateTime.UtcNow.ConvertFromUtc(security.Exchange.TimeZone).Date; var factory = new BaseDataSubscriptionFactory(config, currentLocalDate, true); var source = sourceProvider.GetSource(config, currentLocalDate, true); var factoryReadEnumerator = factory.Read(source).GetEnumerator(); var maximumDataAge = TimeSpan.FromTicks(Math.Max(config.Increment.Ticks, TimeSpan.FromSeconds(5).Ticks)); var fastForward = new FastForwardEnumerator(factoryReadEnumerator, _timeProvider, security.Exchange.TimeZone, maximumDataAge); return new FrontierAwareEnumerator(fastForward, _timeProvider, timeZoneOffsetProvider); }); // rate limit the refreshing of the stack to the requested interval var minimumTimeBetweenCalls = Math.Min(config.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks); var rateLimit = new RateLimitEnumerator(refresher, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls)); _customExchange.AddEnumerator(rateLimit); var enqueable = new EnqueableEnumerator<BaseData>(); _customExchange.SetHandler(config.Symbol, data => { enqueable.Enqueue(data); if (subscription != null) subscription.RealtimePrice = data.Value; }); enumerator = enqueable; } else if (config.Resolution != Resolution.Tick) { // this enumerator allows the exchange to pump ticks into the 'back' of the enumerator, // and the time sync loop can pull aggregated trade bars off the front var aggregator = new TradeBarBuilderEnumerator(config.Increment, security.Exchange.TimeZone, _timeProvider); _exchange.SetHandler(config.Symbol, data => { aggregator.ProcessData((Tick) data); if (subscription != null) subscription.RealtimePrice = data.Value; }); enumerator = aggregator; } else { // tick subscriptions can pass right through var tickEnumerator = new EnqueableEnumerator<BaseData>(); _exchange.SetHandler(config.Symbol, data => { tickEnumerator.Enqueue(data); if (subscription != null) subscription.RealtimePrice = data.Value; }); enumerator = tickEnumerator; } if (config.FillDataForward) { // TODO : Properly resolve fill forward resolution like in FileSystemDataFeed (make considerations for universe-only) enumerator = new LiveFillForwardEnumerator(_frontierTimeProvider, enumerator, security.Exchange, _fillForwardResolution, config.ExtendedMarketHours, localEndTime, config.Increment); } // define market hours and user filters to incoming data enumerator = new SubscriptionFilterEnumerator(enumerator, security, localEndTime); // finally, make our subscriptions aware of the frontier of the data feed, this will help enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, timeZoneOffsetProvider); subscription = new Subscription(universe, security, enumerator, timeZoneOffsetProvider, utcStartTime, utcEndTime, false); } catch (Exception err) { Log.Error(err); } return subscription; }
/// <summary> /// Creates a new subscription for universe selection /// </summary> /// <param name="universe">The universe to add a subscription for</param> /// <param name="startTimeUtc">The start time of the subscription in utc</param> /// <param name="endTimeUtc">The end time of the subscription in utc</param> protected virtual Subscription CreateUniverseSubscription(Universe universe, DateTime startTimeUtc, DateTime endTimeUtc) { // TODO : Consider moving the creating of universe subscriptions to a separate, testable class // grab the relevant exchange hours var config = universe.Configuration; var marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); var exchangeHours = marketHoursDatabase.GetExchangeHours(config); // create a canonical security object var security = new Security(exchangeHours, config, _algorithm.Portfolio.CashBook[CashBook.AccountCurrency], SymbolProperties.GetDefault(CashBook.AccountCurrency)); var tzOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, startTimeUtc, endTimeUtc); IEnumerator<BaseData> enumerator; var userDefined = universe as UserDefinedUniverse; if (userDefined != null) { Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating user defined universe: " + config.Symbol.ToString()); // spoof a tick on the requested interval to trigger the universe selection function enumerator = userDefined.GetTriggerTimes(startTimeUtc, endTimeUtc, marketHoursDatabase) .Select(dt => new Tick { Time = dt }).GetEnumerator(); enumerator = new FrontierAwareEnumerator(enumerator, _timeProvider, tzOffsetProvider); var enqueueable = new EnqueueableEnumerator<BaseData>(); _customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable)); enumerator = enqueueable; } else if (config.Type == typeof (CoarseFundamental)) { Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating coarse universe: " + config.Symbol.ToString()); // since we're binding to the data queue exchange we'll need to let him // know that we expect this data _dataQueueHandler.Subscribe(_job, new[] {security.Symbol}); var enqueable = new EnqueueableEnumerator<BaseData>(); _exchange.SetDataHandler(config.Symbol, data => { enqueable.Enqueue(data); }); enumerator = enqueable; } else { Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating custom universe: " + config.Symbol.ToString()); // each time we exhaust we'll new up this enumerator stack var refresher = new RefreshEnumerator<BaseDataCollection>(() => { var sourceProvider = (BaseData)Activator.CreateInstance(config.Type); var dateInDataTimeZone = DateTime.UtcNow.ConvertFromUtc(config.DataTimeZone).Date; var source = sourceProvider.GetSource(config, dateInDataTimeZone, true); var factory = SubscriptionFactory.ForSource(source, config, dateInDataTimeZone, false); var factorEnumerator = factory.Read(source).GetEnumerator(); var fastForward = new FastForwardEnumerator(factorEnumerator, _timeProvider, security.Exchange.TimeZone, config.Increment); var frontierAware = new FrontierAwareEnumerator(fastForward, _frontierTimeProvider, tzOffsetProvider); return new BaseDataCollectionAggregatorEnumerator(frontierAware, config.Symbol); }); // rate limit the refreshing of the stack to the requested interval var minimumTimeBetweenCalls = Math.Min(config.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks); var rateLimit = new RateLimitEnumerator(refresher, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls)); var enqueueable = new EnqueueableEnumerator<BaseData>(); _customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, rateLimit, enqueueable)); enumerator = enqueueable; } // create the subscription var subscription = new Subscription(universe, security, enumerator, tzOffsetProvider, startTimeUtc, endTimeUtc, true); return subscription; }