public void PassesTicksStraightThrough() { var enumerator = new EnqueableEnumerator<Tick>(); // add some ticks var currentTime = new DateTime(2015, 10, 08); // returns true even if no data present until stop is called Assert.IsTrue(enumerator.MoveNext()); Assert.IsNull(enumerator.Current); var tick1 = new Tick(currentTime, "SPY", 199.55m, 199, 200) {Quantity = 10}; enumerator.Enqueue(tick1); Assert.IsTrue(enumerator.MoveNext()); Assert.AreEqual(tick1, enumerator.Current); Assert.IsTrue(enumerator.MoveNext()); Assert.IsNull(enumerator.Current); var tick2 = new Tick(currentTime, "SPY", 199.56m, 199.21m, 200.02m) {Quantity = 5}; enumerator.Enqueue(tick2); Assert.IsTrue(enumerator.MoveNext()); Assert.AreEqual(tick2, enumerator.Current); enumerator.Stop(); Assert.IsFalse(enumerator.MoveNext()); Assert.IsNull(enumerator.Current); }
public void PassesTicksStraightThrough() { var enumerator = new EnqueableEnumerator <Tick>(); // add some ticks var currentTime = new DateTime(2015, 10, 08); // returns true even if no data present until stop is called Assert.IsTrue(enumerator.MoveNext()); Assert.IsNull(enumerator.Current); var tick1 = new Tick(currentTime, "SPY", 199.55m, 199, 200) { Quantity = 10 }; enumerator.Enqueue(tick1); Assert.IsTrue(enumerator.MoveNext()); Assert.AreEqual(tick1, enumerator.Current); Assert.IsTrue(enumerator.MoveNext()); Assert.IsNull(enumerator.Current); var tick2 = new Tick(currentTime, "SPY", 199.56m, 199.21m, 200.02m) { Quantity = 5 }; enumerator.Enqueue(tick2); Assert.IsTrue(enumerator.MoveNext()); Assert.AreEqual(tick2, enumerator.Current); enumerator.Stop(); Assert.IsFalse(enumerator.MoveNext()); Assert.IsNull(enumerator.Current); }
/// <summary> /// Creates a new subscription for universe selection /// </summary> /// <param name="universe">The universe to add a subscription for</param> /// <param name="startTimeUtc">The start time of the subscription in utc</param> /// <param name="endTimeUtc">The end time of the subscription in utc</param> protected virtual Subscription CreateUniverseSubscription(Universe universe, DateTime startTimeUtc, DateTime endTimeUtc) { // TODO : Consider moving the creating of universe subscriptions to a separate, testable class // grab the relevant exchange hours var config = universe.Configuration; var localStartTime = startTimeUtc.ConvertFromUtc(config.TimeZone); var localEndTime = endTimeUtc.ConvertFromUtc(config.TimeZone); var exchangeHours = SecurityExchangeHoursProvider.FromDataFolder().GetExchangeHours(config); // create a canonical security object var security = new Security(exchangeHours, config, universe.SubscriptionSettings.Leverage); IEnumerator <BaseData> enumerator; var userDefined = universe as UserDefinedUniverse; if (userDefined != null) { // spoof a tick on the requested interval to trigger the universe selection function enumerator = LinqExtensions.Range(localStartTime, localEndTime, dt => dt + userDefined.Interval) .Where(dt => security.Exchange.IsOpenDuringBar(dt, dt + userDefined.Interval, config.ExtendedMarketHours)) .Select(dt => new Tick { Time = dt }).GetEnumerator(); } else if (config.Type == typeof(CoarseFundamental)) { // since we're binding to the data queue exchange we'll need to let him // know that we expect this data _dataQueueHandler.Subscribe(_job, new Dictionary <SecurityType, List <string> > { { config.SecurityType, new List <string> { config.Symbol } } }); var enqueable = new EnqueableEnumerator <BaseData>(); _exchange.SetHandler(config.Symbol, data => { var universeData = data as BaseDataCollection; if (universeData != null) { enqueable.EnqueueRange(universeData.Data); } }); enumerator = enqueable; } else { // each time we exhaust we'll new up this enumerator stack var refresher = new RefreshEnumerator <BaseDataCollection>(() => { var sourceProvider = (BaseData)Activator.CreateInstance(config.Type); var currentLocalDate = DateTime.UtcNow.ConvertFromUtc(config.TimeZone).Date; var factory = new BaseDataSubscriptionFactory(config, currentLocalDate, true); var source = sourceProvider.GetSource(config, currentLocalDate, true); var factorEnumerator = factory.Read(source).GetEnumerator(); var fastForward = new FastForwardEnumerator(factorEnumerator, _timeProvider, config.TimeZone, config.Increment); var timeZoneOffsetProvider = new TimeZoneOffsetProvider(config.TimeZone, startTimeUtc, endTimeUtc); var frontierAware = new FrontierAwareEnumerator(fastForward, _frontierTimeProvider, timeZoneOffsetProvider); return(new BaseDataCollectionAggregatorEnumerator(frontierAware, config.Symbol)); }); // rate limit the refreshing of the stack to the requested interval var minimumTimeBetweenCalls = Math.Min(config.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks); var rateLimit = new RateLimitEnumerator(refresher, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls)); _customExchange.AddEnumerator(rateLimit); var enqueable = new EnqueableEnumerator <BaseData>(); _customExchange.SetHandler(config.Symbol, data => { var universeData = data as BaseDataCollection; if (universeData != null) { enqueable.EnqueueRange(universeData.Data); } else { enqueable.Enqueue(data); } }); enumerator = enqueable; } // create the subscription var subscription = new Subscription(universe, security, enumerator, new TimeZoneOffsetProvider(security.SubscriptionDataConfig.TimeZone, startTimeUtc, endTimeUtc), startTimeUtc, endTimeUtc, true); return(subscription); }
/// <summary> /// Creates a new subscription for the specified security /// </summary> /// <param name="universe"></param> /// <param name="security">The security to create a subscription for</param> /// <param name="utcStartTime">The start time of the subscription in UTC</param> /// <param name="utcEndTime">The end time of the subscription in UTC</param> /// <returns>A new subscription instance of the specified security</returns> protected Subscription CreateSubscription(Universe universe, Security security, DateTime utcStartTime, DateTime utcEndTime) { Subscription subscription = null; try { var config = security.SubscriptionDataConfig; var localStartTime = utcStartTime.ConvertFromUtc(config.TimeZone); var localEndTime = utcEndTime.ConvertFromUtc(config.TimeZone); var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.SubscriptionDataConfig.TimeZone, utcStartTime, utcEndTime); IEnumerator <BaseData> enumerator; if (config.IsCustomData) { // each time we exhaust we'll new up this enumerator stack var refresher = new RefreshEnumerator <BaseData>(() => { var sourceProvider = (BaseData)Activator.CreateInstance(config.Type); var currentLocalDate = DateTime.UtcNow.ConvertFromUtc(config.TimeZone).Date; var factory = new BaseDataSubscriptionFactory(config, currentLocalDate, true); var source = sourceProvider.GetSource(config, currentLocalDate, true); var factoryReadEnumerator = factory.Read(source).GetEnumerator(); var maximumDataAge = TimeSpan.FromTicks(Math.Max(config.Increment.Ticks, TimeSpan.FromSeconds(5).Ticks)); var fastForward = new FastForwardEnumerator(factoryReadEnumerator, _timeProvider, config.TimeZone, maximumDataAge); return(new FrontierAwareEnumerator(fastForward, _timeProvider, timeZoneOffsetProvider)); }); // rate limit the refreshing of the stack to the requested interval var minimumTimeBetweenCalls = Math.Min(config.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks); var rateLimit = new RateLimitEnumerator(refresher, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls)); _customExchange.AddEnumerator(rateLimit); var enqueable = new EnqueableEnumerator <BaseData>(); _customExchange.SetHandler(config.Symbol, data => { enqueable.Enqueue(data); if (subscription != null) { subscription.RealtimePrice = data.Value; } }); enumerator = enqueable; } else if (config.Resolution != Resolution.Tick) { // this enumerator allows the exchange to pump ticks into the 'back' of the enumerator, // and the time sync loop can pull aggregated trade bars off the front var aggregator = new TradeBarBuilderEnumerator(config.Increment, config.TimeZone, _timeProvider); _exchange.SetHandler(config.Symbol, data => { aggregator.ProcessData((Tick)data); if (subscription != null) { subscription.RealtimePrice = data.Value; } }); enumerator = aggregator; } else { // tick subscriptions can pass right through var tickEnumerator = new EnqueableEnumerator <BaseData>(); _exchange.SetHandler(config.Symbol, data => { tickEnumerator.Enqueue(data); if (subscription != null) { subscription.RealtimePrice = data.Value; } }); enumerator = tickEnumerator; } if (config.FillDataForward) { // TODO : Properly resolve fill forward resolution like in FileSystemDataFeed (make considerations for universe-only) enumerator = new LiveFillForwardEnumerator(_frontierTimeProvider, enumerator, security.Exchange, _fillForwardResolution, config.ExtendedMarketHours, localEndTime, config.Increment); } // define market hours and user filters to incoming data enumerator = new SubscriptionFilterEnumerator(enumerator, security, localEndTime); // finally, make our subscriptions aware of the frontier of the data feed, this will help enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, timeZoneOffsetProvider); subscription = new Subscription(universe, security, enumerator, timeZoneOffsetProvider, utcStartTime, utcEndTime, false); } catch (Exception err) { Log.Error(err); } return(subscription); }
/// <summary> /// Creates a new subscription for universe selection /// </summary> /// <param name="universe">The universe to add a subscription for</param> /// <param name="startTimeUtc">The start time of the subscription in utc</param> /// <param name="endTimeUtc">The end time of the subscription in utc</param> protected virtual Subscription CreateUniverseSubscription( IUniverse universe, DateTime startTimeUtc, DateTime endTimeUtc ) { // grab the relevant exchange hours var config = universe.Configuration; var exchangeHours = SecurityExchangeHoursProvider.FromDataFolder() .GetExchangeHours(config.Market, null, config.SecurityType); // create a canonical security object var security = new Security(exchangeHours, config, universe.SubscriptionSettings.Leverage); IEnumerator<BaseData> enumerator; if (config.Type == typeof (CoarseFundamental)) { // since we're binding to the data queue exchange we'll need to let him // know that we expect this data _dataQueueHandler.Subscribe(_job, new Dictionary<SecurityType, List<string>> { {config.SecurityType, new List<string>{config.Symbol}} }); var enqueable = new EnqueableEnumerator<BaseData>(); _exchange.SetHandler(config.Symbol, data => { var universeData = data as BaseDataCollection; if (universeData != null) { enqueable.EnqueueRange(universeData.Data); } }); enumerator = enqueable; } else { var localStartTime = startTimeUtc.ConvertFromUtc(config.TimeZone); var localEndTime = endTimeUtc.ConvertFromUtc(config.TimeZone); // define our data enumerator var tradeableDates = Time.EachTradeableDay(security, localStartTime, localEndTime); var reader = new SubscriptionDataReader(config, localStartTime, localEndTime, _resultHandler, tradeableDates, true); _customExchange.AddEnumerator(reader); var enqueable = new EnqueableEnumerator<BaseData>(); _customExchange.SetHandler(config.Symbol, data => { var universeData = data as BaseDataCollection; if (universeData != null) { enqueable.EnqueueRange(universeData.Data); } else { enqueable.Enqueue(data); } }); enumerator = enqueable; } // create the subscription var subscription = new Subscription(universe, security, enumerator, new TimeZoneOffsetProvider(security.SubscriptionDataConfig.TimeZone, startTimeUtc, endTimeUtc), startTimeUtc, endTimeUtc); return subscription; }
/// <summary> /// Creates a new subscription for the specified security /// </summary> /// <param name="security">The security to create a subscription for</param> /// <param name="utcStartTime">The start time of the subscription in UTC</param> /// <param name="utcEndTime">The end time of the subscription in UTC</param> /// <param name="isUserDefinedSubscription">True for subscriptions manually added by user via AddSecurity</param> /// <returns>A new subscription instance of the specified security</returns> protected Subscription CreateSubscription(Security security, DateTime utcStartTime, DateTime utcEndTime, bool isUserDefinedSubscription) { Subscription subscription = null; try { var config = security.SubscriptionDataConfig; var localStartTime = utcStartTime.ConvertFromUtc(config.TimeZone); var localEndTime = utcEndTime.ConvertFromUtc(config.TimeZone); IEnumerator<BaseData> enumerator; if (config.IsCustomData) { // custom data uses backtest readers var tradeableDates = Time.EachTradeableDay(security, localStartTime, localEndTime); var reader = new SubscriptionDataReader(config, localStartTime, localEndTime, _resultHandler, tradeableDates, true, false); // apply fast forwarding, this is especially important for RemoteFile types that // can send in large chunks of old, irrelevant data var fastForward = new FastForwardEnumerator(reader, _timeProvider, config.TimeZone, config.Increment); // apply rate limits (1x per increment, max 30 minutes between calls) // TODO : Pull limits from config file? var minimumTimeBetweenCalls = Math.Min(config.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks); var rateLimit = new RateLimitEnumerator(fastForward, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls)); // add the enumerator to the exchange _customExchange.AddEnumerator(rateLimit); // this enumerator just allows the exchange to directly dump data into the 'back' of the enumerator var enqueable = new EnqueableEnumerator<BaseData>(); _customExchange.SetHandler(config.Symbol, data => { enqueable.Enqueue(data); if (subscription != null) subscription.RealtimePrice = data.Value; }); enumerator = enqueable; } else if (config.Resolution != Resolution.Tick) { // this enumerator allows the exchange to pump ticks into the 'back' of the enumerator, // and the time sync loop can pull aggregated trade bars off the front var aggregator = new TradeBarBuilderEnumerator(config.Increment, config.TimeZone, _timeProvider); _exchange.SetHandler(config.Symbol, data => { aggregator.ProcessData((Tick) data); if (subscription != null) subscription.RealtimePrice = data.Value; }); enumerator = aggregator; } else { // tick subscriptions can pass right through var tickEnumerator = new EnqueableEnumerator<BaseData>(); _exchange.SetHandler(config.Symbol, data => { tickEnumerator.Enqueue(data); if (subscription != null) subscription.RealtimePrice = data.Value; }); enumerator = tickEnumerator; } if (config.FillDataForward) { // TODO : Properly resolve fill forward resolution like in FileSystemDataFeed (make considerations for universe-only) enumerator = new LiveFillForwardEnumerator(_frontierTimeProvider, enumerator, security.Exchange, _fillForwardResolution.ToTimeSpan(), config.ExtendedMarketHours, localEndTime, config.Increment); } // define market hours and user filters to incoming data enumerator = new SubscriptionFilterEnumerator(enumerator, security, localEndTime); // finally, make our subscriptions aware of the frontier of the data feed, this will help var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.SubscriptionDataConfig.TimeZone, utcStartTime, utcEndTime); enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, timeZoneOffsetProvider); subscription = new Subscription(security, enumerator, timeZoneOffsetProvider, utcStartTime, utcEndTime, isUserDefinedSubscription); } catch (Exception err) { Log.Error(err); } return subscription; }
/// <summary> /// Creates a new subscription for universe selection /// </summary> /// <param name="universe">The universe to add a subscription for</param> /// <param name="startTimeUtc">The start time of the subscription in utc</param> /// <param name="endTimeUtc">The end time of the subscription in utc</param> protected virtual Subscription CreateUniverseSubscription(Universe universe, DateTime startTimeUtc, DateTime endTimeUtc) { // TODO : Consider moving the creating of universe subscriptions to a separate, testable class // grab the relevant exchange hours var config = universe.Configuration; var exchangeHours = MarketHoursDatabase.FromDataFolder().GetExchangeHours(universe.Market, config.Symbol, universe.SecurityType); var localStartTime = startTimeUtc.ConvertFromUtc(exchangeHours.TimeZone); var localEndTime = endTimeUtc.ConvertFromUtc(exchangeHours.TimeZone); // create a canonical security object var security = new Security(exchangeHours, config, universe.SubscriptionSettings.Leverage); IEnumerator<BaseData> enumerator; var userDefined = universe as UserDefinedUniverse; if (userDefined != null) { // spoof a tick on the requested interval to trigger the universe selection function enumerator = LinqExtensions.Range(localStartTime, localEndTime, dt => dt + userDefined.Interval) .Where(dt => security.Exchange.IsOpenDuringBar(dt, dt + userDefined.Interval, config.ExtendedMarketHours)) .Select(dt => new Tick { Time = dt }).GetEnumerator(); } else if (config.Type == typeof (CoarseFundamental)) { // since we're binding to the data queue exchange we'll need to let him // know that we expect this data _dataQueueHandler.Subscribe(_job, new Dictionary<SecurityType, List<Symbol>> { {config.SecurityType, new List<Symbol>{config.Symbol}} }); var enqueable = new EnqueableEnumerator<BaseData>(); _exchange.SetHandler(config.Symbol, data => { var universeData = data as BaseDataCollection; if (universeData != null) { enqueable.EnqueueRange(universeData.Data); } }); enumerator = enqueable; } else { // each time we exhaust we'll new up this enumerator stack var refresher = new RefreshEnumerator<BaseDataCollection>(() => { var sourceProvider = (BaseData)Activator.CreateInstance(config.Type); var currentLocalDate = DateTime.UtcNow.ConvertFromUtc(security.Exchange.TimeZone).Date; var factory = new BaseDataSubscriptionFactory(config, currentLocalDate, true); var source = sourceProvider.GetSource(config, currentLocalDate, true); var factorEnumerator = factory.Read(source).GetEnumerator(); var fastForward = new FastForwardEnumerator(factorEnumerator, _timeProvider, security.Exchange.TimeZone, config.Increment); var tzOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, startTimeUtc, endTimeUtc); var frontierAware = new FrontierAwareEnumerator(fastForward, _frontierTimeProvider, tzOffsetProvider); return new BaseDataCollectionAggregatorEnumerator(frontierAware, config.Symbol); }); // rate limit the refreshing of the stack to the requested interval var minimumTimeBetweenCalls = Math.Min(config.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks); var rateLimit = new RateLimitEnumerator(refresher, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls)); _customExchange.AddEnumerator(rateLimit); var enqueable = new EnqueableEnumerator<BaseData>(); _customExchange.SetHandler(config.Symbol, data => { var universeData = data as BaseDataCollection; if (universeData != null) { enqueable.EnqueueRange(universeData.Data); } else { enqueable.Enqueue(data); } }); enumerator = enqueable; } // create the subscription var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, startTimeUtc, endTimeUtc); var subscription = new Subscription(universe, security, enumerator, timeZoneOffsetProvider, startTimeUtc, endTimeUtc, true); return subscription; }
/// <summary> /// Creates a new subscription for the specified security /// </summary> /// <param name="universe"></param> /// <param name="security">The security to create a subscription for</param> /// <param name="utcStartTime">The start time of the subscription in UTC</param> /// <param name="utcEndTime">The end time of the subscription in UTC</param> /// <returns>A new subscription instance of the specified security</returns> protected Subscription CreateSubscription(Universe universe, Security security, DateTime utcStartTime, DateTime utcEndTime) { Subscription subscription = null; try { var config = security.SubscriptionDataConfig; var localEndTime = utcEndTime.ConvertFromUtc(security.Exchange.TimeZone); var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, utcStartTime, utcEndTime); IEnumerator<BaseData> enumerator; if (config.IsCustomData) { // each time we exhaust we'll new up this enumerator stack var refresher = new RefreshEnumerator<BaseData>(() => { var sourceProvider = (BaseData)Activator.CreateInstance(config.Type); var currentLocalDate = DateTime.UtcNow.ConvertFromUtc(security.Exchange.TimeZone).Date; var factory = new BaseDataSubscriptionFactory(config, currentLocalDate, true); var source = sourceProvider.GetSource(config, currentLocalDate, true); var factoryReadEnumerator = factory.Read(source).GetEnumerator(); var maximumDataAge = TimeSpan.FromTicks(Math.Max(config.Increment.Ticks, TimeSpan.FromSeconds(5).Ticks)); var fastForward = new FastForwardEnumerator(factoryReadEnumerator, _timeProvider, security.Exchange.TimeZone, maximumDataAge); return new FrontierAwareEnumerator(fastForward, _timeProvider, timeZoneOffsetProvider); }); // rate limit the refreshing of the stack to the requested interval var minimumTimeBetweenCalls = Math.Min(config.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks); var rateLimit = new RateLimitEnumerator(refresher, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls)); _customExchange.AddEnumerator(rateLimit); var enqueable = new EnqueableEnumerator<BaseData>(); _customExchange.SetHandler(config.Symbol, data => { enqueable.Enqueue(data); if (subscription != null) subscription.RealtimePrice = data.Value; }); enumerator = enqueable; } else if (config.Resolution != Resolution.Tick) { // this enumerator allows the exchange to pump ticks into the 'back' of the enumerator, // and the time sync loop can pull aggregated trade bars off the front var aggregator = new TradeBarBuilderEnumerator(config.Increment, security.Exchange.TimeZone, _timeProvider); _exchange.SetHandler(config.Symbol, data => { aggregator.ProcessData((Tick) data); if (subscription != null) subscription.RealtimePrice = data.Value; }); enumerator = aggregator; } else { // tick subscriptions can pass right through var tickEnumerator = new EnqueableEnumerator<BaseData>(); _exchange.SetHandler(config.Symbol, data => { tickEnumerator.Enqueue(data); if (subscription != null) subscription.RealtimePrice = data.Value; }); enumerator = tickEnumerator; } if (config.FillDataForward) { // TODO : Properly resolve fill forward resolution like in FileSystemDataFeed (make considerations for universe-only) enumerator = new LiveFillForwardEnumerator(_frontierTimeProvider, enumerator, security.Exchange, _fillForwardResolution, config.ExtendedMarketHours, localEndTime, config.Increment); } // define market hours and user filters to incoming data enumerator = new SubscriptionFilterEnumerator(enumerator, security, localEndTime); // finally, make our subscriptions aware of the frontier of the data feed, this will help enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, timeZoneOffsetProvider); subscription = new Subscription(universe, security, enumerator, timeZoneOffsetProvider, utcStartTime, utcEndTime, false); } catch (Exception err) { Log.Error(err); } return subscription; }