Пример #1
0
        /// <summary>
        /// Creates a new subscription for the specified security
        /// </summary>
        /// <param name="request">The subscription request</param>
        /// <returns>A new subscription instance of the specified security</returns>
        protected Subscription CreateSubscription(SubscriptionRequest request)
        {
            Subscription subscription = null;

            try
            {
                var localEndTime           = request.EndTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone);
                var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc);

                IEnumerator <BaseData> enumerator;
                if (request.Configuration.IsCustomData)
                {
                    if (!Quandl.IsAuthCodeSet)
                    {
                        // we're not using the SubscriptionDataReader, so be sure to set the auth token here
                        Quandl.SetAuthCode(Config.Get("quandl-auth-token"));
                    }

                    // each time we exhaust we'll new up this enumerator stack
                    var refresher = new RefreshEnumerator <BaseData>(() =>
                    {
                        var dateInDataTimeZone    = DateTime.UtcNow.ConvertFromUtc(request.Configuration.DataTimeZone).Date;
                        var enumeratorFactory     = new BaseDataSubscriptionEnumeratorFactory(r => new[] { dateInDataTimeZone });
                        var factoryReadEnumerator = enumeratorFactory.CreateEnumerator(request, _dataFileProvider);
                        var maximumDataAge        = TimeSpan.FromTicks(Math.Max(request.Configuration.Increment.Ticks, TimeSpan.FromSeconds(5).Ticks));
                        var fastForward           = new FastForwardEnumerator(factoryReadEnumerator, _timeProvider, request.Security.Exchange.TimeZone, maximumDataAge);
                        return(new FrontierAwareEnumerator(fastForward, _frontierTimeProvider, timeZoneOffsetProvider));
                    });

                    // rate limit the refreshing of the stack to the requested interval
                    // At Tick resolution, it will refresh at full speed
                    // At Second and Minute resolution, it will refresh every second and minute respectively
                    // At Hour and Daily resolutions, it will refresh every 30 minutes
                    var minimumTimeBetweenCalls = Math.Min(request.Configuration.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks);
                    var rateLimit = new RateLimitEnumerator(refresher, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls));
                    _customExchange.AddEnumerator(request.Configuration.Symbol, rateLimit);

                    var enqueable = new EnqueueableEnumerator <BaseData>();
                    _customExchange.SetDataHandler(request.Configuration.Symbol, data =>
                    {
                        enqueable.Enqueue(data);
                        if (subscription != null)
                        {
                            subscription.RealtimePrice = data.Value;
                        }
                    });
                    enumerator = enqueable;
                }
                else if (request.Configuration.Resolution != Resolution.Tick)
                {
                    // this enumerator allows the exchange to pump ticks into the 'back' of the enumerator,
                    // and the time sync loop can pull aggregated trade bars off the front
                    switch (request.Configuration.TickType)
                    {
                    case TickType.Quote:
                        var quoteBarAggregator = new QuoteBarBuilderEnumerator(request.Configuration.Increment, request.Security.Exchange.TimeZone, _timeProvider);
                        _exchange.AddDataHandler(request.Configuration.Symbol, data =>
                        {
                            var tick = data as Tick;

                            if (tick.TickType == TickType.Quote)
                            {
                                quoteBarAggregator.ProcessData(tick);
                                if (subscription != null)
                                {
                                    subscription.RealtimePrice = data.Value;
                                }
                            }
                        });
                        enumerator = quoteBarAggregator;
                        break;

                    case TickType.Trade:
                    default:
                        var tradeBarAggregator = new TradeBarBuilderEnumerator(request.Configuration.Increment, request.Security.Exchange.TimeZone, _timeProvider);
                        _exchange.AddDataHandler(request.Configuration.Symbol, data =>
                        {
                            var tick = data as Tick;

                            if (tick.TickType == TickType.Trade)
                            {
                                tradeBarAggregator.ProcessData(tick);
                                if (subscription != null)
                                {
                                    subscription.RealtimePrice = data.Value;
                                }
                            }
                        });
                        enumerator = tradeBarAggregator;
                        break;

                    case TickType.OpenInterest:
                        var oiAggregator = new OpenInterestEnumerator(request.Configuration.Increment, request.Security.Exchange.TimeZone, _timeProvider);
                        _exchange.AddDataHandler(request.Configuration.Symbol, data =>
                        {
                            var tick = data as Tick;

                            if (tick.TickType == TickType.OpenInterest)
                            {
                                oiAggregator.ProcessData(tick);
                            }
                        });
                        enumerator = oiAggregator;
                        break;
                    }
                }
                else
                {
                    // tick subscriptions can pass right through
                    var tickEnumerator = new EnqueueableEnumerator <BaseData>();
                    _exchange.SetDataHandler(request.Configuration.Symbol, data =>
                    {
                        tickEnumerator.Enqueue(data);
                        if (subscription != null)
                        {
                            subscription.RealtimePrice = data.Value;
                        }
                    });
                    enumerator = tickEnumerator;
                }

                if (request.Configuration.FillDataForward)
                {
                    var subscriptionConfigs = _subscriptions.Select(x => x.Configuration).Concat(new[] { request.Configuration });

                    UpdateFillForwardResolution(subscriptionConfigs);

                    enumerator = new LiveFillForwardEnumerator(_frontierTimeProvider, enumerator, request.Security.Exchange, _fillForwardResolution, request.Configuration.ExtendedMarketHours, localEndTime, request.Configuration.Increment);
                }

                // define market hours and user filters to incoming data
                if (request.Configuration.IsFilteredSubscription)
                {
                    enumerator = new SubscriptionFilterEnumerator(enumerator, request.Security, localEndTime);
                }

                // finally, make our subscriptions aware of the frontier of the data feed, prevents future data from spewing into the feed
                enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, timeZoneOffsetProvider);

                subscription = new Subscription(request.Universe, request.Security, request.Configuration, enumerator, timeZoneOffsetProvider, request.StartTimeUtc, request.EndTimeUtc, false);
            }
            catch (Exception err)
            {
                Log.Error(err);
            }

            return(subscription);
        }
Пример #2
0
        /// <summary>
        /// Creates a new subscription for the specified security
        /// </summary>
        /// <param name="request">The subscription request</param>
        /// <returns>A new subscription instance of the specified security</returns>
        protected Subscription CreateDataSubscription(SubscriptionRequest request)
        {
            Subscription subscription = null;

            try
            {
                var localEndTime           = request.EndTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone);
                var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc);

                IEnumerator <BaseData> enumerator;
                if (!_channelProvider.ShouldStreamSubscription(request.Configuration))
                {
                    if (!Quandl.IsAuthCodeSet)
                    {
                        // we're not using the SubscriptionDataReader, so be sure to set the auth token here
                        Quandl.SetAuthCode(Config.Get("quandl-auth-token"));
                    }

                    if (!Tiingo.IsAuthCodeSet)
                    {
                        // we're not using the SubscriptionDataReader, so be sure to set the auth token here
                        Tiingo.SetAuthCode(Config.Get("tiingo-auth-token"));
                    }

                    if (!USEnergyAPI.IsAuthCodeSet)
                    {
                        // we're not using the SubscriptionDataReader, so be sure to set the auth token here
                        USEnergyAPI.SetAuthCode(Config.Get("us-energy-information-auth-token"));
                    }

                    if (!FredApi.IsAuthCodeSet)
                    {
                        // we're not using the SubscriptionDataReader, so be sure to set the auth token here
                        FredApi.SetAuthCode(Config.Get("fred-auth-token"));
                    }

                    var factory         = new LiveCustomDataSubscriptionEnumeratorFactory(_timeProvider);
                    var enumeratorStack = factory.CreateEnumerator(request, _dataProvider);

                    _customExchange.AddEnumerator(request.Configuration.Symbol, enumeratorStack);

                    var enqueable = new EnqueueableEnumerator <BaseData>();
                    _customExchange.SetDataHandler(request.Configuration.Symbol, data =>
                    {
                        enqueable.Enqueue(data);

                        subscription.OnNewDataAvailable();

                        UpdateSubscriptionRealTimePrice(
                            subscription,
                            timeZoneOffsetProvider,
                            request.Security.Exchange.Hours,
                            data);
                    });
                    enumerator = enqueable;
                }
                else
                {
                    // this enumerator allows the exchange to pump ticks into the 'back' of the enumerator,
                    // and the time sync loop can pull aggregated trade bars off the front
                    switch (request.Configuration.Type.Name)
                    {
                    case nameof(QuoteBar):
                        var quoteBarAggregator = new QuoteBarBuilderEnumerator(
                            request.Configuration.Increment,
                            request.Security.Exchange.TimeZone,
                            _timeProvider,
                            true,
                            (sender, args) => subscription.OnNewDataAvailable());

                        _exchange.AddDataHandler(request.Configuration.Symbol, data =>
                        {
                            var tick = data as Tick;

                            if (tick?.TickType == TickType.Quote && !tick.Suspicious)
                            {
                                quoteBarAggregator.ProcessData(tick);

                                UpdateSubscriptionRealTimePrice(
                                    subscription,
                                    timeZoneOffsetProvider,
                                    request.Security.Exchange.Hours,
                                    data);
                            }
                        });
                        enumerator = quoteBarAggregator;
                        break;

                    case nameof(TradeBar):
                        var tradeBarAggregator = new TradeBarBuilderEnumerator(
                            request.Configuration.Increment,
                            request.Security.Exchange.TimeZone,
                            _timeProvider,
                            true,
                            (sender, args) => subscription.OnNewDataAvailable());

                        var auxDataEnumerator = new LiveAuxiliaryDataEnumerator(
                            request.Security.Exchange.TimeZone,
                            _timeProvider);

                        _exchange.AddDataHandler(
                            request.Configuration.Symbol,
                            data =>
                        {
                            if (data.DataType == MarketDataType.Auxiliary)
                            {
                                auxDataEnumerator.Enqueue(data);

                                subscription.OnNewDataAvailable();
                            }
                            else
                            {
                                var tick = data as Tick;
                                if (tick?.TickType == TickType.Trade && !tick.Suspicious)
                                {
                                    tradeBarAggregator.ProcessData(tick);

                                    UpdateSubscriptionRealTimePrice(
                                        subscription,
                                        timeZoneOffsetProvider,
                                        request.Security.Exchange.Hours,
                                        data);
                                }
                            }
                        });

                        enumerator = request.Configuration.SecurityType == SecurityType.Equity
                                ? (IEnumerator <BaseData>) new LiveEquityDataSynchronizingEnumerator(_frontierTimeProvider, request.Security.Exchange.TimeZone, auxDataEnumerator, tradeBarAggregator)
                                : tradeBarAggregator;
                        break;

                    case nameof(OpenInterest):
                        var oiAggregator = new OpenInterestEnumerator(
                            request.Configuration.Increment,
                            request.Security.Exchange.TimeZone,
                            _timeProvider,
                            true,
                            (sender, args) => subscription.OnNewDataAvailable());

                        _exchange.AddDataHandler(request.Configuration.Symbol, data =>
                        {
                            var tick = data as Tick;

                            if (tick?.TickType == TickType.OpenInterest && !tick.Suspicious)
                            {
                                oiAggregator.ProcessData(tick);
                            }
                        });
                        enumerator = oiAggregator;
                        break;

                    case nameof(Tick):
                    default:
                        // tick or streaming custom data subscriptions can pass right through
                        var tickEnumerator = new EnqueueableEnumerator <BaseData>();

                        _exchange.AddDataHandler(
                            request.Configuration.Symbol,
                            data =>
                        {
                            var tick = data as Tick;
                            if (tick != null)
                            {
                                if (tick.TickType == request.Configuration.TickType)
                                {
                                    tickEnumerator.Enqueue(data);
                                    subscription.OnNewDataAvailable();
                                    if (tick.TickType != TickType.OpenInterest)
                                    {
                                        UpdateSubscriptionRealTimePrice(
                                            subscription,
                                            timeZoneOffsetProvider,
                                            request.Security.Exchange.Hours,
                                            data);
                                    }
                                }
                            }
                            else
                            {
                                tickEnumerator.Enqueue(data);
                                subscription.OnNewDataAvailable();
                            }
                        });

                        enumerator = tickEnumerator;
                        break;
                    }
                }

                if (request.Configuration.FillDataForward)
                {
                    var fillForwardResolution = _subscriptions.UpdateAndGetFillForwardResolution(request.Configuration);

                    enumerator = new LiveFillForwardEnumerator(_frontierTimeProvider, enumerator, request.Security.Exchange, fillForwardResolution, request.Configuration.ExtendedMarketHours, localEndTime, request.Configuration.Increment, request.Configuration.DataTimeZone, request.StartTimeLocal);
                }

                // define market hours and user filters to incoming data
                if (request.Configuration.IsFilteredSubscription)
                {
                    enumerator = new SubscriptionFilterEnumerator(enumerator, request.Security, localEndTime);
                }

                // finally, make our subscriptions aware of the frontier of the data feed, prevents future data from spewing into the feed
                enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, timeZoneOffsetProvider);

                var subscriptionDataEnumerator = new SubscriptionDataEnumerator(request.Configuration, request.Security.Exchange.Hours, timeZoneOffsetProvider, enumerator);
                subscription = new Subscription(request, subscriptionDataEnumerator, timeZoneOffsetProvider);
            }
            catch (Exception err)
            {
                Log.Error(err);
            }

            return(subscription);
        }
Пример #3
0
        public void LastCloseAndCurrentOpenPriceShouldBeSameConsolidated()
        {
            var timeProvider = new ManualTimeProvider(TimeZones.NewYork);
            var enumerator   = new QuoteBarBuilderEnumerator(Time.OneSecond, TimeZones.NewYork, timeProvider, false);

            // noon new york time
            var currentTime = new DateTime(2015, 10, 08, 12, 0, 0);

            timeProvider.SetCurrentTime(currentTime);

            var reference = DateTime.Today;
            var tick1     = new Tick
            {
                Symbol   = Symbols.SPY,
                Time     = reference,
                TickType = TickType.Quote,
                AskPrice = 0,
                BidPrice = 24,
            };

            enumerator.ProcessData(tick1);

            var tick2 = new Tick
            {
                Symbol   = Symbols.SPY,
                Time     = reference.AddSeconds(1),
                TickType = TickType.Quote,
                AskPrice = 25,
                BidPrice = 0,
            };

            enumerator.ProcessData(tick2);
            currentTime = currentTime.AddSeconds(1);
            timeProvider.SetCurrentTime(currentTime);

            Assert.IsTrue(enumerator.MoveNext());

            var quoteBar = enumerator.Current as QuoteBar;

            // bar 1 emitted
            Assert.AreEqual(tick2.AskPrice, quoteBar.Ask.Open);
            Assert.AreEqual(tick1.BidPrice, quoteBar.Bid.Open);
            Assert.AreEqual(tick2.AskPrice, quoteBar.Ask.Close);
            Assert.AreEqual(tick1.BidPrice, quoteBar.Bid.Close);

            var tick3 = new Tick
            {
                Symbol   = Symbols.SPY,
                Time     = reference.AddSeconds(1),
                TickType = TickType.Quote,
                AskPrice = 36,
                BidPrice = 35,
            };

            enumerator.ProcessData(tick3);

            currentTime = currentTime.AddSeconds(1);
            timeProvider.SetCurrentTime(currentTime);
            Assert.IsTrue(enumerator.MoveNext());
            quoteBar = enumerator.Current as QuoteBar;

            // bar 2 emitted
            Assert.AreEqual(tick2.AskPrice, quoteBar.Ask.Open, "Ask Open not equal to Previous Close");
            Assert.AreEqual(tick1.BidPrice, quoteBar.Bid.Open, "Bid Open not equal to Previous Close");
            Assert.AreEqual(tick3.AskPrice, quoteBar.Ask.Close, "Ask Close incorrect");
            Assert.AreEqual(tick3.BidPrice, quoteBar.Bid.Close, "Bid Close incorrect");

            enumerator.Dispose();
        }
Пример #4
0
        /// <summary>
        /// Creates a new subscription for the specified security
        /// </summary>
        /// <param name="request">The subscription request</param>
        /// <returns>A new subscription instance of the specified security</returns>
        protected Subscription CreateSubscription(SubscriptionRequest request)
        {
            Subscription subscription = null;

            try
            {
                var localEndTime           = request.EndTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone);
                var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc);

                IEnumerator <BaseData> enumerator;
                if (request.Configuration.IsCustomData)
                {
                    if (!Quandl.IsAuthCodeSet)
                    {
                        // we're not using the SubscriptionDataReader, so be sure to set the auth token here
                        Quandl.SetAuthCode(Config.Get("quandl-auth-token"));
                    }

                    var factory         = new LiveCustomDataSubscriptionEnumeratorFactory(_timeProvider);
                    var enumeratorStack = factory.CreateEnumerator(request, _dataProvider);

                    _customExchange.AddEnumerator(request.Configuration.Symbol, enumeratorStack);

                    var enqueable = new EnqueueableEnumerator <BaseData>();
                    _customExchange.SetDataHandler(request.Configuration.Symbol, data =>
                    {
                        enqueable.Enqueue(data);
                        if (SubscriptionShouldUpdateRealTimePrice(subscription, timeZoneOffsetProvider))
                        {
                            subscription.RealtimePrice = data.Value;
                        }
                    });
                    enumerator = enqueable;
                }
                else if (request.Configuration.Resolution != Resolution.Tick)
                {
                    // this enumerator allows the exchange to pump ticks into the 'back' of the enumerator,
                    // and the time sync loop can pull aggregated trade bars off the front
                    switch (request.Configuration.TickType)
                    {
                    case TickType.Quote:
                        var quoteBarAggregator = new QuoteBarBuilderEnumerator(request.Configuration.Increment, request.Security.Exchange.TimeZone, _timeProvider);
                        _exchange.AddDataHandler(request.Configuration.Symbol, data =>
                        {
                            var tick = data as Tick;

                            if (tick.TickType == TickType.Quote)
                            {
                                quoteBarAggregator.ProcessData(tick);
                                if (SubscriptionShouldUpdateRealTimePrice(subscription, timeZoneOffsetProvider))
                                {
                                    subscription.RealtimePrice = data.Value;
                                }
                            }
                        });
                        enumerator = quoteBarAggregator;
                        break;

                    case TickType.Trade:
                    default:
                        var tradeBarAggregator = new TradeBarBuilderEnumerator(request.Configuration.Increment, request.Security.Exchange.TimeZone, _timeProvider);
                        _exchange.AddDataHandler(request.Configuration.Symbol, data =>
                        {
                            var tick = data as Tick;

                            if (tick.TickType == TickType.Trade)
                            {
                                tradeBarAggregator.ProcessData(tick);
                                if (SubscriptionShouldUpdateRealTimePrice(subscription, timeZoneOffsetProvider))
                                {
                                    subscription.RealtimePrice = data.Value;
                                }
                            }
                        });
                        enumerator = tradeBarAggregator;
                        break;

                    case TickType.OpenInterest:
                        var oiAggregator = new OpenInterestEnumerator(request.Configuration.Increment, request.Security.Exchange.TimeZone, _timeProvider);
                        _exchange.AddDataHandler(request.Configuration.Symbol, data =>
                        {
                            var tick = data as Tick;

                            if (tick.TickType == TickType.OpenInterest)
                            {
                                oiAggregator.ProcessData(tick);
                            }
                        });
                        enumerator = oiAggregator;
                        break;
                    }
                }
                else
                {
                    // tick subscriptions can pass right through
                    var tickEnumerator = new EnqueueableEnumerator <BaseData>();
                    _exchange.SetDataHandler(request.Configuration.Symbol, data =>
                    {
                        tickEnumerator.Enqueue(data);
                        if (SubscriptionShouldUpdateRealTimePrice(subscription, timeZoneOffsetProvider))
                        {
                            subscription.RealtimePrice = data.Value;
                        }
                    });
                    enumerator = tickEnumerator;
                }

                if (request.Configuration.FillDataForward)
                {
                    var subscriptionConfigs = _subscriptions.Select(x => x.Configuration).Concat(new[] { request.Configuration });

                    UpdateFillForwardResolution(subscriptionConfigs);

                    enumerator = new LiveFillForwardEnumerator(_frontierTimeProvider, enumerator, request.Security.Exchange, _fillForwardResolution, request.Configuration.ExtendedMarketHours, localEndTime, request.Configuration.Increment, request.Configuration.DataTimeZone);
                }

                // define market hours and user filters to incoming data
                if (request.Configuration.IsFilteredSubscription)
                {
                    enumerator = new SubscriptionFilterEnumerator(enumerator, request.Security, localEndTime);
                }

                // finally, make our subscriptions aware of the frontier of the data feed, prevents future data from spewing into the feed
                enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, timeZoneOffsetProvider);

                var subscriptionDataEnumerator = SubscriptionData.Enumerator(request.Configuration, request.Security, timeZoneOffsetProvider, enumerator);
                subscription = new Subscription(request.Universe, request.Security, request.Configuration, subscriptionDataEnumerator, timeZoneOffsetProvider, request.StartTimeUtc, request.EndTimeUtc, false);
            }
            catch (Exception err)
            {
                Log.Error(err);
            }

            return(subscription);
        }
Пример #5
0
        public void AggregatesTicksIntoSecondBars()
        {
            var timeProvider = new ManualTimeProvider(TimeZones.NewYork);
            var enumerator   = new QuoteBarBuilderEnumerator(Time.OneSecond, TimeZones.NewYork, timeProvider, false);

            // noon new york time
            var currentTime = new DateTime(2015, 10, 08, 12, 0, 0);

            timeProvider.SetCurrentTime(currentTime);

            // add some ticks
            var ticks = new List <Tick>
            {
                new Tick(currentTime, Symbols.SPY, 199, 200)
                {
                    Quantity = 10
                },
                new Tick(currentTime, Symbols.SPY, 199.21m, 200.02m)
                {
                    Quantity = 5
                },
                new Tick(currentTime, Symbols.SPY, 198.77m, 199.75m)
                {
                    Quantity = 20
                },
                new Tick(currentTime, Symbols.SPY, 198.77m, 199.75m)
                {
                    Quantity = 0
                },
                new Tick(currentTime, Symbols.SPY, 198.77m, 199.75m)
                {
                    Quantity = 20
                },
                new Tick(currentTime, Symbols.SPY, 198.77m, 199.75m)
                {
                    Quantity = 0
                },
            };

            foreach (var tick in ticks)
            {
                enumerator.ProcessData(tick);
            }

            // even though no data is here, it will still return true
            Assert.IsTrue(enumerator.MoveNext());
            Assert.IsNull(enumerator.Current);

            // advance a second
            currentTime = currentTime.AddSeconds(1);
            timeProvider.SetCurrentTime(currentTime);

            Assert.IsTrue(enumerator.MoveNext());
            Assert.IsNotNull(enumerator.Current);

            var bar = (QuoteBar)enumerator.Current;

            Assert.AreEqual(currentTime.AddSeconds(-1), bar.Time);
            Assert.AreEqual(currentTime, bar.EndTime);
            Assert.AreEqual(Symbols.SPY, bar.Symbol);
            Assert.AreEqual(ticks.First().LastPrice, bar.Open);
            Assert.AreEqual(ticks.Max(x => x.LastPrice), bar.High);
            Assert.AreEqual(ticks.Min(x => x.LastPrice), bar.Low);
            Assert.AreEqual(ticks.Last().LastPrice, bar.Close);

            enumerator.Dispose();
        }