Exemple #1
0
        public void SynchronizesData(int timeOffsetSeconds, int testTimeSeconds)
        {
            var start = DateTime.UtcNow;
            var end   = start.AddSeconds(testTimeSeconds);

            var time      = start;
            var tickList1 = Enumerable.Range(0, 10).Select(x => new Tick {
                Time = time.AddSeconds(x * 1 + timeOffsetSeconds), Value = x
            }).ToList();
            var tickList2 = Enumerable.Range(0, 5).Select(x => new Tick {
                Time = time.AddSeconds(x * 2 + timeOffsetSeconds), Value = x + 100
            }).ToList();
            var stream1 = tickList1.GetEnumerator();
            var stream2 = tickList2.GetEnumerator();

            var count1       = 0;
            var count2       = 0;
            var previous     = DateTime.MinValue;
            var synchronizer = new LiveAuxiliaryDataSynchronizingEnumerator(new RealTimeProvider(), DateTimeZone.Utc, stream1, new List <IEnumerator <BaseData> > {
                stream2
            });

            while (synchronizer.MoveNext() && DateTime.UtcNow < end)
            {
                if (synchronizer.Current != null)
                {
                    if (synchronizer.Current.Value < 100)
                    {
                        Assert.AreEqual(count1, synchronizer.Current.Value);
                        count1++;
                    }
                    else
                    {
                        Assert.AreEqual(count2 + 100, synchronizer.Current.Value);
                        count2++;
                    }

                    Assert.That(synchronizer.Current.EndTime, Is.GreaterThanOrEqualTo(previous));
                    previous = synchronizer.Current.EndTime;

                    Log.Trace($"Data point emitted: {synchronizer.Current.EndTime:O} - {synchronizer.Current}");
                }
            }

            Log.Trace($"Total point count: {count1 + count2}");

            Assert.AreEqual(tickList1.Count, count1);
            Assert.AreEqual(tickList2.Count, count2);
            synchronizer.Dispose();
        }
Exemple #2
0
        /// <summary>
        /// Creates a new subscription for the specified security
        /// </summary>
        /// <param name="request">The subscription request</param>
        /// <returns>A new subscription instance of the specified security</returns>
        protected Subscription CreateDataSubscription(SubscriptionRequest request)
        {
            Subscription subscription = null;

            try
            {
                var localEndTime           = request.EndTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone);
                var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Configuration.ExchangeTimeZone, request.StartTimeUtc, request.EndTimeUtc);

                IEnumerator <BaseData> enumerator;
                if (!_channelProvider.ShouldStreamSubscription(request.Configuration))
                {
                    if (!Quandl.IsAuthCodeSet)
                    {
                        // we're not using the SubscriptionDataReader, so be sure to set the auth token here
                        Quandl.SetAuthCode(Config.Get("quandl-auth-token"));
                    }

                    if (!Tiingo.IsAuthCodeSet)
                    {
                        // we're not using the SubscriptionDataReader, so be sure to set the auth token here
                        Tiingo.SetAuthCode(Config.Get("tiingo-auth-token"));
                    }

                    var factory         = new LiveCustomDataSubscriptionEnumeratorFactory(_timeProvider);
                    var enumeratorStack = factory.CreateEnumerator(request, _dataProvider);

                    _customExchange.AddEnumerator(request.Configuration.Symbol, enumeratorStack);

                    var enqueable = new EnqueueableEnumerator <BaseData>();
                    _customExchange.SetDataHandler(request.Configuration.Symbol, data =>
                    {
                        enqueable.Enqueue(data);

                        subscription.OnNewDataAvailable();
                    });
                    enumerator = enqueable;
                }
                else
                {
                    var auxEnumerators = new List <IEnumerator <BaseData> >();

                    if (LiveAuxiliaryDataEnumerator.TryCreate(request.Configuration, _timeProvider, _dataQueueHandler,
                                                              request.Security.Cache, _mapFileProvider, _factorFileProvider, request.StartTimeLocal, out var auxDataEnumator))
                    {
                        auxEnumerators.Add(auxDataEnumator);
                    }

                    EventHandler handler = (_, _) => subscription?.OnNewDataAvailable();
                    enumerator = Subscribe(request.Configuration, handler);

                    if (request.Configuration.EmitSplitsAndDividends())
                    {
                        auxEnumerators.Add(Subscribe(new SubscriptionDataConfig(request.Configuration, typeof(Dividend)), handler));
                        auxEnumerators.Add(Subscribe(new SubscriptionDataConfig(request.Configuration, typeof(Split)), handler));
                    }

                    if (auxEnumerators.Count > 0)
                    {
                        enumerator = new LiveAuxiliaryDataSynchronizingEnumerator(_timeProvider, request.Configuration.ExchangeTimeZone, enumerator, auxEnumerators);
                    }
                }

                if (request.Configuration.FillDataForward)
                {
                    var fillForwardResolution = _subscriptions.UpdateAndGetFillForwardResolution(request.Configuration);

                    enumerator = new LiveFillForwardEnumerator(_frontierTimeProvider, enumerator, request.Security.Exchange, fillForwardResolution, request.Configuration.ExtendedMarketHours, localEndTime, request.Configuration.Increment, request.Configuration.DataTimeZone);
                }

                // define market hours and user filters to incoming data
                if (request.Configuration.IsFilteredSubscription)
                {
                    enumerator = new SubscriptionFilterEnumerator(enumerator, request.Security, localEndTime, request.Configuration.ExtendedMarketHours, true, request.ExchangeHours);
                }

                // finally, make our subscriptions aware of the frontier of the data feed, prevents future data from spewing into the feed
                enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, timeZoneOffsetProvider);

                var subscriptionDataEnumerator = new SubscriptionDataEnumerator(request.Configuration, request.Security.Exchange.Hours, timeZoneOffsetProvider, enumerator, request.IsUniverseSubscription);
                subscription = new Subscription(request, subscriptionDataEnumerator, timeZoneOffsetProvider);
            }
            catch (Exception err)
            {
                Log.Error(err);
            }

            return(subscription);
        }
Exemple #3
0
        /// <summary>
        /// Creates a new subscription for the specified security
        /// </summary>
        /// <param name="request">The subscription request</param>
        /// <returns>A new subscription instance of the specified security</returns>
        private Subscription CreateDataSubscription(SubscriptionRequest request)
        {
            Subscription subscription = null;

            try
            {
                var localEndTime           = request.EndTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone);
                var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Configuration.ExchangeTimeZone, request.StartTimeUtc, request.EndTimeUtc);

                IEnumerator <BaseData> enumerator = null;
                // during warmup we might get requested to add some asset which has already expired in which case the live enumerator will be empty
                if (!IsExpired(request.Configuration))
                {
                    if (!_channelProvider.ShouldStreamSubscription(request.Configuration))
                    {
                        if (!Tiingo.IsAuthCodeSet)
                        {
                            // we're not using the SubscriptionDataReader, so be sure to set the auth token here
                            Tiingo.SetAuthCode(Config.Get("tiingo-auth-token"));
                        }

                        var factory         = new LiveCustomDataSubscriptionEnumeratorFactory(_timeProvider);
                        var enumeratorStack = factory.CreateEnumerator(request, _dataProvider);

                        var enqueable = new EnqueueableEnumerator <BaseData>();
                        _customExchange.AddEnumerator(request.Configuration.Symbol, enumeratorStack, handleData: data =>
                        {
                            enqueable.Enqueue(data);

                            subscription?.OnNewDataAvailable();
                        });

                        enumerator = enqueable;
                    }
                    else
                    {
                        var auxEnumerators = new List <IEnumerator <BaseData> >();

                        if (LiveAuxiliaryDataEnumerator.TryCreate(request.Configuration, _timeProvider, _dataQueueHandler,
                                                                  request.Security.Cache, _mapFileProvider, _factorFileProvider, request.StartTimeLocal, out var auxDataEnumator))
                        {
                            auxEnumerators.Add(auxDataEnumator);
                        }

                        EventHandler handler = (_, _) => subscription?.OnNewDataAvailable();
                        enumerator = Subscribe(request.Configuration, handler);

                        if (request.Configuration.EmitSplitsAndDividends())
                        {
                            auxEnumerators.Add(Subscribe(new SubscriptionDataConfig(request.Configuration, typeof(Dividend)), handler));
                            auxEnumerators.Add(Subscribe(new SubscriptionDataConfig(request.Configuration, typeof(Split)), handler));
                        }

                        if (auxEnumerators.Count > 0)
                        {
                            enumerator = new LiveAuxiliaryDataSynchronizingEnumerator(_timeProvider, request.Configuration.ExchangeTimeZone, enumerator, auxEnumerators);
                        }
                    }

                    // scale prices before 'SubscriptionFilterEnumerator' since it updates securities realtime price
                    // and before fill forwarding so we don't happen to apply twice the factor
                    if (request.Configuration.PricesShouldBeScaled(liveMode: true))
                    {
                        enumerator = new PriceScaleFactorEnumerator(
                            enumerator,
                            request.Configuration,
                            _factorFileProvider,
                            liveMode: true);
                    }

                    if (request.Configuration.FillDataForward)
                    {
                        var fillForwardResolution = _subscriptions.UpdateAndGetFillForwardResolution(request.Configuration);

                        enumerator = new LiveFillForwardEnumerator(_frontierTimeProvider, enumerator, request.Security.Exchange, fillForwardResolution, request.Configuration.ExtendedMarketHours, localEndTime, request.Configuration.Increment, request.Configuration.DataTimeZone);
                    }

                    // make our subscriptions aware of the frontier of the data feed, prevents future data from spewing into the feed
                    enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, timeZoneOffsetProvider);

                    // define market hours and user filters to incoming data after the frontier enumerator so during warmup we avoid any realtime data making it's way into the securities
                    if (request.Configuration.IsFilteredSubscription)
                    {
                        enumerator = new SubscriptionFilterEnumerator(enumerator, request.Security, localEndTime, request.Configuration.ExtendedMarketHours, true, request.ExchangeHours);
                    }
                }
                else
                {
                    enumerator = Enumerable.Empty <BaseData>().GetEnumerator();
                }

                enumerator = GetWarmupEnumerator(request, enumerator);

                var subscriptionDataEnumerator = new SubscriptionDataEnumerator(request.Configuration, request.Security.Exchange.Hours, timeZoneOffsetProvider, enumerator, request.IsUniverseSubscription);
                subscription = new Subscription(request, subscriptionDataEnumerator, timeZoneOffsetProvider);
            }
            catch (Exception err)
            {
                Log.Error(err);
            }

            return(subscription);
        }