예제 #1
0
        private Subscription CreateSubscription(Universe universe, Security security, SubscriptionDataConfig config, DateTime startTimeUtc, DateTime endTimeUtc)
        {
            var localStartTime = startTimeUtc.ConvertFromUtc(security.Exchange.TimeZone);
            var localEndTime   = endTimeUtc.ConvertFromUtc(security.Exchange.TimeZone);

            var tradeableDates = Time.EachTradeableDayInTimeZone(security.Exchange.Hours, localStartTime, localEndTime, config.DataTimeZone, config.ExtendedMarketHours);

            // ReSharper disable once PossibleMultipleEnumeration
            if (!tradeableDates.Any())
            {
                _algorithm.Error(string.Format("No data loaded for {0} because there were no tradeable dates for this security.", security.Symbol));
                return(null);
            }

            // get the map file resolver for this market
            var mapFileResolver = MapFileResolver.Empty;

            if (config.SecurityType == SecurityType.Equity)
            {
                mapFileResolver = _mapFileProvider.Get(config.Market);
            }

            // ReSharper disable once PossibleMultipleEnumeration
            var enumerator = CreateSubscriptionEnumerator(security, config, localStartTime, localEndTime, mapFileResolver, tradeableDates);

            var enqueueable = new EnqueueableEnumerator <BaseData>(true);

            // add this enumerator to our exchange
            ScheduleEnumerator(enumerator, enqueueable, GetLowerThreshold(config.Resolution), GetUpperThreshold(config.Resolution));

            var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, startTimeUtc, endTimeUtc);
            var subscription           = new Subscription(universe, security, enqueueable, timeZoneOffsetProvider, startTimeUtc, endTimeUtc, false);

            return(subscription);
        }
예제 #2
0
        private Subscription CreateDataSubscription(SubscriptionRequest request)
        {
            // ReSharper disable once PossibleMultipleEnumeration
            if (!request.TradableDays.Any())
            {
                _algorithm.Error(
                    $"No data loaded for {request.Security.Symbol} because there were no tradeable dates for this security."
                    );
                return(null);
            }

            // ReSharper disable once PossibleMultipleEnumeration
            var enumeratorFactory = GetEnumeratorFactory(request);
            var enumerator        = enumeratorFactory.CreateEnumerator(request, _dataProvider);

            enumerator = ConfigureEnumerator(request, false, enumerator);

            var enqueueable            = new EnqueueableEnumerator <SubscriptionData>(true);
            var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc);
            var subscription           = new Subscription(request, enqueueable, timeZoneOffsetProvider);

            // add this enumerator to our exchange
            ScheduleEnumerator(subscription,
                               enumerator,
                               enqueueable,
                               GetLowerThreshold(request.Configuration.Resolution),
                               GetUpperThreshold(request.Configuration.Resolution),
                               request.Security.Exchange.Hours);

            return(subscription);
        }
예제 #3
0
        public void PassesTicksStraightThrough()
        {
            var enumerator = new EnqueueableEnumerator<Tick>();

            // add some ticks
            var currentTime = new DateTime(2015, 10, 08);

            // returns true even if no data present until stop is called
            Assert.IsTrue(enumerator.MoveNext());
            Assert.IsNull(enumerator.Current);

            var tick1 = new Tick(currentTime, Symbols.SPY, 199.55m, 199, 200) {Quantity = 10};
            enumerator.Enqueue(tick1);
            Assert.IsTrue(enumerator.MoveNext());
            Assert.AreEqual(tick1, enumerator.Current);

            Assert.IsTrue(enumerator.MoveNext());
            Assert.IsNull(enumerator.Current);

            var tick2 = new Tick(currentTime, Symbols.SPY, 199.56m, 199.21m, 200.02m) {Quantity = 5};
            enumerator.Enqueue(tick2);
            Assert.IsTrue(enumerator.MoveNext());
            Assert.AreEqual(tick2, enumerator.Current);

            enumerator.Stop();

            Assert.IsFalse(enumerator.MoveNext());
            Assert.IsNull(enumerator.Current);
        }
예제 #4
0
        /// <summary>
        /// Adds a new subscription for universe selection
        /// </summary>
        /// <param name="request">The subscription request</param>
        private Subscription CreateUniverseSubscription(SubscriptionRequest request)
        {
            // grab the relevant exchange hours
            var config = request.Configuration;

            // define our data enumerator
            var enumerator = GetEnumeratorFactory(request).CreateEnumerator(request, _dataProvider);

            var firstLoopCount = 5;
            var lowerThreshold = GetLowerThreshold(config.Resolution);
            var upperThreshold = GetUpperThreshold(config.Resolution);

            if (config.Type == typeof(CoarseFundamental))
            {
                firstLoopCount = 2;
                lowerThreshold = 5;
                upperThreshold = 100000;
            }

            var enqueueable            = new EnqueueableEnumerator <SubscriptionData>(true);
            var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc);
            var subscription           = new Subscription(request.Universe, request.Security, config, enqueueable, timeZoneOffsetProvider, request.StartTimeUtc, request.EndTimeUtc, true);

            // add this enumerator to our exchange
            ScheduleEnumerator(subscription, enumerator, enqueueable, lowerThreshold, upperThreshold, firstLoopCount);

            return(subscription);
        }
예제 #5
0
        public void PassesTicksStraightThrough()
        {
            var enumerator = new EnqueueableEnumerator<Tick>();

            // add some ticks
            var currentTime = new DateTime(2015, 10, 08);

            // returns true even if no data present until stop is called
            Assert.IsTrue(enumerator.MoveNext());
            Assert.IsNull(enumerator.Current);

            var tick1 = new Tick(currentTime, Symbols.SPY, 199.55m, 199, 200) {Quantity = 10};
            enumerator.Enqueue(tick1);
            Assert.IsTrue(enumerator.MoveNext());
            Assert.AreEqual(tick1, enumerator.Current);

            Assert.IsTrue(enumerator.MoveNext());
            Assert.IsNull(enumerator.Current);

            var tick2 = new Tick(currentTime, Symbols.SPY, 199.56m, 199.21m, 200.02m) {Quantity = 5};
            enumerator.Enqueue(tick2);
            Assert.IsTrue(enumerator.MoveNext());
            Assert.AreEqual(tick2, enumerator.Current);

            enumerator.Stop();

            Assert.IsFalse(enumerator.MoveNext());
            Assert.IsNull(enumerator.Current);
        }
예제 #6
0
        /// <summary>
        /// Adds a new subscription for universe selection
        /// </summary>
        /// <param name="request">The subscription request</param>
        private Subscription CreateUniverseSubscription(SubscriptionRequest request)
        {
            // grab the relevant exchange hours
            var config = request.Configuration;

            // define our data enumerator
            var enumerator = GetEnumeratorFactory(request).CreateEnumerator(request, _dataProvider);

            var firstLoopCount = 5;
            var lowerThreshold = GetLowerThreshold(config.Resolution);
            var upperThreshold = GetUpperThreshold(config.Resolution);

            if (config.Type == typeof(CoarseFundamental))
            {
                firstLoopCount = 2;
                // the lower threshold will be when we start the worker again, if he is stopped
                lowerThreshold = 200;
                // the upper threshold will stop the worker from loading more data. This is roughly 1 GB
                upperThreshold = 500;
            }

            var enqueueable            = new EnqueueableEnumerator <SubscriptionData>(true);
            var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc);
            var subscription           = new Subscription(request, enqueueable, timeZoneOffsetProvider);

            // add this enumerator to our exchange
            ScheduleEnumerator(subscription, enumerator, enqueueable, lowerThreshold, upperThreshold, request.Security.Exchange.Hours, firstLoopCount);

            return(subscription);
        }
예제 #7
0
        private Subscription CreateSubscription(Resolution resolution, string symbol = "AAPL", bool isInternalFeed = false,
                                                SecurityType type = SecurityType.Equity, TickType tickType         = TickType.Trade)
        {
            var      start = DateTime.UtcNow;
            var      end   = start.AddSeconds(10);
            Security security;
            Symbol   _symbol;

            if (type == SecurityType.Equity)
            {
                _symbol  = new Symbol(SecurityIdentifier.GenerateEquity(DateTime.Now, symbol, Market.USA), symbol);
                security = new Equity(
                    _symbol,
                    SecurityExchangeHours.AlwaysOpen(DateTimeZone.Utc),
                    new Cash(Currencies.USD, 0, 1),
                    SymbolProperties.GetDefault(Currencies.USD),
                    ErrorCurrencyConverter.Instance,
                    RegisteredSecurityDataTypesProvider.Null
                    );
            }
            else if (type == SecurityType.Option)
            {
                _symbol = new Symbol(SecurityIdentifier.GenerateOption(DateTime.Now,
                                                                       SecurityIdentifier.GenerateEquity(DateTime.Now, symbol, Market.USA),
                                                                       Market.USA, 0.0m, OptionRight.Call, OptionStyle.American), symbol);
                security = new Option(
                    _symbol,
                    SecurityExchangeHours.AlwaysOpen(DateTimeZone.Utc),
                    new Cash(Currencies.USD, 0, 1),
                    new OptionSymbolProperties(SymbolProperties.GetDefault(Currencies.USD)),
                    ErrorCurrencyConverter.Instance,
                    RegisteredSecurityDataTypesProvider.Null
                    );
            }
            else if (type == SecurityType.Future)
            {
                _symbol  = new Symbol(SecurityIdentifier.GenerateFuture(DateTime.Now, symbol, Market.USA), symbol);
                security = new Future(
                    _symbol,
                    SecurityExchangeHours.AlwaysOpen(DateTimeZone.Utc),
                    new Cash(Currencies.USD, 0, 1),
                    SymbolProperties.GetDefault(Currencies.USD),
                    ErrorCurrencyConverter.Instance,
                    RegisteredSecurityDataTypesProvider.Null
                    );
            }
            else
            {
                throw new Exception("SecurityType not implemented");
            }
            var config = new SubscriptionDataConfig(typeof(TradeBar), _symbol, resolution, DateTimeZone.Utc, DateTimeZone.Utc, true, false, isInternalFeed, false, tickType);
            var timeZoneOffsetProvider     = new TimeZoneOffsetProvider(DateTimeZone.Utc, start, end);
            var enumerator                 = new EnqueueableEnumerator <BaseData>();
            var subscriptionDataEnumerator = new SubscriptionDataEnumerator(config, security.Exchange.Hours, timeZoneOffsetProvider, enumerator);
            var subscriptionRequest        = new SubscriptionRequest(false, null, security, config, start, end);

            return(new Subscription(subscriptionRequest, subscriptionDataEnumerator, timeZoneOffsetProvider));
        }
        /// <summary>
        /// Setups a new <see cref="Subscription"/> which will consume a blocking <see cref="EnqueueableEnumerator{T}"/>
        /// that will be feed by a worker task
        /// </summary>
        /// <param name="request">The subscription data request</param>
        /// <param name="enumerator">The data enumerator stack</param>
        /// <returns>A new subscription instance ready to consume</returns>
        public static Subscription CreateAndScheduleWorker(
            SubscriptionRequest request,
            IEnumerator <BaseData> enumerator)
        {
            var exchangeHours          = request.Security.Exchange.Hours;
            var enqueueable            = new EnqueueableEnumerator <SubscriptionData>(true);
            var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc);
            var subscription           = new Subscription(request, enqueueable, timeZoneOffsetProvider);

            Func <int, bool> produce = (workBatchSize) =>
            {
                try
                {
                    var count = 0;
                    while (enumerator.MoveNext())
                    {
                        // subscription has been removed, no need to continue enumerating
                        if (enqueueable.HasFinished)
                        {
                            enumerator.DisposeSafely();
                            return(false);
                        }

                        var subscriptionData = SubscriptionData.Create(subscription.Configuration, exchangeHours,
                                                                       subscription.OffsetProvider, enumerator.Current);

                        // drop the data into the back of the enqueueable
                        enqueueable.Enqueue(subscriptionData);

                        count++;
                        // stop executing if added more data than the work batch size, we don't want to fill the ram
                        if (count > workBatchSize)
                        {
                            return(true);
                        }
                    }
                }
                catch (Exception exception)
                {
                    Log.Error(exception, $"Subscription worker task exception {request.Configuration}.");
                }

                // we made it here because MoveNext returned false or we exploded, stop the enqueueable
                enqueueable.Stop();
                // we have to dispose of the enumerator
                enumerator.DisposeSafely();
                return(false);
            };

            WeightedWorkScheduler.Instance.QueueWork(produce,
                                                     // if the subscription finished we return 0, so the work is prioritized and gets removed
                                                     () => enqueueable.HasFinished ? 0 : enqueueable.Count);

            return(subscription);
        }
예제 #9
0
        /// <summary>
        /// Setups a new <see cref="Subscription"/> which will consume a blocking <see cref="EnqueueableEnumerator{T}"/>
        /// that will be feed by a worker task
        /// </summary>
        /// <param name="request">The subscription data request</param>
        /// <param name="enumerator">The data enumerator stack</param>
        /// <param name="lowerThreshold">The lower threshold for the worker task, for which the consumer will trigger the worker
        /// if it has stopped <see cref="EnqueueableEnumerator{T}.TriggerProducer"/></param>
        /// <param name="upperThreshold">The upper threshold for the worker task, after which it will stop producing until requested
        /// by the consumer <see cref="EnqueueableEnumerator{T}.TriggerProducer"/></param>
        /// <returns>A new subscription instance ready to consume</returns>
        public static Subscription CreateAndScheduleWorker(
            SubscriptionRequest request,
            IEnumerator <BaseData> enumerator,
            int lowerThreshold,
            int upperThreshold)
        {
            var exchangeHours          = request.Security.Exchange.Hours;
            var enqueueable            = new EnqueueableEnumerator <SubscriptionData>(true);
            var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc);
            var subscription           = new Subscription(request, enqueueable, timeZoneOffsetProvider);

            Action produce = () =>
            {
                var count = 0;
                while (enumerator.MoveNext())
                {
                    // subscription has been removed, no need to continue enumerating
                    if (enqueueable.HasFinished)
                    {
                        enumerator.Dispose();
                        return;
                    }

                    var subscriptionData = SubscriptionData.Create(subscription.Configuration, exchangeHours, subscription.OffsetProvider, enumerator.Current);

                    // drop the data into the back of the enqueueable
                    enqueueable.Enqueue(subscriptionData);

                    count++;

                    // stop executing if we have more data than the upper threshold in the enqueueable, we don't want to fill the ram
                    if (count > upperThreshold)
                    {
                        // we use local count for the outside if, for performance, and adjust here
                        count = enqueueable.Count;
                        if (count > upperThreshold)
                        {
                            // we will be re scheduled to run by the consumer, see EnqueueableEnumerator
                            return;
                        }
                    }
                }

                // we made it here because MoveNext returned false, stop the enqueueable
                enqueueable.Stop();
                // we have to dispose of the enumerator
                enumerator.Dispose();
            };

            enqueueable.SetProducer(produce, lowerThreshold);

            return(subscription);
        }
        public void MoveNextBlocks()
        {
            var finished   = new ManualResetEvent(false);
            var enumerator = new EnqueueableEnumerator <Tick>(true);

            // producer
            int count = 0;

            Task.Run(() =>
            {
                while (!finished.WaitOne(TimeSpan.FromMilliseconds(50)))
                {
                    enumerator.Enqueue(new Tick(DateTime.Now, Symbols.SPY, 100, 101));
                    count++;

                    // 5 data points is plenty
                    if (count > 5)
                    {
                        finished.Set();
                        enumerator.Stop();
                    }
                }
            });

            // consumer
            int  dequeuedCount        = 0;
            bool encounteredError     = false;
            var  consumerTaskFinished = new ManualResetEvent(false);

            Task.Run(() =>
            {
                while (enumerator.MoveNext())
                {
                    dequeuedCount++;
                    if (enumerator.Current == null)
                    {
                        encounteredError = true;
                    }
                }
                consumerTaskFinished.Set();
            });

            finished.WaitOne(Timeout.Infinite);
            consumerTaskFinished.WaitOne(Timeout.Infinite);

            Assert.IsFalse(enumerator.MoveNext());
            Assert.IsFalse(encounteredError);
            Assert.AreEqual(count, dequeuedCount);

            enumerator.Dispose();
        }
예제 #11
0
        public void SetErrorHandlerExitsOnTrueReturn()
        {
            var enqueable = new EnqueueableEnumerator <BaseData>();
            var exchange  = new BaseDataExchange("test");

            var cancellationToken = new CancellationTokenSource();

            Task.Factory.StartNew(() =>
            {
                while (!cancellationToken.IsCancellationRequested)
                {
                    Thread.Sleep(10);
                    enqueable.Enqueue(new Tick {
                        Symbol = Symbols.SPY, Time = DateTime.UtcNow
                    });
                }
            });

            var      first       = true;
            var      errorCaught = new AutoResetEvent(true);
            BaseData last        = null;

            exchange.SetErrorHandler(error =>
            {
                errorCaught.Set();
                return(true);
            });

            exchange.AddEnumerator(Symbols.SPY, enqueable, handleData: spy =>
            {
                if (first)
                {
                    first = false;
                    throw new Exception();
                }
                last = spy;
            });

            Task.Run(() => exchange.Start());

            Assert.IsTrue(errorCaught.WaitOne(DefaultTimeout));

            exchange.Stop();

            Assert.IsNull(last);

            enqueable.Dispose();
            cancellationToken.Cancel();
        }
        private void ScheduleEnumerator(IEnumerator <BaseData> enumerator, EnqueueableEnumerator <BaseData> enqueueable, int lowerThreshold, int upperThreshold, int firstLoopCount = 5)
        {
            // schedule the work on the controller
            var firstLoop = true;
            FuncParallelRunnerWorkItem workItem = null;

            workItem = new FuncParallelRunnerWorkItem(() => enqueueable.Count < lowerThreshold, () =>
            {
                var count = 0;
                while (enumerator.MoveNext())
                {
                    // subscription has been removed, no need to continue enumerating
                    if (enqueueable.HasFinished)
                    {
                        enumerator.Dispose();
                        return;
                    }

                    // drop the data into the back of the enqueueable
                    enqueueable.Enqueue(enumerator.Current);

                    count++;

                    // special behavior for first loop to spool up quickly
                    if (firstLoop && count > firstLoopCount)
                    {
                        // there's more data in the enumerator, reschedule to run again
                        firstLoop = false;
                        _controller.Schedule(workItem);
                        return;
                    }

                    // stop executing if we've dequeued more than the lower threshold or have
                    // more total that upper threshold in the enqueueable's queue
                    if (count > lowerThreshold || enqueueable.Count > upperThreshold)
                    {
                        // there's more data in the enumerator, reschedule to run again
                        _controller.Schedule(workItem);
                        return;
                    }
                }

                // we made it here because MoveNext returned false, stop the enqueueable and don't reschedule
                enqueueable.Stop();
            });
            _controller.Schedule(workItem);
        }
예제 #13
0
        private Subscription CreateSubscription(Universe universe, IResultHandler resultHandler, Security security, DateTime startTimeUtc, DateTime endTimeUtc, IReadOnlyRef <TimeSpan> fillForwardResolution)
        {
            var config         = security.SubscriptionDataConfig;
            var localStartTime = startTimeUtc.ConvertFromUtc(security.Exchange.TimeZone);
            var localEndTime   = endTimeUtc.ConvertFromUtc(security.Exchange.TimeZone);

            var tradeableDates = Time.EachTradeableDay(security, localStartTime, localEndTime);

            // ReSharper disable once PossibleMultipleEnumeration
            if (!tradeableDates.Any())
            {
                _algorithm.Error(string.Format("No data loaded for {0} because there were no tradeable dates for this security.", security.Symbol));
                return(null);
            }

            // get the map file resolver for this market
            var mapFileResolver = MapFileResolver.Empty;

            if (config.SecurityType == SecurityType.Equity)
            {
                mapFileResolver = _mapFileProvider.Get(config.Market);
            }

            // ReSharper disable once PossibleMultipleEnumeration
            IEnumerator <BaseData> enumerator = new SubscriptionDataReader(config, localStartTime, localEndTime, resultHandler, mapFileResolver, _factorFileProvider, tradeableDates, false);

            // optionally apply fill forward logic, but never for tick data
            if (config.FillDataForward && config.Resolution != Resolution.Tick)
            {
                enumerator = new FillForwardEnumerator(enumerator, security.Exchange, fillForwardResolution,
                                                       security.IsExtendedMarketHours, localEndTime, config.Resolution.ToTimeSpan());
            }

            // finally apply exchange/user filters
            enumerator = SubscriptionFilterEnumerator.WrapForDataFeed(resultHandler, enumerator, security, localEndTime);

            var enqueueable = new EnqueueableEnumerator <BaseData>(true);

            // add this enumerator to our exchange
            ScheduleEnumerator(enumerator, enqueueable, GetLowerThreshold(config.Resolution), GetUpperThreshold(config.Resolution));

            var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, startTimeUtc, endTimeUtc);
            var subscription           = new Subscription(universe, security, enqueueable, timeZoneOffsetProvider, startTimeUtc, endTimeUtc, false);

            return(subscription);
        }
예제 #14
0
        public void EndsQueueConsumption()
        {
            var enqueable = new EnqueueableEnumerator <BaseData>();
            var exchange  = new BaseDataExchange("test");

            var cancellationToken = new CancellationTokenSource();

            Task.Run(() =>
            {
                while (!cancellationToken.IsCancellationRequested)
                {
                    Thread.Sleep(10);
                    enqueable.Enqueue(new Tick {
                        Symbol = Symbols.SPY, Time = DateTime.UtcNow
                    });
                }
            });

            BaseData last        = null;
            var      lastUpdated = new AutoResetEvent(false);

            exchange.AddEnumerator(Symbols.SPY, enqueable, handleData: spy =>
            {
                last = spy;
                lastUpdated.Set();
            });

            var finishedRunning = new AutoResetEvent(false);

            Task.Run(() => { exchange.Start(); finishedRunning.Set(); });

            Assert.IsTrue(lastUpdated.WaitOne(DefaultTimeout));

            exchange.Stop();
            cancellationToken.Cancel();

            Assert.IsTrue(finishedRunning.WaitOne(DefaultTimeout));

            var endTime = DateTime.UtcNow;

            Assert.IsNotNull(last);
            Assert.IsTrue(last.Time <= endTime);
            enqueable.Dispose();
        }
            public void Step1D(decimal min, decimal max, decimal step)
            {
                var param = new OptimizationStepParameter("ema-fast", min, max, step);
                var set   = new HashSet <OptimizationParameter>()
                {
                    param
                };

                _strategy.Initialize(new Target("Profit", new Maximization(), null), new List <Constraint>(), set, new StepBaseOptimizationStrategySettings());
                var counter = 0;

                using (var enumerator = new EnqueueableEnumerator <ParameterSet>())
                {
                    _strategy.NewParameterSet += (s, parameterSet) =>
                    {
                        enumerator.Enqueue(parameterSet);
                    };

                    _strategy.PushNewResults(OptimizationResult.Initial);

                    using (var paramEnumerator = new OptimizationStepParameterEnumerator(param))
                    {
                        while (paramEnumerator.MoveNext())
                        {
                            var value = paramEnumerator.Current;
                            counter++;
                            Assert.IsTrue(enumerator.MoveNext());

                            var suggestion = enumerator.Current;

                            Assert.IsNotNull(suggestion);
                            Assert.IsTrue(suggestion.Value.All(s => set.Any(arg => arg.Name == s.Key)));
                            Assert.AreEqual(1, suggestion.Value.Count);
                            Assert.AreEqual(value, suggestion.Value["ema-fast"]);
                        }
                    }

                    Assert.AreEqual(0, enumerator.Count);
                }

                Assert.Greater(counter, 0);
                Assert.AreEqual(Math.Floor((param.MaxValue - param.MinValue) / param.Step.Value) + 1, counter);
            }
예제 #16
0
        public void RecordsInternalQueueCount()
        {
            var enumerator = new EnqueueableEnumerator<Tick>();

            var currentTime = new DateTime(2015, 12, 01);
            var tick = new Tick(currentTime, Symbols.SPY, 100, 101);
            enumerator.Enqueue(tick);
            Assert.AreEqual(1, enumerator.Count);

            tick = new Tick(currentTime, Symbols.SPY, 100, 101);
            enumerator.Enqueue(tick);
            Assert.AreEqual(2, enumerator.Count);

            enumerator.MoveNext();
            Assert.AreEqual(1, enumerator.Count);

            enumerator.MoveNext();
            Assert.AreEqual(0, enumerator.Count);
        }
예제 #17
0
        public void RecordsInternalQueueCount()
        {
            var enumerator = new EnqueueableEnumerator<Tick>();

            var currentTime = new DateTime(2015, 12, 01);
            var tick = new Tick(currentTime, Symbols.SPY, 100, 101);
            enumerator.Enqueue(tick);
            Assert.AreEqual(1, enumerator.Count);

            tick = new Tick(currentTime, Symbols.SPY, 100, 101);
            enumerator.Enqueue(tick);
            Assert.AreEqual(2, enumerator.Count);

            enumerator.MoveNext();
            Assert.AreEqual(1, enumerator.Count);

            enumerator.MoveNext();
            Assert.AreEqual(0, enumerator.Count);
        }
예제 #18
0
        public void RecordsMostRecentlyEnqueuedItem()
        {
            var enumerator = new EnqueueableEnumerator<Tick>();

            var currentTime = new DateTime(2015, 12, 01);
            var tick1 = new Tick(currentTime, Symbols.SPY, 100, 101);
            enumerator.Enqueue(tick1);
            Assert.AreEqual(null, enumerator.Current);
            Assert.AreEqual(tick1, enumerator.LastEnqueued);

            var tick2 = new Tick(currentTime, Symbols.SPY, 100, 101);
            enumerator.Enqueue(tick2);
            Assert.AreEqual(tick2, enumerator.LastEnqueued);

            enumerator.MoveNext();
            Assert.AreEqual(tick1, enumerator.Current);

            enumerator.MoveNext();
            Assert.AreEqual(tick2, enumerator.Current);
        }
예제 #19
0
        public void RecordsMostRecentlyEnqueuedItem()
        {
            var enumerator = new EnqueueableEnumerator<Tick>();

            var currentTime = new DateTime(2015, 12, 01);
            var tick1 = new Tick(currentTime, Symbols.SPY, 100, 101);
            enumerator.Enqueue(tick1);
            Assert.AreEqual(null, enumerator.Current);
            Assert.AreEqual(tick1, enumerator.LastEnqueued);

            var tick2 = new Tick(currentTime, Symbols.SPY, 100, 101);
            enumerator.Enqueue(tick2);
            Assert.AreEqual(tick2, enumerator.LastEnqueued);

            enumerator.MoveNext();
            Assert.AreEqual(tick1, enumerator.Current);

            enumerator.MoveNext();
            Assert.AreEqual(tick2, enumerator.Current);
        }
예제 #20
0
        public void DefaultErrorHandlerDoesNotStopQueueConsumption()
        {
            var enqueable = new EnqueueableEnumerator <BaseData>();
            var exchange  = new BaseDataExchange("test");

            var cancellationToken = new CancellationTokenSource();

            Task.Run(() =>
            {
                while (!cancellationToken.IsCancellationRequested)
                {
                    Thread.Sleep(10);
                    enqueable.Enqueue(new Tick {
                        Symbol = Symbols.SPY, Time = DateTime.UtcNow
                    });
                }
            });

            var      first       = true;
            BaseData last        = null;
            var      lastUpdated = new AutoResetEvent(false);

            exchange.AddEnumerator(Symbols.SPY, enqueable, handleData: spy =>
            {
                if (first)
                {
                    first = false;
                    throw new Exception("This exception should be swalloed by the exchange!");
                }
                last = spy;
                lastUpdated.Set();
            });

            Task.Run(() => exchange.Start());

            Assert.IsTrue(lastUpdated.WaitOne(DefaultTimeout));

            exchange.Stop();
            cancellationToken.Cancel();
            enqueable.Dispose();
        }
예제 #21
0
        /// <summary>
        /// Creates a new subscription for the specified security
        /// </summary>
        /// <param name="request">The subscription request</param>
        /// <returns>A new subscription instance of the specified security</returns>
        protected Subscription CreateDataSubscription(SubscriptionRequest request)
        {
            Subscription subscription = null;

            try
            {
                var localEndTime           = request.EndTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone);
                var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc);

                IEnumerator <BaseData> enumerator;
                if (!_channelProvider.ShouldStreamSubscription(request.Configuration))
                {
                    if (!Quandl.IsAuthCodeSet)
                    {
                        // we're not using the SubscriptionDataReader, so be sure to set the auth token here
                        Quandl.SetAuthCode(Config.Get("quandl-auth-token"));
                    }

                    if (!Tiingo.IsAuthCodeSet)
                    {
                        // we're not using the SubscriptionDataReader, so be sure to set the auth token here
                        Tiingo.SetAuthCode(Config.Get("tiingo-auth-token"));
                    }

                    if (!USEnergyAPI.IsAuthCodeSet)
                    {
                        // we're not using the SubscriptionDataReader, so be sure to set the auth token here
                        USEnergyAPI.SetAuthCode(Config.Get("us-energy-information-auth-token"));
                    }

                    if (!FredApi.IsAuthCodeSet)
                    {
                        // we're not using the SubscriptionDataReader, so be sure to set the auth token here
                        FredApi.SetAuthCode(Config.Get("fred-auth-token"));
                    }

                    var factory         = new LiveCustomDataSubscriptionEnumeratorFactory(_timeProvider);
                    var enumeratorStack = factory.CreateEnumerator(request, _dataProvider);

                    _customExchange.AddEnumerator(request.Configuration.Symbol, enumeratorStack);

                    var enqueable = new EnqueueableEnumerator <BaseData>();
                    _customExchange.SetDataHandler(request.Configuration.Symbol, data =>
                    {
                        enqueable.Enqueue(data);

                        subscription.OnNewDataAvailable();

                        UpdateSubscriptionRealTimePrice(
                            subscription,
                            timeZoneOffsetProvider,
                            request.Security.Exchange.Hours,
                            data);
                    });
                    enumerator = enqueable;
                }
                else
                {
                    // this enumerator allows the exchange to pump ticks into the 'back' of the enumerator,
                    // and the time sync loop can pull aggregated trade bars off the front
                    switch (request.Configuration.Type.Name)
                    {
                    case nameof(QuoteBar):
                        var quoteBarAggregator = new QuoteBarBuilderEnumerator(
                            request.Configuration.Increment,
                            request.Security.Exchange.TimeZone,
                            _timeProvider,
                            true,
                            (sender, args) => subscription.OnNewDataAvailable());

                        _exchange.AddDataHandler(request.Configuration.Symbol, data =>
                        {
                            var tick = data as Tick;

                            if (tick?.TickType == TickType.Quote && !tick.Suspicious)
                            {
                                quoteBarAggregator.ProcessData(tick);

                                UpdateSubscriptionRealTimePrice(
                                    subscription,
                                    timeZoneOffsetProvider,
                                    request.Security.Exchange.Hours,
                                    data);
                            }
                        });
                        enumerator = quoteBarAggregator;
                        break;

                    case nameof(TradeBar):
                        var tradeBarAggregator = new TradeBarBuilderEnumerator(
                            request.Configuration.Increment,
                            request.Security.Exchange.TimeZone,
                            _timeProvider,
                            true,
                            (sender, args) => subscription.OnNewDataAvailable());

                        var auxDataEnumerator = new LiveAuxiliaryDataEnumerator(
                            request.Security.Exchange.TimeZone,
                            _timeProvider);

                        _exchange.AddDataHandler(
                            request.Configuration.Symbol,
                            data =>
                        {
                            if (data.DataType == MarketDataType.Auxiliary)
                            {
                                auxDataEnumerator.Enqueue(data);

                                subscription.OnNewDataAvailable();
                            }
                            else
                            {
                                var tick = data as Tick;
                                if (tick?.TickType == TickType.Trade && !tick.Suspicious)
                                {
                                    tradeBarAggregator.ProcessData(tick);

                                    UpdateSubscriptionRealTimePrice(
                                        subscription,
                                        timeZoneOffsetProvider,
                                        request.Security.Exchange.Hours,
                                        data);
                                }
                            }
                        });

                        enumerator = request.Configuration.SecurityType == SecurityType.Equity
                                ? (IEnumerator <BaseData>) new LiveEquityDataSynchronizingEnumerator(_frontierTimeProvider, request.Security.Exchange.TimeZone, auxDataEnumerator, tradeBarAggregator)
                                : tradeBarAggregator;
                        break;

                    case nameof(OpenInterest):
                        var oiAggregator = new OpenInterestEnumerator(
                            request.Configuration.Increment,
                            request.Security.Exchange.TimeZone,
                            _timeProvider,
                            true,
                            (sender, args) => subscription.OnNewDataAvailable());

                        _exchange.AddDataHandler(request.Configuration.Symbol, data =>
                        {
                            var tick = data as Tick;

                            if (tick?.TickType == TickType.OpenInterest && !tick.Suspicious)
                            {
                                oiAggregator.ProcessData(tick);
                            }
                        });
                        enumerator = oiAggregator;
                        break;

                    case nameof(Tick):
                    default:
                        // tick or streaming custom data subscriptions can pass right through
                        var tickEnumerator = new EnqueueableEnumerator <BaseData>();

                        _exchange.AddDataHandler(
                            request.Configuration.Symbol,
                            data =>
                        {
                            var tick = data as Tick;
                            if (tick != null)
                            {
                                if (tick.TickType == request.Configuration.TickType)
                                {
                                    tickEnumerator.Enqueue(data);
                                    subscription.OnNewDataAvailable();
                                    if (tick.TickType != TickType.OpenInterest)
                                    {
                                        UpdateSubscriptionRealTimePrice(
                                            subscription,
                                            timeZoneOffsetProvider,
                                            request.Security.Exchange.Hours,
                                            data);
                                    }
                                }
                            }
                            else
                            {
                                tickEnumerator.Enqueue(data);
                                subscription.OnNewDataAvailable();
                            }
                        });

                        enumerator = tickEnumerator;
                        break;
                    }
                }

                if (request.Configuration.FillDataForward)
                {
                    var fillForwardResolution = _subscriptions.UpdateAndGetFillForwardResolution(request.Configuration);

                    enumerator = new LiveFillForwardEnumerator(_frontierTimeProvider, enumerator, request.Security.Exchange, fillForwardResolution, request.Configuration.ExtendedMarketHours, localEndTime, request.Configuration.Increment, request.Configuration.DataTimeZone, request.StartTimeLocal);
                }

                // define market hours and user filters to incoming data
                if (request.Configuration.IsFilteredSubscription)
                {
                    enumerator = new SubscriptionFilterEnumerator(enumerator, request.Security, localEndTime);
                }

                // finally, make our subscriptions aware of the frontier of the data feed, prevents future data from spewing into the feed
                enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, timeZoneOffsetProvider);

                var subscriptionDataEnumerator = new SubscriptionDataEnumerator(request.Configuration, request.Security.Exchange.Hours, timeZoneOffsetProvider, enumerator);
                subscription = new Subscription(request, subscriptionDataEnumerator, timeZoneOffsetProvider);
            }
            catch (Exception err)
            {
                Log.Error(err);
            }

            return(subscription);
        }
예제 #22
0
        /// <summary>
        /// Creates a new subscription for universe selection
        /// </summary>
        /// <param name="universe">The universe to add a subscription for</param>
        /// <param name="startTimeUtc">The start time of the subscription in utc</param>
        /// <param name="endTimeUtc">The end time of the subscription in utc</param>
        protected virtual Subscription CreateUniverseSubscription(Universe universe, DateTime startTimeUtc, DateTime endTimeUtc)
        {
            // TODO : Consider moving the creating of universe subscriptions to a separate, testable class

            // grab the relevant exchange hours
            var config = universe.Configuration;

            var marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
            var exchangeHours       = marketHoursDatabase.GetExchangeHours(config);

            Security security;

            if (!_algorithm.Securities.TryGetValue(config.Symbol, out security))
            {
                // create a canonical security object
                security = new Security(exchangeHours, config, _algorithm.Portfolio.CashBook[CashBook.AccountCurrency], SymbolProperties.GetDefault(CashBook.AccountCurrency));
            }

            var tzOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, startTimeUtc, endTimeUtc);

            IEnumerator <BaseData> enumerator;

            var userDefined = universe as UserDefinedUniverse;

            if (userDefined != null)
            {
                Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating user defined universe: " + config.Symbol.ToString());

                // spoof a tick on the requested interval to trigger the universe selection function
                enumerator = userDefined.GetTriggerTimes(startTimeUtc, endTimeUtc, marketHoursDatabase)
                             .Select(dt => new Tick {
                    Time = dt
                }).GetEnumerator();

                enumerator = new FrontierAwareEnumerator(enumerator, _timeProvider, tzOffsetProvider);

                var enqueueable = new EnqueueableEnumerator <BaseData>();
                _customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable));
                enumerator = enqueueable;
            }
            else if (config.Type == typeof(CoarseFundamental))
            {
                Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating coarse universe: " + config.Symbol.ToString());

                // since we're binding to the data queue exchange we'll need to let him
                // know that we expect this data
                _dataQueueHandler.Subscribe(_job, new[] { security.Symbol });

                var enqueable = new EnqueueableEnumerator <BaseData>();
                _exchange.SetDataHandler(config.Symbol, data =>
                {
                    enqueable.Enqueue(data);
                });
                enumerator = enqueable;
            }
            else
            {
                Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating custom universe: " + config.Symbol.ToString());

                // each time we exhaust we'll new up this enumerator stack
                var refresher = new RefreshEnumerator <BaseDataCollection>(() =>
                {
                    var sourceProvider     = (BaseData)Activator.CreateInstance(config.Type);
                    var dateInDataTimeZone = DateTime.UtcNow.ConvertFromUtc(config.DataTimeZone).Date;
                    var source             = sourceProvider.GetSource(config, dateInDataTimeZone, true);
                    var factory            = SubscriptionFactory.ForSource(source, config, dateInDataTimeZone, false);
                    var factorEnumerator   = factory.Read(source).GetEnumerator();
                    var fastForward        = new FastForwardEnumerator(factorEnumerator, _timeProvider, security.Exchange.TimeZone, config.Increment);
                    var frontierAware      = new FrontierAwareEnumerator(fastForward, _frontierTimeProvider, tzOffsetProvider);
                    return(new BaseDataCollectionAggregatorEnumerator(frontierAware, config.Symbol));
                });

                // rate limit the refreshing of the stack to the requested interval
                var minimumTimeBetweenCalls = Math.Min(config.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks);
                var rateLimit   = new RateLimitEnumerator(refresher, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls));
                var enqueueable = new EnqueueableEnumerator <BaseData>();
                _customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, rateLimit, enqueueable));
                enumerator = enqueueable;
            }

            // create the subscription
            var subscription = new Subscription(universe, security, config, enumerator, tzOffsetProvider, startTimeUtc, endTimeUtc, true);

            return(subscription);
        }
예제 #23
0
        /// <summary>
        /// Creates a new subscription for the specified security
        /// </summary>
        /// <param name="universe"></param>
        /// <param name="security">The security to create a subscription for</param>
        /// <param name="config">The subscription config to be added</param>
        /// <param name="utcStartTime">The start time of the subscription in UTC</param>
        /// <param name="utcEndTime">The end time of the subscription in UTC</param>
        /// <returns>A new subscription instance of the specified security</returns>
        protected Subscription CreateSubscription(Universe universe, Security security, SubscriptionDataConfig config, DateTime utcStartTime, DateTime utcEndTime)
        {
            Subscription subscription = null;

            try
            {
                var localEndTime           = utcEndTime.ConvertFromUtc(security.Exchange.TimeZone);
                var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, utcStartTime, utcEndTime);

                IEnumerator <BaseData> enumerator;
                if (config.IsCustomData)
                {
                    if (!Quandl.IsAuthCodeSet)
                    {
                        // we're not using the SubscriptionDataReader, so be sure to set the auth token here
                        Quandl.SetAuthCode(Config.Get("quandl-auth-token"));
                    }

                    // each time we exhaust we'll new up this enumerator stack
                    var refresher = new RefreshEnumerator <BaseData>(() =>
                    {
                        var sourceProvider        = (BaseData)Activator.CreateInstance(config.Type);
                        var dateInDataTimeZone    = DateTime.UtcNow.ConvertFromUtc(config.DataTimeZone).Date;
                        var source                = sourceProvider.GetSource(config, dateInDataTimeZone, true);
                        var factory               = SubscriptionFactory.ForSource(source, config, dateInDataTimeZone, false);
                        var factoryReadEnumerator = factory.Read(source).GetEnumerator();
                        var maximumDataAge        = TimeSpan.FromTicks(Math.Max(config.Increment.Ticks, TimeSpan.FromSeconds(5).Ticks));
                        var fastForward           = new FastForwardEnumerator(factoryReadEnumerator, _timeProvider, security.Exchange.TimeZone, maximumDataAge);
                        return(new FrontierAwareEnumerator(fastForward, _timeProvider, timeZoneOffsetProvider));
                    });

                    // rate limit the refreshing of the stack to the requested interval
                    var minimumTimeBetweenCalls = Math.Min(config.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks);
                    var rateLimit = new RateLimitEnumerator(refresher, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls));
                    _customExchange.AddEnumerator(config.Symbol, rateLimit);

                    var enqueable = new EnqueueableEnumerator <BaseData>();
                    _customExchange.SetDataHandler(config.Symbol, data =>
                    {
                        enqueable.Enqueue(data);
                        if (subscription != null)
                        {
                            subscription.RealtimePrice = data.Value;
                        }
                    });
                    enumerator = enqueable;
                }
                else if (config.Resolution != Resolution.Tick)
                {
                    // this enumerator allows the exchange to pump ticks into the 'back' of the enumerator,
                    // and the time sync loop can pull aggregated trade bars off the front
                    var aggregator = new TradeBarBuilderEnumerator(config.Increment, security.Exchange.TimeZone, _timeProvider);
                    _exchange.SetDataHandler(config.Symbol, data =>
                    {
                        aggregator.ProcessData((Tick)data);
                        if (subscription != null)
                        {
                            subscription.RealtimePrice = data.Value;
                        }
                    });
                    enumerator = aggregator;
                }
                else
                {
                    // tick subscriptions can pass right through
                    var tickEnumerator = new EnqueueableEnumerator <BaseData>();
                    _exchange.SetDataHandler(config.Symbol, data =>
                    {
                        tickEnumerator.Enqueue(data);
                        if (subscription != null)
                        {
                            subscription.RealtimePrice = data.Value;
                        }
                    });
                    enumerator = tickEnumerator;
                }

                if (config.FillDataForward)
                {
                    enumerator = new LiveFillForwardEnumerator(_frontierTimeProvider, enumerator, security.Exchange, _fillForwardResolution, config.ExtendedMarketHours, localEndTime, config.Increment);
                }

                // define market hours and user filters to incoming data
                if (config.IsFilteredSubscription)
                {
                    enumerator = new SubscriptionFilterEnumerator(enumerator, security, localEndTime);
                }

                // finally, make our subscriptions aware of the frontier of the data feed, prevents future data from spewing into the feed
                enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, timeZoneOffsetProvider);

                subscription = new Subscription(universe, security, config, enumerator, timeZoneOffsetProvider, utcStartTime, utcEndTime, false);
            }
            catch (Exception err)
            {
                Log.Error(err);
            }

            return(subscription);
        }
예제 #24
0
        /// <summary>
        /// Adds a new subscription for universe selection
        /// </summary>
        /// <param name="universe">The universe to add a subscription for</param>
        /// <param name="startTimeUtc">The start time of the subscription in utc</param>
        /// <param name="endTimeUtc">The end time of the subscription in utc</param>
        public void AddUniverseSubscription(Universe universe, DateTime startTimeUtc, DateTime endTimeUtc)
        {
            // TODO : Consider moving the creating of universe subscriptions to a separate, testable class

            // grab the relevant exchange hours
            var config = universe.Configuration;

            var marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
            var exchangeHours       = marketHoursDatabase.GetExchangeHours(config);

            // create a canonical security object
            var security = new Security(exchangeHours, config, _algorithm.Portfolio.CashBook[CashBook.AccountCurrency], SymbolProperties.GetDefault(CashBook.AccountCurrency));

            var localStartTime = startTimeUtc.ConvertFromUtc(security.Exchange.TimeZone);
            var localEndTime   = endTimeUtc.ConvertFromUtc(security.Exchange.TimeZone);

            // define our data enumerator
            IEnumerator <BaseData> enumerator;

            var tradeableDates = Time.EachTradeableDay(security, localStartTime, localEndTime);

            var userDefined = universe as UserDefinedUniverse;

            if (userDefined != null)
            {
                // spoof a tick on the requested interval to trigger the universe selection function
                enumerator = userDefined.GetTriggerTimes(startTimeUtc, endTimeUtc, marketHoursDatabase)
                             .Select(x => new Tick {
                    Time = x, Symbol = config.Symbol
                }).GetEnumerator();

                // route these custom subscriptions through the exchange for buffering
                var enqueueable = new EnqueueableEnumerator <BaseData>(true);

                // add this enumerator to our exchange
                ScheduleEnumerator(enumerator, enqueueable, GetLowerThreshold(config.Resolution), GetUpperThreshold(config.Resolution));

                enumerator = enqueueable;
            }
            else if (config.Type == typeof(CoarseFundamental))
            {
                var cf = new CoarseFundamental();

                var enqueueable = new EnqueueableEnumerator <BaseData>(true);

                // load coarse data day by day
                var coarse = from date in Time.EachTradeableDay(security, _algorithm.StartDate, _algorithm.EndDate)
                             let dateInDataTimeZone = date.ConvertTo(config.ExchangeTimeZone, config.DataTimeZone).Date
                                                      let source = cf.GetSource(config, dateInDataTimeZone, false)
                                                                   let factory = SubscriptionFactory.ForSource(source, config, dateInDataTimeZone, false)
                                                                                 let coarseFundamentalForDate = factory.Read(source)
                                                                                                                select new BaseDataCollection(date, config.Symbol, coarseFundamentalForDate);


                ScheduleEnumerator(coarse.GetEnumerator(), enqueueable, 5, 100000, 2);

                enumerator = enqueueable;
            }
            else
            {
                // normal reader for all others
                enumerator = new SubscriptionDataReader(config, localStartTime, localEndTime, _resultHandler, MapFileResolver.Empty, _factorFileProvider, tradeableDates, false);

                // route these custom subscriptions through the exchange for buffering
                var enqueueable = new EnqueueableEnumerator <BaseData>(true);

                // add this enumerator to our exchange
                ScheduleEnumerator(enumerator, enqueueable, GetLowerThreshold(config.Resolution), GetUpperThreshold(config.Resolution));

                enumerator = enqueueable;
            }

            // create the subscription
            var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, startTimeUtc, endTimeUtc);
            var subscription           = new Subscription(universe, security, enumerator, timeZoneOffsetProvider, startTimeUtc, endTimeUtc, true);

            _subscriptions.AddOrUpdate(subscription.Security.Symbol, subscription);
        }
예제 #25
0
 public EnumeratorHandler(IEnumerator<BaseData> enumerator, EnqueueableEnumerator<BaseData> enqueueable)
     : base(enumerator, true)
 {
     _enqueueable = enqueueable;
 }
예제 #26
0
        /// <summary>
        /// Creates a new subscription for the specified security
        /// </summary>
        /// <param name="universe"></param>
        /// <param name="security">The security to create a subscription for</param>
        /// <param name="utcStartTime">The start time of the subscription in UTC</param>
        /// <param name="utcEndTime">The end time of the subscription in UTC</param>
        /// <returns>A new subscription instance of the specified security</returns>
        protected Subscription CreateSubscription(Universe universe, Security security, DateTime utcStartTime, DateTime utcEndTime)
        {
            Subscription subscription = null;
            try
            {
                var config = security.SubscriptionDataConfig;
                var localEndTime = utcEndTime.ConvertFromUtc(security.Exchange.TimeZone);
                var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, utcStartTime, utcEndTime);

                IEnumerator<BaseData> enumerator;
                if (config.IsCustomData)
                {
                    if (!Quandl.IsAuthCodeSet)
                    {
                        // we're not using the SubscriptionDataReader, so be sure to set the auth token here
                        Quandl.SetAuthCode(Config.Get("quandl-auth-token"));
                    }

                    // each time we exhaust we'll new up this enumerator stack
                    var refresher = new RefreshEnumerator<BaseData>(() =>
                    {
                        var sourceProvider = (BaseData)Activator.CreateInstance(config.Type);
                        var currentLocalDate = DateTime.UtcNow.ConvertFromUtc(security.Exchange.TimeZone).Date;
                        var factory = new BaseDataSubscriptionFactory(config, currentLocalDate, true);
                        var source = sourceProvider.GetSource(config, currentLocalDate, true);
                        var factoryReadEnumerator = factory.Read(source).GetEnumerator();
                        var maximumDataAge = TimeSpan.FromTicks(Math.Max(config.Increment.Ticks, TimeSpan.FromSeconds(5).Ticks));
                        var fastForward = new FastForwardEnumerator(factoryReadEnumerator, _timeProvider, security.Exchange.TimeZone, maximumDataAge);
                        return new FrontierAwareEnumerator(fastForward, _timeProvider, timeZoneOffsetProvider);
                    });

                    // rate limit the refreshing of the stack to the requested interval
                    var minimumTimeBetweenCalls = Math.Min(config.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks);
                    var rateLimit = new RateLimitEnumerator(refresher, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls));
                    _customExchange.AddEnumerator(rateLimit);

                    var enqueable = new EnqueueableEnumerator<BaseData>();
                    _customExchange.SetDataHandler(config.Symbol, data =>
                    {
                        enqueable.Enqueue(data);
                        if (subscription != null) subscription.RealtimePrice = data.Value;
                    });
                    enumerator = enqueable;
                }
                else if (config.Resolution != Resolution.Tick)
                {
                    // this enumerator allows the exchange to pump ticks into the 'back' of the enumerator,
                    // and the time sync loop can pull aggregated trade bars off the front
                    var aggregator = new TradeBarBuilderEnumerator(config.Increment, security.Exchange.TimeZone, _timeProvider);
                    _exchange.SetDataHandler(config.Symbol, data =>
                    {
                        aggregator.ProcessData((Tick) data);
                        if (subscription != null) subscription.RealtimePrice = data.Value;
                    });
                    enumerator = aggregator;
                }
                else
                {
                    // tick subscriptions can pass right through
                    var tickEnumerator = new EnqueueableEnumerator<BaseData>();
                    _exchange.SetDataHandler(config.Symbol, data =>
                    {
                        tickEnumerator.Enqueue(data);
                        if (subscription != null) subscription.RealtimePrice = data.Value;
                    });
                    enumerator = tickEnumerator;
                }

                if (config.FillDataForward)
                {
                    // TODO : Properly resolve fill forward resolution like in FileSystemDataFeed (make considerations for universe-only)
                    enumerator = new LiveFillForwardEnumerator(_frontierTimeProvider, enumerator, security.Exchange, _fillForwardResolution, config.ExtendedMarketHours, localEndTime, config.Increment);
                }

                // define market hours and user filters to incoming data
                enumerator = new SubscriptionFilterEnumerator(enumerator, security, localEndTime);

                // finally, make our subscriptions aware of the frontier of the data feed, this will help
                enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, timeZoneOffsetProvider);


                subscription = new Subscription(universe, security, enumerator, timeZoneOffsetProvider, utcStartTime, utcEndTime, false);
            }
            catch (Exception err)
            {
                Log.Error(err);
            }

            return subscription;
        }
예제 #27
0
        /// <summary>
        /// Creates a new subscription for universe selection
        /// </summary>
        /// <param name="request">The subscription request</param>
        private Subscription CreateUniverseSubscription(SubscriptionRequest request)
        {
            Subscription subscription = null;

            // TODO : Consider moving the creating of universe subscriptions to a separate, testable class

            // grab the relevant exchange hours
            var config           = request.Universe.Configuration;
            var localEndTime     = request.EndTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone);
            var tzOffsetProvider = new TimeZoneOffsetProvider(request.Configuration.ExchangeTimeZone, request.StartTimeUtc, request.EndTimeUtc);

            IEnumerator <BaseData> enumerator = null;

            var timeTriggered = request.Universe as ITimeTriggeredUniverse;

            if (timeTriggered != null)
            {
                Log.Trace($"LiveTradingDataFeed.CreateUniverseSubscription(): Creating user defined universe: {config.Symbol.ID}");

                // spoof a tick on the requested interval to trigger the universe selection function
                var enumeratorFactory = new TimeTriggeredUniverseSubscriptionEnumeratorFactory(timeTriggered, MarketHoursDatabase.FromDataFolder(), _frontierTimeProvider);
                enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider);

                enumerator = new FrontierAwareEnumerator(enumerator, _timeProvider, tzOffsetProvider);

                var enqueueable = new EnqueueableEnumerator <BaseData>();
                _customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable));
                enumerator = enqueueable;
            }
            else if (config.Type == typeof(CoarseFundamental) || config.Type == typeof(ETFConstituentData))
            {
                Log.Trace($"LiveTradingDataFeed.CreateUniverseSubscription(): Creating {config.Type.Name} universe: {config.Symbol.ID}");

                // Will try to pull data from the data folder every 10min, file with yesterdays date.
                // If lean is started today it will trigger initial coarse universe selection
                var factory = new LiveCustomDataSubscriptionEnumeratorFactory(_timeProvider,
                                                                              // we adjust time to the previous tradable date
                                                                              time => Time.GetStartTimeForTradeBars(request.Security.Exchange.Hours, time, Time.OneDay, 1, false, config.DataTimeZone),
                                                                              TimeSpan.FromMinutes(10)
                                                                              );
                var enumeratorStack = factory.CreateEnumerator(request, _dataProvider);

                // aggregates each coarse data point into a single BaseDataCollection
                var aggregator = new BaseDataCollectionAggregatorEnumerator(enumeratorStack, config.Symbol, true);
                _customExchange.AddEnumerator(config.Symbol, aggregator);

                var enqueable = new EnqueueableEnumerator <BaseData>();
                _customExchange.SetDataHandler(config.Symbol, data =>
                {
                    enqueable.Enqueue(data);
                    subscription.OnNewDataAvailable();
                });
                enumerator = GetConfiguredFrontierAwareEnumerator(enqueable, tzOffsetProvider,
                                                                  // advance time if before 23pm or after 5am and not on Saturdays
                                                                  time => time.Hour < 23 && time.Hour > 5 && time.DayOfWeek != DayOfWeek.Saturday);
            }
            else if (request.Universe is OptionChainUniverse)
            {
                Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating option chain universe: " + config.Symbol.ID);

                Func <SubscriptionRequest, IEnumerator <BaseData> > configure = (subRequest) =>
                {
                    var fillForwardResolution = _subscriptions.UpdateAndGetFillForwardResolution(subRequest.Configuration);
                    var input = Subscribe(subRequest.Configuration, (sender, args) => subscription.OnNewDataAvailable());
                    return(new LiveFillForwardEnumerator(_frontierTimeProvider, input, subRequest.Security.Exchange, fillForwardResolution, subRequest.Configuration.ExtendedMarketHours, localEndTime, subRequest.Configuration.Increment, subRequest.Configuration.DataTimeZone));
                };

                var symbolUniverse = _dataQueueHandler as IDataQueueUniverseProvider;
                if (symbolUniverse == null)
                {
                    throw new NotSupportedException("The DataQueueHandler does not support Options.");
                }

                var enumeratorFactory = new OptionChainUniverseSubscriptionEnumeratorFactory(configure, symbolUniverse, _timeProvider);
                enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider);

                enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, tzOffsetProvider);
            }
            else if (request.Universe is FuturesChainUniverse)
            {
                Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating futures chain universe: " + config.Symbol.ID);

                var symbolUniverse = _dataQueueHandler as IDataQueueUniverseProvider;
                if (symbolUniverse == null)
                {
                    throw new NotSupportedException("The DataQueueHandler does not support Futures.");
                }

                var enumeratorFactory = new FuturesChainUniverseSubscriptionEnumeratorFactory(symbolUniverse, _timeProvider);
                enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider);

                enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, tzOffsetProvider);
            }
            else
            {
                Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating custom universe: " + config.Symbol.ID);

                var factory         = new LiveCustomDataSubscriptionEnumeratorFactory(_timeProvider);
                var enumeratorStack = factory.CreateEnumerator(request, _dataProvider);
                enumerator = new BaseDataCollectionAggregatorEnumerator(enumeratorStack, config.Symbol, liveMode: true);

                var enqueueable = new EnqueueableEnumerator <BaseData>();
                _customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable));
                enumerator = enqueueable;
            }

            // create the subscription
            var subscriptionDataEnumerator = new SubscriptionDataEnumerator(request.Configuration, request.Security.Exchange.Hours, tzOffsetProvider, enumerator, request.IsUniverseSubscription);

            subscription = new Subscription(request, subscriptionDataEnumerator, tzOffsetProvider);

            // send the subscription for the new symbol through to the data queuehandler
            if (_channelProvider.ShouldStreamSubscription(subscription.Configuration))
            {
                Subscribe(request.Configuration, (sender, args) => subscription.OnNewDataAvailable());
            }

            return(subscription);
        }
예제 #28
0
 public EnumeratorHandler(Symbol symbol, IEnumerator<BaseData> enumerator, EnqueueableEnumerator<BaseData> enqueueable)
     : base(symbol, enumerator, true)
 {
     _enqueueable = enqueueable;
 }
예제 #29
0
        /// <summary>
        /// Creates a new subscription for the specified security
        /// </summary>
        /// <param name="request">The subscription request</param>
        /// <returns>A new subscription instance of the specified security</returns>
        protected Subscription CreateSubscription(SubscriptionRequest request)
        {
            Subscription subscription = null;
            try
            {
                var localEndTime = request.EndTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone);
                var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc);

                IEnumerator<BaseData> enumerator;
                if (request.Configuration.IsCustomData)
                {
                    if (!Quandl.IsAuthCodeSet)
                    {
                        // we're not using the SubscriptionDataReader, so be sure to set the auth token here
                        Quandl.SetAuthCode(Config.Get("quandl-auth-token"));
                    }

                    // each time we exhaust we'll new up this enumerator stack
                    var refresher = new RefreshEnumerator<BaseData>(() =>
                    {
                        var dateInDataTimeZone = DateTime.UtcNow.ConvertFromUtc(request.Configuration.DataTimeZone).Date;
                        var enumeratorFactory = new BaseDataCollectionSubscriptionEnumeratorFactory(r => new [] {dateInDataTimeZone});
                        var factoryReadEnumerator = enumeratorFactory.CreateEnumerator(request);
                        var maximumDataAge = TimeSpan.FromTicks(Math.Max(request.Configuration.Increment.Ticks, TimeSpan.FromSeconds(5).Ticks));
                        return new FastForwardEnumerator(factoryReadEnumerator, _timeProvider, request.Security.Exchange.TimeZone, maximumDataAge);
                    });

                    // rate limit the refreshing of the stack to the requested interval
                    var minimumTimeBetweenCalls = Math.Min(request.Configuration.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks);
                    var rateLimit = new RateLimitEnumerator(refresher, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls));
                    var frontierAware = new FrontierAwareEnumerator(rateLimit, _timeProvider, timeZoneOffsetProvider);
                    _customExchange.AddEnumerator(request.Configuration.Symbol, frontierAware);

                    var enqueable = new EnqueueableEnumerator<BaseData>();
                    _customExchange.SetDataHandler(request.Configuration.Symbol, data =>
                    {
                        enqueable.Enqueue(data);
                        if (subscription != null) subscription.RealtimePrice = data.Value;
                    });
                    enumerator = enqueable;
                }
                else if (request.Configuration.Resolution != Resolution.Tick)
                {
                    // this enumerator allows the exchange to pump ticks into the 'back' of the enumerator,
                    // and the time sync loop can pull aggregated trade bars off the front
                    var aggregator = new TradeBarBuilderEnumerator(request.Configuration.Increment, request.Security.Exchange.TimeZone, _timeProvider);
                    _exchange.SetDataHandler(request.Configuration.Symbol, data =>
                    {
                        aggregator.ProcessData((Tick) data);
                        if (subscription != null) subscription.RealtimePrice = data.Value;
                    });
                    enumerator = aggregator;
                }
                else
                {
                    // tick subscriptions can pass right through
                    var tickEnumerator = new EnqueueableEnumerator<BaseData>();
                    _exchange.SetDataHandler(request.Configuration.Symbol, data =>
                    {
                        tickEnumerator.Enqueue(data);
                        if (subscription != null) subscription.RealtimePrice = data.Value;
                    });
                    enumerator = tickEnumerator;
                }

                if (request.Configuration.FillDataForward)
                {
                    enumerator = new LiveFillForwardEnumerator(_frontierTimeProvider, enumerator, request.Security.Exchange, _fillForwardResolution, request.Configuration.ExtendedMarketHours, localEndTime, request.Configuration.Increment);
                }

                // define market hours and user filters to incoming data
                if (request.Configuration.IsFilteredSubscription)
                {
                    enumerator = new SubscriptionFilterEnumerator(enumerator, request.Security, localEndTime);
                }

                // finally, make our subscriptions aware of the frontier of the data feed, prevents future data from spewing into the feed
                enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, timeZoneOffsetProvider);

                subscription = new Subscription(request.Universe, request.Security, request.Configuration, enumerator, timeZoneOffsetProvider, request.StartTimeUtc, request.EndTimeUtc, false);
            }
            catch (Exception err)
            {
                Log.Error(err);
            }

            return subscription;
        }
예제 #30
0
        /// <summary>
        /// Creates a new subscription for universe selection
        /// </summary>
        /// <param name="universe">The universe to add a subscription for</param>
        /// <param name="startTimeUtc">The start time of the subscription in utc</param>
        /// <param name="endTimeUtc">The end time of the subscription in utc</param>
        protected virtual Subscription CreateUniverseSubscription(Universe universe, DateTime startTimeUtc, DateTime endTimeUtc)
        {
            // TODO : Consider moving the creating of universe subscriptions to a separate, testable class

            // grab the relevant exchange hours
            var config = universe.Configuration;

            var marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
            var exchangeHours = marketHoursDatabase.GetExchangeHours(config);

            // create a canonical security object
            var security = new Security(exchangeHours, config, _algorithm.Portfolio.CashBook[CashBook.AccountCurrency], SymbolProperties.GetDefault(CashBook.AccountCurrency));
            var tzOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, startTimeUtc, endTimeUtc);

            IEnumerator<BaseData> enumerator;
            
            var userDefined = universe as UserDefinedUniverse;
            if (userDefined != null)
            {
                Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating user defined universe: " + config.Symbol.ToString());

                // spoof a tick on the requested interval to trigger the universe selection function
                enumerator = userDefined.GetTriggerTimes(startTimeUtc, endTimeUtc, marketHoursDatabase)
                    .Select(dt => new Tick { Time = dt }).GetEnumerator();

                enumerator = new FrontierAwareEnumerator(enumerator, _timeProvider, tzOffsetProvider);

                var enqueueable = new EnqueueableEnumerator<BaseData>();
                _customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable));
                enumerator = enqueueable;
            }
            else if (config.Type == typeof (CoarseFundamental))
            {
                Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating coarse universe: " + config.Symbol.ToString());

                // since we're binding to the data queue exchange we'll need to let him
                // know that we expect this data
                _dataQueueHandler.Subscribe(_job, new[] {security.Symbol});

                var enqueable = new EnqueueableEnumerator<BaseData>();
                _exchange.SetDataHandler(config.Symbol, data =>
                {
                    enqueable.Enqueue(data);
                });
                enumerator = enqueable;
            }
            else
            {
                Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating custom universe: " + config.Symbol.ToString());

                // each time we exhaust we'll new up this enumerator stack
                var refresher = new RefreshEnumerator<BaseDataCollection>(() =>
                {
                    var sourceProvider = (BaseData)Activator.CreateInstance(config.Type);
                    var dateInDataTimeZone = DateTime.UtcNow.ConvertFromUtc(config.DataTimeZone).Date;
                    var source = sourceProvider.GetSource(config, dateInDataTimeZone, true);
                    var factory = SubscriptionFactory.ForSource(source, config, dateInDataTimeZone, false);
                    var factorEnumerator = factory.Read(source).GetEnumerator();
                    var fastForward = new FastForwardEnumerator(factorEnumerator, _timeProvider, security.Exchange.TimeZone, config.Increment);
                    var frontierAware = new FrontierAwareEnumerator(fastForward, _frontierTimeProvider, tzOffsetProvider);
                    return new BaseDataCollectionAggregatorEnumerator(frontierAware, config.Symbol);
                });
                
                // rate limit the refreshing of the stack to the requested interval
                var minimumTimeBetweenCalls = Math.Min(config.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks);
                var rateLimit = new RateLimitEnumerator(refresher, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls));
                var enqueueable = new EnqueueableEnumerator<BaseData>();
                _customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, rateLimit, enqueueable));
                enumerator = enqueueable;
            }

            // create the subscription
            var subscription = new Subscription(universe, security, enumerator, tzOffsetProvider, startTimeUtc, endTimeUtc, true);

            return subscription;
        }
예제 #31
0
        /// <summary>
        /// Adds a new subscription for universe selection
        /// </summary>
        /// <param name="request">The subscription request</param>
        private Subscription CreateUniverseSubscription(SubscriptionRequest request)
        {
            // grab the relevant exchange hours
            var config = request.Configuration;

            // define our data enumerator
            var enumerator = GetEnumeratorFactory(request).CreateEnumerator(request);

            var firstLoopCount = 5;
            var lowerThreshold = GetLowerThreshold(config.Resolution);
            var upperThreshold = GetUpperThreshold(config.Resolution);
            if (config.Type == typeof (CoarseFundamental))
            {
                firstLoopCount = 2;
                lowerThreshold = 5;
                upperThreshold = 100000;
            }

            var enqueueable = new EnqueueableEnumerator<BaseData>(true);
            ScheduleEnumerator(enumerator, enqueueable, lowerThreshold, upperThreshold, firstLoopCount);
            enumerator = enqueueable;

            // create the subscription
            var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc);
            return new Subscription(request.Universe, request.Security, config, enumerator, timeZoneOffsetProvider, request.StartTimeUtc, request.EndTimeUtc, true);
        }
예제 #32
0
        private void ScheduleEnumerator(IEnumerator<BaseData> enumerator, EnqueueableEnumerator<BaseData> enqueueable, int lowerThreshold, int upperThreshold, int firstLoopCount = 5)
        {
            // schedule the work on the controller
            var firstLoop = true;
            FuncParallelRunnerWorkItem workItem = null;
            workItem = new FuncParallelRunnerWorkItem(() => enqueueable.Count < lowerThreshold, () =>
            {
                var count = 0;
                while (enumerator.MoveNext())
                {
                    // drop the data into the back of the enqueueable
                    enqueueable.Enqueue(enumerator.Current);

                    count++;

                    // special behavior for first loop to spool up quickly
                    if (firstLoop && count > firstLoopCount)
                    {
                        // there's more data in the enumerator, reschedule to run again
                        firstLoop = false;
                        _controller.Schedule(workItem);
                        return;
                    }

                    // stop executing if we've dequeued more than the lower threshold or have
                    // more total that upper threshold in the enqueueable's queue
                    if (count > lowerThreshold || enqueueable.Count > upperThreshold)
                    {
                        // there's more data in the enumerator, reschedule to run again
                        _controller.Schedule(workItem);
                        return;
                    }
                }

                // we made it here because MoveNext returned false, stop the enqueueable and don't reschedule
                enqueueable.Stop();
            });
            _controller.Schedule(workItem);
        }
예제 #33
0
        private Subscription CreateSubscription(SubscriptionRequest request)
        {
            // ReSharper disable once PossibleMultipleEnumeration
            if (!request.TradableDays.Any())
            {
                _algorithm.Error(string.Format("No data loaded for {0} because there were no tradeable dates for this security.", request.Security.Symbol));
                return null;
            }

            // ReSharper disable once PossibleMultipleEnumeration
            var enumeratorFactory = GetEnumeratorFactory(request);
            var enumerator = enumeratorFactory.CreateEnumerator(request);
            enumerator = ConfigureEnumerator(request, false, enumerator);

            var enqueueable = new EnqueueableEnumerator<BaseData>(true);

            // add this enumerator to our exchange
            ScheduleEnumerator(enumerator, enqueueable, GetLowerThreshold(request.Configuration.Resolution), GetUpperThreshold(request.Configuration.Resolution));

            var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc);
            var subscription = new Subscription(request.Universe, request.Security, request.Configuration, enqueueable, timeZoneOffsetProvider, request.StartTimeUtc, request.EndTimeUtc, false);
            return subscription;
        }
예제 #34
0
        /// <summary>
        /// Creates a new subscription for universe selection
        /// </summary>
        /// <param name="request">The subscription request</param>
        private Subscription CreateUniverseSubscription(SubscriptionRequest request)
        {
            Subscription subscription = null;

            // TODO : Consider moving the creating of universe subscriptions to a separate, testable class

            // grab the relevant exchange hours
            var config           = request.Universe.Configuration;
            var localEndTime     = request.EndTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone);
            var tzOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc);

            IEnumerator <BaseData> enumerator;

            var timeTriggered = request.Universe as ITimeTriggeredUniverse;

            if (timeTriggered != null)
            {
                Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating user defined universe: " + config.Symbol.ToString());

                // spoof a tick on the requested interval to trigger the universe selection function
                var enumeratorFactory = new TimeTriggeredUniverseSubscriptionEnumeratorFactory(timeTriggered, MarketHoursDatabase.FromDataFolder());
                enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider);

                enumerator = new FrontierAwareEnumerator(enumerator, _timeProvider, tzOffsetProvider);

                var enqueueable = new EnqueueableEnumerator <BaseData>();
                _customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable));
                enumerator = enqueueable;

                // Trigger universe selection when security added/removed after Initialize
                if (timeTriggered is UserDefinedUniverse)
                {
                    var userDefined = (UserDefinedUniverse)timeTriggered;
                    userDefined.CollectionChanged += (sender, args) =>
                    {
                        var items =
                            args.Action == NotifyCollectionChangedAction.Add ? args.NewItems :
                            args.Action == NotifyCollectionChangedAction.Remove ? args.OldItems : null;

                        var currentFrontierUtcTime = _frontierTimeProvider.GetUtcNow();
                        if (items == null || currentFrontierUtcTime == DateTime.MinValue)
                        {
                            return;
                        }

                        var symbol = items.OfType <Symbol>().FirstOrDefault();
                        if (symbol == null)
                        {
                            return;
                        }

                        var collection = new BaseDataCollection(currentFrontierUtcTime, symbol);
                        var changes    = _universeSelection.ApplyUniverseSelection(userDefined, currentFrontierUtcTime, collection);
                        _algorithm.OnSecuritiesChanged(changes);

                        subscription.OnNewDataAvailable();
                    };
                }
            }
            else if (config.Type == typeof(CoarseFundamental))
            {
                Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating coarse universe: " + config.Symbol.ToString());

                // we subscribe using a normalized symbol, without a random GUID,
                // since the ticker plant will send the coarse data using this symbol
                var normalizedSymbol = CoarseFundamental.CreateUniverseSymbol(config.Symbol.ID.Market, false);

                // since we're binding to the data queue exchange we'll need to let him
                // know that we expect this data
                _dataQueueHandler.Subscribe(_job, new[] { normalizedSymbol });

                var enqueable = new EnqueueableEnumerator <BaseData>();
                // We `AddDataHandler` not `Set` so we can have multiple handlers for the coarse data
                _exchange.AddDataHandler(normalizedSymbol, data =>
                {
                    enqueable.Enqueue(data);

                    subscription.OnNewDataAvailable();
                });

                enumerator = GetConfiguredFrontierAwareEnumerator(enqueable, tzOffsetProvider,
                                                                  // advance time if before 23pm or after 5am and not on Saturdays
                                                                  time => time.Hour < 23 && time.Hour > 5 && time.DayOfWeek != DayOfWeek.Saturday);
            }
            else if (request.Universe is OptionChainUniverse)
            {
                Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating option chain universe: " + config.Symbol.ToString());

                Func <SubscriptionRequest, IEnumerator <BaseData>, IEnumerator <BaseData> > configure = (subRequest, input) =>
                {
                    // we check if input enumerator is an underlying enumerator. If yes, we subscribe it to the data.
                    var aggregator = input as TradeBarBuilderEnumerator;

                    if (aggregator != null)
                    {
                        _exchange.SetDataHandler(request.Configuration.Symbol, data =>
                        {
                            aggregator.ProcessData((Tick)data);
                        });
                    }

                    var fillForwardResolution = _subscriptions.UpdateAndGetFillForwardResolution(request.Configuration);

                    return(new LiveFillForwardEnumerator(_frontierTimeProvider, input, request.Security.Exchange, fillForwardResolution, request.Configuration.ExtendedMarketHours, localEndTime, request.Configuration.Increment, request.Configuration.DataTimeZone, request.StartTimeLocal));
                };

                var symbolUniverse = _dataQueueHandler as IDataQueueUniverseProvider;
                if (symbolUniverse == null)
                {
                    throw new NotSupportedException("The DataQueueHandler does not support Options.");
                }

                var enumeratorFactory = new OptionChainUniverseSubscriptionEnumeratorFactory(configure, symbolUniverse, _timeProvider);
                enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider);

                enumerator = GetConfiguredFrontierAwareEnumerator(enumerator, tzOffsetProvider,
                                                                  time => symbolUniverse.CanAdvanceTime(config.SecurityType));
            }
            else if (request.Universe is FuturesChainUniverse)
            {
                Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating futures chain universe: " + config.Symbol.ToString());

                var symbolUniverse = _dataQueueHandler as IDataQueueUniverseProvider;
                if (symbolUniverse == null)
                {
                    throw new NotSupportedException("The DataQueueHandler does not support Futures.");
                }

                var enumeratorFactory = new FuturesChainUniverseSubscriptionEnumeratorFactory(symbolUniverse, _timeProvider);
                enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider);

                enumerator = GetConfiguredFrontierAwareEnumerator(enumerator, tzOffsetProvider,
                                                                  time => symbolUniverse.CanAdvanceTime(config.SecurityType));
            }
            else
            {
                Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating custom universe: " + config.Symbol.ToString());

                var factory         = new LiveCustomDataSubscriptionEnumeratorFactory(_timeProvider);
                var enumeratorStack = factory.CreateEnumerator(request, _dataProvider);
                enumerator = new BaseDataCollectionAggregatorEnumerator(enumeratorStack, config.Symbol, liveMode: true);

                var enqueueable = new EnqueueableEnumerator <BaseData>();
                _customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable));
                enumerator = enqueueable;
            }

            // create the subscription
            var subscriptionDataEnumerator = new SubscriptionDataEnumerator(request.Configuration, request.Security.Exchange.Hours, tzOffsetProvider, enumerator);

            subscription = new Subscription(request, subscriptionDataEnumerator, tzOffsetProvider);

            return(subscription);
        }
예제 #35
0
        /// <summary>
        /// Creates a new subscription for universe selection
        /// </summary>
        /// <param name="request">The subscription request</param>
        private Subscription CreateUniverseSubscription(SubscriptionRequest request)
        {
            // TODO : Consider moving the creating of universe subscriptions to a separate, testable class

            // grab the relevant exchange hours
            var config           = request.Universe.Configuration;
            var localEndTime     = request.EndTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone);
            var tzOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc);

            IEnumerator <BaseData> enumerator;

            var userDefined = request.Universe as UserDefinedUniverse;

            if (userDefined != null)
            {
                Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating user defined universe: " + config.Symbol.ToString());

                // spoof a tick on the requested interval to trigger the universe selection function
                var enumeratorFactory = new UserDefinedUniverseSubscriptionEnumeratorFactory(userDefined, MarketHoursDatabase.FromDataFolder());
                enumerator = enumeratorFactory.CreateEnumerator(request, _dataFileProvider);

                enumerator = new FrontierAwareEnumerator(enumerator, _timeProvider, tzOffsetProvider);

                var enqueueable = new EnqueueableEnumerator <BaseData>();
                _customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable));
                enumerator = enqueueable;

                // Trigger universe selection when security added/removed after Initialize
                userDefined.CollectionChanged += (sender, args) =>
                {
                    var items =
                        args.Action == NotifyCollectionChangedAction.Add ? args.NewItems :
                        args.Action == NotifyCollectionChangedAction.Remove ? args.OldItems : null;

                    if (items == null || _frontierUtc == DateTime.MinValue)
                    {
                        return;
                    }

                    var symbol = items.OfType <Symbol>().FirstOrDefault();
                    if (symbol == null)
                    {
                        return;
                    }

                    var collection = new BaseDataCollection(_frontierUtc, symbol);
                    var changes    = _universeSelection.ApplyUniverseSelection(userDefined, _frontierUtc, collection);
                    _algorithm.OnSecuritiesChanged(changes);
                };
            }
            else if (config.Type == typeof(CoarseFundamental))
            {
                Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating coarse universe: " + config.Symbol.ToString());

                // since we're binding to the data queue exchange we'll need to let him
                // know that we expect this data
                _dataQueueHandler.Subscribe(_job, new[] { request.Security.Symbol });

                var enqueable = new EnqueueableEnumerator <BaseData>();
                _exchange.SetDataHandler(config.Symbol, data =>
                {
                    enqueable.Enqueue(data);
                });
                enumerator = enqueable;
            }
            else if (request.Universe is OptionChainUniverse)
            {
                Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating option chain universe: " + config.Symbol.ToString());

                Func <SubscriptionRequest, IEnumerator <BaseData>, IEnumerator <BaseData> > configure = (subRequest, input) =>
                {
                    // we check if input enumerator is an underlying enumerator. If yes, we subscribe it to the data.
                    var aggregator = input as TradeBarBuilderEnumerator;

                    if (aggregator != null)
                    {
                        _exchange.SetDataHandler(request.Configuration.Symbol, data =>
                        {
                            aggregator.ProcessData((Tick)data);
                        });
                    }

                    var subscriptionConfigs = _subscriptions.Select(x => x.Configuration).Concat(new[] { request.Configuration });

                    UpdateFillForwardResolution(subscriptionConfigs);

                    return(new LiveFillForwardEnumerator(_frontierTimeProvider, input, request.Security.Exchange, _fillForwardResolution, request.Configuration.ExtendedMarketHours, localEndTime, request.Configuration.Increment));
                };

                var symbolUniverse = _dataQueueHandler as IDataQueueUniverseProvider;

                var enumeratorFactory = new OptionChainUniverseSubscriptionEnumeratorFactory(configure, symbolUniverse, _timeProvider);
                enumerator = enumeratorFactory.CreateEnumerator(request, _dataFileProvider);

                enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, tzOffsetProvider);
            }
            else if (request.Universe is FuturesChainUniverse)
            {
                Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating futures chain universe: " + config.Symbol.ToString());

                var symbolUniverse = _dataQueueHandler as IDataQueueUniverseProvider;

                var enumeratorFactory = new FuturesChainUniverseSubscriptionEnumeratorFactory(symbolUniverse, _timeProvider);
                enumerator = enumeratorFactory.CreateEnumerator(request, _dataFileProvider);

                enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, tzOffsetProvider);
            }
            else
            {
                Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating custom universe: " + config.Symbol.ToString());

                // each time we exhaust we'll new up this enumerator stack
                var refresher = new RefreshEnumerator <BaseDataCollection>(() =>
                {
                    var sourceProvider     = (BaseData)Activator.CreateInstance(config.Type);
                    var dateInDataTimeZone = DateTime.UtcNow.ConvertFromUtc(config.DataTimeZone).Date;
                    var source             = sourceProvider.GetSource(config, dateInDataTimeZone, true);
                    var factory            = SubscriptionDataSourceReader.ForSource(source, _dataFileProvider, config, dateInDataTimeZone, false);
                    var factorEnumerator   = factory.Read(source).GetEnumerator();
                    var fastForward        = new FastForwardEnumerator(factorEnumerator, _timeProvider, request.Security.Exchange.TimeZone, config.Increment);
                    var frontierAware      = new FrontierAwareEnumerator(fastForward, _frontierTimeProvider, tzOffsetProvider);
                    return(new BaseDataCollectionAggregatorEnumerator(frontierAware, config.Symbol));
                });

                // rate limit the refreshing of the stack to the requested interval
                var minimumTimeBetweenCalls = Math.Min(config.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks);
                var rateLimit   = new RateLimitEnumerator(refresher, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls));
                var enqueueable = new EnqueueableEnumerator <BaseData>();
                _customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, rateLimit, enqueueable));
                enumerator = enqueueable;
            }

            // create the subscription
            var subscription = new Subscription(request.Universe, request.Security, config, enumerator, tzOffsetProvider, request.StartTimeUtc, request.EndTimeUtc, true);

            return(subscription);
        }
예제 #36
0
        /// <summary>
        /// Adds a new subscription for universe selection
        /// </summary>
        /// <param name="universe">The universe to add a subscription for</param>
        /// <param name="startTimeUtc">The start time of the subscription in utc</param>
        /// <param name="endTimeUtc">The end time of the subscription in utc</param>
        public void AddUniverseSubscription(Universe universe, DateTime startTimeUtc, DateTime endTimeUtc)
        {
            // TODO : Consider moving the creating of universe subscriptions to a separate, testable class

            // grab the relevant exchange hours
            var config = universe.Configuration;

            var marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
            var exchangeHours = marketHoursDatabase.GetExchangeHours(config);

            // create a canonical security object
            var security = new Security(exchangeHours, config);

            var localStartTime = startTimeUtc.ConvertFromUtc(security.Exchange.TimeZone);
            var localEndTime = endTimeUtc.ConvertFromUtc(security.Exchange.TimeZone);

            // define our data enumerator
            IEnumerator<BaseData> enumerator;

            var tradeableDates = Time.EachTradeableDay(security, localStartTime, localEndTime);

            var userDefined = universe as UserDefinedUniverse;
            if (userDefined != null)
            {
                // spoof a tick on the requested interval to trigger the universe selection function
                enumerator = userDefined.GetTriggerTimes(startTimeUtc, endTimeUtc, marketHoursDatabase)
                    .Select(x => new Tick { Time = x, Symbol = config.Symbol }).GetEnumerator();

                // route these custom subscriptions through the exchange for buffering
                var enqueueable = new EnqueueableEnumerator<BaseData>(true);

                // add this enumerator to our exchange
                ScheduleEnumerator(enumerator, enqueueable, GetLowerThreshold(config.Resolution), GetUpperThreshold(config.Resolution));

                enumerator = enqueueable;
            }
            else if (config.Type == typeof (CoarseFundamental))
            {
                var cf = new CoarseFundamental();

                var enqueueable = new EnqueueableEnumerator<BaseData>(true);

                // load coarse data day by day
                var coarse = from date in Time.EachTradeableDay(security, _algorithm.StartDate, _algorithm.EndDate)
                             let dateInDataTimeZone = date.ConvertTo(config.ExchangeTimeZone, config.DataTimeZone).Date
                             let factory = new BaseDataSubscriptionFactory(config, dateInDataTimeZone, false)
                             let source = cf.GetSource(config, dateInDataTimeZone, false)
                             let coarseFundamentalForDate = factory.Read(source)
                             select new BaseDataCollection(date, config.Symbol, coarseFundamentalForDate);

                
                ScheduleEnumerator(coarse.GetEnumerator(), enqueueable, 5, 100000, 2);

                enumerator = enqueueable;
            }
            else
            {
                // normal reader for all others
                enumerator = new SubscriptionDataReader(config, localStartTime, localEndTime, _resultHandler, MapFileResolver.Empty, _factorFileProvider, tradeableDates, false);

                // route these custom subscriptions through the exchange for buffering
                var enqueueable = new EnqueueableEnumerator<BaseData>(true);

                // add this enumerator to our exchange
                ScheduleEnumerator(enumerator, enqueueable, GetLowerThreshold(config.Resolution), GetUpperThreshold(config.Resolution));

                enumerator = enqueueable;
            }

            // create the subscription
            var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, startTimeUtc, endTimeUtc);
            var subscription = new Subscription(universe, security, enumerator, timeZoneOffsetProvider, startTimeUtc, endTimeUtc, true);
            _subscriptions.AddOrUpdate(subscription.Security.Symbol, subscription);
        }
예제 #37
0
        /// <summary>
        /// Creates a new subscription for universe selection
        /// </summary>
        /// <param name="request">The subscription request</param>
        private Subscription CreateUniverseSubscription(SubscriptionRequest request)
        {
            // TODO : Consider moving the creating of universe subscriptions to a separate, testable class

            // grab the relevant exchange hours
            var config           = request.Universe.Configuration;
            var localEndTime     = request.EndTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone);
            var tzOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc);

            IEnumerator <BaseData> enumerator;

            var timeTriggered = request.Universe as ITimeTriggeredUniverse;

            if (timeTriggered != null)
            {
                Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating user defined universe: " + config.Symbol.ToString());

                // spoof a tick on the requested interval to trigger the universe selection function
                var enumeratorFactory = new TimeTriggeredUniverseSubscriptionEnumeratorFactory(timeTriggered, MarketHoursDatabase.FromDataFolder());
                enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider);

                enumerator = new FrontierAwareEnumerator(enumerator, _timeProvider, tzOffsetProvider);

                var enqueueable = new EnqueueableEnumerator <BaseData>();
                _customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable));
                enumerator = enqueueable;

                // Trigger universe selection when security added/removed after Initialize
                if (timeTriggered is UserDefinedUniverse)
                {
                    var userDefined = (UserDefinedUniverse)timeTriggered;
                    userDefined.CollectionChanged += (sender, args) =>
                    {
                        var items =
                            args.Action == NotifyCollectionChangedAction.Add ? args.NewItems :
                            args.Action == NotifyCollectionChangedAction.Remove ? args.OldItems : null;

                        if (items == null || _frontierUtc == DateTime.MinValue)
                        {
                            return;
                        }

                        var symbol = items.OfType <Symbol>().FirstOrDefault();
                        if (symbol == null)
                        {
                            return;
                        }

                        var collection = new BaseDataCollection(_frontierUtc, symbol);
                        var changes    = _universeSelection.ApplyUniverseSelection(userDefined, _frontierUtc, collection);
                        _algorithm.OnSecuritiesChanged(changes);
                    };
                }
            }
            else if (config.Type == typeof(CoarseFundamental))
            {
                Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating coarse universe: " + config.Symbol.ToString());

                // since we're binding to the data queue exchange we'll need to let him
                // know that we expect this data
                _dataQueueHandler.Subscribe(_job, new[] { request.Security.Symbol });

                var enqueable = new EnqueueableEnumerator <BaseData>();
                _exchange.SetDataHandler(config.Symbol, data =>
                {
                    enqueable.Enqueue(data);
                });
                enumerator = enqueable;
            }
            else if (request.Universe is OptionChainUniverse)
            {
                Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating option chain universe: " + config.Symbol.ToString());

                Func <SubscriptionRequest, IEnumerator <BaseData>, IEnumerator <BaseData> > configure = (subRequest, input) =>
                {
                    // we check if input enumerator is an underlying enumerator. If yes, we subscribe it to the data.
                    var aggregator = input as TradeBarBuilderEnumerator;

                    if (aggregator != null)
                    {
                        _exchange.SetDataHandler(request.Configuration.Symbol, data =>
                        {
                            aggregator.ProcessData((Tick)data);
                        });
                    }

                    var subscriptionConfigs = _subscriptions.Select(x => x.Configuration).Concat(new[] { request.Configuration });

                    UpdateFillForwardResolution(subscriptionConfigs);

                    return(new LiveFillForwardEnumerator(_frontierTimeProvider, input, request.Security.Exchange, _fillForwardResolution, request.Configuration.ExtendedMarketHours, localEndTime, request.Configuration.Increment, request.Configuration.DataTimeZone));
                };

                var symbolUniverse = _dataQueueHandler as IDataQueueUniverseProvider;
                if (symbolUniverse == null)
                {
                    throw new NotSupportedException("The DataQueueHandler does not support Options.");
                }

                var enumeratorFactory = new OptionChainUniverseSubscriptionEnumeratorFactory(configure, symbolUniverse, _timeProvider);
                enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider);

                enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, tzOffsetProvider);
            }
            else if (request.Universe is FuturesChainUniverse)
            {
                Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating futures chain universe: " + config.Symbol.ToString());

                var symbolUniverse = _dataQueueHandler as IDataQueueUniverseProvider;
                if (symbolUniverse == null)
                {
                    throw new NotSupportedException("The DataQueueHandler does not support Futures.");
                }

                var enumeratorFactory = new FuturesChainUniverseSubscriptionEnumeratorFactory(symbolUniverse, _timeProvider);
                enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider);

                enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, tzOffsetProvider);
            }
            else
            {
                Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating custom universe: " + config.Symbol.ToString());

                var factory         = new LiveCustomDataSubscriptionEnumeratorFactory(_timeProvider);
                var enumeratorStack = factory.CreateEnumerator(request, _dataProvider);
                enumerator = new BaseDataCollectionAggregatorEnumerator(enumeratorStack, config.Symbol);

                var enqueueable = new EnqueueableEnumerator <BaseData>();
                _customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable));
                enumerator = enqueueable;
            }

            // create the subscription
            var subscriptionDataEnumerator = SubscriptionData.Enumerator(request.Configuration, request.Security, tzOffsetProvider, enumerator);
            var subscription = new Subscription(request.Universe, request.Security, config, subscriptionDataEnumerator, tzOffsetProvider, request.StartTimeUtc, request.EndTimeUtc, true);

            return(subscription);
        }
예제 #38
0
        private Subscription CreateSubscription(Universe universe, Security security, DateTime startTimeUtc, DateTime endTimeUtc)
        {
            var config = security.SubscriptionDataConfig;
            var localStartTime = startTimeUtc.ConvertFromUtc(security.Exchange.TimeZone);
            var localEndTime = endTimeUtc.ConvertFromUtc(security.Exchange.TimeZone);

            var tradeableDates = Time.EachTradeableDay(security, localStartTime, localEndTime);

            // ReSharper disable once PossibleMultipleEnumeration
            if (!tradeableDates.Any())
            {
                _algorithm.Error(string.Format("No data loaded for {0} because there were no tradeable dates for this security.", security.Symbol));
                return null;
            }

            // get the map file resolver for this market
            var mapFileResolver = MapFileResolver.Empty;
            if (config.SecurityType == SecurityType.Equity) mapFileResolver = _mapFileProvider.Get(config.Market);

            // ReSharper disable once PossibleMultipleEnumeration
            var enumerator = CreateSubscriptionEnumerator(security, config, localStartTime, localEndTime, mapFileResolver, tradeableDates);

            var enqueueable = new EnqueueableEnumerator<BaseData>(true);

            // add this enumerator to our exchange
            ScheduleEnumerator(enumerator, enqueueable, GetLowerThreshold(config.Resolution), GetUpperThreshold(config.Resolution));

            var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, startTimeUtc, endTimeUtc);
            var subscription = new Subscription(universe, security, enqueueable, timeZoneOffsetProvider, startTimeUtc, endTimeUtc, false);
            return subscription;
        }
예제 #39
0
        /// <summary>
        /// Creates a new subscription for universe selection
        /// </summary>
        /// <param name="request">The subscription request</param>
        private Subscription CreateUniverseSubscription(SubscriptionRequest request)
        {
            // TODO : Consider moving the creating of universe subscriptions to a separate, testable class

            // grab the relevant exchange hours
            var config = request.Universe.Configuration;

            var tzOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc);

            IEnumerator<BaseData> enumerator;
            
            var userDefined = request.Universe as UserDefinedUniverse;
            if (userDefined != null)
            {
                Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating user defined universe: " + config.Symbol.ToString());

                // spoof a tick on the requested interval to trigger the universe selection function
                var enumeratorFactory = new UserDefinedUniverseSubscriptionEnumeratorFactory(userDefined, MarketHoursDatabase.FromDataFolder());
                enumerator = enumeratorFactory.CreateEnumerator(request);

                enumerator = new FrontierAwareEnumerator(enumerator, _timeProvider, tzOffsetProvider);

                var enqueueable = new EnqueueableEnumerator<BaseData>();
                _customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable));
                enumerator = enqueueable;

                // Trigger universe selection when security added/removed after Initialize
                userDefined.CollectionChanged += (sender, args) =>
                {
                    var items =
                           args.Action == NotifyCollectionChangedAction.Add ? args.NewItems :
                           args.Action == NotifyCollectionChangedAction.Remove ? args.OldItems : null;

                    if (items == null || _frontierUtc == DateTime.MinValue) return;

                    var symbol = items.OfType<Symbol>().FirstOrDefault();
                    if (symbol == null) return;

                    var collection = new BaseDataCollection(_frontierUtc, symbol);
                    var changes = _universeSelection.ApplyUniverseSelection(userDefined, _frontierUtc, collection);
                    _algorithm.OnSecuritiesChanged(changes);
                };
            }
            else if (config.Type == typeof (CoarseFundamental))
            {
                Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating coarse universe: " + config.Symbol.ToString());

                // since we're binding to the data queue exchange we'll need to let him
                // know that we expect this data
                _dataQueueHandler.Subscribe(_job, new[] {request.Security.Symbol});

                var enqueable = new EnqueueableEnumerator<BaseData>();
                _exchange.SetDataHandler(config.Symbol, data =>
                {
                    enqueable.Enqueue(data);
                });
                enumerator = enqueable;
            }
            else
            {
                Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating custom universe: " + config.Symbol.ToString());

                // each time we exhaust we'll new up this enumerator stack
                var refresher = new RefreshEnumerator<BaseDataCollection>(() =>
                {
                    var sourceProvider = (BaseData)Activator.CreateInstance(config.Type);
                    var dateInDataTimeZone = DateTime.UtcNow.ConvertFromUtc(config.DataTimeZone).Date;
                    var source = sourceProvider.GetSource(config, dateInDataTimeZone, true);
                    var factory = SubscriptionDataSourceReader.ForSource(source, config, dateInDataTimeZone, false);
                    var factorEnumerator = factory.Read(source).GetEnumerator();
                    var fastForward = new FastForwardEnumerator(factorEnumerator, _timeProvider, request.Security.Exchange.TimeZone, config.Increment);
                    var frontierAware = new FrontierAwareEnumerator(fastForward, _frontierTimeProvider, tzOffsetProvider);
                    return new BaseDataCollectionAggregatorEnumerator(frontierAware, config.Symbol);
                });
                
                // rate limit the refreshing of the stack to the requested interval
                var minimumTimeBetweenCalls = Math.Min(config.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks);
                var rateLimit = new RateLimitEnumerator(refresher, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls));
                var enqueueable = new EnqueueableEnumerator<BaseData>();
                _customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, rateLimit, enqueueable));
                enumerator = enqueueable;
            }

            // create the subscription
            var subscription = new Subscription(request.Universe, request.Security, config, enumerator, tzOffsetProvider, request.StartTimeUtc, request.EndTimeUtc, true);

            return subscription;
        }
예제 #40
0
        /// <summary>
        /// Creates a new subscription for the specified security
        /// </summary>
        /// <param name="request">The subscription request</param>
        /// <returns>A new subscription instance of the specified security</returns>
        protected Subscription CreateSubscription(SubscriptionRequest request)
        {
            Subscription subscription = null;

            try
            {
                var localEndTime           = request.EndTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone);
                var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc);

                IEnumerator <BaseData> enumerator;
                if (request.Configuration.IsCustomData)
                {
                    if (!Quandl.IsAuthCodeSet)
                    {
                        // we're not using the SubscriptionDataReader, so be sure to set the auth token here
                        Quandl.SetAuthCode(Config.Get("quandl-auth-token"));
                    }

                    var factory         = new LiveCustomDataSubscriptionEnumeratorFactory(_timeProvider);
                    var enumeratorStack = factory.CreateEnumerator(request, _dataProvider);

                    _customExchange.AddEnumerator(request.Configuration.Symbol, enumeratorStack);

                    var enqueable = new EnqueueableEnumerator <BaseData>();
                    _customExchange.SetDataHandler(request.Configuration.Symbol, data =>
                    {
                        enqueable.Enqueue(data);
                        if (SubscriptionShouldUpdateRealTimePrice(subscription, timeZoneOffsetProvider))
                        {
                            subscription.RealtimePrice = data.Value;
                        }
                    });
                    enumerator = enqueable;
                }
                else if (request.Configuration.Resolution != Resolution.Tick)
                {
                    // this enumerator allows the exchange to pump ticks into the 'back' of the enumerator,
                    // and the time sync loop can pull aggregated trade bars off the front
                    switch (request.Configuration.TickType)
                    {
                    case TickType.Quote:
                        var quoteBarAggregator = new QuoteBarBuilderEnumerator(request.Configuration.Increment, request.Security.Exchange.TimeZone, _timeProvider);
                        _exchange.AddDataHandler(request.Configuration.Symbol, data =>
                        {
                            var tick = data as Tick;

                            if (tick.TickType == TickType.Quote)
                            {
                                quoteBarAggregator.ProcessData(tick);
                                if (SubscriptionShouldUpdateRealTimePrice(subscription, timeZoneOffsetProvider))
                                {
                                    subscription.RealtimePrice = data.Value;
                                }
                            }
                        });
                        enumerator = quoteBarAggregator;
                        break;

                    case TickType.Trade:
                    default:
                        var tradeBarAggregator = new TradeBarBuilderEnumerator(request.Configuration.Increment, request.Security.Exchange.TimeZone, _timeProvider);
                        _exchange.AddDataHandler(request.Configuration.Symbol, data =>
                        {
                            var tick = data as Tick;

                            if (tick.TickType == TickType.Trade)
                            {
                                tradeBarAggregator.ProcessData(tick);
                                if (SubscriptionShouldUpdateRealTimePrice(subscription, timeZoneOffsetProvider))
                                {
                                    subscription.RealtimePrice = data.Value;
                                }
                            }
                        });
                        enumerator = tradeBarAggregator;
                        break;

                    case TickType.OpenInterest:
                        var oiAggregator = new OpenInterestEnumerator(request.Configuration.Increment, request.Security.Exchange.TimeZone, _timeProvider);
                        _exchange.AddDataHandler(request.Configuration.Symbol, data =>
                        {
                            var tick = data as Tick;

                            if (tick.TickType == TickType.OpenInterest)
                            {
                                oiAggregator.ProcessData(tick);
                            }
                        });
                        enumerator = oiAggregator;
                        break;
                    }
                }
                else
                {
                    // tick subscriptions can pass right through
                    var tickEnumerator = new EnqueueableEnumerator <BaseData>();
                    _exchange.SetDataHandler(request.Configuration.Symbol, data =>
                    {
                        tickEnumerator.Enqueue(data);
                        if (SubscriptionShouldUpdateRealTimePrice(subscription, timeZoneOffsetProvider))
                        {
                            subscription.RealtimePrice = data.Value;
                        }
                    });
                    enumerator = tickEnumerator;
                }

                if (request.Configuration.FillDataForward)
                {
                    var subscriptionConfigs = _subscriptions.Select(x => x.Configuration).Concat(new[] { request.Configuration });

                    UpdateFillForwardResolution(subscriptionConfigs);

                    enumerator = new LiveFillForwardEnumerator(_frontierTimeProvider, enumerator, request.Security.Exchange, _fillForwardResolution, request.Configuration.ExtendedMarketHours, localEndTime, request.Configuration.Increment, request.Configuration.DataTimeZone);
                }

                // define market hours and user filters to incoming data
                if (request.Configuration.IsFilteredSubscription)
                {
                    enumerator = new SubscriptionFilterEnumerator(enumerator, request.Security, localEndTime);
                }

                // finally, make our subscriptions aware of the frontier of the data feed, prevents future data from spewing into the feed
                enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, timeZoneOffsetProvider);

                var subscriptionDataEnumerator = SubscriptionData.Enumerator(request.Configuration, request.Security, timeZoneOffsetProvider, enumerator);
                subscription = new Subscription(request.Universe, request.Security, request.Configuration, subscriptionDataEnumerator, timeZoneOffsetProvider, request.StartTimeUtc, request.EndTimeUtc, false);
            }
            catch (Exception err)
            {
                Log.Error(err);
            }

            return(subscription);
        }
예제 #41
0
        /// <summary>
        /// Creates a new subscription for the specified security
        /// </summary>
        /// <param name="request">The subscription request</param>
        /// <returns>A new subscription instance of the specified security</returns>
        protected Subscription CreateDataSubscription(SubscriptionRequest request)
        {
            Subscription subscription = null;

            try
            {
                var localEndTime           = request.EndTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone);
                var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Configuration.ExchangeTimeZone, request.StartTimeUtc, request.EndTimeUtc);

                IEnumerator <BaseData> enumerator;
                if (!_channelProvider.ShouldStreamSubscription(request.Configuration))
                {
                    if (!Quandl.IsAuthCodeSet)
                    {
                        // we're not using the SubscriptionDataReader, so be sure to set the auth token here
                        Quandl.SetAuthCode(Config.Get("quandl-auth-token"));
                    }

                    if (!Tiingo.IsAuthCodeSet)
                    {
                        // we're not using the SubscriptionDataReader, so be sure to set the auth token here
                        Tiingo.SetAuthCode(Config.Get("tiingo-auth-token"));
                    }

                    var factory         = new LiveCustomDataSubscriptionEnumeratorFactory(_timeProvider);
                    var enumeratorStack = factory.CreateEnumerator(request, _dataProvider);

                    _customExchange.AddEnumerator(request.Configuration.Symbol, enumeratorStack);

                    var enqueable = new EnqueueableEnumerator <BaseData>();
                    _customExchange.SetDataHandler(request.Configuration.Symbol, data =>
                    {
                        enqueable.Enqueue(data);

                        subscription.OnNewDataAvailable();
                    });
                    enumerator = enqueable;
                }
                else
                {
                    var auxEnumerators = new List <IEnumerator <BaseData> >();

                    if (LiveAuxiliaryDataEnumerator.TryCreate(request.Configuration, _timeProvider, _dataQueueHandler,
                                                              request.Security.Cache, _mapFileProvider, _factorFileProvider, request.StartTimeLocal, out var auxDataEnumator))
                    {
                        auxEnumerators.Add(auxDataEnumator);
                    }

                    EventHandler handler = (_, _) => subscription?.OnNewDataAvailable();
                    enumerator = Subscribe(request.Configuration, handler);

                    if (request.Configuration.EmitSplitsAndDividends())
                    {
                        auxEnumerators.Add(Subscribe(new SubscriptionDataConfig(request.Configuration, typeof(Dividend)), handler));
                        auxEnumerators.Add(Subscribe(new SubscriptionDataConfig(request.Configuration, typeof(Split)), handler));
                    }

                    if (auxEnumerators.Count > 0)
                    {
                        enumerator = new LiveAuxiliaryDataSynchronizingEnumerator(_timeProvider, request.Configuration.ExchangeTimeZone, enumerator, auxEnumerators);
                    }
                }

                if (request.Configuration.FillDataForward)
                {
                    var fillForwardResolution = _subscriptions.UpdateAndGetFillForwardResolution(request.Configuration);

                    enumerator = new LiveFillForwardEnumerator(_frontierTimeProvider, enumerator, request.Security.Exchange, fillForwardResolution, request.Configuration.ExtendedMarketHours, localEndTime, request.Configuration.Increment, request.Configuration.DataTimeZone);
                }

                // define market hours and user filters to incoming data
                if (request.Configuration.IsFilteredSubscription)
                {
                    enumerator = new SubscriptionFilterEnumerator(enumerator, request.Security, localEndTime, request.Configuration.ExtendedMarketHours, true, request.ExchangeHours);
                }

                // finally, make our subscriptions aware of the frontier of the data feed, prevents future data from spewing into the feed
                enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, timeZoneOffsetProvider);

                var subscriptionDataEnumerator = new SubscriptionDataEnumerator(request.Configuration, request.Security.Exchange.Hours, timeZoneOffsetProvider, enumerator, request.IsUniverseSubscription);
                subscription = new Subscription(request, subscriptionDataEnumerator, timeZoneOffsetProvider);
            }
            catch (Exception err)
            {
                Log.Error(err);
            }

            return(subscription);
        }
예제 #42
0
        /// <summary>
        /// Setups a new <see cref="Subscription"/> which will consume a blocking <see cref="EnqueueableEnumerator{T}"/>
        /// that will be feed by a worker task
        /// </summary>
        /// <param name="request">The subscription data request</param>
        /// <param name="enumerator">The data enumerator stack</param>
        /// <param name="firstLoopLimit">The first loop data point count for which the worker will stop</param>
        /// <returns>A new subscription instance ready to consume</returns>
        public static Subscription CreateAndScheduleWorker(
            SubscriptionRequest request,
            IEnumerator <BaseData> enumerator,
            int firstLoopLimit = 50)
        {
            var upperThreshold = GetUpperThreshold(request.Configuration.Resolution);
            var lowerThreshold = GetLowerThreshold(request.Configuration.Resolution);

            if (request.Configuration.Type == typeof(CoarseFundamental))
            {
                // the lower threshold will be when we start the worker again, if he is stopped
                lowerThreshold = 200;
                // the upper threshold will stop the worker from loading more data. This is roughly 1 GB
                upperThreshold = 500;
            }

            var exchangeHours          = request.Security.Exchange.Hours;
            var enqueueable            = new EnqueueableEnumerator <SubscriptionData>(true);
            var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc);
            var subscription           = new Subscription(request, enqueueable, timeZoneOffsetProvider);

            // The first loop of a backtest can load hundreds of subscription feeds, resulting in long delays while the thresholds
            // for the buffer are reached. For the first loop start up with just 50 items in the buffer.
            var    firstLoop = Ref.Create(true);
            Action produce   = () =>
            {
                try
                {
                    var count = 0;
                    while (enumerator.MoveNext())
                    {
                        // subscription has been removed, no need to continue enumerating
                        if (enqueueable.HasFinished)
                        {
                            enumerator.DisposeSafely();
                            return;
                        }

                        var subscriptionData = SubscriptionData.Create(subscription.Configuration, exchangeHours,
                                                                       subscription.OffsetProvider, enumerator.Current);

                        // drop the data into the back of the enqueueable
                        enqueueable.Enqueue(subscriptionData);

                        count++;

                        // stop executing if we have more data than the upper threshold in the enqueueable, we don't want to fill the ram
                        if (count > upperThreshold || count > firstLoopLimit && firstLoop.Value)
                        {
                            // we use local count for the outside if, for performance, and adjust here
                            count = enqueueable.Count;
                            if (count > upperThreshold || firstLoop.Value)
                            {
                                firstLoop.Value = false;
                                // we will be re scheduled to run by the consumer, see EnqueueableEnumerator

                                // if the consumer is already waiting for us wake him up, he will rescheduled us if required
                                enqueueable.CancellationTokenSource.Cancel();
                                return;
                            }
                        }
                    }
                }
                catch (Exception exception)
                {
                    Log.Error(exception, $"Subscription worker task exception {request.Configuration}.");
                }

                // we made it here because MoveNext returned false or we exploded, stop the enqueueable
                enqueueable.Stop();
                // we have to dispose of the enumerator
                enumerator.DisposeSafely();
            };

            enqueueable.SetProducer(produce, lowerThreshold);

            return(subscription);
        }
예제 #43
0
        /// <summary>
        /// Creates a new subscription for the specified security
        /// </summary>
        /// <param name="request">The subscription request</param>
        /// <returns>A new subscription instance of the specified security</returns>
        protected Subscription CreateSubscription(SubscriptionRequest request)
        {
            Subscription subscription = null;

            try
            {
                var localEndTime           = request.EndTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone);
                var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc);

                IEnumerator <BaseData> enumerator;
                if (request.Configuration.IsCustomData)
                {
                    if (!Quandl.IsAuthCodeSet)
                    {
                        // we're not using the SubscriptionDataReader, so be sure to set the auth token here
                        Quandl.SetAuthCode(Config.Get("quandl-auth-token"));
                    }

                    // each time we exhaust we'll new up this enumerator stack
                    var refresher = new RefreshEnumerator <BaseData>(() =>
                    {
                        var dateInDataTimeZone    = DateTime.UtcNow.ConvertFromUtc(request.Configuration.DataTimeZone).Date;
                        var enumeratorFactory     = new BaseDataSubscriptionEnumeratorFactory(r => new[] { dateInDataTimeZone });
                        var factoryReadEnumerator = enumeratorFactory.CreateEnumerator(request, _dataFileProvider);
                        var maximumDataAge        = TimeSpan.FromTicks(Math.Max(request.Configuration.Increment.Ticks, TimeSpan.FromSeconds(5).Ticks));
                        var fastForward           = new FastForwardEnumerator(factoryReadEnumerator, _timeProvider, request.Security.Exchange.TimeZone, maximumDataAge);
                        return(new FrontierAwareEnumerator(fastForward, _frontierTimeProvider, timeZoneOffsetProvider));
                    });

                    // rate limit the refreshing of the stack to the requested interval
                    // At Tick resolution, it will refresh at full speed
                    // At Second and Minute resolution, it will refresh every second and minute respectively
                    // At Hour and Daily resolutions, it will refresh every 30 minutes
                    var minimumTimeBetweenCalls = Math.Min(request.Configuration.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks);
                    var rateLimit = new RateLimitEnumerator(refresher, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls));
                    _customExchange.AddEnumerator(request.Configuration.Symbol, rateLimit);

                    var enqueable = new EnqueueableEnumerator <BaseData>();
                    _customExchange.SetDataHandler(request.Configuration.Symbol, data =>
                    {
                        enqueable.Enqueue(data);
                        if (subscription != null)
                        {
                            subscription.RealtimePrice = data.Value;
                        }
                    });
                    enumerator = enqueable;
                }
                else if (request.Configuration.Resolution != Resolution.Tick)
                {
                    // this enumerator allows the exchange to pump ticks into the 'back' of the enumerator,
                    // and the time sync loop can pull aggregated trade bars off the front
                    switch (request.Configuration.TickType)
                    {
                    case TickType.Quote:
                        var quoteBarAggregator = new QuoteBarBuilderEnumerator(request.Configuration.Increment, request.Security.Exchange.TimeZone, _timeProvider);
                        _exchange.AddDataHandler(request.Configuration.Symbol, data =>
                        {
                            var tick = data as Tick;

                            if (tick.TickType == TickType.Quote)
                            {
                                quoteBarAggregator.ProcessData(tick);
                                if (subscription != null)
                                {
                                    subscription.RealtimePrice = data.Value;
                                }
                            }
                        });
                        enumerator = quoteBarAggregator;
                        break;

                    case TickType.Trade:
                    default:
                        var tradeBarAggregator = new TradeBarBuilderEnumerator(request.Configuration.Increment, request.Security.Exchange.TimeZone, _timeProvider);
                        _exchange.AddDataHandler(request.Configuration.Symbol, data =>
                        {
                            var tick = data as Tick;

                            if (tick.TickType == TickType.Trade)
                            {
                                tradeBarAggregator.ProcessData(tick);
                                if (subscription != null)
                                {
                                    subscription.RealtimePrice = data.Value;
                                }
                            }
                        });
                        enumerator = tradeBarAggregator;
                        break;

                    case TickType.OpenInterest:
                        var oiAggregator = new OpenInterestEnumerator(request.Configuration.Increment, request.Security.Exchange.TimeZone, _timeProvider);
                        _exchange.AddDataHandler(request.Configuration.Symbol, data =>
                        {
                            var tick = data as Tick;

                            if (tick.TickType == TickType.OpenInterest)
                            {
                                oiAggregator.ProcessData(tick);
                            }
                        });
                        enumerator = oiAggregator;
                        break;
                    }
                }
                else
                {
                    // tick subscriptions can pass right through
                    var tickEnumerator = new EnqueueableEnumerator <BaseData>();
                    _exchange.SetDataHandler(request.Configuration.Symbol, data =>
                    {
                        tickEnumerator.Enqueue(data);
                        if (subscription != null)
                        {
                            subscription.RealtimePrice = data.Value;
                        }
                    });
                    enumerator = tickEnumerator;
                }

                if (request.Configuration.FillDataForward)
                {
                    var subscriptionConfigs = _subscriptions.Select(x => x.Configuration).Concat(new[] { request.Configuration });

                    UpdateFillForwardResolution(subscriptionConfigs);

                    enumerator = new LiveFillForwardEnumerator(_frontierTimeProvider, enumerator, request.Security.Exchange, _fillForwardResolution, request.Configuration.ExtendedMarketHours, localEndTime, request.Configuration.Increment);
                }

                // define market hours and user filters to incoming data
                if (request.Configuration.IsFilteredSubscription)
                {
                    enumerator = new SubscriptionFilterEnumerator(enumerator, request.Security, localEndTime);
                }

                // finally, make our subscriptions aware of the frontier of the data feed, prevents future data from spewing into the feed
                enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, timeZoneOffsetProvider);

                subscription = new Subscription(request.Universe, request.Security, request.Configuration, enumerator, timeZoneOffsetProvider, request.StartTimeUtc, request.EndTimeUtc, false);
            }
            catch (Exception err)
            {
                Log.Error(err);
            }

            return(subscription);
        }
예제 #44
0
        public void MoveNextBlocks()
        {
            var finished = new ManualResetEvent(false);
            var enumerator = new EnqueueableEnumerator<Tick>(true);

            // producer
            int count = 0;
            Task.Run(() =>
            {
                while (!finished.WaitOne(TimeSpan.FromMilliseconds(50)))
                {
                    enumerator.Enqueue(new Tick(DateTime.Now, Symbols.SPY, 100, 101));
                    count++;

                    // 5 data points is plenty
                    if (count > 5)
                    {
                        finished.Set();
                        enumerator.Stop();
                    }
                }
            });

            // consumer
            int dequeuedCount = 0;
            bool encounteredError = false;
            var consumerTaskFinished = new ManualResetEvent(false);
            Task.Run(() =>
            {
                while (enumerator.MoveNext())
                {
                    dequeuedCount++;
                    if (enumerator.Current == null)
                    {
                        encounteredError = true;
                    }
                }
                consumerTaskFinished.Set();
            });

            finished.WaitOne(Timeout.Infinite);
            consumerTaskFinished.WaitOne(Timeout.Infinite);

            Assert.IsFalse(enumerator.MoveNext());
            Assert.IsFalse(encounteredError);
            Assert.AreEqual(count, dequeuedCount);
        }
예제 #45
0
 public EnumeratorHandler(Symbol symbol, IEnumerator <BaseData> enumerator, EnqueueableEnumerator <BaseData> enqueueable)
     : base(symbol, enumerator, true)
 {
     _enqueueable = enqueueable;
 }
예제 #46
0
        private Subscription CreateSubscription(Universe universe, IResultHandler resultHandler, Security security, DateTime startTimeUtc, DateTime endTimeUtc, IReadOnlyRef<TimeSpan> fillForwardResolution)
        {
            var config = security.SubscriptionDataConfig;
            var localStartTime = startTimeUtc.ConvertFromUtc(security.Exchange.TimeZone);
            var localEndTime = endTimeUtc.ConvertFromUtc(security.Exchange.TimeZone);

            var tradeableDates = Time.EachTradeableDay(security, localStartTime, localEndTime);

            // ReSharper disable once PossibleMultipleEnumeration
            if (!tradeableDates.Any())
            {
                _algorithm.Error(string.Format("No data loaded for {0} because there were no tradeable dates for this security.", security.Symbol));
                return null;
            }

            // get the map file resolver for this market
            var mapFileResolver = MapFileResolver.Empty;
            if (config.SecurityType == SecurityType.Equity) mapFileResolver = _mapFileProvider.Get(config.Market);

            // ReSharper disable once PossibleMultipleEnumeration
            IEnumerator<BaseData> enumerator = new SubscriptionDataReader(config, localStartTime, localEndTime, resultHandler, mapFileResolver, _factorFileProvider, tradeableDates, false);

            // optionally apply fill forward logic, but never for tick data
            if (config.FillDataForward && config.Resolution != Resolution.Tick)
            {
                enumerator = new FillForwardEnumerator(enumerator, security.Exchange, fillForwardResolution,
                    security.IsExtendedMarketHours, localEndTime, config.Resolution.ToTimeSpan());
            }

            // finally apply exchange/user filters
            enumerator = SubscriptionFilterEnumerator.WrapForDataFeed(resultHandler, enumerator, security, localEndTime);

            var enqueueable = new EnqueueableEnumerator<BaseData>(true);

            // add this enumerator to our exchange
            ScheduleEnumerator(enumerator, enqueueable, GetLowerThreshold(config.Resolution), GetUpperThreshold(config.Resolution));

            var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, startTimeUtc, endTimeUtc);
            var subscription = new Subscription(universe, security, enqueueable, timeZoneOffsetProvider, startTimeUtc, endTimeUtc, false);
            return subscription;
        }