Exemple #1
0
        /// <summary>
        /// Dequeues the current working bar
        /// </summary>
        /// <param name="utcTriggerTime">The current trigger time in UTC</param>
        /// <returns>The base data instance, or null, if no data is to be emitted</returns>
        public BaseData TriggerArchive(DateTime utcTriggerTime)
        {
            BaseData bar;
            if (!_queue.TryDequeue(out bar))
            {
                // if a bar wasn't ready, check for fill forward
                if (_previous != null && _config.FillDataForward)
                {
                    // exchanges hours are in local time, so convert to local before checking if exchange is open
                    var localTriggerTime = utcTriggerTime.ConvertFromUtc(_config.TimeZone);

                    // only perform fill forward behavior if the exchange is considered open
                    var barStartTime = localTriggerTime - _increment;
                    if (_security.Exchange.IsOpenDuringBar(barStartTime, localTriggerTime, _config.ExtendedMarketHours))
                    {
                        bar = _previous.Clone(true);
                        bar.Time = barStartTime.ExchangeRoundDown(_increment, _security.Exchange.Hours, _security.IsExtendedMarketHours);
                    }
                }
            }

            // we don't have data, so just return null
            if (bar == null) return null;

            // reset the previous bar for fill forward
            _previous = bar.Clone();

            return bar;
        }
        /// <summary>
        /// Applies cash settlement rules
        /// </summary>
        /// <param name="portfolio">The algorithm's portfolio</param>
        /// <param name="security">The fill's security</param>
        /// <param name="applicationTimeUtc">The fill time (in UTC)</param>
        /// <param name="currency">The currency symbol</param>
        /// <param name="amount">The amount of cash to apply</param>
        public void ApplyFunds(SecurityPortfolioManager portfolio, Security security, DateTime applicationTimeUtc, string currency, decimal amount)
        {
            if (amount > 0)
            {
                // positive amount: sell order filled

                portfolio.UnsettledCashBook[currency].AddAmount(amount);

                // find the correct settlement date (usually T+3 or T+1)
                var settlementDate = applicationTimeUtc.ConvertFromUtc(security.Exchange.TimeZone).Date;
                for (var i = 0; i < _numberOfDays; i++)
                {
                    settlementDate = settlementDate.AddDays(1);

                    // only count days when market is open
                    if (!security.Exchange.Hours.IsDateOpen(settlementDate))
                        i--;
                }

                // use correct settlement time
                var settlementTimeUtc = settlementDate.Add(_timeOfDay).ConvertToUtc(security.Exchange.Hours.TimeZone);

                portfolio.AddUnsettledCashAmount(new UnsettledCashAmount(settlementTimeUtc, currency, amount));
            }
            else
            {
                // negative amount: buy order filled

                portfolio.CashBook[currency].AddAmount(amount);
            }
        }
Exemple #3
0
        /// <summary>
        /// Creates a new <see cref="TimeSlice"/> for the specified time using the specified data
        /// </summary>
        /// <param name="utcDateTime">The UTC frontier date time</param>
        /// <param name="algorithmTimeZone">The algorithm's time zone, required for computing algorithm and slice time</param>
        /// <param name="cashBook">The algorithm's cash book, required for generating cash update pairs</param>
        /// <param name="data">The data in this <see cref="TimeSlice"/></param>
        /// <param name="changes">The new changes that are seen in this time slice as a result of universe selection</param>
        /// <returns>A new <see cref="TimeSlice"/> containing the specified data</returns>
        public static TimeSlice Create(DateTime utcDateTime, DateTimeZone algorithmTimeZone, CashBook cashBook, List<KeyValuePair<Security, List<BaseData>>> data, SecurityChanges changes)
        {
            int count = 0;
            var security = new List<KeyValuePair<Security, BaseData>>();
            var custom = new List<KeyValuePair<Security, List<BaseData>>>();
            var consolidator = new List<KeyValuePair<SubscriptionDataConfig, List<BaseData>>>();
            var allDataForAlgorithm = new List<BaseData>(data.Count);
            var cash = new List<KeyValuePair<Cash, BaseData>>(cashBook.Count);

            var cashSecurities = new HashSet<Symbol>();
            foreach (var cashItem in cashBook.Values)
            {
                cashSecurities.Add(cashItem.SecuritySymbol);
            }

            Split split;
            Dividend dividend;
            Delisting delisting;
            SymbolChangedEvent symbolChange;

            var algorithmTime = utcDateTime.ConvertFromUtc(algorithmTimeZone);
            var tradeBars = new TradeBars(algorithmTime);
            var ticks = new Ticks(algorithmTime);
            var splits = new Splits(algorithmTime);
            var dividends = new Dividends(algorithmTime);
            var delistings = new Delistings(algorithmTime);
            var symbolChanges = new SymbolChangedEvents(algorithmTime);

            foreach (var kvp in data)
            {
                var list = kvp.Value;
                var symbol = kvp.Key.Symbol;
                
                // keep count of all data points
                if (list.Count == 1 && list[0] is BaseDataCollection)
                {
                    count += ((BaseDataCollection) list[0]).Data.Count;
                }
                else
                {
                    count += list.Count;
                }

                BaseData update = null;
                var consolidatorUpdate = new List<BaseData>(list.Count);
                for (int i = 0; i < list.Count; i++)
                {
                    var baseData = list[i];
                    if (!kvp.Key.SubscriptionDataConfig.IsInternalFeed)
                    {
                        // this is all the data that goes into the algorithm
                        allDataForAlgorithm.Add(baseData);
                        if (kvp.Key.SubscriptionDataConfig.IsCustomData)
                        {
                            // this is all the custom data
                            custom.Add(kvp);
                        }
                    }
                    // don't add internal feed data to ticks/bars objects
                    if (baseData.DataType != MarketDataType.Auxiliary)
                    {
                        if (!kvp.Key.SubscriptionDataConfig.IsInternalFeed)
                        {
                            // populate ticks and tradebars dictionaries with no aux data
                            if (baseData.DataType == MarketDataType.Tick)
                            {
                                List<Tick> ticksList;
                                if (!ticks.TryGetValue(symbol, out ticksList))
                                {
                                    ticksList = new List<Tick> {(Tick) baseData};
                                    ticks[symbol] = ticksList;
                                }
                                ticksList.Add((Tick) baseData);
                            }
                            else if (baseData.DataType == MarketDataType.TradeBar)
                            {
                                tradeBars[symbol] = (TradeBar) baseData;
                            }

                            // this is data used to update consolidators
                            consolidatorUpdate.Add(baseData);
                        }

                        // this is the data used set market prices
                        update = baseData;
                    }
                    // include checks for various aux types so we don't have to construct the dictionaries in Slice
                    else if ((delisting = baseData as Delisting) != null)
                    {
                        delistings[symbol] = delisting;
                    }
                    else if ((dividend = baseData as Dividend) != null)
                    {
                        dividends[symbol] = dividend;
                    }
                    else if ((split = baseData as Split) != null)
                    {
                        splits[symbol] = split;
                    }
                    else if ((symbolChange = baseData as SymbolChangedEvent) != null)
                    {
                        // symbol changes is keyed by the requested symbol
                        symbolChanges[kvp.Key.SubscriptionDataConfig.Symbol] = symbolChange;
                    }
                }

                // check for 'cash securities' if we found valid update data for this symbol
                // and we need this data to update cash conversion rates, long term we should
                // have Cash hold onto it's security, then he can update himself, or rather, just
                // patch through calls to conversion rate to compue it on the fly using Security.Price
                if (update != null && cashSecurities.Contains(kvp.Key.Symbol))
                {
                    foreach (var cashKvp in cashBook)
                    {
                        if (cashKvp.Value.SecuritySymbol == kvp.Key.Symbol)
                        {
                            cash.Add(new KeyValuePair<Cash, BaseData>(cashKvp.Value, update));
                        }
                    }
                }

                security.Add(new KeyValuePair<Security, BaseData>(kvp.Key, update));
                consolidator.Add(new KeyValuePair<SubscriptionDataConfig, List<BaseData>>(kvp.Key.SubscriptionDataConfig, consolidatorUpdate));
            }

            var slice = new Slice(utcDateTime.ConvertFromUtc(algorithmTimeZone), allDataForAlgorithm, tradeBars, ticks, splits, dividends, delistings, symbolChanges, allDataForAlgorithm.Count > 0);

            return new TimeSlice(utcDateTime, count, slice, data, cash, security, consolidator, custom, changes);
        }
Exemple #4
0
        /// <summary>
        /// A time period has lapsed, trigger a save/queue of the current value of data.
        /// </summary>
        /// <param name="utcTriggerTime">The time we're triggering this archive for</param>
        /// <param name="fillForward">Data stream is a fillforward type</param>
        public void TriggerArchive(DateTime utcTriggerTime, bool fillForward)
        {
            var localTriggerTime = utcTriggerTime.ConvertFromUtc(_config.TimeZone);
            lock (_lock)
            {
                try
                {
                    //When there's nothing to do:
                    if (_data == null && !fillForward)
                    {
                        Log.Debug("StreamStore.TriggerArchive(): No data to store, and not fill forward: " + Symbol);
                    }

                    if (_data != null)
                    {
                        //Create clone and reset original
                        Log.Debug("StreamStore.TriggerArchive(): Enqueued new data: S:" + _data.Symbol + " V:" + _data.Value);
                        _previousData = _data.Clone();
                        _queue.Enqueue(_data.Clone());
                        _data = null;
                    }
                    else if (fillForward && _data == null && _previousData != null)
                    {
                        // the time is actually the end time of a bar, check to see if the start time
                        // is within market hours, which is really just checking the _previousData's EndTime
                        if (!_security.Exchange.IsOpenDuringBar(localTriggerTime - _increment, localTriggerTime, _config.ExtendedMarketHours))
                        {
                            Log.Debug("StreamStore.TriggerArchive(): Exchange is closed: " + Symbol);
                            return;
                        }

                        //There was no other data in this timer period, and this is a fillforward subscription:
                        Log.Debug("StreamStore.TriggerArchive(): Fillforward, Previous Enqueued: S:" + _previousData.Symbol + " V:" + _previousData.Value);
                        var cloneForward = _previousData.Clone(true);
                        cloneForward.Time = _previousData.Time.Add(_increment);
                        _queue.Enqueue(cloneForward);

                        _previousData = cloneForward.Clone();
                    }
                }
                catch (Exception err)
                {
                    Log.Error("StreamStore.TriggerAchive(fillforward): Failed to archive: " + err.Message);
                }
            }
        }
        /// <summary>
        /// Creates a new subscription for universe selection
        /// </summary>
        /// <param name="universe">The universe to add a subscription for</param>
        /// <param name="startTimeUtc">The start time of the subscription in utc</param>
        /// <param name="endTimeUtc">The end time of the subscription in utc</param>
        protected virtual Subscription CreateUniverseSubscription(Universe universe, DateTime startTimeUtc, DateTime endTimeUtc)
        {
            // TODO : Consider moving the creating of universe subscriptions to a separate, testable class

            // grab the relevant exchange hours
            var config = universe.Configuration;

            var exchangeHours = MarketHoursDatabase.FromDataFolder().GetExchangeHours(universe.Market, config.Symbol, universe.SecurityType);

            var localStartTime = startTimeUtc.ConvertFromUtc(exchangeHours.TimeZone);
            var localEndTime = endTimeUtc.ConvertFromUtc(exchangeHours.TimeZone);

            // create a canonical security object
            var security = new Security(exchangeHours, config, universe.SubscriptionSettings.Leverage);

            IEnumerator<BaseData> enumerator;
            
            var userDefined = universe as UserDefinedUniverse;
            if (userDefined != null)
            {
                // spoof a tick on the requested interval to trigger the universe selection function
                enumerator = LinqExtensions.Range(localStartTime, localEndTime, dt => dt + userDefined.Interval)
                    .Where(dt => security.Exchange.IsOpenDuringBar(dt, dt + userDefined.Interval, config.ExtendedMarketHours))
                    .Select(dt => new Tick { Time = dt }).GetEnumerator();
            }
            else if (config.Type == typeof (CoarseFundamental))
            {
                // since we're binding to the data queue exchange we'll need to let him
                // know that we expect this data
                _dataQueueHandler.Subscribe(_job, new Dictionary<SecurityType, List<Symbol>>
                {
                    {config.SecurityType, new List<Symbol>{config.Symbol}}
                });

                var enqueable = new EnqueableEnumerator<BaseData>();
                _exchange.SetHandler(config.Symbol, data =>
                {
                    var universeData = data as BaseDataCollection;
                    if (universeData != null)
                    {
                        enqueable.EnqueueRange(universeData.Data);
                    }
                });
                enumerator = enqueable;
            }
            else
            {
                // each time we exhaust we'll new up this enumerator stack
                var refresher = new RefreshEnumerator<BaseDataCollection>(() =>
                {
                    var sourceProvider = (BaseData)Activator.CreateInstance(config.Type);
                    var currentLocalDate = DateTime.UtcNow.ConvertFromUtc(security.Exchange.TimeZone).Date;
                    var factory = new BaseDataSubscriptionFactory(config, currentLocalDate, true);
                    var source = sourceProvider.GetSource(config, currentLocalDate, true);
                    var factorEnumerator = factory.Read(source).GetEnumerator();
                    var fastForward = new FastForwardEnumerator(factorEnumerator, _timeProvider, security.Exchange.TimeZone, config.Increment);
                    var tzOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, startTimeUtc, endTimeUtc);
                    var frontierAware = new FrontierAwareEnumerator(fastForward, _frontierTimeProvider, tzOffsetProvider);
                    return new BaseDataCollectionAggregatorEnumerator(frontierAware, config.Symbol);
                });
                
                // rate limit the refreshing of the stack to the requested interval
                var minimumTimeBetweenCalls = Math.Min(config.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks);
                var rateLimit = new RateLimitEnumerator(refresher, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls));
                _customExchange.AddEnumerator(rateLimit);

                var enqueable = new EnqueableEnumerator<BaseData>();
                _customExchange.SetHandler(config.Symbol, data =>
                {
                    var universeData = data as BaseDataCollection;
                    if (universeData != null)
                    {
                        enqueable.EnqueueRange(universeData.Data);
                    }
                    else
                    {
                        enqueable.Enqueue(data);
                    }
                });
                enumerator = enqueable;
            }

            // create the subscription
            var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, startTimeUtc, endTimeUtc);
            var subscription = new Subscription(universe, security, enumerator, timeZoneOffsetProvider, startTimeUtc, endTimeUtc, true);

            return subscription;
        }
        /// <summary>
        /// Creates a new subscription for the specified security
        /// </summary>
        /// <param name="universe"></param>
        /// <param name="security">The security to create a subscription for</param>
        /// <param name="utcStartTime">The start time of the subscription in UTC</param>
        /// <param name="utcEndTime">The end time of the subscription in UTC</param>
        /// <returns>A new subscription instance of the specified security</returns>
        protected Subscription CreateSubscription(Universe universe, Security security, DateTime utcStartTime, DateTime utcEndTime)
        {
            Subscription subscription = null;
            try
            {
                var config = security.SubscriptionDataConfig;
                var localEndTime = utcEndTime.ConvertFromUtc(security.Exchange.TimeZone);
                var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, utcStartTime, utcEndTime);

                IEnumerator<BaseData> enumerator;
                if (config.IsCustomData)
                {
                    // each time we exhaust we'll new up this enumerator stack
                    var refresher = new RefreshEnumerator<BaseData>(() =>
                    {
                        var sourceProvider = (BaseData)Activator.CreateInstance(config.Type);
                        var currentLocalDate = DateTime.UtcNow.ConvertFromUtc(security.Exchange.TimeZone).Date;
                        var factory = new BaseDataSubscriptionFactory(config, currentLocalDate, true);
                        var source = sourceProvider.GetSource(config, currentLocalDate, true);
                        var factoryReadEnumerator = factory.Read(source).GetEnumerator();
                        var maximumDataAge = TimeSpan.FromTicks(Math.Max(config.Increment.Ticks, TimeSpan.FromSeconds(5).Ticks));
                        var fastForward = new FastForwardEnumerator(factoryReadEnumerator, _timeProvider, security.Exchange.TimeZone, maximumDataAge);
                        return new FrontierAwareEnumerator(fastForward, _timeProvider, timeZoneOffsetProvider);
                    });

                    // rate limit the refreshing of the stack to the requested interval
                    var minimumTimeBetweenCalls = Math.Min(config.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks);
                    var rateLimit = new RateLimitEnumerator(refresher, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls));
                    _customExchange.AddEnumerator(rateLimit);

                    var enqueable = new EnqueableEnumerator<BaseData>();
                    _customExchange.SetHandler(config.Symbol, data =>
                    {
                        enqueable.Enqueue(data);
                        if (subscription != null) subscription.RealtimePrice = data.Value;
                    });
                    enumerator = enqueable;
                }
                else if (config.Resolution != Resolution.Tick)
                {
                    // this enumerator allows the exchange to pump ticks into the 'back' of the enumerator,
                    // and the time sync loop can pull aggregated trade bars off the front
                    var aggregator = new TradeBarBuilderEnumerator(config.Increment, security.Exchange.TimeZone, _timeProvider);
                    _exchange.SetHandler(config.Symbol, data =>
                    {
                        aggregator.ProcessData((Tick) data);
                        if (subscription != null) subscription.RealtimePrice = data.Value;
                    });
                    enumerator = aggregator;
                }
                else
                {
                    // tick subscriptions can pass right through
                    var tickEnumerator = new EnqueableEnumerator<BaseData>();
                    _exchange.SetHandler(config.Symbol, data =>
                    {
                        tickEnumerator.Enqueue(data);
                        if (subscription != null) subscription.RealtimePrice = data.Value;
                    });
                    enumerator = tickEnumerator;
                }

                if (config.FillDataForward)
                {
                    // TODO : Properly resolve fill forward resolution like in FileSystemDataFeed (make considerations for universe-only)
                    enumerator = new LiveFillForwardEnumerator(_frontierTimeProvider, enumerator, security.Exchange, _fillForwardResolution, config.ExtendedMarketHours, localEndTime, config.Increment);
                }

                // define market hours and user filters to incoming data
                enumerator = new SubscriptionFilterEnumerator(enumerator, security, localEndTime);

                // finally, make our subscriptions aware of the frontier of the data feed, this will help
                enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, timeZoneOffsetProvider);


                subscription = new Subscription(universe, security, enumerator, timeZoneOffsetProvider, utcStartTime, utcEndTime, false);
            }
            catch (Exception err)
            {
                Log.Error(err);
            }

            return subscription;
        }
        /// <summary>
        /// Initializes a new instance of the <see cref="SubscriptionRequest"/> class
        /// </summary>
        public SubscriptionRequest(bool isUniverseSubscription,
            Universe universe,
            Security security,
            SubscriptionDataConfig configuration,
            DateTime startTimeUtc,
            DateTime endTimeUtc)
        {
            IsUniverseSubscription = isUniverseSubscription;
            Universe = universe;
            Security = security;
            Configuration = configuration;
            StartTimeUtc = startTimeUtc;
            EndTimeUtc = endTimeUtc;

            _localStartTime = new Lazy<DateTime>(() => StartTimeUtc.ConvertFromUtc(Configuration.ExchangeTimeZone));
            _localEndTime = new Lazy<DateTime>(() => EndTimeUtc.ConvertFromUtc(Configuration.ExchangeTimeZone));
        }
        /// <summary>
        /// Adds a new subscription for universe selection
        /// </summary>
        /// <param name="universe">The universe to add a subscription for</param>
        /// <param name="startTimeUtc">The start time of the subscription in utc</param>
        /// <param name="endTimeUtc">The end time of the subscription in utc</param>
        public void AddUniverseSubscription(
            IUniverse universe,
            DateTime startTimeUtc,
            DateTime endTimeUtc
            )
        {
            // grab the relevant exchange hours
            SubscriptionDataConfig config = universe.Configuration;

            var exchangeHours = SecurityExchangeHoursProvider.FromDataFolder()
                .GetExchangeHours(config.Market, null, config.SecurityType);

            // create a canonical security object
            var security = new Security(exchangeHours, config, universe.SubscriptionSettings.Leverage);

            var localStartTime = startTimeUtc.ConvertFromUtc(config.TimeZone);
            var localEndTime = endTimeUtc.ConvertFromUtc(config.TimeZone);

            // define our data enumerator
            var tradeableDates = Time.EachTradeableDay(security, localStartTime, localEndTime);
            var enumerator = new SubscriptionDataReader(config, localStartTime, localEndTime, _resultHandler, tradeableDates, false);

            // create the subscription
            var subscription = new LiveSubscription(universe, security, enumerator, startTimeUtc, endTimeUtc);

            // only message the user if it's one of their universe types
            _subscriptions.AddOrUpdate(new SymbolSecurityType(subscription), subscription);
        }
        /// <summary>
        /// Adds a new subscription for universe selection
        /// </summary>
        /// <param name="universe">The universe to add a subscription for</param>
        /// <param name="startTimeUtc">The start time of the subscription in utc</param>
        /// <param name="endTimeUtc">The end time of the subscription in utc</param>
        public void AddUniverseSubscription(Universe universe, DateTime startTimeUtc, DateTime endTimeUtc)
        {
            // TODO : Consider moving the creating of universe subscriptions to a separate, testable class

            // grab the relevant exchange hours
            var config = universe.Configuration;

            var marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
            var exchangeHours = marketHoursDatabase.GetExchangeHours(config);

            // create a canonical security object
            var security = new Security(exchangeHours, config);

            var localStartTime = startTimeUtc.ConvertFromUtc(security.Exchange.TimeZone);
            var localEndTime = endTimeUtc.ConvertFromUtc(security.Exchange.TimeZone);

            // define our data enumerator
            IEnumerator<BaseData> enumerator;

            var tradeableDates = Time.EachTradeableDay(security, localStartTime, localEndTime);

            var userDefined = universe as UserDefinedUniverse;
            if (userDefined != null)
            {
                // spoof a tick on the requested interval to trigger the universe selection function
                enumerator = userDefined.GetTriggerTimes(startTimeUtc, endTimeUtc, marketHoursDatabase)
                    .Select(x => new Tick { Time = x, Symbol = config.Symbol }).GetEnumerator();

                // route these custom subscriptions through the exchange for buffering
                var enqueueable = new EnqueueableEnumerator<BaseData>(true);

                // add this enumerator to our exchange
                ScheduleEnumerator(enumerator, enqueueable, GetLowerThreshold(config.Resolution), GetUpperThreshold(config.Resolution));

                enumerator = enqueueable;
            }
            else if (config.Type == typeof (CoarseFundamental))
            {
                var cf = new CoarseFundamental();

                var enqueueable = new EnqueueableEnumerator<BaseData>(true);

                // load coarse data day by day
                var coarse = from date in Time.EachTradeableDay(security, _algorithm.StartDate, _algorithm.EndDate)
                             let dateInDataTimeZone = date.ConvertTo(config.ExchangeTimeZone, config.DataTimeZone).Date
                             let factory = new BaseDataSubscriptionFactory(config, dateInDataTimeZone, false)
                             let source = cf.GetSource(config, dateInDataTimeZone, false)
                             let coarseFundamentalForDate = factory.Read(source)
                             select new BaseDataCollection(date, config.Symbol, coarseFundamentalForDate);

                
                ScheduleEnumerator(coarse.GetEnumerator(), enqueueable, 5, 100000, 2);

                enumerator = enqueueable;
            }
            else
            {
                // normal reader for all others
                enumerator = new SubscriptionDataReader(config, localStartTime, localEndTime, _resultHandler, MapFileResolver.Empty, _factorFileProvider, tradeableDates, false);

                // route these custom subscriptions through the exchange for buffering
                var enqueueable = new EnqueueableEnumerator<BaseData>(true);

                // add this enumerator to our exchange
                ScheduleEnumerator(enumerator, enqueueable, GetLowerThreshold(config.Resolution), GetUpperThreshold(config.Resolution));

                enumerator = enqueueable;
            }

            // create the subscription
            var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, startTimeUtc, endTimeUtc);
            var subscription = new Subscription(universe, security, enumerator, timeZoneOffsetProvider, startTimeUtc, endTimeUtc, true);
            _subscriptions.AddOrUpdate(subscription.Security.Symbol, subscription);
        }
Exemple #10
0
        /// <summary>
        /// Creates a new <see cref="TimeSlice"/> for the specified time using the specified data
        /// </summary>
        /// <param name="utcDateTime">The UTC frontier date time</param>
        /// <param name="algorithmTimeZone">The algorithm's time zone, required for computing algorithm and slice time</param>
        /// <param name="cashBook">The algorithm's cash book, required for generating cash update pairs</param>
        /// <param name="data">The data in this <see cref="TimeSlice"/></param>
        /// <param name="changes">The new changes that are seen in this time slice as a result of universe selection</param>
        /// <returns>A new <see cref="TimeSlice"/> containing the specified data</returns>
        public static TimeSlice Create(DateTime utcDateTime, DateTimeZone algorithmTimeZone, CashBook cashBook, List<DataFeedPacket> data, SecurityChanges changes)
        {
            int count = 0;
            var security = new List<UpdateData<Security>>();
            var custom = new List<UpdateData<Security>>();
            var consolidator = new List<UpdateData<SubscriptionDataConfig>>();
            var allDataForAlgorithm = new List<BaseData>(data.Count);
            var cash = new List<UpdateData<Cash>>(cashBook.Count);

            var cashSecurities = new HashSet<Symbol>();
            foreach (var cashItem in cashBook.Values)
            {
                cashSecurities.Add(cashItem.SecuritySymbol);
            }

            Split split;
            Dividend dividend;
            Delisting delisting;
            SymbolChangedEvent symbolChange;

            // we need to be able to reference the slice being created in order to define the
            // evaluation of option price models, so we define a 'future' that can be referenced
            // in the option price model evaluation delegates for each contract
            Slice slice = null;
            var sliceFuture = new Lazy<Slice>(() => slice);

            var algorithmTime = utcDateTime.ConvertFromUtc(algorithmTimeZone);
            var tradeBars = new TradeBars(algorithmTime);
            var quoteBars = new QuoteBars(algorithmTime);
            var ticks = new Ticks(algorithmTime);
            var splits = new Splits(algorithmTime);
            var dividends = new Dividends(algorithmTime);
            var delistings = new Delistings(algorithmTime);
            var optionChains = new OptionChains(algorithmTime);
            var symbolChanges = new SymbolChangedEvents(algorithmTime);

            foreach (var packet in data)
            {
                var list = packet.Data;
                var symbol = packet.Security.Symbol;

                if (list.Count == 0) continue;
                
                // keep count of all data points
                if (list.Count == 1 && list[0] is BaseDataCollection)
                {
                    var baseDataCollectionCount = ((BaseDataCollection)list[0]).Data.Count;
                    if (baseDataCollectionCount == 0)
                    {
                        continue;
                    }
                    count += baseDataCollectionCount;
                }
                else
                {
                    count += list.Count;
                }

                if (!packet.Configuration.IsInternalFeed && packet.Configuration.IsCustomData)
                {
                    // This is all the custom data
                    custom.Add(new UpdateData<Security>(packet.Security, packet.Configuration.Type, list));
                }

                var securityUpdate = new List<BaseData>(list.Count);
                var consolidatorUpdate = new List<BaseData>(list.Count);
                for (int i = 0; i < list.Count; i++)
                {
                    var baseData = list[i];
                    if (!packet.Configuration.IsInternalFeed)
                    {
                        // this is all the data that goes into the algorithm
                        allDataForAlgorithm.Add(baseData);
                    }
                    // don't add internal feed data to ticks/bars objects
                    if (baseData.DataType != MarketDataType.Auxiliary)
                    {
                        if (!packet.Configuration.IsInternalFeed)
                        {
                            PopulateDataDictionaries(baseData, ticks, tradeBars, quoteBars, optionChains);

                            // special handling of options data to build the option chain
                            if (packet.Security.Type == SecurityType.Option)
                            {
                                if (baseData.DataType == MarketDataType.OptionChain)
                                {
                                    optionChains[baseData.Symbol] = (OptionChain) baseData;
                                }
                                else if (!HandleOptionData(algorithmTime, baseData, optionChains, packet.Security, sliceFuture))
                                {
                                    continue;
                                }
                            }

                            // this is data used to update consolidators
                            consolidatorUpdate.Add(baseData);
                        }

                        // this is the data used set market prices
                        securityUpdate.Add(baseData);
                    }
                    // include checks for various aux types so we don't have to construct the dictionaries in Slice
                    else if ((delisting = baseData as Delisting) != null)
                    {
                        delistings[symbol] = delisting;
                    }
                    else if ((dividend = baseData as Dividend) != null)
                    {
                        dividends[symbol] = dividend;
                    }
                    else if ((split = baseData as Split) != null)
                    {
                        splits[symbol] = split;
                    }
                    else if ((symbolChange = baseData as SymbolChangedEvent) != null)
                    {
                        // symbol changes is keyed by the requested symbol
                        symbolChanges[packet.Configuration.Symbol] = symbolChange;
                    }
                }

                if (securityUpdate.Count > 0)
                {
                    // check for 'cash securities' if we found valid update data for this symbol
                    // and we need this data to update cash conversion rates, long term we should
                    // have Cash hold onto it's security, then he can update himself, or rather, just
                    // patch through calls to conversion rate to compue it on the fly using Security.Price
                    if (cashSecurities.Contains(packet.Security.Symbol))
                    {
                        foreach (var cashKvp in cashBook)
                        {
                            if (cashKvp.Value.SecuritySymbol == packet.Security.Symbol)
                            {
                                var cashUpdates = new List<BaseData> {securityUpdate[securityUpdate.Count - 1]};
                                cash.Add(new UpdateData<Cash>(cashKvp.Value, packet.Configuration.Type, cashUpdates));
                            }
                        }
                    }

                    security.Add(new UpdateData<Security>(packet.Security, packet.Configuration.Type, securityUpdate));
                }
                if (consolidatorUpdate.Count > 0)
                {
                    consolidator.Add(new UpdateData<SubscriptionDataConfig>(packet.Configuration, packet.Configuration.Type, consolidatorUpdate));
                }
            }

            slice = new Slice(algorithmTime, allDataForAlgorithm, tradeBars, quoteBars, ticks, optionChains, splits, dividends, delistings, symbolChanges, allDataForAlgorithm.Count > 0);

            return new TimeSlice(utcDateTime, count, slice, data, cash, security, consolidator, custom, changes);
        }
        private Subscription CreateSubscription(Universe universe, IResultHandler resultHandler, Security security, DateTime startTimeUtc, DateTime endTimeUtc, IReadOnlyRef<TimeSpan> fillForwardResolution)
        {
            var config = security.SubscriptionDataConfig;
            var localStartTime = startTimeUtc.ConvertFromUtc(security.Exchange.TimeZone);
            var localEndTime = endTimeUtc.ConvertFromUtc(security.Exchange.TimeZone);

            var tradeableDates = Time.EachTradeableDay(security, localStartTime, localEndTime);

            // ReSharper disable once PossibleMultipleEnumeration
            if (!tradeableDates.Any())
            {
                _algorithm.Error(string.Format("No data loaded for {0} because there were no tradeable dates for this security.", security.Symbol));
                return null;
            }

            // get the map file resolver for this market
            var mapFileResolver = MapFileResolver.Empty;
            if (config.SecurityType == SecurityType.Equity) mapFileResolver = _mapFileProvider.Get(config.Market);

            // ReSharper disable once PossibleMultipleEnumeration
            IEnumerator<BaseData> enumerator = new SubscriptionDataReader(config, localStartTime, localEndTime, resultHandler, mapFileResolver, _factorFileProvider, tradeableDates, false);

            // optionally apply fill forward logic, but never for tick data
            if (config.FillDataForward && config.Resolution != Resolution.Tick)
            {
                enumerator = new FillForwardEnumerator(enumerator, security.Exchange, fillForwardResolution,
                    security.IsExtendedMarketHours, localEndTime, config.Resolution.ToTimeSpan());
            }

            // finally apply exchange/user filters
            enumerator = SubscriptionFilterEnumerator.WrapForDataFeed(resultHandler, enumerator, security, localEndTime);

            var enqueueable = new EnqueueableEnumerator<BaseData>(true);

            // add this enumerator to our exchange
            ScheduleEnumerator(enumerator, enqueueable, GetLowerThreshold(config.Resolution), GetUpperThreshold(config.Resolution));

            var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, startTimeUtc, endTimeUtc);
            var subscription = new Subscription(universe, security, enqueueable, timeZoneOffsetProvider, startTimeUtc, endTimeUtc, false);
            return subscription;
        }
        /// <summary>
        /// Determines whether or not the specified security can be removed from
        /// this universe. This is useful to prevent securities from being taken
        /// out of a universe before the algorithm has had enough time to make
        /// decisions on the security
        /// </summary>
        /// <param name="utcTime">The current utc time</param>
        /// <param name="security">The security to check if its ok to remove</param>
        /// <returns>True if we can remove the security, false otherwise</returns>
        public override bool CanRemoveMember(DateTime utcTime, Security security)
        {
            // if we haven't begun receiving data for this security then it's safe to remove
            var lastData = security.Cache.GetData();
            if (lastData == null)
            {
                return true;
            }

            // only remove members on day changes, this prevents us from needing to
            // fast forward contracts continuously as price moves and out filtered
            // contracts change thoughout the day
            var localTime = utcTime.ConvertFromUtc(security.Exchange.TimeZone);
            if (localTime.Date != lastData.Time.Date)
            {
                return true;
            }
            return false;
        }
        private Subscription CreateSubscription(Universe universe, Security security, DateTime startTimeUtc, DateTime endTimeUtc)
        {
            var config = security.SubscriptionDataConfig;
            var localStartTime = startTimeUtc.ConvertFromUtc(security.Exchange.TimeZone);
            var localEndTime = endTimeUtc.ConvertFromUtc(security.Exchange.TimeZone);

            var tradeableDates = Time.EachTradeableDay(security, localStartTime, localEndTime);

            // ReSharper disable once PossibleMultipleEnumeration
            if (!tradeableDates.Any())
            {
                _algorithm.Error(string.Format("No data loaded for {0} because there were no tradeable dates for this security.", security.Symbol));
                return null;
            }

            // get the map file resolver for this market
            var mapFileResolver = MapFileResolver.Empty;
            if (config.SecurityType == SecurityType.Equity) mapFileResolver = _mapFileProvider.Get(config.Market);

            // ReSharper disable once PossibleMultipleEnumeration
            var enumerator = CreateSubscriptionEnumerator(security, config, localStartTime, localEndTime, mapFileResolver, tradeableDates);

            var enqueueable = new EnqueueableEnumerator<BaseData>(true);

            // add this enumerator to our exchange
            ScheduleEnumerator(enumerator, enqueueable, GetLowerThreshold(config.Resolution), GetUpperThreshold(config.Resolution));

            var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, startTimeUtc, endTimeUtc);
            var subscription = new Subscription(universe, security, enqueueable, timeZoneOffsetProvider, startTimeUtc, endTimeUtc, false);
            return subscription;
        }
        /// <summary>
        /// Adds a new subscription for universe selection
        /// </summary>
        /// <param name="universe">The universe to add a subscription for</param>
        /// <param name="startTimeUtc">The start time of the subscription in utc</param>
        /// <param name="endTimeUtc">The end time of the subscription in utc</param>
        public void AddUniverseSubscription(Universe universe, DateTime startTimeUtc, DateTime endTimeUtc)
        {
            // TODO : Consider moving the creating of universe subscriptions to a separate, testable class

            // grab the relevant exchange hours
            var config = universe.Configuration;

            var exchangeHours = MarketHoursDatabase.FromDataFolder().GetExchangeHours(config);

            // create a canonical security object
            var security = new Security(exchangeHours, config, universe.SubscriptionSettings.Leverage);

            var localStartTime = startTimeUtc.ConvertFromUtc(security.Exchange.TimeZone);
            var localEndTime = endTimeUtc.ConvertFromUtc(security.Exchange.TimeZone);

            // define our data enumerator
            IEnumerator<BaseData> enumerator;

            var tradeableDates = Time.EachTradeableDay(security, localStartTime, localEndTime);

            var userDefined = universe as UserDefinedUniverse;
            if (userDefined != null)
            {
                // spoof a tick on the requested interval to trigger the universe selection function
                enumerator = LinqExtensions.Range(localStartTime, localEndTime, dt => dt + userDefined.Interval)
                    .Where(dt => security.Exchange.IsOpenDuringBar(dt, dt + userDefined.Interval, config.ExtendedMarketHours))
                    .Select(dt => new Tick {Time = dt}).GetEnumerator();
            }
            else
            {
                // normal reader for all others
                enumerator = new SubscriptionDataReader(config, localStartTime, localEndTime, _resultHandler, MapFileResolver.Empty, tradeableDates, false);
            }

            // create the subscription
            var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, startTimeUtc, endTimeUtc);
            var subscription = new Subscription(universe, security, enumerator, timeZoneOffsetProvider, startTimeUtc, endTimeUtc, true);

            // only message the user if it's one of their universe types
            var messageUser = subscription.Configuration.Type != typeof(CoarseFundamental);
            PrimeSubscriptionPump(subscription, messageUser);
            _subscriptions.AddOrUpdate(subscription.Security.Symbol, subscription);
        }
        /// <summary>
        /// Creates a new subscription for the specified security
        /// </summary>
        /// <param name="security">The security to create a subscription for</param>
        /// <param name="utcStartTime">The start time of the subscription in UTC</param>
        /// <param name="utcEndTime">The end time of the subscription in UTC</param>
        /// <param name="isUserDefinedSubscription">True for subscriptions manually added by user via AddSecurity</param>
        /// <returns>A new subscription instance of the specified security</returns>
        protected Subscription CreateSubscription(Security security, DateTime utcStartTime, DateTime utcEndTime, bool isUserDefinedSubscription)
        {
            Subscription subscription = null;
            try
            {
                var config = security.SubscriptionDataConfig;
                var localStartTime = utcStartTime.ConvertFromUtc(config.TimeZone);
                var localEndTime = utcEndTime.ConvertFromUtc(config.TimeZone);

                IEnumerator<BaseData> enumerator;
                if (config.IsCustomData)
                {
                    // custom data uses backtest readers
                    var tradeableDates = Time.EachTradeableDay(security, localStartTime, localEndTime);
                    var reader = new SubscriptionDataReader(config, localStartTime, localEndTime, _resultHandler, tradeableDates, true, false);

                    // apply fast forwarding, this is especially important for RemoteFile types that
                    // can send in large chunks of old, irrelevant data
                    var fastForward = new FastForwardEnumerator(reader, _timeProvider, config.TimeZone, config.Increment);

                    // apply rate limits (1x per increment, max 30 minutes between calls)
                    // TODO : Pull limits from config file?
                    var minimumTimeBetweenCalls = Math.Min(config.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks);
                    var rateLimit = new RateLimitEnumerator(fastForward, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls));

                    // add the enumerator to the exchange
                    _customExchange.AddEnumerator(rateLimit);

                    // this enumerator just allows the exchange to directly dump data into the 'back' of the enumerator
                    var enqueable = new EnqueableEnumerator<BaseData>();
                    _customExchange.SetHandler(config.Symbol, data =>
                    {
                        enqueable.Enqueue(data);
                        if (subscription != null) subscription.RealtimePrice = data.Value;
                    });
                    enumerator = enqueable;
                }
                else if (config.Resolution != Resolution.Tick)
                {
                    // this enumerator allows the exchange to pump ticks into the 'back' of the enumerator,
                    // and the time sync loop can pull aggregated trade bars off the front
                    var aggregator = new TradeBarBuilderEnumerator(config.Increment, config.TimeZone, _timeProvider);
                    _exchange.SetHandler(config.Symbol, data =>
                    {
                        aggregator.ProcessData((Tick) data);
                        if (subscription != null) subscription.RealtimePrice = data.Value;
                    });
                    enumerator = aggregator;
                }
                else
                {
                    // tick subscriptions can pass right through
                    var tickEnumerator = new EnqueableEnumerator<BaseData>();
                    _exchange.SetHandler(config.Symbol, data =>
                    {
                        tickEnumerator.Enqueue(data);
                        if (subscription != null) subscription.RealtimePrice = data.Value;
                    });
                    enumerator = tickEnumerator;
                }

                if (config.FillDataForward)
                {
                    // TODO : Properly resolve fill forward resolution like in FileSystemDataFeed (make considerations for universe-only)
                    enumerator = new LiveFillForwardEnumerator(_frontierTimeProvider, enumerator, security.Exchange, _fillForwardResolution.ToTimeSpan(), config.ExtendedMarketHours, localEndTime, config.Increment);
                }

                // define market hours and user filters to incoming data
                enumerator = new SubscriptionFilterEnumerator(enumerator, security, localEndTime);

                // finally, make our subscriptions aware of the frontier of the data feed, this will help
                var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.SubscriptionDataConfig.TimeZone, utcStartTime, utcEndTime);
                enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, timeZoneOffsetProvider);

                subscription = new Subscription(security, enumerator, timeZoneOffsetProvider, utcStartTime, utcEndTime, isUserDefinedSubscription);
            }
            catch (Exception err)
            {
                Log.Error(err);
            }

            return subscription;
        }
        /// <summary>
        /// Returns an enumerator that defines when this user defined universe will be invoked
        /// </summary>
        /// <returns>An enumerator of DateTime that defines when this universe will be invoked</returns>
        public virtual IEnumerable<DateTime> GetTriggerTimes(DateTime startTimeUtc, DateTime endTimeUtc, MarketHoursDatabase marketHoursDatabase)
        {
            var exchangeHours = marketHoursDatabase.GetExchangeHours(Configuration);
            var localStartTime = startTimeUtc.ConvertFromUtc(exchangeHours.TimeZone);
            var localEndTime = endTimeUtc.ConvertFromUtc(exchangeHours.TimeZone);

            var first = true;
            foreach (var dateTime in LinqExtensions.Range(localStartTime, localEndTime, dt => dt + Interval))
            {
                if (first)
                {
                    yield return dateTime;
                    first = false;
                }
                if (exchangeHours.IsOpen(dateTime, dateTime + Interval, Configuration.ExtendedMarketHours))
                {
                    yield return dateTime;
                }
            }
        }
        /// <summary>
        /// Creates a new subscription for universe selection
        /// </summary>
        /// <param name="universe">The universe to add a subscription for</param>
        /// <param name="startTimeUtc">The start time of the subscription in utc</param>
        /// <param name="endTimeUtc">The end time of the subscription in utc</param>
        protected virtual Subscription CreateUniverseSubscription(
            IUniverse universe,
            DateTime startTimeUtc,
            DateTime endTimeUtc
            )
        {
            // grab the relevant exchange hours
            var config = universe.Configuration;

            var exchangeHours = SecurityExchangeHoursProvider.FromDataFolder()
                .GetExchangeHours(config.Market, null, config.SecurityType);

            // create a canonical security object
            var security = new Security(exchangeHours, config, universe.SubscriptionSettings.Leverage);

            IEnumerator<BaseData> enumerator;
            if (config.Type == typeof (CoarseFundamental))
            {
                // since we're binding to the data queue exchange we'll need to let him
                // know that we expect this data
                _dataQueueHandler.Subscribe(_job, new Dictionary<SecurityType, List<string>>
                {
                    {config.SecurityType, new List<string>{config.Symbol}}
                });

                var enqueable = new EnqueableEnumerator<BaseData>();
                _exchange.SetHandler(config.Symbol, data =>
                {
                    var universeData = data as BaseDataCollection;
                    if (universeData != null)
                    {
                        enqueable.EnqueueRange(universeData.Data);
                    }
                });
                enumerator = enqueable;
            }
            else
            {
                var localStartTime = startTimeUtc.ConvertFromUtc(config.TimeZone);
                var localEndTime = endTimeUtc.ConvertFromUtc(config.TimeZone);

                // define our data enumerator
                var tradeableDates = Time.EachTradeableDay(security, localStartTime, localEndTime);
                var reader = new SubscriptionDataReader(config, localStartTime, localEndTime, _resultHandler, tradeableDates, true);
                _customExchange.AddEnumerator(reader);

                var enqueable = new EnqueableEnumerator<BaseData>();
                _customExchange.SetHandler(config.Symbol, data =>
                {
                    var universeData = data as BaseDataCollection;
                    if (universeData != null)
                    {
                        enqueable.EnqueueRange(universeData.Data);
                    }
                    else
                    {
                        enqueable.Enqueue(data);
                    }
                });
                enumerator = enqueable;
            }

            // create the subscription
            var subscription = new Subscription(universe, security, enumerator, new TimeZoneOffsetProvider(security.SubscriptionDataConfig.TimeZone, startTimeUtc, endTimeUtc), startTimeUtc, endTimeUtc);

            return subscription;
        }
Exemple #18
0
        /// <summary>
        /// Creates a new subscription for the specified security
        /// </summary>
        /// <param name="universe"></param>
        /// <param name="security">The security to create a subscription for</param>
        /// <param name="config">The subscription config to be added</param>
        /// <param name="utcStartTime">The start time of the subscription in UTC</param>
        /// <param name="utcEndTime">The end time of the subscription in UTC</param>
        /// <returns>A new subscription instance of the specified security</returns>
        protected Subscription CreateSubscription(Universe universe, Security security, SubscriptionDataConfig config, DateTime utcStartTime, DateTime utcEndTime)
        {
            Subscription subscription = null;
            try
            {
                var localEndTime = utcEndTime.ConvertFromUtc(security.Exchange.TimeZone);
                var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, utcStartTime, utcEndTime);

                IEnumerator<BaseData> enumerator;
                if (config.IsCustomData)
                {
                    if (!Quandl.IsAuthCodeSet)
                    {
                        // we're not using the SubscriptionDataReader, so be sure to set the auth token here
                        Quandl.SetAuthCode(Config.Get("quandl-auth-token"));
                    }

                    // each time we exhaust we'll new up this enumerator stack
                    var refresher = new RefreshEnumerator<BaseData>(() =>
                    {
                        var sourceProvider = (BaseData)Activator.CreateInstance(config.Type);
                        var dateInDataTimeZone = DateTime.UtcNow.ConvertFromUtc(config.DataTimeZone).Date;
                        var source = sourceProvider.GetSource(config, dateInDataTimeZone, true);
                        var factory = SubscriptionDataSourceReader.ForSource(source, config, dateInDataTimeZone, false);
                        var factoryReadEnumerator = factory.Read(source).GetEnumerator();
                        var maximumDataAge = TimeSpan.FromTicks(Math.Max(config.Increment.Ticks, TimeSpan.FromSeconds(5).Ticks));
                        return new FastForwardEnumerator(factoryReadEnumerator, _timeProvider, security.Exchange.TimeZone, maximumDataAge);
                    });

                    // rate limit the refreshing of the stack to the requested interval
                    var minimumTimeBetweenCalls = Math.Min(config.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks);
                    var rateLimit = new RateLimitEnumerator(refresher, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls));
                    var frontierAware = new FrontierAwareEnumerator(rateLimit, _timeProvider, timeZoneOffsetProvider);
                    _customExchange.AddEnumerator(config.Symbol, frontierAware);

                    var enqueable = new EnqueueableEnumerator<BaseData>();
                    _customExchange.SetDataHandler(config.Symbol, data =>
                    {
                        enqueable.Enqueue(data);
                        if (subscription != null) subscription.RealtimePrice = data.Value;
                    });
                    enumerator = enqueable;
                }
                else if (config.Resolution != Resolution.Tick)
                {
                    // this enumerator allows the exchange to pump ticks into the 'back' of the enumerator,
                    // and the time sync loop can pull aggregated trade bars off the front
                    var aggregator = new TradeBarBuilderEnumerator(config.Increment, security.Exchange.TimeZone, _timeProvider);
                    _exchange.SetDataHandler(config.Symbol, data =>
                    {
                        aggregator.ProcessData((Tick) data);
                        if (subscription != null) subscription.RealtimePrice = data.Value;
                    });
                    enumerator = aggregator;
                }
                else
                {
                    // tick subscriptions can pass right through
                    var tickEnumerator = new EnqueueableEnumerator<BaseData>();
                    _exchange.SetDataHandler(config.Symbol, data =>
                    {
                        tickEnumerator.Enqueue(data);
                        if (subscription != null) subscription.RealtimePrice = data.Value;
                    });
                    enumerator = tickEnumerator;
                }

                if (config.FillDataForward)
                {
                    enumerator = new LiveFillForwardEnumerator(_frontierTimeProvider, enumerator, security.Exchange, _fillForwardResolution, config.ExtendedMarketHours, localEndTime, config.Increment);
                }

                // define market hours and user filters to incoming data
                if (config.IsFilteredSubscription)
                {
                    enumerator = new SubscriptionFilterEnumerator(enumerator, security, localEndTime);
                }

                // finally, make our subscriptions aware of the frontier of the data feed, prevents future data from spewing into the feed
                enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, timeZoneOffsetProvider);

                subscription = new Subscription(universe, security, config, enumerator, timeZoneOffsetProvider, utcStartTime, utcEndTime, false);
            }
            catch (Exception err)
            {
                Log.Error(err);
            }

            return subscription;
        }