예제 #1
0
        private ISubscriptionDataSourceReader CreateSubscriptionFactory(SubscriptionDataSource source, BaseData baseDataInstance)
        {
            var factory = SubscriptionDataSourceReader.ForSource(source, _dataCacheProvider, _config, _tradeableDates.Current, _isLiveMode, baseDataInstance);

            AttachEventHandlers(factory, source);
            return(factory);
        }
예제 #2
0
        /// <summary>
        /// Reads the specified <paramref name="source"/>
        /// </summary>
        /// <param name="source">The source to be read</param>
        /// <returns>An <see cref="IEnumerable{BaseData}"/> that contains the data in the source</returns>
        public IEnumerable <BaseData> Read(SubscriptionDataSource source)
        {
            SubscriptionDataSourceReader.CheckRemoteFileCache();

            IStreamReader reader = null;

            try
            {
                switch (source.TransportMedium)
                {
                default:
                case SubscriptionTransportMedium.Rest:
                    reader = new RestSubscriptionStreamReader(source.Source);
                    break;

                case SubscriptionTransportMedium.LocalFile:
                    reader = new LocalFileSubscriptionStreamReader(_dataCacheProvider, source.Source);
                    break;

                case SubscriptionTransportMedium.RemoteFile:
                    reader = new RemoteFileSubscriptionStreamReader(_dataCacheProvider, source.Source, Globals.Cache);
                    break;
                }

                var raw = "";
                while (!reader.EndOfStream)
                {
                    BaseDataCollection instances;
                    try
                    {
                        raw = reader.ReadLine();
                        var result = _factory.Reader(_config, raw, _date, _isLiveMode);
                        instances = result as BaseDataCollection;
                        if (instances == null)
                        {
                            OnInvalidSource(source, new Exception("Reader must generate a BaseDataCollection with the FileFormat.Collection"));
                            continue;
                        }
                    }
                    catch (Exception err)
                    {
                        OnReaderError(raw, err);
                        continue;
                    }

                    yield return(instances);
                }
            }
            finally
            {
                if (reader != null)
                {
                    reader.Dispose();
                }
            }
        }
예제 #3
0
        /// <summary>
        /// Reads the specified <paramref name="source"/>
        /// </summary>
        /// <param name="source">The source to be read</param>
        /// <returns>An <see cref="IEnumerable{BaseData}"/> that contains the data in the source</returns>
        public override IEnumerable <BaseData> Read(SubscriptionDataSource source)
        {
            // handles zip or text files
            using (var reader = CreateStreamReader(source))
            {
                // if the reader doesn't have data then we're done with this subscription
                if (reader == null || reader.EndOfStream)
                {
                    OnInvalidSource(source, new Exception($"The reader was empty for source: ${source.Source}"));
                    yield break;
                }

                // while the reader has data
                while (!reader.EndOfStream)
                {
                    // read a line and pass it to the base data factory
                    var line = reader.ReadLine();
                    if (line.IsNullOrEmpty())
                    {
                        continue;
                    }

                    SubscriptionDataSource dataSource;
                    try
                    {
                        dataSource = _factory.GetSourceForAnIndex(_config, _date, line, IsLiveMode);
                    }
                    catch
                    {
                        OnInvalidSource(source, new Exception("Factory.GetSourceForAnIndex() failed to return a valid source"));
                        yield break;
                    }

                    if (dataSource != null)
                    {
                        var dataReader = SubscriptionDataSourceReader.ForSource(
                            dataSource,
                            DataCacheProvider,
                            _config,
                            _date,
                            IsLiveMode,
                            _factory,
                            _dataProvider);

                        var enumerator = dataReader.Read(dataSource).GetEnumerator();
                        while (enumerator.MoveNext())
                        {
                            yield return(enumerator.Current);
                        }
                        enumerator.DisposeSafely();
                    }
                }
            }
        }
예제 #4
0
        /// <summary>
        /// Opens up an IStreamReader for a remote file source
        /// </summary>
        protected IStreamReader HandleRemoteSourceFile(SubscriptionDataSource source)
        {
            SubscriptionDataSourceReader.CheckRemoteFileCache();

            try
            {
                // this will fire up a web client in order to download the 'source' file to the cache
                return(new RemoteFileSubscriptionStreamReader(DataCacheProvider, source.Source, Globals.Cache, source.Headers));
            }
            catch (Exception)
            {
                return(null);
            }
        }
예제 #5
0
        /// <summary>
        /// Opens up an IStreamReader for a remote file source
        /// </summary>
        private IStreamReader HandleRemoteSourceFile(SubscriptionDataSource source)
        {
            SubscriptionDataSourceReader.CheckRemoteFileCache();

            try
            {
                // this will fire up a web client in order to download the 'source' file to the cache
                return(new RemoteFileSubscriptionStreamReader(_dataCacheProvider, source.Source, Globals.Cache));
            }
            catch (Exception err)
            {
                OnInvalidSource(source, err);
                return(null);
            }
        }
예제 #6
0
        /// <summary>
        /// Creates an enumerator for the specified security/configuration
        /// </summary>
        private IEnumerator <BaseData> CreateSubscriptionEnumerator(Security security,
                                                                    SubscriptionDataConfig config,
                                                                    DateTime localStartTime,
                                                                    DateTime localEndTime,
                                                                    MapFileResolver mapFileResolver,
                                                                    IEnumerable <DateTime> tradeableDates,
                                                                    bool useSubscriptionDataReader,
                                                                    bool aggregate)
        {
            IEnumerator <BaseData> enumerator;

            if (useSubscriptionDataReader)
            {
                enumerator = new SubscriptionDataReader(config, localStartTime, localEndTime, _resultHandler, mapFileResolver,
                                                        _factorFileProvider, tradeableDates, false);
            }
            else
            {
                var sourceFactory = (BaseData)Activator.CreateInstance(config.Type);
                enumerator = (from date in tradeableDates
                              let source = sourceFactory.GetSource(config, date, false)
                                           let factory = SubscriptionDataSourceReader.ForSource(source, config, date, false)
                                                         let entriesForDate = factory.Read(source)
                                                                              from entry in entriesForDate
                                                                              select entry).GetEnumerator();
            }

            if (aggregate)
            {
                enumerator = new BaseDataCollectionAggregatorEnumerator(enumerator, config.Symbol);
            }

            // optionally apply fill forward logic, but never for tick data
            if (config.FillDataForward && config.Resolution != Resolution.Tick)
            {
                enumerator = new FillForwardEnumerator(enumerator, security.Exchange, _fillForwardResolution,
                                                       security.IsExtendedMarketHours, localEndTime, config.Resolution.ToTimeSpan());
            }

            // optionally apply exchange/user filters
            if (config.IsFilteredSubscription)
            {
                enumerator = SubscriptionFilterEnumerator.WrapForDataFeed(_resultHandler, enumerator, security, localEndTime);
            }
            return(enumerator);
        }
예제 #7
0
        /// <summary>
        /// Creates a new subscription for universe selection
        /// </summary>
        /// <param name="request">The subscription request</param>
        private Subscription CreateUniverseSubscription(SubscriptionRequest request)
        {
            // TODO : Consider moving the creating of universe subscriptions to a separate, testable class

            // grab the relevant exchange hours
            var config           = request.Universe.Configuration;
            var localEndTime     = request.EndTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone);
            var tzOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc);

            IEnumerator <BaseData> enumerator;

            var userDefined = request.Universe as UserDefinedUniverse;

            if (userDefined != null)
            {
                Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating user defined universe: " + config.Symbol.ToString());

                // spoof a tick on the requested interval to trigger the universe selection function
                var enumeratorFactory = new UserDefinedUniverseSubscriptionEnumeratorFactory(userDefined, MarketHoursDatabase.FromDataFolder());
                enumerator = enumeratorFactory.CreateEnumerator(request, _dataFileProvider);

                enumerator = new FrontierAwareEnumerator(enumerator, _timeProvider, tzOffsetProvider);

                var enqueueable = new EnqueueableEnumerator <BaseData>();
                _customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable));
                enumerator = enqueueable;

                // Trigger universe selection when security added/removed after Initialize
                userDefined.CollectionChanged += (sender, args) =>
                {
                    var items =
                        args.Action == NotifyCollectionChangedAction.Add ? args.NewItems :
                        args.Action == NotifyCollectionChangedAction.Remove ? args.OldItems : null;

                    if (items == null || _frontierUtc == DateTime.MinValue)
                    {
                        return;
                    }

                    var symbol = items.OfType <Symbol>().FirstOrDefault();
                    if (symbol == null)
                    {
                        return;
                    }

                    var collection = new BaseDataCollection(_frontierUtc, symbol);
                    var changes    = _universeSelection.ApplyUniverseSelection(userDefined, _frontierUtc, collection);
                    _algorithm.OnSecuritiesChanged(changes);
                };
            }
            else if (config.Type == typeof(CoarseFundamental))
            {
                Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating coarse universe: " + config.Symbol.ToString());

                // since we're binding to the data queue exchange we'll need to let him
                // know that we expect this data
                _dataQueueHandler.Subscribe(_job, new[] { request.Security.Symbol });

                var enqueable = new EnqueueableEnumerator <BaseData>();
                _exchange.SetDataHandler(config.Symbol, data =>
                {
                    enqueable.Enqueue(data);
                });
                enumerator = enqueable;
            }
            else if (request.Universe is OptionChainUniverse)
            {
                Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating option chain universe: " + config.Symbol.ToString());

                Func <SubscriptionRequest, IEnumerator <BaseData>, IEnumerator <BaseData> > configure = (subRequest, input) =>
                {
                    // we check if input enumerator is an underlying enumerator. If yes, we subscribe it to the data.
                    var aggregator = input as TradeBarBuilderEnumerator;

                    if (aggregator != null)
                    {
                        _exchange.SetDataHandler(request.Configuration.Symbol, data =>
                        {
                            aggregator.ProcessData((Tick)data);
                        });
                    }

                    var subscriptionConfigs = _subscriptions.Select(x => x.Configuration).Concat(new[] { request.Configuration });

                    UpdateFillForwardResolution(subscriptionConfigs);

                    return(new LiveFillForwardEnumerator(_frontierTimeProvider, input, request.Security.Exchange, _fillForwardResolution, request.Configuration.ExtendedMarketHours, localEndTime, request.Configuration.Increment));
                };

                var symbolUniverse = _dataQueueHandler as IDataQueueUniverseProvider;

                var enumeratorFactory = new OptionChainUniverseSubscriptionEnumeratorFactory(configure, symbolUniverse, _timeProvider);
                enumerator = enumeratorFactory.CreateEnumerator(request, _dataFileProvider);

                enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, tzOffsetProvider);
            }
            else if (request.Universe is FuturesChainUniverse)
            {
                Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating futures chain universe: " + config.Symbol.ToString());

                var symbolUniverse = _dataQueueHandler as IDataQueueUniverseProvider;

                var enumeratorFactory = new FuturesChainUniverseSubscriptionEnumeratorFactory(symbolUniverse, _timeProvider);
                enumerator = enumeratorFactory.CreateEnumerator(request, _dataFileProvider);

                enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, tzOffsetProvider);
            }
            else
            {
                Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating custom universe: " + config.Symbol.ToString());

                // each time we exhaust we'll new up this enumerator stack
                var refresher = new RefreshEnumerator <BaseDataCollection>(() =>
                {
                    var sourceProvider     = (BaseData)Activator.CreateInstance(config.Type);
                    var dateInDataTimeZone = DateTime.UtcNow.ConvertFromUtc(config.DataTimeZone).Date;
                    var source             = sourceProvider.GetSource(config, dateInDataTimeZone, true);
                    var factory            = SubscriptionDataSourceReader.ForSource(source, _dataFileProvider, config, dateInDataTimeZone, false);
                    var factorEnumerator   = factory.Read(source).GetEnumerator();
                    var fastForward        = new FastForwardEnumerator(factorEnumerator, _timeProvider, request.Security.Exchange.TimeZone, config.Increment);
                    var frontierAware      = new FrontierAwareEnumerator(fastForward, _frontierTimeProvider, tzOffsetProvider);
                    return(new BaseDataCollectionAggregatorEnumerator(frontierAware, config.Symbol));
                });

                // rate limit the refreshing of the stack to the requested interval
                var minimumTimeBetweenCalls = Math.Min(config.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks);
                var rateLimit   = new RateLimitEnumerator(refresher, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls));
                var enqueueable = new EnqueueableEnumerator <BaseData>();
                _customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, rateLimit, enqueueable));
                enumerator = enqueueable;
            }

            // create the subscription
            var subscription = new Subscription(request.Universe, request.Security, config, enumerator, tzOffsetProvider, request.StartTimeUtc, request.EndTimeUtc, true);

            return(subscription);
        }
예제 #8
0
        /// <summary>
        /// Creates a new subscription for universe selection
        /// </summary>
        /// <param name="universe">The universe to add a subscription for</param>
        /// <param name="startTimeUtc">The start time of the subscription in utc</param>
        /// <param name="endTimeUtc">The end time of the subscription in utc</param>
        protected virtual Subscription CreateUniverseSubscription(Universe universe, DateTime startTimeUtc, DateTime endTimeUtc)
        {
            // TODO : Consider moving the creating of universe subscriptions to a separate, testable class

            // grab the relevant exchange hours
            var config = universe.Configuration;

            var marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
            var exchangeHours       = marketHoursDatabase.GetExchangeHours(config);

            Security security;

            if (!_algorithm.Securities.TryGetValue(config.Symbol, out security))
            {
                // create a canonical security object
                security = new Security(exchangeHours, config, _algorithm.Portfolio.CashBook[CashBook.AccountCurrency], SymbolProperties.GetDefault(CashBook.AccountCurrency));
            }

            var tzOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, startTimeUtc, endTimeUtc);

            IEnumerator <BaseData> enumerator;

            var userDefined = universe as UserDefinedUniverse;

            if (userDefined != null)
            {
                Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating user defined universe: " + config.Symbol.ToString());

                // spoof a tick on the requested interval to trigger the universe selection function
                enumerator = userDefined.GetTriggerTimes(startTimeUtc, endTimeUtc, marketHoursDatabase)
                             .Select(dt => new Tick {
                    Time = dt
                }).GetEnumerator();

                enumerator = new FrontierAwareEnumerator(enumerator, _timeProvider, tzOffsetProvider);

                var enqueueable = new EnqueueableEnumerator <BaseData>();
                _customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable));
                enumerator = enqueueable;
            }
            else if (config.Type == typeof(CoarseFundamental))
            {
                Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating coarse universe: " + config.Symbol.ToString());

                // since we're binding to the data queue exchange we'll need to let him
                // know that we expect this data
                _dataQueueHandler.Subscribe(_job, new[] { security.Symbol });

                var enqueable = new EnqueueableEnumerator <BaseData>();
                _exchange.SetDataHandler(config.Symbol, data =>
                {
                    enqueable.Enqueue(data);
                });
                enumerator = enqueable;
            }
            else
            {
                Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating custom universe: " + config.Symbol.ToString());

                // each time we exhaust we'll new up this enumerator stack
                var refresher = new RefreshEnumerator <BaseDataCollection>(() =>
                {
                    var sourceProvider     = (BaseData)Activator.CreateInstance(config.Type);
                    var dateInDataTimeZone = DateTime.UtcNow.ConvertFromUtc(config.DataTimeZone).Date;
                    var source             = sourceProvider.GetSource(config, dateInDataTimeZone, true);
                    var factory            = SubscriptionDataSourceReader.ForSource(source, config, dateInDataTimeZone, false);
                    var factorEnumerator   = factory.Read(source).GetEnumerator();
                    var fastForward        = new FastForwardEnumerator(factorEnumerator, _timeProvider, security.Exchange.TimeZone, config.Increment);
                    var frontierAware      = new FrontierAwareEnumerator(fastForward, _frontierTimeProvider, tzOffsetProvider);
                    return(new BaseDataCollectionAggregatorEnumerator(frontierAware, config.Symbol));
                });

                // rate limit the refreshing of the stack to the requested interval
                var minimumTimeBetweenCalls = Math.Min(config.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks);
                var rateLimit   = new RateLimitEnumerator(refresher, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls));
                var enqueueable = new EnqueueableEnumerator <BaseData>();
                _customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, rateLimit, enqueueable));
                enumerator = enqueueable;
            }

            // create the subscription
            var subscription = new Subscription(universe, security, config, enumerator, tzOffsetProvider, startTimeUtc, endTimeUtc, true);

            return(subscription);
        }
예제 #9
0
        /// <summary>
        /// Creates a new subscription for the specified security
        /// </summary>
        /// <param name="universe"></param>
        /// <param name="security">The security to create a subscription for</param>
        /// <param name="config">The subscription config to be added</param>
        /// <param name="utcStartTime">The start time of the subscription in UTC</param>
        /// <param name="utcEndTime">The end time of the subscription in UTC</param>
        /// <returns>A new subscription instance of the specified security</returns>
        protected Subscription CreateSubscription(Universe universe, Security security, SubscriptionDataConfig config, DateTime utcStartTime, DateTime utcEndTime)
        {
            Subscription subscription = null;

            try
            {
                var localEndTime           = utcEndTime.ConvertFromUtc(security.Exchange.TimeZone);
                var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, utcStartTime, utcEndTime);

                IEnumerator <BaseData> enumerator;
                if (config.IsCustomData)
                {
                    if (!Quandl.IsAuthCodeSet)
                    {
                        // we're not using the SubscriptionDataReader, so be sure to set the auth token here
                        Quandl.SetAuthCode(Config.Get("quandl-auth-token"));
                    }

                    // each time we exhaust we'll new up this enumerator stack
                    var refresher = new RefreshEnumerator <BaseData>(() =>
                    {
                        var sourceProvider        = (BaseData)Activator.CreateInstance(config.Type);
                        var dateInDataTimeZone    = DateTime.UtcNow.ConvertFromUtc(config.DataTimeZone).Date;
                        var source                = sourceProvider.GetSource(config, dateInDataTimeZone, true);
                        var factory               = SubscriptionDataSourceReader.ForSource(source, config, dateInDataTimeZone, false);
                        var factoryReadEnumerator = factory.Read(source).GetEnumerator();
                        var maximumDataAge        = TimeSpan.FromTicks(Math.Max(config.Increment.Ticks, TimeSpan.FromSeconds(5).Ticks));
                        return(new FastForwardEnumerator(factoryReadEnumerator, _timeProvider, security.Exchange.TimeZone, maximumDataAge));
                    });

                    // rate limit the refreshing of the stack to the requested interval
                    var minimumTimeBetweenCalls = Math.Min(config.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks);
                    var rateLimit     = new RateLimitEnumerator(refresher, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls));
                    var frontierAware = new FrontierAwareEnumerator(rateLimit, _timeProvider, timeZoneOffsetProvider);
                    _customExchange.AddEnumerator(config.Symbol, frontierAware);

                    var enqueable = new EnqueueableEnumerator <BaseData>();
                    _customExchange.SetDataHandler(config.Symbol, data =>
                    {
                        enqueable.Enqueue(data);
                        if (subscription != null)
                        {
                            subscription.RealtimePrice = data.Value;
                        }
                    });
                    enumerator = enqueable;
                }
                else if (config.Resolution != Resolution.Tick)
                {
                    // this enumerator allows the exchange to pump ticks into the 'back' of the enumerator,
                    // and the time sync loop can pull aggregated trade bars off the front
                    var aggregator = new TradeBarBuilderEnumerator(config.Increment, security.Exchange.TimeZone, _timeProvider);
                    _exchange.SetDataHandler(config.Symbol, data =>
                    {
                        aggregator.ProcessData((Tick)data);
                        if (subscription != null)
                        {
                            subscription.RealtimePrice = data.Value;
                        }
                    });
                    enumerator = aggregator;
                }
                else
                {
                    // tick subscriptions can pass right through
                    var tickEnumerator = new EnqueueableEnumerator <BaseData>();
                    _exchange.SetDataHandler(config.Symbol, data =>
                    {
                        tickEnumerator.Enqueue(data);
                        if (subscription != null)
                        {
                            subscription.RealtimePrice = data.Value;
                        }
                    });
                    enumerator = tickEnumerator;
                }

                if (config.FillDataForward)
                {
                    enumerator = new LiveFillForwardEnumerator(_frontierTimeProvider, enumerator, security.Exchange, _fillForwardResolution, config.ExtendedMarketHours, localEndTime, config.Increment);
                }

                // define market hours and user filters to incoming data
                if (config.IsFilteredSubscription)
                {
                    enumerator = new SubscriptionFilterEnumerator(enumerator, security, localEndTime);
                }

                // finally, make our subscriptions aware of the frontier of the data feed, prevents future data from spewing into the feed
                enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, timeZoneOffsetProvider);

                subscription = new Subscription(universe, security, config, enumerator, timeZoneOffsetProvider, utcStartTime, utcEndTime, false);
            }
            catch (Exception err)
            {
                Log.Error(err);
            }

            return(subscription);
        }
예제 #10
0
        /// <summary>
        /// Adds a new subscription for universe selection
        /// </summary>
        /// <param name="universe">The universe to add a subscription for</param>
        /// <param name="startTimeUtc">The start time of the subscription in utc</param>
        /// <param name="endTimeUtc">The end time of the subscription in utc</param>
        public void AddUniverseSubscription(Universe universe, DateTime startTimeUtc, DateTime endTimeUtc)
        {
            // TODO : Consider moving the creating of universe subscriptions to a separate, testable class

            // grab the relevant exchange hours
            var config = universe.Configuration;

            var marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
            var exchangeHours       = marketHoursDatabase.GetExchangeHours(config);

            Security security;

            if (!_algorithm.Securities.TryGetValue(config.Symbol, out security))
            {
                // create a canonical security object if it doesn't exist
                security = new Security(exchangeHours, config, _algorithm.Portfolio.CashBook[CashBook.AccountCurrency], SymbolProperties.GetDefault(CashBook.AccountCurrency));
            }

            var localStartTime = startTimeUtc.ConvertFromUtc(security.Exchange.TimeZone);
            var localEndTime   = endTimeUtc.ConvertFromUtc(security.Exchange.TimeZone);

            // define our data enumerator
            IEnumerator <BaseData> enumerator;

            var tradeableDates = Time.EachTradeableDayInTimeZone(security.Exchange.Hours, localStartTime, localEndTime, config.DataTimeZone, config.ExtendedMarketHours);

            var userDefined = universe as UserDefinedUniverse;

            if (userDefined != null)
            {
                // spoof a tick on the requested interval to trigger the universe selection function
                enumerator = userDefined.GetTriggerTimes(startTimeUtc, endTimeUtc, marketHoursDatabase)
                             .Select(x => new Tick {
                    Time = x, Symbol = config.Symbol
                }).GetEnumerator();

                // route these custom subscriptions through the exchange for buffering
                var enqueueable = new EnqueueableEnumerator <BaseData>(true);

                // add this enumerator to our exchange
                ScheduleEnumerator(enumerator, enqueueable, GetLowerThreshold(config.Resolution), GetUpperThreshold(config.Resolution));

                enumerator = enqueueable;
            }
            else if (config.Type == typeof(CoarseFundamental))
            {
                var cf = new CoarseFundamental();

                // load coarse data day by day
                enumerator = (from date in Time.EachTradeableDayInTimeZone(security.Exchange.Hours, _algorithm.StartDate, _algorithm.EndDate, config.DataTimeZone, config.ExtendedMarketHours)
                              let source = cf.GetSource(config, date, false)
                                           let factory = SubscriptionDataSourceReader.ForSource(source, config, date, false)
                                                         let coarseFundamentalForDate = factory.Read(source)
                                                                                        select new BaseDataCollection(date.AddDays(1), config.Symbol, coarseFundamentalForDate)
                              ).GetEnumerator();

                var enqueueable = new EnqueueableEnumerator <BaseData>(true);
                ScheduleEnumerator(enumerator, enqueueable, 5, 100000, 2);

                enumerator = enqueueable;
            }
            else if (config.SecurityType == SecurityType.Option && security is Option)
            {
                var configs     = universe.GetSubscriptions(security);
                var enumerators = configs.Select(c =>
                                                 CreateSubscriptionEnumerator(security, c, localStartTime, localEndTime, _mapFileProvider.Get(c.Market), tradeableDates, false, true)
                                                 ).ToList();

                var sync = new SynchronizingEnumerator(enumerators);
                enumerator = new OptionChainUniverseDataCollectionAggregatorEnumerator(sync, config.Symbol);

                var enqueueable = new EnqueueableEnumerator <BaseData>(true);

                // add this enumerator to our exchange
                ScheduleEnumerator(enumerator, enqueueable, GetLowerThreshold(config.Resolution), GetUpperThreshold(config.Resolution));

                enumerator = enqueueable;
            }
            else
            {
                // normal reader for all others
                enumerator = CreateSubscriptionEnumerator(security, config, localStartTime, localEndTime, MapFileResolver.Empty, tradeableDates, true, false);

                // route these custom subscriptions through the exchange for buffering
                var enqueueable = new EnqueueableEnumerator <BaseData>(true);

                // add this enumerator to our exchange
                ScheduleEnumerator(enumerator, enqueueable, GetLowerThreshold(config.Resolution), GetUpperThreshold(config.Resolution));

                enumerator = enqueueable;
            }

            // create the subscription
            var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, startTimeUtc, endTimeUtc);
            var subscription           = new Subscription(universe, security, config, enumerator, timeZoneOffsetProvider, startTimeUtc, endTimeUtc, true);

            _subscriptions.TryAdd(subscription);

            UpdateFillForwardResolution();
        }
        /// <summary>
        /// Reads the specified <paramref name="source"/>
        /// </summary>
        /// <param name="source">The source to be read</param>
        /// <returns>An <see cref="IEnumerable{BaseData}"/> that contains the data in the source</returns>
        public override IEnumerable <BaseData> Read(SubscriptionDataSource source)
        {
            SubscriptionDataSourceReader.CheckRemoteFileCache();

            IStreamReader reader = null;

            try
            {
                reader = CreateStreamReader(source);
                if (reader == null)
                {
                    yield break;
                }

                var raw = "";
                while (!reader.EndOfStream)
                {
                    BaseDataCollection instances = null;
                    try
                    {
                        raw = reader.ReadLine();
                        var result = _factory.Reader(_config, raw, _date, IsLiveMode);
                        instances = result as BaseDataCollection;
                        if (instances == null && !reader.ShouldBeRateLimited)
                        {
                            OnInvalidSource(source, new Exception("Reader must generate a BaseDataCollection with the FileFormat.Collection"));
                            continue;
                        }
                    }
                    catch (Exception err)
                    {
                        OnReaderError(raw, err);
                        if (!reader.ShouldBeRateLimited)
                        {
                            continue;
                        }
                    }

                    if (IsLiveMode
                        // this shouldn't happen, rest reader is the only one to be rate limited
                        // and in live mode, but just in case...
                        || instances == null && reader.ShouldBeRateLimited)
                    {
                        // in live trading these data points will be unrolled at the
                        // 'LiveCustomDataSubscriptionEnumeratorFactory' level
                        yield return(instances);
                    }
                    else
                    {
                        foreach (var instance in instances.Data)
                        {
                            if (instance != null && instance.EndTime != default(DateTime))
                            {
                                yield return(instance);
                            }
                        }
                    }
                }
            }
            finally
            {
                reader.DisposeSafely();
            }
        }
        /// <summary>
        /// Reads the specified <paramref name="source"/>
        /// </summary>
        /// <param name="source">The source to be read</param>
        /// <returns>An <see cref="IEnumerable{BaseData}"/> that contains the data in the source</returns>
        public IEnumerable <BaseData> Read(SubscriptionDataSource source)
        {
            SubscriptionDataSourceReader.CheckRemoteFileCache();

            IStreamReader reader = null;

            try
            {
                try
                {
                    switch (source.TransportMedium)
                    {
                    default:
                    case SubscriptionTransportMedium.Rest:
                        reader = new RestSubscriptionStreamReader(source.Source, source.Headers, _isLiveMode);
                        break;

                    case SubscriptionTransportMedium.LocalFile:
                        reader = new LocalFileSubscriptionStreamReader(_dataCacheProvider, source.Source);
                        break;

                    case SubscriptionTransportMedium.RemoteFile:
                        reader = new RemoteFileSubscriptionStreamReader(_dataCacheProvider, source.Source, Globals.Cache, source.Headers);
                        break;
                    }
                }
                catch (Exception e)
                {
                    OnInvalidSource(source, e);
                    yield break;
                }

                var raw = "";
                while (!reader.EndOfStream)
                {
                    BaseDataCollection instances = null;
                    try
                    {
                        raw = reader.ReadLine();
                        var result = _factory.Reader(_config, raw, _date, _isLiveMode);
                        instances = result as BaseDataCollection;
                        if (instances == null && !reader.ShouldBeRateLimited)
                        {
                            OnInvalidSource(source, new Exception("Reader must generate a BaseDataCollection with the FileFormat.Collection"));
                            continue;
                        }
                    }
                    catch (Exception err)
                    {
                        OnReaderError(raw, err);
                        if (!reader.ShouldBeRateLimited)
                        {
                            continue;
                        }
                    }

                    if (_isLiveMode
                        // this shouldn't happen, rest reader is the only one to be rate limited
                        // and in live mode, but just in case...
                        || instances == null && reader.ShouldBeRateLimited)
                    {
                        yield return(instances);
                    }
                    else
                    {
                        foreach (var instance in instances.Data)
                        {
                            if (instance != null && instance.EndTime != default(DateTime))
                            {
                                yield return(instance);
                            }
                        }
                    }
                }
            }
            finally
            {
                if (reader != null)
                {
                    reader.Dispose();
                }
            }
        }