public void DoesNotLeakMemory()
        {
            var symbol   = Symbols.AAPL;
            var config   = new SubscriptionDataConfig(typeof(TradeBar), symbol, Resolution.Daily, TimeZones.NewYork, TimeZones.NewYork, false, false, false, false, TickType.Trade, false);
            var security = new Security(
                SecurityExchangeHours.AlwaysOpen(TimeZones.NewYork),
                config,
                new Cash(Currencies.USD, 0, 1),
                SymbolProperties.GetDefault(Currencies.USD),
                ErrorCurrencyConverter.Instance,
                RegisteredSecurityDataTypesProvider.Null,
                new SecurityCache()
                );

            var mapFileProvider    = new LocalDiskMapFileProvider();
            var factorFileProvider = new LocalDiskFactorFileProvider(mapFileProvider);
            var mapFileResolver    = mapFileProvider.Get(security.Symbol.ID.Market);
            var fileProvider       = new DefaultDataProvider();

            var factory = new BaseDataSubscriptionEnumeratorFactory(false, mapFileResolver, factorFileProvider);

            GC.Collect();
            var ramUsageBeforeLoop = OS.TotalPhysicalMemoryUsed;

            var date = new DateTime(1998, 1, 1);

            const int iterations = 1000;

            for (var i = 0; i < iterations; i++)
            {
                var request = new SubscriptionRequest(false, null, security, config, date, date);
                using (var enumerator = factory.CreateEnumerator(request, fileProvider))
                {
                    enumerator.MoveNext();
                }
                date = date.AddDays(1);
            }

            GC.Collect();
            var ramUsageAfterLoop = OS.TotalPhysicalMemoryUsed;

            Log.Trace($"RAM usage - before: {ramUsageBeforeLoop} MB, after: {ramUsageAfterLoop} MB");

            Assert.IsTrue(ramUsageAfterLoop - ramUsageBeforeLoop < 10);
        }
Example #2
0
        /// <summary>
        /// Adds a new subscription for universe selection
        /// </summary>
        /// <param name="request">The subscription request</param>
        private Subscription CreateUniverseSubscription(SubscriptionRequest request)
        {
            ISubscriptionEnumeratorFactory factory = _subscriptionFactory;

            if (request.Universe is ITimeTriggeredUniverse)
            {
                factory = new TimeTriggeredUniverseSubscriptionEnumeratorFactory(request.Universe as ITimeTriggeredUniverse,
                                                                                 MarketHoursDatabase.FromDataFolder(),
                                                                                 _timeProvider);

                if (request.Universe is UserDefinedUniverse)
                {
                    // for user defined universe we do not use a worker task, since calls to AddData can happen in any moment
                    // and we have to be able to inject selection data points into the enumerator
                    return(SubscriptionUtils.Create(request, factory.CreateEnumerator(request, _dataProvider)));
                }
            }
            if (request.Configuration.Type == typeof(CoarseFundamental))
            {
                factory = new BaseDataCollectionSubscriptionEnumeratorFactory();
            }
            if (request.Universe is OptionChainUniverse)
            {
                factory = new OptionChainUniverseSubscriptionEnumeratorFactory((req) =>
                {
                    if (!req.Configuration.SecurityType.IsOption())
                    {
                        var enumerator = _subscriptionFactory.CreateEnumerator(req, _dataProvider);
                        enumerator     = new FilterEnumerator <BaseData>(enumerator, data => data.DataType != MarketDataType.Auxiliary);
                        return(ConfigureEnumerator(req, true, enumerator));
                    }
                    var underlyingFactory = new BaseDataSubscriptionEnumeratorFactory(false, _mapFileProvider, _cacheProvider);
                    return(ConfigureEnumerator(req, true, underlyingFactory.CreateEnumerator(req, _dataProvider)));
                });
            }
            if (request.Universe is FuturesChainUniverse)
            {
                factory = new FuturesChainUniverseSubscriptionEnumeratorFactory((req, e) => ConfigureEnumerator(req, true, e), _cacheProvider);
            }

            // define our data enumerator
            var enumerator = factory.CreateEnumerator(request, _dataProvider);

            return(SubscriptionUtils.CreateAndScheduleWorker(request, enumerator, _factorFileProvider, true));
        }
Example #3
0
        protected IEnumerator <BaseData> CreateUniverseEnumerator(SubscriptionRequest request, Func <SubscriptionRequest, IEnumerator <BaseData> > createUnderlyingEnumerator)
        {
            ISubscriptionEnumeratorFactory factory = _subscriptionFactory;

            if (request.Universe is ITimeTriggeredUniverse)
            {
                factory = new TimeTriggeredUniverseSubscriptionEnumeratorFactory(request.Universe as ITimeTriggeredUniverse,
                                                                                 MarketHoursDatabase.FromDataFolder(),
                                                                                 _timeProvider);

                if (request.Universe is UserDefinedUniverse)
                {
                    return(factory.CreateEnumerator(request, _dataProvider));
                }
            }
            else if (request.Configuration.Type == typeof(CoarseFundamental))
            {
                factory = new BaseDataCollectionSubscriptionEnumeratorFactory();
            }
            else if (request.Configuration.Type == typeof(ZipEntryName))
            {
                // TODO: subscription should already come in correctly built
                var resolution = request.Configuration.Resolution == Resolution.Tick ? Resolution.Second : request.Configuration.Resolution;

                // TODO: subscription should already come in as fill forward true
                request = new SubscriptionRequest(request, configuration: new SubscriptionDataConfig(request.Configuration, fillForward: true, resolution: resolution));

                var result = new BaseDataSubscriptionEnumeratorFactory(_algorithm.OptionChainProvider, _algorithm.FutureChainProvider)
                             .CreateEnumerator(request, _dataProvider);
                result = ConfigureEnumerator(request, true, result);
                return(TryAppendUnderlyingEnumerator(request, result, createUnderlyingEnumerator));
            }

            // define our data enumerator
            var enumerator = factory.CreateEnumerator(request, _dataProvider);

            return(enumerator);
        }
Example #4
0
        /// <summary>
        /// Creates a new subscription for the specified security
        /// </summary>
        /// <param name="request">The subscription request</param>
        /// <returns>A new subscription instance of the specified security</returns>
        protected Subscription CreateSubscription(SubscriptionRequest request)
        {
            Subscription subscription = null;

            try
            {
                var localEndTime           = request.EndTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone);
                var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc);

                IEnumerator <BaseData> enumerator;
                if (request.Configuration.IsCustomData)
                {
                    if (!Quandl.IsAuthCodeSet)
                    {
                        // we're not using the SubscriptionDataReader, so be sure to set the auth token here
                        Quandl.SetAuthCode(Config.Get("quandl-auth-token"));
                    }

                    // each time we exhaust we'll new up this enumerator stack
                    var refresher = new RefreshEnumerator <BaseData>(() =>
                    {
                        var dateInDataTimeZone    = DateTime.UtcNow.ConvertFromUtc(request.Configuration.DataTimeZone).Date;
                        var enumeratorFactory     = new BaseDataSubscriptionEnumeratorFactory(r => new[] { dateInDataTimeZone });
                        var factoryReadEnumerator = enumeratorFactory.CreateEnumerator(request, _dataFileProvider);
                        var maximumDataAge        = TimeSpan.FromTicks(Math.Max(request.Configuration.Increment.Ticks, TimeSpan.FromSeconds(5).Ticks));
                        var fastForward           = new FastForwardEnumerator(factoryReadEnumerator, _timeProvider, request.Security.Exchange.TimeZone, maximumDataAge);
                        return(new FrontierAwareEnumerator(fastForward, _frontierTimeProvider, timeZoneOffsetProvider));
                    });

                    // rate limit the refreshing of the stack to the requested interval
                    // At Tick resolution, it will refresh at full speed
                    // At Second and Minute resolution, it will refresh every second and minute respectively
                    // At Hour and Daily resolutions, it will refresh every 30 minutes
                    var minimumTimeBetweenCalls = Math.Min(request.Configuration.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks);
                    var rateLimit = new RateLimitEnumerator(refresher, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls));
                    _customExchange.AddEnumerator(request.Configuration.Symbol, rateLimit);

                    var enqueable = new EnqueueableEnumerator <BaseData>();
                    _customExchange.SetDataHandler(request.Configuration.Symbol, data =>
                    {
                        enqueable.Enqueue(data);
                        if (subscription != null)
                        {
                            subscription.RealtimePrice = data.Value;
                        }
                    });
                    enumerator = enqueable;
                }
                else if (request.Configuration.Resolution != Resolution.Tick)
                {
                    // this enumerator allows the exchange to pump ticks into the 'back' of the enumerator,
                    // and the time sync loop can pull aggregated trade bars off the front
                    switch (request.Configuration.TickType)
                    {
                    case TickType.Quote:
                        var quoteBarAggregator = new QuoteBarBuilderEnumerator(request.Configuration.Increment, request.Security.Exchange.TimeZone, _timeProvider);
                        _exchange.AddDataHandler(request.Configuration.Symbol, data =>
                        {
                            var tick = data as Tick;

                            if (tick.TickType == TickType.Quote)
                            {
                                quoteBarAggregator.ProcessData(tick);
                                if (subscription != null)
                                {
                                    subscription.RealtimePrice = data.Value;
                                }
                            }
                        });
                        enumerator = quoteBarAggregator;
                        break;

                    case TickType.Trade:
                    default:
                        var tradeBarAggregator = new TradeBarBuilderEnumerator(request.Configuration.Increment, request.Security.Exchange.TimeZone, _timeProvider);
                        _exchange.AddDataHandler(request.Configuration.Symbol, data =>
                        {
                            var tick = data as Tick;

                            if (tick.TickType == TickType.Trade)
                            {
                                tradeBarAggregator.ProcessData(tick);
                                if (subscription != null)
                                {
                                    subscription.RealtimePrice = data.Value;
                                }
                            }
                        });
                        enumerator = tradeBarAggregator;
                        break;

                    case TickType.OpenInterest:
                        var oiAggregator = new OpenInterestEnumerator(request.Configuration.Increment, request.Security.Exchange.TimeZone, _timeProvider);
                        _exchange.AddDataHandler(request.Configuration.Symbol, data =>
                        {
                            var tick = data as Tick;

                            if (tick.TickType == TickType.OpenInterest)
                            {
                                oiAggregator.ProcessData(tick);
                            }
                        });
                        enumerator = oiAggregator;
                        break;
                    }
                }
                else
                {
                    // tick subscriptions can pass right through
                    var tickEnumerator = new EnqueueableEnumerator <BaseData>();
                    _exchange.SetDataHandler(request.Configuration.Symbol, data =>
                    {
                        tickEnumerator.Enqueue(data);
                        if (subscription != null)
                        {
                            subscription.RealtimePrice = data.Value;
                        }
                    });
                    enumerator = tickEnumerator;
                }

                if (request.Configuration.FillDataForward)
                {
                    var subscriptionConfigs = _subscriptions.Select(x => x.Configuration).Concat(new[] { request.Configuration });

                    UpdateFillForwardResolution(subscriptionConfigs);

                    enumerator = new LiveFillForwardEnumerator(_frontierTimeProvider, enumerator, request.Security.Exchange, _fillForwardResolution, request.Configuration.ExtendedMarketHours, localEndTime, request.Configuration.Increment);
                }

                // define market hours and user filters to incoming data
                if (request.Configuration.IsFilteredSubscription)
                {
                    enumerator = new SubscriptionFilterEnumerator(enumerator, request.Security, localEndTime);
                }

                // finally, make our subscriptions aware of the frontier of the data feed, prevents future data from spewing into the feed
                enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, timeZoneOffsetProvider);

                subscription = new Subscription(request.Universe, request.Security, request.Configuration, enumerator, timeZoneOffsetProvider, request.StartTimeUtc, request.EndTimeUtc, false);
            }
            catch (Exception err)
            {
                Log.Error(err);
            }

            return(subscription);
        }
 /// <summary>
 /// Initializes a new instance of the <see cref="OptionChainUniverseSubscriptionEnumeratorFactory"/> class
 /// </summary>
 /// <param name="enumeratorConfigurator">Function used to configure the sub-enumerators before sync (fill-forward/filter/ect...)</param>
 public OptionChainUniverseSubscriptionEnumeratorFactory(Func<SubscriptionRequest, IEnumerator<BaseData>, IEnumerator<BaseData>> enumeratorConfigurator)
 {
     _enumeratorConfigurator = enumeratorConfigurator;
     _factory = new BaseDataSubscriptionEnumeratorFactory();
 }
        public void DoesNotEmitInvalidData()
        {
            var startTime = new DateTime(2014, 06, 06, 0, 0, 0);
            var endTime   = new DateTime(2014, 06, 09, 20, 0, 0);

            var canonicalSymbol = Symbol.Create("AAPL", SecurityType.Option, Market.USA, "?AAPL");

            var quoteCurrency = new Cash(Currencies.USD, 0, 1);
            var exchangeHours = MarketHoursDatabase.FromDataFolder().GetExchangeHours(Market.USA, canonicalSymbol, SecurityType.Option);
            var config        = new SubscriptionDataConfig(
                typeof(ZipEntryName),
                canonicalSymbol,
                Resolution.Minute,
                TimeZones.Utc,
                TimeZones.NewYork,
                true,
                false,
                false,
                false,
                TickType.Quote,
                false,
                DataNormalizationMode.Raw
                );

            var option = new Option(
                canonicalSymbol,
                exchangeHours,
                quoteCurrency,
                new OptionSymbolProperties(SymbolProperties.GetDefault(Currencies.USD)),
                ErrorCurrencyConverter.Instance,
                RegisteredSecurityDataTypesProvider.Null,
                new SecurityCache()
                );

            var enumeratorFactory     = new BaseDataSubscriptionEnumeratorFactory(false, MapFileResolver.Create(Globals.DataFolder, Market.USA), new LocalDiskFactorFileProvider(new LocalDiskMapFileProvider()));
            var fillForwardResolution = Ref.CreateReadOnly(() => Resolution.Minute.ToTimeSpan());
            Func <SubscriptionRequest, IEnumerator <BaseData> > underlyingEnumeratorFunc = (req) =>
            {
                var input = enumeratorFactory.CreateEnumerator(req, new DefaultDataProvider());

                input = new BaseDataCollectionAggregatorEnumerator(input, req.Configuration.Symbol);
                return(new FillForwardEnumerator(
                           input,
                           option.Exchange,
                           fillForwardResolution,
                           false,
                           endTime,
                           Resolution.Minute.ToTimeSpan(),
                           TimeZones.Utc,
                           startTime));
            };
            var factory = new OptionChainUniverseSubscriptionEnumeratorFactory(underlyingEnumeratorFunc);

            var request    = new SubscriptionRequest(true, null, option, config, startTime, endTime);
            var enumerator = factory.CreateEnumerator(request, new DefaultDataProvider());

            var emittedCount = 0;

            foreach (var data in enumerator.AsEnumerable())
            {
                emittedCount++;
                var optionData = data as OptionChainUniverseDataCollection;

                Assert.IsNotNull(optionData);
                Assert.IsNotNull(optionData.Underlying);
                Assert.AreNotEqual(0, optionData.Data.Count);
            }

            // 9:30 to 15:59 -> 6.5 hours * 60 => 390 minutes * 2 days = 780
            Assert.AreEqual(780, emittedCount);
        }
Example #7
0
        /// <summary>
        /// Creates a new subscription for the specified security
        /// </summary>
        /// <param name="request">The subscription request</param>
        /// <returns>A new subscription instance of the specified security</returns>
        protected Subscription CreateSubscription(SubscriptionRequest request)
        {
            Subscription subscription = null;
            try
            {
                var localEndTime = request.EndTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone);
                var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc);

                IEnumerator<BaseData> enumerator;
                if (request.Configuration.IsCustomData)
                {
                    if (!Quandl.IsAuthCodeSet)
                    {
                        // we're not using the SubscriptionDataReader, so be sure to set the auth token here
                        Quandl.SetAuthCode(Config.Get("quandl-auth-token"));
                    }

                    // each time we exhaust we'll new up this enumerator stack
                    var refresher = new RefreshEnumerator<BaseData>(() =>
                    {
                        var dateInDataTimeZone = DateTime.UtcNow.ConvertFromUtc(request.Configuration.DataTimeZone).Date;
                        var enumeratorFactory = new BaseDataSubscriptionEnumeratorFactory(r => new[] { dateInDataTimeZone });
                        var factoryReadEnumerator = enumeratorFactory.CreateEnumerator(request, _dataFileProvider);
                        var maximumDataAge = TimeSpan.FromTicks(Math.Max(request.Configuration.Increment.Ticks, TimeSpan.FromSeconds(5).Ticks));
                        var fastForward = new FastForwardEnumerator(factoryReadEnumerator, _timeProvider, request.Security.Exchange.TimeZone, maximumDataAge);
                        return new FrontierAwareEnumerator(fastForward, _frontierTimeProvider, timeZoneOffsetProvider);
                    });

                    // rate limit the refreshing of the stack to the requested interval
                    // At Tick resolution, it will refresh at full speed
                    // At Second and Minute resolution, it will refresh every second and minute respectively
                    // At Hour and Daily resolutions, it will refresh every 30 minutes
                    var minimumTimeBetweenCalls = Math.Min(request.Configuration.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks);
                    var rateLimit = new RateLimitEnumerator(refresher, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls));
                    _customExchange.AddEnumerator(request.Configuration.Symbol, rateLimit);

                    var enqueable = new EnqueueableEnumerator<BaseData>();
                    _customExchange.SetDataHandler(request.Configuration.Symbol, data =>
                    {
                        enqueable.Enqueue(data);
                        if (subscription != null) subscription.RealtimePrice = data.Value;
                    });
                    enumerator = enqueable;
                }
                else if (request.Configuration.Resolution != Resolution.Tick)
                {
                    // this enumerator allows the exchange to pump ticks into the 'back' of the enumerator,
                    // and the time sync loop can pull aggregated trade bars off the front
                    var aggregator = new TradeBarBuilderEnumerator(request.Configuration.Increment, request.Security.Exchange.TimeZone, _timeProvider);
                    _exchange.SetDataHandler(request.Configuration.Symbol, data =>
                    {
                        aggregator.ProcessData((Tick) data);
                        if (subscription != null) subscription.RealtimePrice = data.Value;
                    });
                    enumerator = aggregator;
                }
                else
                {
                    // tick subscriptions can pass right through
                    var tickEnumerator = new EnqueueableEnumerator<BaseData>();
                    _exchange.SetDataHandler(request.Configuration.Symbol, data =>
                    {
                        tickEnumerator.Enqueue(data);
                        if (subscription != null) subscription.RealtimePrice = data.Value;
                    });
                    enumerator = tickEnumerator;
                }

                if (request.Configuration.FillDataForward)
                {
                    enumerator = new LiveFillForwardEnumerator(_frontierTimeProvider, enumerator, request.Security.Exchange, _fillForwardResolution, request.Configuration.ExtendedMarketHours, localEndTime, request.Configuration.Increment);
                }

                // define market hours and user filters to incoming data
                if (request.Configuration.IsFilteredSubscription)
                {
                    enumerator = new SubscriptionFilterEnumerator(enumerator, request.Security, localEndTime);
                }

                // finally, make our subscriptions aware of the frontier of the data feed, prevents future data from spewing into the feed
                enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, timeZoneOffsetProvider);

                subscription = new Subscription(request.Universe, request.Security, request.Configuration, enumerator, timeZoneOffsetProvider, request.StartTimeUtc, request.EndTimeUtc, false);
            }
            catch (Exception err)
            {
                Log.Error(err);
            }

            return subscription;
        }