public void DoesNotLeakMemory() { var symbol = Symbols.AAPL; var config = new SubscriptionDataConfig(typeof(TradeBar), symbol, Resolution.Daily, TimeZones.NewYork, TimeZones.NewYork, false, false, false, false, TickType.Trade, false); var security = new Security( SecurityExchangeHours.AlwaysOpen(TimeZones.NewYork), config, new Cash(Currencies.USD, 0, 1), SymbolProperties.GetDefault(Currencies.USD), ErrorCurrencyConverter.Instance, RegisteredSecurityDataTypesProvider.Null, new SecurityCache() ); var mapFileProvider = new LocalDiskMapFileProvider(); var factorFileProvider = new LocalDiskFactorFileProvider(mapFileProvider); var mapFileResolver = mapFileProvider.Get(security.Symbol.ID.Market); var fileProvider = new DefaultDataProvider(); var factory = new BaseDataSubscriptionEnumeratorFactory(false, mapFileResolver, factorFileProvider); GC.Collect(); var ramUsageBeforeLoop = OS.TotalPhysicalMemoryUsed; var date = new DateTime(1998, 1, 1); const int iterations = 1000; for (var i = 0; i < iterations; i++) { var request = new SubscriptionRequest(false, null, security, config, date, date); using (var enumerator = factory.CreateEnumerator(request, fileProvider)) { enumerator.MoveNext(); } date = date.AddDays(1); } GC.Collect(); var ramUsageAfterLoop = OS.TotalPhysicalMemoryUsed; Log.Trace($"RAM usage - before: {ramUsageBeforeLoop} MB, after: {ramUsageAfterLoop} MB"); Assert.IsTrue(ramUsageAfterLoop - ramUsageBeforeLoop < 10); }
/// <summary> /// Adds a new subscription for universe selection /// </summary> /// <param name="request">The subscription request</param> private Subscription CreateUniverseSubscription(SubscriptionRequest request) { ISubscriptionEnumeratorFactory factory = _subscriptionFactory; if (request.Universe is ITimeTriggeredUniverse) { factory = new TimeTriggeredUniverseSubscriptionEnumeratorFactory(request.Universe as ITimeTriggeredUniverse, MarketHoursDatabase.FromDataFolder(), _timeProvider); if (request.Universe is UserDefinedUniverse) { // for user defined universe we do not use a worker task, since calls to AddData can happen in any moment // and we have to be able to inject selection data points into the enumerator return(SubscriptionUtils.Create(request, factory.CreateEnumerator(request, _dataProvider))); } } if (request.Configuration.Type == typeof(CoarseFundamental)) { factory = new BaseDataCollectionSubscriptionEnumeratorFactory(); } if (request.Universe is OptionChainUniverse) { factory = new OptionChainUniverseSubscriptionEnumeratorFactory((req) => { if (!req.Configuration.SecurityType.IsOption()) { var enumerator = _subscriptionFactory.CreateEnumerator(req, _dataProvider); enumerator = new FilterEnumerator <BaseData>(enumerator, data => data.DataType != MarketDataType.Auxiliary); return(ConfigureEnumerator(req, true, enumerator)); } var underlyingFactory = new BaseDataSubscriptionEnumeratorFactory(false, _mapFileProvider, _cacheProvider); return(ConfigureEnumerator(req, true, underlyingFactory.CreateEnumerator(req, _dataProvider))); }); } if (request.Universe is FuturesChainUniverse) { factory = new FuturesChainUniverseSubscriptionEnumeratorFactory((req, e) => ConfigureEnumerator(req, true, e), _cacheProvider); } // define our data enumerator var enumerator = factory.CreateEnumerator(request, _dataProvider); return(SubscriptionUtils.CreateAndScheduleWorker(request, enumerator, _factorFileProvider, true)); }
protected IEnumerator <BaseData> CreateUniverseEnumerator(SubscriptionRequest request, Func <SubscriptionRequest, IEnumerator <BaseData> > createUnderlyingEnumerator) { ISubscriptionEnumeratorFactory factory = _subscriptionFactory; if (request.Universe is ITimeTriggeredUniverse) { factory = new TimeTriggeredUniverseSubscriptionEnumeratorFactory(request.Universe as ITimeTriggeredUniverse, MarketHoursDatabase.FromDataFolder(), _timeProvider); if (request.Universe is UserDefinedUniverse) { return(factory.CreateEnumerator(request, _dataProvider)); } } else if (request.Configuration.Type == typeof(CoarseFundamental)) { factory = new BaseDataCollectionSubscriptionEnumeratorFactory(); } else if (request.Configuration.Type == typeof(ZipEntryName)) { // TODO: subscription should already come in correctly built var resolution = request.Configuration.Resolution == Resolution.Tick ? Resolution.Second : request.Configuration.Resolution; // TODO: subscription should already come in as fill forward true request = new SubscriptionRequest(request, configuration: new SubscriptionDataConfig(request.Configuration, fillForward: true, resolution: resolution)); var result = new BaseDataSubscriptionEnumeratorFactory(_algorithm.OptionChainProvider, _algorithm.FutureChainProvider) .CreateEnumerator(request, _dataProvider); result = ConfigureEnumerator(request, true, result); return(TryAppendUnderlyingEnumerator(request, result, createUnderlyingEnumerator)); } // define our data enumerator var enumerator = factory.CreateEnumerator(request, _dataProvider); return(enumerator); }
/// <summary> /// Creates a new subscription for the specified security /// </summary> /// <param name="request">The subscription request</param> /// <returns>A new subscription instance of the specified security</returns> protected Subscription CreateSubscription(SubscriptionRequest request) { Subscription subscription = null; try { var localEndTime = request.EndTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone); var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc); IEnumerator <BaseData> enumerator; if (request.Configuration.IsCustomData) { if (!Quandl.IsAuthCodeSet) { // we're not using the SubscriptionDataReader, so be sure to set the auth token here Quandl.SetAuthCode(Config.Get("quandl-auth-token")); } // each time we exhaust we'll new up this enumerator stack var refresher = new RefreshEnumerator <BaseData>(() => { var dateInDataTimeZone = DateTime.UtcNow.ConvertFromUtc(request.Configuration.DataTimeZone).Date; var enumeratorFactory = new BaseDataSubscriptionEnumeratorFactory(r => new[] { dateInDataTimeZone }); var factoryReadEnumerator = enumeratorFactory.CreateEnumerator(request, _dataFileProvider); var maximumDataAge = TimeSpan.FromTicks(Math.Max(request.Configuration.Increment.Ticks, TimeSpan.FromSeconds(5).Ticks)); var fastForward = new FastForwardEnumerator(factoryReadEnumerator, _timeProvider, request.Security.Exchange.TimeZone, maximumDataAge); return(new FrontierAwareEnumerator(fastForward, _frontierTimeProvider, timeZoneOffsetProvider)); }); // rate limit the refreshing of the stack to the requested interval // At Tick resolution, it will refresh at full speed // At Second and Minute resolution, it will refresh every second and minute respectively // At Hour and Daily resolutions, it will refresh every 30 minutes var minimumTimeBetweenCalls = Math.Min(request.Configuration.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks); var rateLimit = new RateLimitEnumerator(refresher, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls)); _customExchange.AddEnumerator(request.Configuration.Symbol, rateLimit); var enqueable = new EnqueueableEnumerator <BaseData>(); _customExchange.SetDataHandler(request.Configuration.Symbol, data => { enqueable.Enqueue(data); if (subscription != null) { subscription.RealtimePrice = data.Value; } }); enumerator = enqueable; } else if (request.Configuration.Resolution != Resolution.Tick) { // this enumerator allows the exchange to pump ticks into the 'back' of the enumerator, // and the time sync loop can pull aggregated trade bars off the front switch (request.Configuration.TickType) { case TickType.Quote: var quoteBarAggregator = new QuoteBarBuilderEnumerator(request.Configuration.Increment, request.Security.Exchange.TimeZone, _timeProvider); _exchange.AddDataHandler(request.Configuration.Symbol, data => { var tick = data as Tick; if (tick.TickType == TickType.Quote) { quoteBarAggregator.ProcessData(tick); if (subscription != null) { subscription.RealtimePrice = data.Value; } } }); enumerator = quoteBarAggregator; break; case TickType.Trade: default: var tradeBarAggregator = new TradeBarBuilderEnumerator(request.Configuration.Increment, request.Security.Exchange.TimeZone, _timeProvider); _exchange.AddDataHandler(request.Configuration.Symbol, data => { var tick = data as Tick; if (tick.TickType == TickType.Trade) { tradeBarAggregator.ProcessData(tick); if (subscription != null) { subscription.RealtimePrice = data.Value; } } }); enumerator = tradeBarAggregator; break; case TickType.OpenInterest: var oiAggregator = new OpenInterestEnumerator(request.Configuration.Increment, request.Security.Exchange.TimeZone, _timeProvider); _exchange.AddDataHandler(request.Configuration.Symbol, data => { var tick = data as Tick; if (tick.TickType == TickType.OpenInterest) { oiAggregator.ProcessData(tick); } }); enumerator = oiAggregator; break; } } else { // tick subscriptions can pass right through var tickEnumerator = new EnqueueableEnumerator <BaseData>(); _exchange.SetDataHandler(request.Configuration.Symbol, data => { tickEnumerator.Enqueue(data); if (subscription != null) { subscription.RealtimePrice = data.Value; } }); enumerator = tickEnumerator; } if (request.Configuration.FillDataForward) { var subscriptionConfigs = _subscriptions.Select(x => x.Configuration).Concat(new[] { request.Configuration }); UpdateFillForwardResolution(subscriptionConfigs); enumerator = new LiveFillForwardEnumerator(_frontierTimeProvider, enumerator, request.Security.Exchange, _fillForwardResolution, request.Configuration.ExtendedMarketHours, localEndTime, request.Configuration.Increment); } // define market hours and user filters to incoming data if (request.Configuration.IsFilteredSubscription) { enumerator = new SubscriptionFilterEnumerator(enumerator, request.Security, localEndTime); } // finally, make our subscriptions aware of the frontier of the data feed, prevents future data from spewing into the feed enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, timeZoneOffsetProvider); subscription = new Subscription(request.Universe, request.Security, request.Configuration, enumerator, timeZoneOffsetProvider, request.StartTimeUtc, request.EndTimeUtc, false); } catch (Exception err) { Log.Error(err); } return(subscription); }
/// <summary> /// Initializes a new instance of the <see cref="OptionChainUniverseSubscriptionEnumeratorFactory"/> class /// </summary> /// <param name="enumeratorConfigurator">Function used to configure the sub-enumerators before sync (fill-forward/filter/ect...)</param> public OptionChainUniverseSubscriptionEnumeratorFactory(Func<SubscriptionRequest, IEnumerator<BaseData>, IEnumerator<BaseData>> enumeratorConfigurator) { _enumeratorConfigurator = enumeratorConfigurator; _factory = new BaseDataSubscriptionEnumeratorFactory(); }
public void DoesNotEmitInvalidData() { var startTime = new DateTime(2014, 06, 06, 0, 0, 0); var endTime = new DateTime(2014, 06, 09, 20, 0, 0); var canonicalSymbol = Symbol.Create("AAPL", SecurityType.Option, Market.USA, "?AAPL"); var quoteCurrency = new Cash(Currencies.USD, 0, 1); var exchangeHours = MarketHoursDatabase.FromDataFolder().GetExchangeHours(Market.USA, canonicalSymbol, SecurityType.Option); var config = new SubscriptionDataConfig( typeof(ZipEntryName), canonicalSymbol, Resolution.Minute, TimeZones.Utc, TimeZones.NewYork, true, false, false, false, TickType.Quote, false, DataNormalizationMode.Raw ); var option = new Option( canonicalSymbol, exchangeHours, quoteCurrency, new OptionSymbolProperties(SymbolProperties.GetDefault(Currencies.USD)), ErrorCurrencyConverter.Instance, RegisteredSecurityDataTypesProvider.Null, new SecurityCache() ); var enumeratorFactory = new BaseDataSubscriptionEnumeratorFactory(false, MapFileResolver.Create(Globals.DataFolder, Market.USA), new LocalDiskFactorFileProvider(new LocalDiskMapFileProvider())); var fillForwardResolution = Ref.CreateReadOnly(() => Resolution.Minute.ToTimeSpan()); Func <SubscriptionRequest, IEnumerator <BaseData> > underlyingEnumeratorFunc = (req) => { var input = enumeratorFactory.CreateEnumerator(req, new DefaultDataProvider()); input = new BaseDataCollectionAggregatorEnumerator(input, req.Configuration.Symbol); return(new FillForwardEnumerator( input, option.Exchange, fillForwardResolution, false, endTime, Resolution.Minute.ToTimeSpan(), TimeZones.Utc, startTime)); }; var factory = new OptionChainUniverseSubscriptionEnumeratorFactory(underlyingEnumeratorFunc); var request = new SubscriptionRequest(true, null, option, config, startTime, endTime); var enumerator = factory.CreateEnumerator(request, new DefaultDataProvider()); var emittedCount = 0; foreach (var data in enumerator.AsEnumerable()) { emittedCount++; var optionData = data as OptionChainUniverseDataCollection; Assert.IsNotNull(optionData); Assert.IsNotNull(optionData.Underlying); Assert.AreNotEqual(0, optionData.Data.Count); } // 9:30 to 15:59 -> 6.5 hours * 60 => 390 minutes * 2 days = 780 Assert.AreEqual(780, emittedCount); }
/// <summary> /// Creates a new subscription for the specified security /// </summary> /// <param name="request">The subscription request</param> /// <returns>A new subscription instance of the specified security</returns> protected Subscription CreateSubscription(SubscriptionRequest request) { Subscription subscription = null; try { var localEndTime = request.EndTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone); var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc); IEnumerator<BaseData> enumerator; if (request.Configuration.IsCustomData) { if (!Quandl.IsAuthCodeSet) { // we're not using the SubscriptionDataReader, so be sure to set the auth token here Quandl.SetAuthCode(Config.Get("quandl-auth-token")); } // each time we exhaust we'll new up this enumerator stack var refresher = new RefreshEnumerator<BaseData>(() => { var dateInDataTimeZone = DateTime.UtcNow.ConvertFromUtc(request.Configuration.DataTimeZone).Date; var enumeratorFactory = new BaseDataSubscriptionEnumeratorFactory(r => new[] { dateInDataTimeZone }); var factoryReadEnumerator = enumeratorFactory.CreateEnumerator(request, _dataFileProvider); var maximumDataAge = TimeSpan.FromTicks(Math.Max(request.Configuration.Increment.Ticks, TimeSpan.FromSeconds(5).Ticks)); var fastForward = new FastForwardEnumerator(factoryReadEnumerator, _timeProvider, request.Security.Exchange.TimeZone, maximumDataAge); return new FrontierAwareEnumerator(fastForward, _frontierTimeProvider, timeZoneOffsetProvider); }); // rate limit the refreshing of the stack to the requested interval // At Tick resolution, it will refresh at full speed // At Second and Minute resolution, it will refresh every second and minute respectively // At Hour and Daily resolutions, it will refresh every 30 minutes var minimumTimeBetweenCalls = Math.Min(request.Configuration.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks); var rateLimit = new RateLimitEnumerator(refresher, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls)); _customExchange.AddEnumerator(request.Configuration.Symbol, rateLimit); var enqueable = new EnqueueableEnumerator<BaseData>(); _customExchange.SetDataHandler(request.Configuration.Symbol, data => { enqueable.Enqueue(data); if (subscription != null) subscription.RealtimePrice = data.Value; }); enumerator = enqueable; } else if (request.Configuration.Resolution != Resolution.Tick) { // this enumerator allows the exchange to pump ticks into the 'back' of the enumerator, // and the time sync loop can pull aggregated trade bars off the front var aggregator = new TradeBarBuilderEnumerator(request.Configuration.Increment, request.Security.Exchange.TimeZone, _timeProvider); _exchange.SetDataHandler(request.Configuration.Symbol, data => { aggregator.ProcessData((Tick) data); if (subscription != null) subscription.RealtimePrice = data.Value; }); enumerator = aggregator; } else { // tick subscriptions can pass right through var tickEnumerator = new EnqueueableEnumerator<BaseData>(); _exchange.SetDataHandler(request.Configuration.Symbol, data => { tickEnumerator.Enqueue(data); if (subscription != null) subscription.RealtimePrice = data.Value; }); enumerator = tickEnumerator; } if (request.Configuration.FillDataForward) { enumerator = new LiveFillForwardEnumerator(_frontierTimeProvider, enumerator, request.Security.Exchange, _fillForwardResolution, request.Configuration.ExtendedMarketHours, localEndTime, request.Configuration.Increment); } // define market hours and user filters to incoming data if (request.Configuration.IsFilteredSubscription) { enumerator = new SubscriptionFilterEnumerator(enumerator, request.Security, localEndTime); } // finally, make our subscriptions aware of the frontier of the data feed, prevents future data from spewing into the feed enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, timeZoneOffsetProvider); subscription = new Subscription(request.Universe, request.Security, request.Configuration, enumerator, timeZoneOffsetProvider, request.StartTimeUtc, request.EndTimeUtc, false); } catch (Exception err) { Log.Error(err); } return subscription; }