public void ReturnsExpectedTimestamps() { var symbol = CoarseFundamental.CreateUniverseSymbol(Market.USA); var config = new SubscriptionDataConfig(typeof(CoarseFundamental), symbol, Resolution.Daily, TimeZones.NewYork, TimeZones.NewYork, false, false, false, false, TickType.Trade, false); var security = new Security( SecurityExchangeHours.AlwaysOpen(TimeZones.NewYork), config, new Cash(Currencies.USD, 0, 1), SymbolProperties.GetDefault(Currencies.USD), ErrorCurrencyConverter.Instance ); var universeSettings = new UniverseSettings(Resolution.Daily, 2m, true, false, TimeSpan.FromDays(1)); var securityInitializer = new BrokerageModelSecurityInitializer(new DefaultBrokerageModel(), SecuritySeeder.Null); var universe = new CoarseFundamentalUniverse(universeSettings, securityInitializer, x => new List <Symbol> { Symbols.AAPL }); var fileProvider = new DefaultDataProvider(); var factory = new BaseDataCollectionSubscriptionEnumeratorFactory(); var dateStart = new DateTime(2014, 3, 25); var dateEnd = new DateTime(2014, 3, 26); var days = (dateEnd - dateStart).Days + 1; var request = new SubscriptionRequest(true, universe, security, config, dateStart, dateEnd); using (var enumerator = factory.CreateEnumerator(request, fileProvider)) { for (var i = 0; i < days; i++) { Assert.IsTrue(enumerator.MoveNext()); var current = enumerator.Current as BaseDataCollection; Assert.IsNotNull(current); Assert.AreEqual(dateStart.AddDays(i), current.Time); Assert.AreEqual(dateStart.AddDays(i), current.EndTime); Assert.AreEqual(dateStart.AddDays(i - 1), current.Data[0].Time); Assert.AreEqual(dateStart.AddDays(i), current.Data[0].EndTime); } Assert.IsFalse(enumerator.MoveNext()); Assert.IsNotNull(enumerator.Current); } }
public void DoesNotLeakMemory() { var symbol = CoarseFundamental.CreateUniverseSymbol(Market.USA); var config = new SubscriptionDataConfig(typeof(CoarseFundamental), symbol, Resolution.Daily, TimeZones.NewYork, TimeZones.NewYork, false, false, false, false, TickType.Trade, false); var security = new Security( SecurityExchangeHours.AlwaysOpen(TimeZones.NewYork), config, new Cash(Currencies.USD, 0, 1), SymbolProperties.GetDefault(Currencies.USD), ErrorCurrencyConverter.Instance, RegisteredSecurityDataTypesProvider.Null, new SecurityCache() ); var universeSettings = new UniverseSettings(Resolution.Daily, 2m, true, false, TimeSpan.FromDays(1)); var securityInitializer = new BrokerageModelSecurityInitializer(new DefaultBrokerageModel(), SecuritySeeder.Null); var universe = new CoarseFundamentalUniverse(universeSettings, x => new List <Symbol> { Symbols.AAPL }); var factory = new BaseDataCollectionSubscriptionEnumeratorFactory(); GC.Collect(); var ramUsageBeforeLoop = OS.TotalPhysicalMemoryUsed; var date = new DateTime(2014, 3, 25); const int iterations = 1000; for (var i = 0; i < iterations; i++) { var request = new SubscriptionRequest(true, universe, security, config, date, date); using (var enumerator = factory.CreateEnumerator(request, TestGlobals.DataProvider)) { enumerator.MoveNext(); } } GC.Collect(); var ramUsageAfterLoop = OS.TotalPhysicalMemoryUsed; Log.Trace($"RAM usage - before: {ramUsageBeforeLoop} MB, after: {ramUsageAfterLoop} MB"); Assert.IsTrue(ramUsageAfterLoop - ramUsageBeforeLoop < 10); }
/// <summary> /// Adds a new subscription for universe selection /// </summary> /// <param name="request">The subscription request</param> private Subscription CreateUniverseSubscription(SubscriptionRequest request) { ISubscriptionEnumeratorFactory factory = _subscriptionFactory; if (request.Universe is ITimeTriggeredUniverse) { factory = new TimeTriggeredUniverseSubscriptionEnumeratorFactory(request.Universe as ITimeTriggeredUniverse, MarketHoursDatabase.FromDataFolder(), _timeProvider); if (request.Universe is UserDefinedUniverse) { // for user defined universe we do not use a worker task, since calls to AddData can happen in any moment // and we have to be able to inject selection data points into the enumerator return(SubscriptionUtils.Create(request, factory.CreateEnumerator(request, _dataProvider))); } } if (request.Configuration.Type == typeof(CoarseFundamental)) { factory = new BaseDataCollectionSubscriptionEnumeratorFactory(); } if (request.Universe is OptionChainUniverse) { factory = new OptionChainUniverseSubscriptionEnumeratorFactory((req) => { if (!req.Configuration.SecurityType.IsOption()) { var enumerator = _subscriptionFactory.CreateEnumerator(req, _dataProvider); enumerator = new FilterEnumerator <BaseData>(enumerator, data => data.DataType != MarketDataType.Auxiliary); return(ConfigureEnumerator(req, true, enumerator)); } var underlyingFactory = new BaseDataSubscriptionEnumeratorFactory(false, _mapFileProvider, _cacheProvider); return(ConfigureEnumerator(req, true, underlyingFactory.CreateEnumerator(req, _dataProvider))); }); } if (request.Universe is FuturesChainUniverse) { factory = new FuturesChainUniverseSubscriptionEnumeratorFactory((req, e) => ConfigureEnumerator(req, true, e), _cacheProvider); } // define our data enumerator var enumerator = factory.CreateEnumerator(request, _dataProvider); return(SubscriptionUtils.CreateAndScheduleWorker(request, enumerator, _factorFileProvider, true)); }
protected IEnumerator <BaseData> CreateUniverseEnumerator(SubscriptionRequest request, Func <SubscriptionRequest, IEnumerator <BaseData> > createUnderlyingEnumerator) { ISubscriptionEnumeratorFactory factory = _subscriptionFactory; if (request.Universe is ITimeTriggeredUniverse) { factory = new TimeTriggeredUniverseSubscriptionEnumeratorFactory(request.Universe as ITimeTriggeredUniverse, MarketHoursDatabase.FromDataFolder(), _timeProvider); if (request.Universe is UserDefinedUniverse) { return(factory.CreateEnumerator(request, _dataProvider)); } } else if (request.Configuration.Type == typeof(CoarseFundamental)) { factory = new BaseDataCollectionSubscriptionEnumeratorFactory(); } else if (request.Configuration.Type == typeof(ZipEntryName)) { // TODO: subscription should already come in correctly built var resolution = request.Configuration.Resolution == Resolution.Tick ? Resolution.Second : request.Configuration.Resolution; // TODO: subscription should already come in as fill forward true request = new SubscriptionRequest(request, configuration: new SubscriptionDataConfig(request.Configuration, fillForward: true, resolution: resolution)); var result = new BaseDataSubscriptionEnumeratorFactory(_algorithm.OptionChainProvider, _algorithm.FutureChainProvider) .CreateEnumerator(request, _dataProvider); result = ConfigureEnumerator(request, true, result); return(TryAppendUnderlyingEnumerator(request, result, createUnderlyingEnumerator)); } // define our data enumerator var enumerator = factory.CreateEnumerator(request, _dataProvider); return(enumerator); }
/// <summary> /// Adds a new subscription for universe selection /// </summary> /// <param name="request">The subscription request</param> private Subscription CreateUniverseSubscription(SubscriptionRequest request) { ISubscriptionEnumeratorFactory factory = _subscriptionFactory; if (request.Universe is ITimeTriggeredUniverse) { factory = new TimeTriggeredUniverseSubscriptionEnumeratorFactory(request.Universe as ITimeTriggeredUniverse, MarketHoursDatabase.FromDataFolder(), _timeProvider); if (request.Universe is UserDefinedUniverse) { // for user defined universe we do not use a worker task, since calls to AddData can happen in any moment // and we have to be able to inject selection data points into the enumerator return(SubscriptionUtils.Create(request, factory.CreateEnumerator(request, _dataProvider))); } } if (request.Configuration.Type == typeof(CoarseFundamental)) { factory = new BaseDataCollectionSubscriptionEnumeratorFactory(); } if (request.Universe is OptionChainUniverse) { factory = new OptionChainUniverseSubscriptionEnumeratorFactory((req, e) => ConfigureEnumerator(req, true, e), _mapFileProvider.Get(request.Security.Symbol.ID.Market), _factorFileProvider); } if (request.Universe is FuturesChainUniverse) { factory = new FuturesChainUniverseSubscriptionEnumeratorFactory((req, e) => ConfigureEnumerator(req, true, e)); } // define our data enumerator var enumerator = factory.CreateEnumerator(request, _dataProvider); return(SubscriptionUtils.CreateAndScheduleWorker(request, enumerator)); }
/// <summary> /// Creates a new subscription for the specified security /// </summary> /// <param name="request">The subscription request</param> /// <returns>A new subscription instance of the specified security</returns> protected Subscription CreateSubscription(SubscriptionRequest request) { Subscription subscription = null; try { var localEndTime = request.EndTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone); var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc); IEnumerator <BaseData> enumerator; if (request.Configuration.IsCustomData) { if (!Quandl.IsAuthCodeSet) { // we're not using the SubscriptionDataReader, so be sure to set the auth token here Quandl.SetAuthCode(Config.Get("quandl-auth-token")); } // each time we exhaust we'll new up this enumerator stack var refresher = new RefreshEnumerator <BaseData>(() => { var dateInDataTimeZone = DateTime.UtcNow.ConvertFromUtc(request.Configuration.DataTimeZone).Date; var enumeratorFactory = new BaseDataCollectionSubscriptionEnumeratorFactory(r => new [] { dateInDataTimeZone }); var factoryReadEnumerator = enumeratorFactory.CreateEnumerator(request); var maximumDataAge = TimeSpan.FromTicks(Math.Max(request.Configuration.Increment.Ticks, TimeSpan.FromSeconds(5).Ticks)); return(new FastForwardEnumerator(factoryReadEnumerator, _timeProvider, request.Security.Exchange.TimeZone, maximumDataAge)); }); // rate limit the refreshing of the stack to the requested interval var minimumTimeBetweenCalls = Math.Min(request.Configuration.Increment.Ticks, TimeSpan.FromMinutes(30).Ticks); var rateLimit = new RateLimitEnumerator(refresher, _timeProvider, TimeSpan.FromTicks(minimumTimeBetweenCalls)); var frontierAware = new FrontierAwareEnumerator(rateLimit, _timeProvider, timeZoneOffsetProvider); _customExchange.AddEnumerator(request.Configuration.Symbol, frontierAware); var enqueable = new EnqueueableEnumerator <BaseData>(); _customExchange.SetDataHandler(request.Configuration.Symbol, data => { enqueable.Enqueue(data); if (subscription != null) { subscription.RealtimePrice = data.Value; } }); enumerator = enqueable; } else if (request.Configuration.Resolution != Resolution.Tick) { // this enumerator allows the exchange to pump ticks into the 'back' of the enumerator, // and the time sync loop can pull aggregated trade bars off the front var aggregator = new TradeBarBuilderEnumerator(request.Configuration.Increment, request.Security.Exchange.TimeZone, _timeProvider); _exchange.SetDataHandler(request.Configuration.Symbol, data => { aggregator.ProcessData((Tick)data); if (subscription != null) { subscription.RealtimePrice = data.Value; } }); enumerator = aggregator; } else { // tick subscriptions can pass right through var tickEnumerator = new EnqueueableEnumerator <BaseData>(); _exchange.SetDataHandler(request.Configuration.Symbol, data => { tickEnumerator.Enqueue(data); if (subscription != null) { subscription.RealtimePrice = data.Value; } }); enumerator = tickEnumerator; } if (request.Configuration.FillDataForward) { enumerator = new LiveFillForwardEnumerator(_frontierTimeProvider, enumerator, request.Security.Exchange, _fillForwardResolution, request.Configuration.ExtendedMarketHours, localEndTime, request.Configuration.Increment); } // define market hours and user filters to incoming data if (request.Configuration.IsFilteredSubscription) { enumerator = new SubscriptionFilterEnumerator(enumerator, request.Security, localEndTime); } // finally, make our subscriptions aware of the frontier of the data feed, prevents future data from spewing into the feed enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, timeZoneOffsetProvider); subscription = new Subscription(request.Universe, request.Security, request.Configuration, enumerator, timeZoneOffsetProvider, request.StartTimeUtc, request.EndTimeUtc, false); } catch (Exception err) { Log.Error(err); } return(subscription); }