/// <summary> /// Initializes the data feed for the specified job and algorithm /// </summary> public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler, IMapFileProvider mapFileProvider, IFactorFileProvider factorFileProvider, IDataProvider dataProvider) { _algorithm = algorithm; _resultHandler = resultHandler; _mapFileProvider = mapFileProvider; _factorFileProvider = factorFileProvider; _dataProvider = dataProvider; _subscriptions = new SubscriptionCollection(); _universeSelection = new UniverseSelection(this, algorithm); _cancellationTokenSource = new CancellationTokenSource(); _subscriptionfactory = new SubscriptionDataReaderSubscriptionEnumeratorFactory(_resultHandler, _mapFileProvider, _factorFileProvider, _dataProvider, false, true); IsActive = true; var threadCount = Math.Max(1, Math.Min(4, Environment.ProcessorCount - 3)); _controller = new ParallelRunnerController(threadCount); _controller.Start(_cancellationTokenSource.Token); var ffres = Time.OneMinute; _fillForwardResolution = Ref.Create(() => ffres, res => ffres = res); // wire ourselves up to receive notifications when universes are added/removed algorithm.UniverseManager.CollectionChanged += (sender, args) => { switch (args.Action) { case NotifyCollectionChangedAction.Add: foreach (var universe in args.NewItems.OfType <Universe>()) { var config = universe.Configuration; var start = _algorithm.UtcTime; var marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); var exchangeHours = marketHoursDatabase.GetExchangeHours(config); Security security; if (!_algorithm.Securities.TryGetValue(config.Symbol, out security)) { // create a canonical security object if it doesn't exist security = new Security(exchangeHours, config, _algorithm.Portfolio.CashBook[CashBook.AccountCurrency], SymbolProperties.GetDefault(CashBook.AccountCurrency)); } var end = _algorithm.EndDate.ConvertToUtc(_algorithm.TimeZone); AddSubscription(new SubscriptionRequest(true, universe, security, config, start, end)); } break; case NotifyCollectionChangedAction.Remove: foreach (var universe in args.OldItems.OfType <Universe>()) { RemoveSubscription(universe.Configuration); } break; default: throw new NotImplementedException("The specified action is not implemented: " + args.Action); } }; }
public void ChecksMapFileFirstDate() { var algorithm = PerformanceBenchmarkAlgorithms.SingleSecurity_Second; algorithm.Initialize(); algorithm.PostInitialize(); var dataProvider = new DefaultDataProvider(); var resultHandler = new TestResultHandler(); var mapFileProvider = new LocalDiskMapFileProvider(); var factorFileProvider = new LocalDiskFactorFileProvider(mapFileProvider); var factory = new SubscriptionDataReaderSubscriptionEnumeratorFactory(resultHandler, mapFileProvider, factorFileProvider, dataProvider, true, enablePriceScaling: false); var universe = algorithm.UniverseManager.Single().Value; var security = algorithm.AddEquity("AAA", Resolution.Daily); var securityConfig = security.Subscriptions.First(); // start date is before the first date in the map file var subscriptionRequest = new SubscriptionRequest(false, universe, security, securityConfig, new DateTime(2001, 12, 1), new DateTime(2016, 11, 1)); var enumerator = factory.CreateEnumerator(subscriptionRequest, dataProvider); // should initialize the data source reader enumerator.MoveNext(); enumerator.Dispose(); factory.DisposeSafely(); resultHandler.Exit(); var message = ((DebugPacket)resultHandler.Messages.Single()).Message; Assert.IsTrue(message.Equals( "The starting date for symbol AAA, 2001-11-30, has been adjusted to match map file first date 2002-05-22.")); }
/// <summary> /// Initializes the data feed for the specified job and algorithm /// </summary> public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler, IMapFileProvider mapFileProvider, IFactorFileProvider factorFileProvider, IDataProvider dataProvider, IDataFeedSubscriptionManager subscriptionManager, IDataFeedTimeProvider dataFeedTimeProvider) { _algorithm = algorithm; _resultHandler = resultHandler; _mapFileProvider = mapFileProvider; _factorFileProvider = factorFileProvider; _dataProvider = dataProvider; _subscriptions = subscriptionManager.DataFeedSubscriptions; _universeSelection = subscriptionManager.UniverseSelection; _cancellationTokenSource = new CancellationTokenSource(); _subscriptionFactory = new SubscriptionDataReaderSubscriptionEnumeratorFactory( _resultHandler, _mapFileProvider, _factorFileProvider, _dataProvider, includeAuxiliaryData: true); IsActive = true; var threadCount = Math.Max(1, Math.Min(4, Environment.ProcessorCount - 3)); }
/// <summary> /// Initializes the data feed for the specified job and algorithm /// </summary> public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler, IMapFileProvider mapFileProvider, IFactorFileProvider factorFileProvider, IDataProvider dataProvider, IDataFeedSubscriptionManager subscriptionManager, IDataFeedTimeProvider dataFeedTimeProvider, IDataChannelProvider dataChannelProvider) { _algorithm = algorithm; _resultHandler = resultHandler; _mapFileProvider = mapFileProvider; _factorFileProvider = factorFileProvider; _dataProvider = dataProvider; _timeProvider = dataFeedTimeProvider.FrontierTimeProvider; _subscriptions = subscriptionManager.DataFeedSubscriptions; _cancellationTokenSource = new CancellationTokenSource(); _subscriptionFactory = new SubscriptionDataReaderSubscriptionEnumeratorFactory( _resultHandler, _mapFileProvider, _factorFileProvider, _dataProvider, includeAuxiliaryData: true, enablePriceScaling: false); IsActive = true; }
public void ChecksMapFileFirstDate() { var algorithm = PerformanceBenchmarkAlgorithms.SingleSecurity_Second; algorithm.Initialize(); algorithm.PostInitialize(); var resultHandler = new TestResultHandler(); using var cache = new ZipDataCacheProvider(TestGlobals.DataProvider); var factory = new SubscriptionDataReaderSubscriptionEnumeratorFactory(resultHandler, TestGlobals.MapFileProvider, TestGlobals.FactorFileProvider, cache, enablePriceScaling: false); var universe = algorithm.UniverseManager.Single().Value; var security = algorithm.AddEquity("AAA", Resolution.Daily); var securityConfig = security.Subscriptions.First(); // start date is before the first date in the map file var subscriptionRequest = new SubscriptionRequest(false, universe, security, securityConfig, new DateTime(2001, 12, 1), new DateTime(2016, 11, 1)); var enumerator = factory.CreateEnumerator(subscriptionRequest, TestGlobals.DataProvider); // should initialize the data source reader enumerator.MoveNext(); enumerator.Dispose(); factory.DisposeSafely(); resultHandler.Exit(); var message = ((DebugPacket)resultHandler.Messages.Single()).Message; Assert.IsTrue(message.Equals( "The starting dates for the following symbols have been adjusted to match their map files first date: [AAA, 2020-09-09]")); }
public void TestDataFeedEnumeratorStackSpeed() { var algorithm = PerformanceBenchmarkAlgorithms.SingleSecurity_Second; algorithm.Initialize(); algorithm.PostInitialize(); var dataProvider = new DefaultDataProvider(); var resultHandler = new BacktestingResultHandler(); var mapFileProvider = new LocalDiskMapFileProvider(); var factorFileProvider = new LocalDiskFactorFileProvider(mapFileProvider); var factory = new SubscriptionDataReaderSubscriptionEnumeratorFactory(resultHandler, mapFileProvider, factorFileProvider, dataProvider, true, enablePriceScaling: false); var universe = algorithm.UniverseManager.Single().Value; var security = algorithm.Securities.Single().Value; var securityConfig = security.Subscriptions.First(); var subscriptionRequest = new SubscriptionRequest(false, universe, security, securityConfig, algorithm.StartDate, algorithm.EndDate); var enumerator = factory.CreateEnumerator(subscriptionRequest, dataProvider); var count = 0; var stopwatch = Stopwatch.StartNew(); var lastMonth = algorithm.StartDate.Month; while (enumerator.MoveNext()) { var current = enumerator.Current; if (current == null) { Log.Trace("ERROR: Current is null"); continue; } if (current.Time.Month != lastMonth) { var elapsed = stopwatch.Elapsed.TotalSeconds; var thousands = count / 1000d; Log.Trace($"{DateTime.Now} - Time: {current.Time}: KPS: {thousands / elapsed}"); lastMonth = current.Time.Month; } count++; } Log.Trace("Count: " + count); stopwatch.Stop(); enumerator.Dispose(); factory.DisposeSafely(); Log.Trace($"Elapsed time: {stopwatch.Elapsed} KPS: {count / 1000d / stopwatch.Elapsed.TotalSeconds}"); }