public void YieldsDataEachDayAsTimePasses() { // previous point is exactly one resolution step behind, so it emits Assert.IsTrue(_enumerator.MoveNext()); Assert.IsNotNull(_enumerator.Current); Assert.AreEqual(_referenceLocal.AddDays(-1), _enumerator.Current.EndTime); VerifyGetSourceInvocation(1); // yields the data for the current time Assert.IsTrue(_enumerator.MoveNext()); Assert.IsNotNull(_enumerator.Current); Assert.AreEqual(_referenceLocal, _enumerator.Current.EndTime); VerifyGetSourceInvocation(0); _timeProvider.Advance(Time.OneDay); // now we can yield the next data point as it has passed frontier time Assert.IsTrue(_enumerator.MoveNext()); Assert.IsNotNull(_enumerator.Current); Assert.AreEqual(_referenceLocal.AddDays(1), _enumerator.Current.EndTime); VerifyGetSourceInvocation(0); // this call exhaused the enumerator stack and yields a null result Assert.IsTrue(_enumerator.MoveNext()); Assert.IsNull(_enumerator.Current); VerifyGetSourceInvocation(0); // this call refrshes the enumerator stack but finds no data ahead of the frontier Assert.IsTrue(_enumerator.MoveNext()); Assert.IsNull(_enumerator.Current); VerifyGetSourceInvocation(1); _timeProvider.Advance(TimeSpan.FromMinutes(30)); // time advances 30 minutes so we'll try to refresh again Assert.IsTrue(_enumerator.MoveNext()); Assert.IsNull(_enumerator.Current); VerifyGetSourceInvocation(1); _timeProvider.Advance(Time.OneDay); // now to the next day, we'll try again and get data _dataPointsAfterReference++; Assert.IsTrue(_enumerator.MoveNext()); Assert.IsNotNull(_enumerator.Current); Assert.AreEqual(_referenceLocal.AddDays(2), _enumerator.Current.EndTime); VerifyGetSourceInvocation(1); _timeProvider.Advance(TimeSpan.FromHours(1)); // out of data Assert.IsTrue(_enumerator.MoveNext()); Assert.IsNull(_enumerator.Current); VerifyGetSourceInvocation(0); _timeProvider.Advance(TimeSpan.FromHours(1)); // time advanced so we'll try to refresh the souce again, but exhaust the stack because no data Assert.IsTrue(_enumerator.MoveNext()); Assert.IsNull(_enumerator.Current); VerifyGetSourceInvocation(1); // move forward to next whole day, midnight _timeProvider.Advance(Time.OneDay.Subtract(TimeSpan.FromHours(2.5))); // the day elapsed but there's still no data available Assert.IsTrue(_enumerator.MoveNext()); Assert.IsNull(_enumerator.Current); VerifyGetSourceInvocation(1); // this is rate limited by the 30 minute guard for daily data _timeProvider.Advance(TimeSpan.FromMinutes(29)); Assert.IsTrue(_enumerator.MoveNext()); Assert.IsNull(_enumerator.Current); VerifyGetSourceInvocation(0); // another 30 minutes elapsed and now there's data available _dataPointsAfterReference++; _timeProvider.Advance(TimeSpan.FromMinutes(1)); Assert.IsTrue(_enumerator.MoveNext()); Assert.IsNotNull(_enumerator.Current); Assert.AreEqual(_referenceLocal.AddDays(3), _enumerator.Current.EndTime); VerifyGetSourceInvocation(1); // exhausted the stack Assert.IsTrue(_enumerator.MoveNext()); Assert.IsNull(_enumerator.Current); VerifyGetSourceInvocation(0); // rate limited Assert.IsTrue(_enumerator.MoveNext()); Assert.IsNull(_enumerator.Current); VerifyGetSourceInvocation(0); }
public void RemoteDataDoesNotIncreaseNumberOfSlices() { Config.Set("quandl-auth-token", "QUANDL-TOKEN"); var startDate = new DateTime(2018, 4, 2); var endDate = new DateTime(2018, 4, 19); var algorithm = new QCAlgorithm(); var dataManager = new DataManagerStub(algorithm); algorithm.SubscriptionManager.SetDataManager(dataManager); var symbols = new List <Symbol> { algorithm.AddData <Quandl>("CBOE/VXV", Resolution.Daily).Symbol, algorithm.AddData <QuandlVix>("CBOE/VIX", Resolution.Daily).Symbol, algorithm.AddEquity("SPY", Resolution.Daily).Symbol, algorithm.AddEquity("AAPL", Resolution.Daily).Symbol }; algorithm.PostInitialize(); var timeProvider = new ManualTimeProvider(TimeZones.NewYork); timeProvider.SetCurrentTime(startDate); var cancellationTokenSource = new CancellationTokenSource(); var dataPointsEmitted = 0; var slicesEmitted = 0; var dataQueueHandler = new FuncDataQueueHandler(fdqh => { var time = timeProvider.GetUtcNow().ConvertFromUtc(TimeZones.NewYork); var tick = new Tick(time, Symbols.SPY, 1.3m, 1.2m, 1.3m) { TickType = TickType.Trade }; var tick2 = new Tick(time, Symbols.AAPL, 1.3m, 1.2m, 1.3m) { TickType = TickType.Trade }; return(new[] { tick, tick2 }); }); RunLiveDataFeed(algorithm, startDate, symbols, timeProvider, dataManager, dataQueueHandler); Thread.Sleep(5000); // Give remote sources a handicap, so the data is available in time // create a timer to advance time much faster than realtime and to simulate live Quandl data file updates var timerInterval = TimeSpan.FromMilliseconds(100); var timer = Ref.Create <Timer>(null); timer.Value = new Timer(state => { // stop the timer to prevent reentrancy timer.Value.Change(Timeout.Infinite, Timeout.Infinite); var currentTime = timeProvider.GetUtcNow().ConvertFromUtc(TimeZones.NewYork); if (currentTime.Date > endDate.Date) { _feed.Exit(); cancellationTokenSource.Cancel(); return; } timeProvider.Advance(TimeSpan.FromHours(3)); // restart the timer timer.Value.Change(timerInterval, timerInterval); }, null, TimeSpan.FromSeconds(2), timerInterval); try { foreach (var timeSlice in _synchronizer.StreamData(cancellationTokenSource.Token)) { if (timeSlice.Slice.HasData) { slicesEmitted++; dataPointsEmitted += timeSlice.Slice.Values.Count; Assert.IsTrue(timeSlice.Slice.Values.Any(x => x.Symbol == symbols[0]), $"Slice doesn't contain {symbols[0]}"); Assert.IsTrue(timeSlice.Slice.Values.Any(x => x.Symbol == symbols[1]), $"Slice doesn't contain {symbols[1]}"); Assert.IsTrue(timeSlice.Slice.Values.Any(x => x.Symbol == symbols[2]), $"Slice doesn't contain {symbols[2]}"); Assert.IsTrue(timeSlice.Slice.Values.Any(x => x.Symbol == symbols[3]), $"Slice doesn't contain {symbols[3]}"); } } } catch (Exception exception) { Log.Trace($"Error: {exception}"); } timer.Value.Dispose(); Assert.AreEqual(14, slicesEmitted); Assert.AreEqual(14 * symbols.Count, dataPointsEmitted); }
public void RefreshesOptionChainUniverseOnDateChange() { var startTime = new DateTime(2018, 10, 19, 10, 0, 0); var timeProvider = new ManualTimeProvider(startTime); var canonicalSymbol = Symbol.Create("SPY", SecurityType.Option, Market.USA, "?SPY"); var quoteCurrency = new Cash(Currencies.USD, 0, 1); var exchangeHours = MarketHoursDatabase.FromDataFolder().GetExchangeHours(Market.USA, canonicalSymbol, SecurityType.Option); var config = new SubscriptionDataConfig( typeof(ZipEntryName), canonicalSymbol, Resolution.Minute, TimeZones.Utc, TimeZones.NewYork, true, false, false, false, TickType.Quote, false, DataNormalizationMode.Raw ); var option = new Option( canonicalSymbol, exchangeHours, quoteCurrency, new OptionSymbolProperties(SymbolProperties.GetDefault(Currencies.USD)), ErrorCurrencyConverter.Instance, RegisteredSecurityDataTypesProvider.Null, new SecurityCache(), null ); var fillForwardResolution = Ref.CreateReadOnly(() => Resolution.Minute.ToTimeSpan()); var symbolUniverse = new TestDataQueueUniverseProvider(timeProvider); EnqueueableEnumerator <BaseData> underlyingEnumerator = null; Func <SubscriptionRequest, IEnumerator <BaseData> > underlyingEnumeratorFunc = (req) => { underlyingEnumerator = new EnqueueableEnumerator <BaseData>(); return(new LiveFillForwardEnumerator( timeProvider, underlyingEnumerator, option.Exchange, fillForwardResolution, false, Time.EndOfTime, Resolution.Minute.ToTimeSpan(), TimeZones.Utc)); }; var factory = new OptionChainUniverseSubscriptionEnumeratorFactory(underlyingEnumeratorFunc, symbolUniverse, timeProvider); var universeSettings = new UniverseSettings(Resolution.Minute, 0, true, false, TimeSpan.Zero); var universe = new OptionChainUniverse(option, universeSettings, true); var request = new SubscriptionRequest(true, universe, option, config, startTime, Time.EndOfTime); var enumerator = (DataQueueOptionChainUniverseDataCollectionEnumerator)factory.CreateEnumerator(request, TestGlobals.DataProvider); // 2018-10-19 10:00 AM UTC underlyingEnumerator.Enqueue(new Tick { Symbol = Symbols.SPY, Value = 280m }); // 2018-10-19 10:01 AM UTC timeProvider.Advance(Time.OneMinute); underlyingEnumerator.Enqueue(new Tick { Symbol = Symbols.SPY, Value = 280m }); Assert.IsTrue(enumerator.MoveNext()); Assert.IsNotNull(enumerator.Current); Assert.AreEqual(1, symbolUniverse.TotalLookupCalls); var data = enumerator.Current; Assert.IsNotNull(data); Assert.AreEqual(1, data.Data.Count); Assert.IsNotNull(data.Underlying); // 2018-10-19 10:02 AM UTC timeProvider.Advance(Time.OneMinute); underlyingEnumerator.Enqueue(new Tick { Symbol = Symbols.SPY, Value = 280m }); Assert.IsTrue(enumerator.MoveNext()); Assert.IsNotNull(enumerator.Current); Assert.AreEqual(1, symbolUniverse.TotalLookupCalls); data = enumerator.Current; Assert.IsNotNull(data); Assert.AreEqual(1, data.Data.Count); Assert.IsNotNull(data.Underlying); // 2018-10-19 10:03 AM UTC timeProvider.Advance(Time.OneMinute); underlyingEnumerator.Enqueue(new Tick { Symbol = Symbols.SPY, Value = 280m }); Assert.IsTrue(enumerator.MoveNext()); Assert.IsNotNull(enumerator.Current); Assert.AreEqual(1, symbolUniverse.TotalLookupCalls); data = enumerator.Current; Assert.IsNotNull(data); Assert.AreEqual(1, data.Data.Count); Assert.IsNotNull(data.Underlying); // 2018-10-20 10:03 AM UTC timeProvider.Advance(Time.OneDay); underlyingEnumerator.Enqueue(new Tick { Symbol = Symbols.SPY, Value = 280m }); Assert.IsTrue(enumerator.MoveNext()); Assert.IsNotNull(enumerator.Current); Assert.AreEqual(2, symbolUniverse.TotalLookupCalls); data = enumerator.Current; Assert.IsNotNull(data); Assert.AreEqual(2, data.Data.Count); Assert.IsNotNull(data.Underlying); // 2018-10-20 10:04 AM UTC timeProvider.Advance(Time.OneMinute); underlyingEnumerator.Enqueue(new Tick { Symbol = Symbols.SPY, Value = 280m }); Assert.IsTrue(enumerator.MoveNext()); Assert.IsNotNull(enumerator.Current); Assert.AreEqual(2, symbolUniverse.TotalLookupCalls); data = enumerator.Current; Assert.IsNotNull(data); Assert.AreEqual(2, data.Data.Count); Assert.IsNotNull(data.Underlying); enumerator.Dispose(); }
public void CoarseUniverseRotatesActiveSecurity() { var startDate = new DateTime(2014, 3, 24); var endDate = new DateTime(2014, 3, 28); var timeProvider = new ManualTimeProvider(TimeZones.NewYork); timeProvider.SetCurrentTime(startDate); var coarseTimes = new List <DateTime> { new DateTime(2014, 3, 25), new DateTime(2014, 3, 25, 23, 0, 0), new DateTime(2014, 3, 27, 1, 0, 0) }.ToHashSet(); var coarseSymbols = new List <Symbol> { Symbols.SPY, Symbols.AAPL, Symbols.MSFT }; var coarseUsaSymbol = CoarseFundamental.CreateUniverseSymbol(Market.USA, false); var coarseDataEmittedCount = 0; var lastTime = DateTime.MinValue; var dataQueueHandler = new FuncDataQueueHandler(fdqh => { var time = timeProvider.GetUtcNow().ConvertFromUtc(TimeZones.NewYork); if (time != lastTime) { lastTime = time; if (coarseTimes.Contains(time)) { // emit coarse data at selected times var coarseData = new BaseDataCollection { Symbol = coarseUsaSymbol }; foreach (var symbol in coarseSymbols) { coarseData.Data.Add( new CoarseFundamental { Symbol = symbol, Time = time, Market = Market.USA, Value = 100 }); } coarseDataEmittedCount++; return(new List <BaseData> { coarseData }); } } return(Enumerable.Empty <BaseData>()); }); var feed = new TestableLiveTradingDataFeed(dataQueueHandler); var algorithm = new AlgorithmStub(feed); algorithm.SetLiveMode(true); var mock = new Mock <ITransactionHandler>(); mock.Setup(m => m.GetOpenOrders(It.IsAny <Func <Order, bool> >())).Returns(new List <Order>()); algorithm.Transactions.SetOrderProcessor(mock.Object); var synchronizer = new TestableLiveSynchronizer(timeProvider); synchronizer.Initialize(algorithm, algorithm.DataManager); var mapFileProvider = new LocalDiskMapFileProvider(); feed.Initialize(algorithm, new LiveNodePacket(), new BacktestingResultHandler(), mapFileProvider, new LocalDiskFactorFileProvider(mapFileProvider), new DefaultDataProvider(), algorithm.DataManager, synchronizer); var symbolIndex = 0; var coarseUniverseSelectionCount = 0; algorithm.AddUniverse( coarse => { coarseUniverseSelectionCount++; // rotate single symbol in universe if (symbolIndex == coarseSymbols.Count) { symbolIndex = 0; } return(new[] { coarseSymbols[symbolIndex++] }); }); algorithm.PostInitialize(); var cancellationTokenSource = new CancellationTokenSource(); Exception exceptionThrown = null; // create a timer to advance time much faster than realtime var timerInterval = TimeSpan.FromMilliseconds(50); var timer = Ref.Create <Timer>(null); timer.Value = new Timer(state => { try { // stop the timer to prevent reentrancy timer.Value.Change(Timeout.Infinite, Timeout.Infinite); var currentTime = timeProvider.GetUtcNow().ConvertFromUtc(TimeZones.NewYork); if (currentTime.Date > endDate.Date) { feed.Exit(); cancellationTokenSource.Cancel(); return; } timeProvider.Advance(TimeSpan.FromHours(1)); var activeSecuritiesCount = algorithm.ActiveSecurities.Count; Assert.That(activeSecuritiesCount <= 1); // restart the timer timer.Value.Change(timerInterval, timerInterval); } catch (Exception exception) { Log.Error(exception); exceptionThrown = exception; feed.Exit(); cancellationTokenSource.Cancel(); } }, null, TimeSpan.FromSeconds(1), timerInterval); foreach (var _ in synchronizer.StreamData(cancellationTokenSource.Token)) { } timer.Value.Dispose(); if (exceptionThrown != null) { throw new Exception("Exception in timer: ", exceptionThrown); } Assert.AreEqual(coarseTimes.Count, coarseDataEmittedCount); Assert.AreEqual(coarseTimes.Count, coarseUniverseSelectionCount); }
public void EmitsDailyQuandlFutureDataOverWeekends() { var tickers = new[] { "CHRIS/CME_ES1", "CHRIS/CME_ES2" }; var startDate = new DateTime(2018, 4, 1); var endDate = new DateTime(2018, 4, 20); // delete temp files foreach (var ticker in tickers) { var fileName = TestableQuandlFuture.GetLocalFileName(ticker, "test"); File.Delete(fileName); } var algorithm = new QCAlgorithm(); var dataManager = new DataManagerStub(algorithm); algorithm.SubscriptionManager.SetDataManager(dataManager); var symbols = tickers.Select(ticker => algorithm.AddData <TestableQuandlFuture>(ticker, Resolution.Daily).Symbol).ToList(); algorithm.PostInitialize(); var timeProvider = new ManualTimeProvider(TimeZones.NewYork); timeProvider.SetCurrentTime(startDate); var dataPointsEmitted = 0; var feed = RunLiveDataFeed(algorithm, startDate, symbols, timeProvider, dataManager); var lastFileWriteDate = DateTime.MinValue; // create a timer to advance time much faster than realtime and to simulate live Quandl data file updates var timerInterval = TimeSpan.FromMilliseconds(100); var timer = Ref.Create <Timer>(null); timer.Value = new Timer(state => { // stop the timer to prevent reentrancy timer.Value.Change(Timeout.Infinite, Timeout.Infinite); var currentTime = timeProvider.GetUtcNow().ConvertFromUtc(TimeZones.NewYork); if (currentTime.Date > endDate.Date) { Log.Trace($"Total data points emitted: {dataPointsEmitted}"); feed.Exit(); return; } if (currentTime.Date > lastFileWriteDate.Date) { foreach (var ticker in tickers) { var source = TestableQuandlFuture.GetLocalFileName(ticker, "csv"); // write new local file including only rows up to current date var outputFileName = TestableQuandlFuture.GetLocalFileName(ticker, "test"); var sb = new StringBuilder(); { using (var reader = new StreamReader(source)) { var firstLine = true; string line; while ((line = reader.ReadLine()) != null) { if (firstLine) { sb.AppendLine(line); firstLine = false; continue; } var csv = line.Split(','); var time = DateTime.ParseExact(csv[0], "yyyy-MM-dd", CultureInfo.InvariantCulture); if (time.Date >= currentTime.Date) { break; } sb.AppendLine(line); } } } if (currentTime.Date.DayOfWeek != DayOfWeek.Saturday && currentTime.Date.DayOfWeek != DayOfWeek.Sunday) { File.WriteAllText(outputFileName, sb.ToString()); Log.Trace($"Time:{currentTime} - Ticker:{ticker} - Files written:{++_countFilesWritten}"); } } lastFileWriteDate = currentTime; } // 30 minutes is the check interval for daily remote files, so we choose a smaller one to advance time timeProvider.Advance(TimeSpan.FromMinutes(15)); //Log.Trace($"Time advanced to: {timeProvider.GetUtcNow().ConvertFromUtc(TimeZones.NewYork)}"); // restart the timer timer.Value.Change(timerInterval, timerInterval); }, null, TimeSpan.FromSeconds(2), timerInterval); try { foreach (var timeSlice in feed) { foreach (var dataPoint in timeSlice.Slice.Values) { Log.Trace($"Data point emitted at {timeSlice.Slice.Time}: {dataPoint.Symbol.Value} {dataPoint.Value} {dataPoint.EndTime}"); dataPointsEmitted++; } } } catch (Exception exception) { Log.Trace($"Error: {exception}"); } timer.Value.Dispose(); Assert.AreEqual(14 * tickers.Length, dataPointsEmitted); }
public void RefreshesFutureChainUniverseOnDateChange() { var startTime = new DateTime(2018, 10, 17, 10, 0, 0); var timeProvider = new ManualTimeProvider(startTime); var symbolUniverse = new TestDataQueueUniverseProvider(timeProvider); var factory = new FuturesChainUniverseSubscriptionEnumeratorFactory(symbolUniverse, timeProvider); var canonicalSymbol = Symbol.Create(Futures.Indices.VIX, SecurityType.Future, Market.CBOE, "/VX"); var quoteCurrency = new Cash(Currencies.USD, 0, 1); var exchangeHours = MarketHoursDatabase.FromDataFolder().GetExchangeHours(Market.CBOE, canonicalSymbol, SecurityType.Future); var config = new SubscriptionDataConfig( typeof(ZipEntryName), canonicalSymbol, Resolution.Minute, TimeZones.Utc, TimeZones.Chicago, true, false, false, false, TickType.Quote, false, DataNormalizationMode.Raw ); var future = new Future( canonicalSymbol, exchangeHours, quoteCurrency, SymbolProperties.GetDefault(Currencies.USD), ErrorCurrencyConverter.Instance, RegisteredSecurityDataTypesProvider.Null, new SecurityCache() ); var universeSettings = new UniverseSettings(Resolution.Minute, 0, true, false, TimeSpan.Zero); var universe = new FuturesChainUniverse(future, universeSettings); var request = new SubscriptionRequest(true, universe, future, config, startTime, Time.EndOfTime); var enumerator = factory.CreateEnumerator(request, new DefaultDataProvider()); Assert.IsTrue(enumerator.MoveNext()); Assert.IsNotNull(enumerator.Current); Assert.AreEqual(1, symbolUniverse.TotalLookupCalls); var data = enumerator.Current as FuturesChainUniverseDataCollection; Assert.IsNotNull(data); Assert.AreEqual(1, data.Data.Count); timeProvider.Advance(Time.OneSecond); Assert.IsTrue(enumerator.MoveNext()); Assert.IsNull(enumerator.Current); Assert.AreEqual(1, symbolUniverse.TotalLookupCalls); timeProvider.Advance(Time.OneMinute); Assert.IsTrue(enumerator.MoveNext()); Assert.IsNull(enumerator.Current); Assert.AreEqual(1, symbolUniverse.TotalLookupCalls); timeProvider.Advance(Time.OneDay); Assert.IsTrue(enumerator.MoveNext()); Assert.IsNotNull(enumerator.Current); Assert.AreEqual(2, symbolUniverse.TotalLookupCalls); data = enumerator.Current as FuturesChainUniverseDataCollection; Assert.IsNotNull(data); Assert.AreEqual(2, data.Data.Count); timeProvider.Advance(Time.OneMinute); Assert.IsTrue(enumerator.MoveNext()); Assert.IsNull(enumerator.Current); Assert.AreEqual(2, symbolUniverse.TotalLookupCalls); enumerator.Dispose(); }
public void EmitsMappingEventsBasedOnCurrentMapFileAndTime(DataMappingMode dataMappingMode, string mappingDate, bool delayed) { var config = new SubscriptionDataConfig(typeof(TradeBar), Symbols.ES_Future_Chain, Resolution.Daily, TimeZones.NewYork, TimeZones.NewYork, true, true, false, dataMappingMode: dataMappingMode); var symbolMaps = new List <SubscriptionDataConfig.NewSymbolEventArgs>(); config.NewSymbol += (sender, args) => symbolMaps.Add(args); var time = new DateTime(2013, 05, 28); var cache = new SecurityCache(); cache.AddData(new Tick(time, config.Symbol, 20, 10)); var timeProvider = new ManualTimeProvider(time); var futureTicker1 = "es vhle2yxr5blt"; TestMapFileResolver.MapFile = new MapFile(Futures.Indices.SP500EMini, new [] { new MapFileRow(Time.BeginningOfTime, Futures.Indices.SP500EMini, Exchange.CME), new MapFileRow(new DateTime(2013, 06, 01), futureTicker1, Exchange.CME, DataMappingMode.FirstDayMonth), new MapFileRow(new DateTime(2013, 06, 15), futureTicker1, Exchange.CME, DataMappingMode.OpenInterest), new MapFileRow(new DateTime(2013, 06, 22), futureTicker1, Exchange.CME, DataMappingMode.LastTradingDay), }); IEnumerator <BaseData> enumerator; Assert.IsTrue(LiveAuxiliaryDataEnumerator.TryCreate(config, timeProvider, null, cache, new TestMapFileProvider(), TestGlobals.FactorFileProvider, time, out enumerator)); // get's mapped right away! Assert.AreEqual(futureTicker1.ToUpper(), config.MappedSymbol); Assert.AreEqual(1, symbolMaps.Count); Assert.AreEqual(Symbols.ES_Future_Chain, symbolMaps[0].Old); Assert.AreEqual(Futures.Indices.SP500EMini, symbolMaps[0].Old.ID.Symbol); Assert.AreEqual(Symbols.ES_Future_Chain, symbolMaps[0].New); Assert.AreEqual(futureTicker1.ToUpper(), symbolMaps[0].New.Underlying.ID.ToString()); Assert.IsTrue(enumerator.MoveNext()); Assert.IsNull(enumerator.Current); var expectedMappingDate = DateTime.ParseExact(mappingDate, DateFormat.EightCharacter, CultureInfo.InvariantCulture); if (delayed) { // we advance to the mapping date, without any new mapFile! timeProvider.Advance(expectedMappingDate.ConvertToUtc(config.ExchangeTimeZone) - timeProvider.GetUtcNow() + LiveAuxiliaryDataEnumerator.TradableDateOffset); } else { // just advance a day to show nothing happens until mapping time timeProvider.Advance(TimeSpan.FromDays(1)); } Assert.IsTrue(enumerator.MoveNext()); Assert.IsNull(enumerator.Current); var futureTicker2 = "es vk2zrh843z7l"; TestMapFileResolver.MapFile = new MapFile(Futures.Indices.SP500EMini, TestMapFileResolver.MapFile.Concat( new [] { new MapFileRow(new DateTime(2013, 09, 01), futureTicker2, Exchange.CME, DataMappingMode.FirstDayMonth), new MapFileRow(new DateTime(2013, 09, 14), futureTicker2, Exchange.CME, DataMappingMode.OpenInterest), new MapFileRow(new DateTime(2013, 09, 21), futureTicker2, Exchange.CME, DataMappingMode.LastTradingDay), })); Assert.IsTrue(enumerator.MoveNext()); Assert.IsNull(enumerator.Current); if (delayed) { // we got a new mapFile! advance the date and expect mapping to have happened timeProvider.Advance(TimeSpan.FromDays(1)); } else { // we advance to the mapping date timeProvider.Advance(expectedMappingDate.ConvertToUtc(config.ExchangeTimeZone) - timeProvider.GetUtcNow() + LiveAuxiliaryDataEnumerator.TradableDateOffset); } Assert.IsTrue(enumerator.MoveNext()); Assert.IsNotNull(enumerator.Current); Assert.AreEqual(2, symbolMaps.Count); Assert.AreEqual(Symbols.ES_Future_Chain, symbolMaps[1].Old); Assert.AreEqual(futureTicker1.ToUpper(), symbolMaps[1].Old.Underlying.ID.ToString()); Assert.AreEqual(Symbols.ES_Future_Chain, symbolMaps[1].New); Assert.AreEqual(futureTicker2.ToUpper(), symbolMaps[1].New.Underlying.ID.ToString()); Assert.AreEqual(futureTicker2.ToUpper(), config.MappedSymbol); Assert.AreEqual(futureTicker2.ToUpper(), (enumerator.Current as SymbolChangedEvent).NewSymbol); Assert.AreEqual(futureTicker1.ToUpper(), (enumerator.Current as SymbolChangedEvent).OldSymbol); Assert.AreEqual(config.Symbol, (enumerator.Current as SymbolChangedEvent).Symbol); Assert.AreEqual(timeProvider.GetUtcNow().Date, (enumerator.Current as SymbolChangedEvent).Time); Assert.IsTrue(enumerator.MoveNext()); Assert.IsNull(enumerator.Current); }
public void EmitsDailyQuandlFutureDataOverWeekends() { RemoteFileSubscriptionStreamReader.SetDownloadProvider(new Api.Api()); var tickers = new[] { "CHRIS/CME_ES1", "CHRIS/CME_ES2" }; var startDate = new DateTime(2018, 4, 1); var endDate = new DateTime(2018, 4, 20); // delete temp files foreach (var ticker in tickers) { var fileName = TestableQuandlFuture.GetLocalFileName(ticker, "test"); File.Delete(fileName); } var algorithm = new QCAlgorithm(); CreateDataFeed(); var dataManager = new DataManagerStub(algorithm, _feed); algorithm.SubscriptionManager.SetDataManager(dataManager); var symbols = tickers.Select(ticker => algorithm.AddData <TestableQuandlFuture>(ticker, Resolution.Daily).Symbol).ToList(); var timeProvider = new ManualTimeProvider(TimeZones.NewYork); timeProvider.SetCurrentTime(startDate); var dataPointsEmitted = 0; RunLiveDataFeed(algorithm, startDate, symbols, timeProvider, dataManager); var cancellationTokenSource = new CancellationTokenSource(); var lastFileWriteDate = DateTime.MinValue; // create a timer to advance time much faster than realtime and to simulate live Quandl data file updates var timerInterval = TimeSpan.FromMilliseconds(20); var timer = Ref.Create <Timer>(null); timer.Value = new Timer(state => { try { var currentTime = timeProvider.GetUtcNow().ConvertFromUtc(TimeZones.NewYork); if (currentTime.Date > endDate.Date) { Log.Trace($"Total data points emitted: {dataPointsEmitted.ToStringInvariant()}"); _feed.Exit(); cancellationTokenSource.Cancel(); return; } if (currentTime.Date > lastFileWriteDate.Date) { foreach (var ticker in tickers) { var source = TestableQuandlFuture.GetLocalFileName(ticker, "csv"); // write new local file including only rows up to current date var outputFileName = TestableQuandlFuture.GetLocalFileName(ticker, "test"); var sb = new StringBuilder(); { using (var reader = new StreamReader(source)) { var firstLine = true; string line; while ((line = reader.ReadLine()) != null) { if (firstLine) { sb.AppendLine(line); firstLine = false; continue; } var csv = line.Split(','); var time = Parse.DateTimeExact(csv[0], "yyyy-MM-dd"); if (time.Date >= currentTime.Date) { break; } sb.AppendLine(line); } } } if (currentTime.Date.DayOfWeek != DayOfWeek.Saturday && currentTime.Date.DayOfWeek != DayOfWeek.Sunday) { var fileContent = sb.ToString(); try { File.WriteAllText(outputFileName, fileContent); } catch (IOException) { Log.Error("IOException: will sleep 200ms and retry once more"); // lets sleep 200ms and retry once more, consumer could be reading the file // this exception happens in travis intermittently, GH issue 3273 Thread.Sleep(200); File.WriteAllText(outputFileName, fileContent); } Log.Trace($"Time:{currentTime} - Ticker:{ticker} - Files written:{++_countFilesWritten}"); } } lastFileWriteDate = currentTime; } // 30 minutes is the check interval for daily remote files, so we choose a smaller one to advance time timeProvider.Advance(TimeSpan.FromMinutes(20)); //Log.Trace($"Time advanced to: {timeProvider.GetUtcNow().ConvertFromUtc(TimeZones.NewYork)}"); // restart the timer timer.Value.Change(timerInterval.Milliseconds, Timeout.Infinite); } catch (Exception exception) { Log.Error(exception); _feed.Exit(); cancellationTokenSource.Cancel(); } }, null, timerInterval.Milliseconds, Timeout.Infinite); try { foreach (var timeSlice in _synchronizer.StreamData(cancellationTokenSource.Token)) { foreach (var dataPoint in timeSlice.Slice.Values) { Log.Trace($"Data point emitted at {timeSlice.Slice.Time.ToStringInvariant()}: " + $"{dataPoint.Symbol.Value} {dataPoint.Value.ToStringInvariant()} " + $"{dataPoint.EndTime.ToStringInvariant()}" ); dataPointsEmitted++; } } } catch (Exception exception) { Log.Trace($"Error: {exception}"); } timer.Value.Dispose(); dataManager.RemoveAllSubscriptions(); Assert.AreEqual(14 * tickers.Length, dataPointsEmitted); }
public void CoarseUniverseRotatesActiveSecurity() { var startDate = new DateTime(2014, 3, 24); var endDate = new DateTime(2014, 3, 29); var timeProvider = new ManualTimeProvider(TimeZones.NewYork); timeProvider.SetCurrentTime(startDate); var coarseTimes = new List <DateTime> { new DateTime(2014, 3, 25, 5, 0, 0, 0), new DateTime(2014, 3, 26, 5, 0, 0, 0), new DateTime(2014, 3, 27, 5, 0, 0, 0), new DateTime(2014, 3, 28, 5, 0, 0, 0), new DateTime(2014, 3, 29, 5, 0, 0, 0) }.ToHashSet(); var coarseSymbols = new List <Symbol> { Symbols.SPY, Symbols.AAPL, Symbols.MSFT }; var emitted = new AutoResetEvent(false); var dataQueueHandler = new FuncDataQueueHandler(fdqh => Enumerable.Empty <BaseData>(), timeProvider); var feed = new TestableLiveTradingDataFeed(dataQueueHandler); var algorithm = new AlgorithmStub(feed); algorithm.SetLiveMode(true); var mock = new Mock <ITransactionHandler>(); mock.Setup(m => m.GetOpenOrders(It.IsAny <Func <Order, bool> >())).Returns(new List <Order>()); algorithm.Transactions.SetOrderProcessor(mock.Object); var synchronizer = new TestableLiveSynchronizer(timeProvider); synchronizer.Initialize(algorithm, algorithm.DataManager); feed.Initialize(algorithm, new LiveNodePacket(), new BacktestingResultHandler(), TestGlobals.MapFileProvider, TestGlobals.FactorFileProvider, TestGlobals.DataProvider, algorithm.DataManager, synchronizer, new DataChannelProvider()); var symbolIndex = 0; var coarseUniverseSelectionCount = 0; algorithm.AddUniverse( coarse => { Log.Trace($"Emitted at {algorithm.Time}. Coarse {coarse.First().Time} to {coarse.First().EndTime}"); Interlocked.Increment(ref coarseUniverseSelectionCount); emitted.Set(); // rotate single symbol in universe if (symbolIndex == coarseSymbols.Count) { symbolIndex = 0; } return(new[] { coarseSymbols[symbolIndex++] }); }); algorithm.PostInitialize(); var cancellationTokenSource = new CancellationTokenSource(); Exception exceptionThrown = null; // create a timer to advance time much faster than realtime var timerInterval = TimeSpan.FromMilliseconds(5); var timer = Ref.Create <Timer>(null); timer.Value = new Timer(state => { try { var currentTime = timeProvider.GetUtcNow().ConvertFromUtc(TimeZones.NewYork); if (currentTime.Date > endDate.Date) { feed.Exit(); cancellationTokenSource.Cancel(); return; } timeProvider.Advance(TimeSpan.FromHours(1)); var time = timeProvider.GetUtcNow().ConvertFromUtc(TimeZones.NewYork); algorithm.SetDateTime(timeProvider.GetUtcNow()); if (coarseTimes.Contains(time)) { // lets wait for coarse to emit if (!emitted.WaitOne(TimeSpan.FromMilliseconds(15000))) { throw new TimeoutException($"Timeout waiting for coarse to emit at {time}"); } } var activeSecuritiesCount = algorithm.ActiveSecurities.Count; Assert.That(activeSecuritiesCount <= 1); // restart the timer timer.Value.Change(timerInterval, Timeout.InfiniteTimeSpan); } catch (Exception exception) { Log.Error(exception); exceptionThrown = exception; feed.Exit(); cancellationTokenSource.Cancel(); } }, null, timerInterval, Timeout.InfiniteTimeSpan); foreach (var _ in synchronizer.StreamData(cancellationTokenSource.Token)) { } timer.Value.DisposeSafely(); algorithm.DataManager.RemoveAllSubscriptions(); dataQueueHandler.DisposeSafely(); synchronizer.DisposeSafely(); emitted.DisposeSafely(); if (exceptionThrown != null) { throw new Exception("Exception in timer: ", exceptionThrown); } Assert.AreEqual(coarseTimes.Count, coarseUniverseSelectionCount, message: "coarseUniverseSelectionCount"); }