Example #1
0
        public void EmitsDailyQuandlFutureDataOverWeekends()
        {
            var tickers   = new[] { "CHRIS/CME_ES1", "CHRIS/CME_ES2" };
            var startDate = new DateTime(2018, 4, 1);
            var endDate   = new DateTime(2018, 4, 20);

            // delete temp files
            foreach (var ticker in tickers)
            {
                var fileName = TestableQuandlFuture.GetLocalFileName(ticker, "test");
                File.Delete(fileName);
            }

            var algorithm = new QCAlgorithm();

            CreateDataFeed();
            var dataManager = new DataManagerStub(algorithm, _feed);

            algorithm.SubscriptionManager.SetDataManager(dataManager);

            var symbols = tickers.Select(ticker => algorithm.AddData <TestableQuandlFuture>(ticker, Resolution.Daily).Symbol).ToList();

            var timeProvider = new ManualTimeProvider(TimeZones.NewYork);

            timeProvider.SetCurrentTime(startDate);

            var dataPointsEmitted = 0;

            RunLiveDataFeed(algorithm, startDate, symbols, timeProvider, dataManager);

            var cancellationTokenSource = new CancellationTokenSource();
            var lastFileWriteDate       = DateTime.MinValue;

            // create a timer to advance time much faster than realtime and to simulate live Quandl data file updates
            var timerInterval = TimeSpan.FromMilliseconds(100);
            var timer         = Ref.Create <Timer>(null);

            timer.Value = new Timer(state =>
            {
                // stop the timer to prevent reentrancy
                timer.Value.Change(Timeout.Infinite, Timeout.Infinite);

                var currentTime = timeProvider.GetUtcNow().ConvertFromUtc(TimeZones.NewYork);

                if (currentTime.Date > endDate.Date)
                {
                    Log.Trace($"Total data points emitted: {dataPointsEmitted}");

                    _feed.Exit();
                    cancellationTokenSource.Cancel();
                    return;
                }

                if (currentTime.Date > lastFileWriteDate.Date)
                {
                    foreach (var ticker in tickers)
                    {
                        var source = TestableQuandlFuture.GetLocalFileName(ticker, "csv");

                        // write new local file including only rows up to current date
                        var outputFileName = TestableQuandlFuture.GetLocalFileName(ticker, "test");

                        var sb = new StringBuilder();
                        {
                            using (var reader = new StreamReader(source))
                            {
                                var firstLine = true;
                                string line;
                                while ((line = reader.ReadLine()) != null)
                                {
                                    if (firstLine)
                                    {
                                        sb.AppendLine(line);
                                        firstLine = false;
                                        continue;
                                    }

                                    var csv  = line.Split(',');
                                    var time = DateTime.ParseExact(csv[0], "yyyy-MM-dd", CultureInfo.InvariantCulture);
                                    if (time.Date >= currentTime.Date)
                                    {
                                        break;
                                    }

                                    sb.AppendLine(line);
                                }
                            }
                        }

                        if (currentTime.Date.DayOfWeek != DayOfWeek.Saturday && currentTime.Date.DayOfWeek != DayOfWeek.Sunday)
                        {
                            File.WriteAllText(outputFileName, sb.ToString());

                            Log.Trace($"Time:{currentTime} - Ticker:{ticker} - Files written:{++_countFilesWritten}");
                        }
                    }

                    lastFileWriteDate = currentTime;
                }

                // 30 minutes is the check interval for daily remote files, so we choose a smaller one to advance time
                timeProvider.Advance(TimeSpan.FromMinutes(15));

                //Log.Trace($"Time advanced to: {timeProvider.GetUtcNow().ConvertFromUtc(TimeZones.NewYork)}");

                // restart the timer
                timer.Value.Change(timerInterval, timerInterval);
            }, null, TimeSpan.FromSeconds(2), timerInterval);

            try
            {
                foreach (var timeSlice in _synchronizer.StreamData(cancellationTokenSource.Token))
                {
                    foreach (var dataPoint in timeSlice.Slice.Values)
                    {
                        Log.Trace($"Data point emitted at {timeSlice.Slice.Time}: {dataPoint.Symbol.Value} {dataPoint.Value} {dataPoint.EndTime}");
                        dataPointsEmitted++;
                    }
                }
            }
            catch (Exception exception)
            {
                Log.Trace($"Error: {exception}");
            }

            timer.Value.Dispose();
            Assert.AreEqual(14 * tickers.Length, dataPointsEmitted);
        }
Example #2
0
 public void SetUp()
 {
     CustomMockedFileBaseData.StartDate = _startDate;
     _manualTimeProvider = new ManualTimeProvider();
     _manualTimeProvider.SetCurrentTimeUtc(_startDate);
 }
        public void FillsForwardOnNulls()
        {
            var reference  = new DateTime(2015, 10, 08);
            var period     = Time.OneSecond;
            var underlying = new List <BaseData>
            {
                // 0 seconds
                new TradeBar(reference, Symbols.SPY, 10, 20, 5, 15, 123456, period),
                // 1 seconds
                null,
                // 3 seconds
                new TradeBar(reference.AddSeconds(2), Symbols.SPY, 100, 200, 50, 150, 1234560, period),
                null,
                null,
                null,
                null
            };

            var timeProvider = new ManualTimeProvider(TimeZones.NewYork);

            timeProvider.SetCurrentTime(reference);
            var exchange    = new SecurityExchange(SecurityExchangeHours.AlwaysOpen(TimeZones.NewYork));
            var fillForward = new LiveFillForwardEnumerator(timeProvider, underlying.GetEnumerator(), exchange, Ref.Create(Time.OneSecond), false, Time.EndOfTime, Time.OneSecond, exchange.TimeZone);

            // first point is always emitted
            Assert.IsTrue(fillForward.MoveNext());
            Assert.AreEqual(underlying[0], fillForward.Current);
            Assert.AreEqual(123456, ((TradeBar)fillForward.Current).Volume);

            // stepping again without advancing time does nothing, but we'll still
            // return true as per IEnumerator contract
            Assert.IsTrue(fillForward.MoveNext());
            Assert.IsNull(fillForward.Current);

            timeProvider.SetCurrentTime(reference.AddSeconds(1));

            // non-null next will fill forward in between
            Assert.IsTrue(fillForward.MoveNext());
            Assert.AreEqual(underlying[0].EndTime, fillForward.Current.Time);
            Assert.AreEqual(underlying[0].Value, fillForward.Current.Value);
            Assert.IsTrue(fillForward.Current.IsFillForward);
            Assert.AreEqual(0, ((TradeBar)fillForward.Current).Volume);

            // even without stepping the time this will advance since non-null data is ready
            Assert.IsTrue(fillForward.MoveNext());
            Assert.AreEqual(underlying[2], fillForward.Current);
            Assert.AreEqual(1234560, ((TradeBar)fillForward.Current).Volume);

            // step ahead into null data territory
            timeProvider.SetCurrentTime(reference.AddSeconds(4));

            Assert.IsTrue(fillForward.MoveNext());
            Assert.AreEqual(underlying[2].Value, fillForward.Current.Value);
            Assert.AreEqual(timeProvider.GetUtcNow().ConvertFromUtc(TimeZones.NewYork), fillForward.Current.EndTime);
            Assert.IsTrue(fillForward.Current.IsFillForward);
            Assert.AreEqual(0, ((TradeBar)fillForward.Current).Volume);

            Assert.IsTrue(fillForward.MoveNext());
            Assert.IsNull(fillForward.Current);

            timeProvider.SetCurrentTime(reference.AddSeconds(5));

            Assert.IsTrue(fillForward.MoveNext());
            Assert.AreEqual(underlying[2].Value, fillForward.Current.Value);
            Assert.AreEqual(timeProvider.GetUtcNow().ConvertFromUtc(TimeZones.NewYork), fillForward.Current.EndTime);
            Assert.IsTrue(fillForward.Current.IsFillForward);
            Assert.AreEqual(0, ((TradeBar)fillForward.Current).Volume);

            timeProvider.SetCurrentTime(reference.AddSeconds(6));

            Assert.IsTrue(fillForward.MoveNext());
            Assert.AreEqual(underlying[2].Value, fillForward.Current.Value);
            Assert.AreEqual(timeProvider.GetUtcNow().ConvertFromUtc(TimeZones.NewYork), fillForward.Current.EndTime);
            Assert.IsTrue(fillForward.Current.IsFillForward);
            Assert.AreEqual(0, ((TradeBar)fillForward.Current).Volume);

            fillForward.Dispose();
        }
Example #4
0
        public void RemoteDataDoesNotIncreaseNumberOfSlices()
        {
            Config.Set("quandl-auth-token", "QUANDL-TOKEN");

            var startDate = new DateTime(2018, 4, 2);
            var endDate   = new DateTime(2018, 4, 19);
            var algorithm = new QCAlgorithm();

            var timeProvider = new ManualTimeProvider(TimeZones.NewYork);

            timeProvider.SetCurrentTime(startDate);
            var dataQueueHandler = new FuncDataQueueHandler(fdqh =>
            {
                var time = timeProvider.GetUtcNow().ConvertFromUtc(TimeZones.NewYork);
                var tick = new Tick(time, Symbols.SPY, 1.3m, 1.2m, 1.3m)
                {
                    TickType = TickType.Trade
                };
                var tick2 = new Tick(time, Symbols.AAPL, 1.3m, 1.2m, 1.3m)
                {
                    TickType = TickType.Trade
                };
                return(new[] { tick, tick2 });
            });

            CreateDataFeed(dataQueueHandler);
            var dataManager = new DataManagerStub(algorithm, _feed);

            algorithm.SubscriptionManager.SetDataManager(dataManager);
            var symbols = new List <Symbol>
            {
                algorithm.AddData <Quandl>("CBOE/VXV", Resolution.Daily).Symbol,
                algorithm.AddData <QuandlVix>("CBOE/VIX", Resolution.Daily).Symbol,
                algorithm.AddEquity("SPY", Resolution.Daily).Symbol,
                algorithm.AddEquity("AAPL", Resolution.Daily).Symbol
            };

            algorithm.PostInitialize();

            var cancellationTokenSource = new CancellationTokenSource();

            var dataPointsEmitted = 0;
            var slicesEmitted     = 0;

            RunLiveDataFeed(algorithm, startDate, symbols, timeProvider, dataManager);
            Thread.Sleep(5000); // Give remote sources a handicap, so the data is available in time

            // create a timer to advance time much faster than realtime and to simulate live Quandl data file updates
            var timerInterval = TimeSpan.FromMilliseconds(100);
            var timer         = Ref.Create <Timer>(null);

            timer.Value = new Timer(state =>
            {
                // stop the timer to prevent reentrancy
                timer.Value.Change(Timeout.Infinite, Timeout.Infinite);

                var currentTime = timeProvider.GetUtcNow().ConvertFromUtc(TimeZones.NewYork);

                if (currentTime.Date > endDate.Date)
                {
                    _feed.Exit();
                    cancellationTokenSource.Cancel();
                    return;
                }

                timeProvider.Advance(TimeSpan.FromHours(3));

                // restart the timer
                timer.Value.Change(timerInterval, timerInterval);
            }, null, TimeSpan.FromSeconds(2), timerInterval);

            try
            {
                foreach (var timeSlice in _synchronizer.StreamData(cancellationTokenSource.Token))
                {
                    if (timeSlice.Slice.HasData)
                    {
                        slicesEmitted++;
                        dataPointsEmitted += timeSlice.Slice.Values.Count;
                        Assert.IsTrue(timeSlice.Slice.Values.Any(x => x.Symbol == symbols[0]), $"Slice doesn't contain {symbols[0]}");
                        Assert.IsTrue(timeSlice.Slice.Values.Any(x => x.Symbol == symbols[1]), $"Slice doesn't contain {symbols[1]}");
                        Assert.IsTrue(timeSlice.Slice.Values.Any(x => x.Symbol == symbols[2]), $"Slice doesn't contain {symbols[2]}");
                        Assert.IsTrue(timeSlice.Slice.Values.Any(x => x.Symbol == symbols[3]), $"Slice doesn't contain {symbols[3]}");
                    }
                }
            }
            catch (Exception exception)
            {
                Log.Trace($"Error: {exception}");
            }

            timer.Value.Dispose();
            Assert.AreEqual(14, slicesEmitted);
            Assert.AreEqual(14 * symbols.Count, dataPointsEmitted);
        }
        public void RefreshesFutureChainUniverseOnDateChange()
        {
            var startTime    = new DateTime(2018, 10, 17, 10, 0, 0);
            var timeProvider = new ManualTimeProvider(startTime);

            var symbolUniverse = new TestDataQueueUniverseProvider(timeProvider);
            var factory        = new FuturesChainUniverseSubscriptionEnumeratorFactory(symbolUniverse, timeProvider);

            var canonicalSymbol = Symbol.Create(Futures.Indices.VIX, SecurityType.Future, Market.CBOE, "/VX");

            var quoteCurrency = new Cash(Currencies.USD, 0, 1);
            var exchangeHours = MarketHoursDatabase.FromDataFolder().GetExchangeHours(Market.CBOE, canonicalSymbol, SecurityType.Future);
            var config        = new SubscriptionDataConfig(
                typeof(ZipEntryName),
                canonicalSymbol,
                Resolution.Minute,
                TimeZones.Utc,
                TimeZones.Chicago,
                true,
                false,
                false,
                false,
                TickType.Quote,
                false,
                DataNormalizationMode.Raw
                );

            var future = new Future(
                canonicalSymbol,
                exchangeHours,
                quoteCurrency,
                SymbolProperties.GetDefault(Currencies.USD),
                ErrorCurrencyConverter.Instance,
                RegisteredSecurityDataTypesProvider.Null,
                new SecurityCache()
                );

            var universeSettings = new UniverseSettings(Resolution.Minute, 0, true, false, TimeSpan.Zero);
            var universe         = new FuturesChainUniverse(future, universeSettings);
            var request          = new SubscriptionRequest(true, universe, future, config, startTime, Time.EndOfTime);
            var enumerator       = factory.CreateEnumerator(request, new DefaultDataProvider());

            Assert.IsTrue(enumerator.MoveNext());
            Assert.IsNotNull(enumerator.Current);
            Assert.AreEqual(1, symbolUniverse.TotalLookupCalls);
            var data = enumerator.Current as FuturesChainUniverseDataCollection;

            Assert.IsNotNull(data);
            Assert.AreEqual(1, data.Data.Count);

            timeProvider.Advance(Time.OneSecond);

            Assert.IsTrue(enumerator.MoveNext());
            Assert.IsNull(enumerator.Current);
            Assert.AreEqual(1, symbolUniverse.TotalLookupCalls);

            timeProvider.Advance(Time.OneMinute);

            Assert.IsTrue(enumerator.MoveNext());
            Assert.IsNull(enumerator.Current);
            Assert.AreEqual(1, symbolUniverse.TotalLookupCalls);

            timeProvider.Advance(Time.OneDay);

            Assert.IsTrue(enumerator.MoveNext());
            Assert.IsNotNull(enumerator.Current);
            Assert.AreEqual(2, symbolUniverse.TotalLookupCalls);
            data = enumerator.Current as FuturesChainUniverseDataCollection;
            Assert.IsNotNull(data);
            Assert.AreEqual(2, data.Data.Count);

            timeProvider.Advance(Time.OneMinute);

            Assert.IsTrue(enumerator.MoveNext());
            Assert.IsNull(enumerator.Current);
            Assert.AreEqual(2, symbolUniverse.TotalLookupCalls);

            enumerator.Dispose();
        }
        public void CoarseUniverseRotatesActiveSecurity()
        {
            var startDate = new DateTime(2014, 3, 24);
            var endDate   = new DateTime(2014, 3, 31);

            var timeProvider = new ManualTimeProvider(TimeZones.NewYork);

            timeProvider.SetCurrentTime(startDate);

            var coarseTimes = new List <DateTime>
            {
                // coarse files go from the 24th (Monday) to the 28th (Friday)
                // they are emitted in the next day, excluding saturday
                new DateTime(2014, 3, 25, 5, 0, 0, 0),
                new DateTime(2014, 3, 26, 5, 0, 0, 0),
                new DateTime(2014, 3, 27, 5, 0, 0, 0),
                new DateTime(2014, 3, 28, 5, 0, 0, 0),
                // 29th is Saturday
                new DateTime(2014, 3, 30, 5, 0, 0, 0)
            }.ToHashSet();

            var coarseSymbols = new List <Symbol> {
                Symbols.SPY, Symbols.AAPL, Symbols.MSFT
            };

            var emitted          = new AutoResetEvent(false);
            var dataQueueHandler = new FuncDataQueueHandler(fdqh => Enumerable.Empty <BaseData>(), timeProvider);

            var feed = new TestableLiveTradingDataFeed(dataQueueHandler);

            var algorithm = new AlgorithmStub(feed);

            algorithm.SetLiveMode(true);

            var mock = new Mock <ITransactionHandler>();

            mock.Setup(m => m.GetOpenOrders(It.IsAny <Func <Order, bool> >())).Returns(new List <Order>());
            algorithm.Transactions.SetOrderProcessor(mock.Object);

            var synchronizer = new TestableLiveSynchronizer(timeProvider);

            synchronizer.Initialize(algorithm, algorithm.DataManager);

            feed.Initialize(algorithm, new LiveNodePacket(), new BacktestingResultHandler(),
                            TestGlobals.MapFileProvider, TestGlobals.FactorFileProvider, TestGlobals.DataProvider, algorithm.DataManager, synchronizer, new DataChannelProvider());

            var symbolIndex = 0;
            var coarseUniverseSelectionCount = 0;

            algorithm.AddUniverse(
                coarse =>
            {
                Log.Trace($"Emitted at {algorithm.Time}. Coarse {coarse.First().Time} to {coarse.First().EndTime}");
                Interlocked.Increment(ref coarseUniverseSelectionCount);
                emitted.Set();

                // rotate single symbol in universe
                if (symbolIndex == coarseSymbols.Count)
                {
                    symbolIndex = 0;
                }

                return(new[] { coarseSymbols[symbolIndex++] });
            });

            algorithm.PostInitialize();

            var cancellationTokenSource = new CancellationTokenSource();

            Exception exceptionThrown = null;

            // create a timer to advance time much faster than realtime
            var timerInterval = TimeSpan.FromMilliseconds(5);
            var timer         = Ref.Create <Timer>(null);

            timer.Value = new Timer(state =>
            {
                try
                {
                    var currentTime = timeProvider.GetUtcNow().ConvertFromUtc(TimeZones.NewYork);

                    if (currentTime.Date > endDate.Date)
                    {
                        feed.Exit();
                        cancellationTokenSource.Cancel();
                        return;
                    }

                    timeProvider.Advance(TimeSpan.FromHours(1));

                    var time = timeProvider.GetUtcNow().ConvertFromUtc(TimeZones.NewYork);
                    algorithm.SetDateTime(timeProvider.GetUtcNow());
                    if (coarseTimes.Contains(time))
                    {
                        // lets wait for coarse to emit
                        if (!emitted.WaitOne(TimeSpan.FromMilliseconds(15000)))
                        {
                            throw new TimeoutException($"Timeout waiting for coarse to emit at {time}");
                        }
                    }
                    var activeSecuritiesCount = algorithm.ActiveSecurities.Count;

                    Assert.That(activeSecuritiesCount <= 1);

                    // restart the timer
                    timer.Value.Change(timerInterval, Timeout.InfiniteTimeSpan);
                }
                catch (Exception exception)
                {
                    Log.Error(exception);
                    exceptionThrown = exception;

                    feed.Exit();
                    cancellationTokenSource.Cancel();
                }
            }, null, timerInterval, Timeout.InfiniteTimeSpan);

            foreach (var _ in synchronizer.StreamData(cancellationTokenSource.Token))
            {
            }

            timer.Value.DisposeSafely();
            algorithm.DataManager.RemoveAllSubscriptions();
            dataQueueHandler.DisposeSafely();
            synchronizer.DisposeSafely();
            emitted.DisposeSafely();

            if (exceptionThrown != null)
            {
                throw new Exception("Exception in timer: ", exceptionThrown);
            }

            Assert.AreEqual(coarseTimes.Count, coarseUniverseSelectionCount, message: "coarseUniverseSelectionCount");
        }
        public void RefreshesOptionChainUniverseOnDateChange()
        {
            var startTime    = new DateTime(2018, 10, 19, 10, 0, 0);
            var timeProvider = new ManualTimeProvider(startTime);

            var canonicalSymbol = Symbol.Create("SPY", SecurityType.Option, Market.USA, "?SPY");

            var quoteCurrency = new Cash(Currencies.USD, 0, 1);
            var exchangeHours = MarketHoursDatabase.FromDataFolder().GetExchangeHours(Market.USA, canonicalSymbol, SecurityType.Option);
            var config        = new SubscriptionDataConfig(
                typeof(ZipEntryName),
                canonicalSymbol,
                Resolution.Minute,
                TimeZones.Utc,
                TimeZones.NewYork,
                true,
                false,
                false,
                false,
                TickType.Quote,
                false,
                DataNormalizationMode.Raw
                );

            var option = new Option(
                canonicalSymbol,
                exchangeHours,
                quoteCurrency,
                new OptionSymbolProperties(SymbolProperties.GetDefault(Currencies.USD)),
                ErrorCurrencyConverter.Instance,
                RegisteredSecurityDataTypesProvider.Null,
                new SecurityCache(),
                null
                );

            var fillForwardResolution = Ref.CreateReadOnly(() => Resolution.Minute.ToTimeSpan());
            var symbolUniverse        = new TestDataQueueUniverseProvider(timeProvider);
            EnqueueableEnumerator <BaseData> underlyingEnumerator = null;
            Func <SubscriptionRequest, IEnumerator <BaseData> > underlyingEnumeratorFunc =
                (req) =>
            {
                underlyingEnumerator = new EnqueueableEnumerator <BaseData>();
                return(new LiveFillForwardEnumerator(
                           timeProvider,
                           underlyingEnumerator,
                           option.Exchange,
                           fillForwardResolution,
                           false,
                           Time.EndOfTime,
                           Resolution.Minute.ToTimeSpan(),
                           TimeZones.Utc));
            };
            var factory = new OptionChainUniverseSubscriptionEnumeratorFactory(underlyingEnumeratorFunc, symbolUniverse, timeProvider);

            var universeSettings = new UniverseSettings(Resolution.Minute, 0, true, false, TimeSpan.Zero);
            var universe         = new OptionChainUniverse(option, universeSettings, true);
            var request          = new SubscriptionRequest(true, universe, option, config, startTime, Time.EndOfTime);
            var enumerator       = (DataQueueOptionChainUniverseDataCollectionEnumerator)factory.CreateEnumerator(request, new DefaultDataProvider());

            // 2018-10-19 10:00 AM UTC
            underlyingEnumerator.Enqueue(new Tick {
                Symbol = Symbols.SPY, Value = 280m
            });

            // 2018-10-19 10:01 AM UTC
            timeProvider.Advance(Time.OneMinute);

            underlyingEnumerator.Enqueue(new Tick {
                Symbol = Symbols.SPY, Value = 280m
            });

            Assert.IsTrue(enumerator.MoveNext());
            Assert.IsNotNull(enumerator.Current);
            Assert.AreEqual(1, symbolUniverse.TotalLookupCalls);
            var data = enumerator.Current;

            Assert.IsNotNull(data);
            Assert.AreEqual(1, data.Data.Count);
            Assert.IsNotNull(data.Underlying);

            // 2018-10-19 10:02 AM UTC
            timeProvider.Advance(Time.OneMinute);

            underlyingEnumerator.Enqueue(new Tick {
                Symbol = Symbols.SPY, Value = 280m
            });

            Assert.IsTrue(enumerator.MoveNext());
            Assert.IsNotNull(enumerator.Current);
            Assert.AreEqual(1, symbolUniverse.TotalLookupCalls);
            data = enumerator.Current;
            Assert.IsNotNull(data);
            Assert.AreEqual(1, data.Data.Count);
            Assert.IsNotNull(data.Underlying);

            // 2018-10-19 10:03 AM UTC
            timeProvider.Advance(Time.OneMinute);

            underlyingEnumerator.Enqueue(new Tick {
                Symbol = Symbols.SPY, Value = 280m
            });

            Assert.IsTrue(enumerator.MoveNext());
            Assert.IsNotNull(enumerator.Current);
            Assert.AreEqual(1, symbolUniverse.TotalLookupCalls);
            data = enumerator.Current;
            Assert.IsNotNull(data);
            Assert.AreEqual(1, data.Data.Count);
            Assert.IsNotNull(data.Underlying);

            // 2018-10-20 10:03 AM UTC
            timeProvider.Advance(Time.OneDay);

            underlyingEnumerator.Enqueue(new Tick {
                Symbol = Symbols.SPY, Value = 280m
            });

            Assert.IsTrue(enumerator.MoveNext());
            Assert.IsNotNull(enumerator.Current);
            Assert.AreEqual(2, symbolUniverse.TotalLookupCalls);
            data = enumerator.Current;
            Assert.IsNotNull(data);
            Assert.AreEqual(2, data.Data.Count);
            Assert.IsNotNull(data.Underlying);

            // 2018-10-20 10:04 AM UTC
            timeProvider.Advance(Time.OneMinute);

            underlyingEnumerator.Enqueue(new Tick {
                Symbol = Symbols.SPY, Value = 280m
            });

            Assert.IsTrue(enumerator.MoveNext());
            Assert.IsNotNull(enumerator.Current);
            Assert.AreEqual(2, symbolUniverse.TotalLookupCalls);
            data = enumerator.Current;
            Assert.IsNotNull(data);
            Assert.AreEqual(2, data.Data.Count);
            Assert.IsNotNull(data.Underlying);

            enumerator.Dispose();
        }
 public void Setup()
 {
     _timeProvider = new ManualTimeProvider(new DateTime(2020, 1, 1, 0, 0, 0, DateTimeKind.Utc));
     TimeService.SetProvider(_timeProvider);
     _dut = new CacheManager();
 }
        public void LiveFillForwardEnumeratorDoesNotStall()
        {
            var now          = DateTime.UtcNow;
            var timeProvider = new ManualTimeProvider(new DateTime(2020, 5, 21, 9, 40, 0, 100), TimeZones.NewYork);

            var enqueueableEnumerator = new EnqueueableEnumerator <BaseData>();
            var fillForwardEnumerator = new LiveFillForwardEnumerator(
                timeProvider,
                enqueueableEnumerator,
                new SecurityExchange(MarketHoursDatabase.FromDataFolder()
                                     .ExchangeHoursListing
                                     .First(kvp => kvp.Key.Market == Market.USA && kvp.Key.SecurityType == SecurityType.Equity)
                                     .Value
                                     .ExchangeHours),
                Ref.CreateReadOnly(() => Resolution.Minute.ToTimeSpan()),
                false,
                now,
                Resolution.Minute.ToTimeSpan(),
                TimeZones.NewYork,
                new DateTime(2020, 5, 21)
                );
            var openingBar = new TradeBar
            {
                Open    = 0.01m,
                High    = 0.01m,
                Low     = 0.01m,
                Close   = 0.01m,
                Volume  = 1,
                EndTime = new DateTime(2020, 5, 21, 9, 40, 0),
                Symbol  = Symbols.AAPL
            };
            var secondBar = new TradeBar
            {
                Open    = 1m,
                High    = 2m,
                Low     = 1m,
                Close   = 2m,
                Volume  = 100,
                EndTime = new DateTime(2020, 5, 21, 9, 42, 0),
                Symbol  = Symbols.AAPL
            };


            // Enqueue the first point, which will be emitted ASAP.
            enqueueableEnumerator.Enqueue(openingBar);
            Assert.IsTrue(fillForwardEnumerator.MoveNext());
            Assert.NotNull(fillForwardEnumerator.Current);
            Assert.AreEqual(openingBar.Open, ((TradeBar)fillForwardEnumerator.Current).Open);
            Assert.AreEqual(openingBar.EndTime, fillForwardEnumerator.Current.EndTime);

            // Advance the time, we expect a fill-forward bar.
            timeProvider.SetCurrentTime(new DateTime(2020, 5, 21, 9, 41, 0, 100));
            Assert.IsTrue(fillForwardEnumerator.MoveNext());
            Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward);
            Assert.AreEqual(openingBar.Open, ((TradeBar)fillForwardEnumerator.Current).Open);
            Assert.AreEqual(openingBar.EndTime.AddMinutes(1), fillForwardEnumerator.Current.EndTime);

            // Now we expect data. The secondBar should be fill-forwarded from here on out after the MoveNext
            timeProvider.SetCurrentTime(new DateTime(2020, 5, 21, 9, 42, 0, 100));
            enqueueableEnumerator.Enqueue(secondBar);
            Assert.IsTrue(fillForwardEnumerator.MoveNext());
            Assert.IsFalse(fillForwardEnumerator.Current.IsFillForward);
        }