public void CreatedEquityIsNotAddedToSymbolCache() { SymbolCache.Clear(); var algorithm = new AlgorithmStub(new MockDataFeed()); algorithm.SetEndDate(Time.EndOfTime); algorithm.SetStartDate(DateTime.UtcNow.Subtract(TimeSpan.FromDays(10))); algorithm.AddUniverse(CoarseSelectionFunction, FineSelectionFunction); var universe = algorithm.UniverseManager.Values.First(); var securityChanges = algorithm.DataManager.UniverseSelection.ApplyUniverseSelection( universe, algorithm.EndDate.ConvertToUtc(algorithm.TimeZone).Subtract(TimeSpan.FromDays(1)), new BaseDataCollection( DateTime.UtcNow, Symbol.Create("GOOG", SecurityType.Equity, Market.USA), new List <BaseData>() ) ); Symbol symbol; Assert.AreEqual(1, securityChanges.AddedSecurities.Count); Assert.AreEqual(Symbol.Create("SPY", SecurityType.Equity, Market.USA), securityChanges.AddedSecurities.First().Symbol); Assert.IsFalse(SymbolCache.TryGetSymbol("GOOG", out symbol)); Assert.IsFalse(SymbolCache.TryGetSymbol("SPY", out symbol)); }
private IDataFeed RunDataFeed(out FuncDataQueueHandler dataQueueHandler, Func <FuncDataQueueHandler, IEnumerable <BaseData> > getNextTicksFunction = null, Resolution resolution = Resolution.Second, List <string> equities = null, List <string> forex = null) { _algorithm.SetStartDate(_startDate); var lastTime = _manualTimeProvider.GetUtcNow(); getNextTicksFunction = getNextTicksFunction ?? (fdqh => { var time = _manualTimeProvider.GetUtcNow(); if (time == lastTime) { return(Enumerable.Empty <BaseData>()); } lastTime = time; return(fdqh.Subscriptions.Where(symbol => !_algorithm.UniverseManager.ContainsKey(symbol)) // its not a universe .Select(symbol => new Tick(lastTime.ConvertFromUtc(TimeZones.NewYork), symbol, 1, 2) { Quantity = 1, // Symbol could not be in the Securities collections for the custom Universe tests. AlgorithmManager is in charge of adding them, and we are not executing that code here. TickType = _algorithm.Securities.ContainsKey(symbol) ? _algorithm.Securities[symbol].SubscriptionDataConfig.TickType : TickType.Trade })); }); // job is used to send into DataQueueHandler var job = new LiveNodePacket(); // result handler is used due to dependency in SubscriptionDataReader var resultHandler = new BacktestingResultHandler(); dataQueueHandler = new FuncDataQueueHandler(getNextTicksFunction); var feed = new TestableLiveTradingDataFeed(dataQueueHandler, _manualTimeProvider); var mapFileProvider = new LocalDiskMapFileProvider(); var fileProvider = new DefaultDataProvider(); var dataManager = new DataManager(feed, new UniverseSelection(feed, _algorithm), _algorithm.Settings, _algorithm.TimeKeeper); _algorithm.SubscriptionManager.SetDataManager(dataManager); _algorithm.AddSecurities(resolution, equities, forex); feed.Initialize(_algorithm, job, resultHandler, mapFileProvider, new LocalDiskFactorFileProvider(mapFileProvider), fileProvider, dataManager); _algorithm.PostInitialize(); Thread.Sleep(150); // small handicap for the data to be pumped so TimeSlices have data of all subscriptions var feedThreadStarted = new ManualResetEvent(false); Task.Factory.StartNew(() => { feedThreadStarted.Set(); feed.Run(); }); // wait for feed.Run to actually begin feedThreadStarted.WaitOne(); return(feed); }
private IDataFeed RunDataFeed(out FuncDataQueueHandler dataQueueHandler, Func <FuncDataQueueHandler, IEnumerable <BaseData> > getNextTicksFunction = null, Resolution resolution = Resolution.Second, List <string> equities = null, List <string> forex = null, List <string> crypto = null) { _algorithm.SetStartDate(_startDate); var lastTime = _manualTimeProvider.GetUtcNow(); getNextTicksFunction = getNextTicksFunction ?? (fdqh => { var time = _manualTimeProvider.GetUtcNow(); if (time == lastTime) { return(Enumerable.Empty <BaseData>()); } lastTime = time; var tickTime = lastTime.AddMinutes(-1).ConvertFromUtc(TimeZones.NewYork); return(fdqh.Subscriptions.Where(symbol => !_algorithm.UniverseManager.ContainsKey(symbol)) // its not a universe .Select(symbol => new Tick(tickTime, symbol, 1, 2) { Quantity = 1, // Symbol could not be in the Securities collections for the custom Universe tests. AlgorithmManager is in charge of adding them, and we are not executing that code here. TickType = _algorithm.Securities.ContainsKey(symbol) ? _algorithm.Securities[symbol].SubscriptionDataConfig.TickType : TickType.Trade }).ToList()); }); // job is used to send into DataQueueHandler var job = new LiveNodePacket(); // result handler is used due to dependency in SubscriptionDataReader var resultHandler = new BacktestingResultHandler(); dataQueueHandler = new FuncDataQueueHandler(getNextTicksFunction); var feed = new TestableLiveTradingDataFeed(dataQueueHandler); var mapFileProvider = new LocalDiskMapFileProvider(); var fileProvider = new DefaultDataProvider(); var marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); var symbolPropertiesDataBase = SymbolPropertiesDatabase.FromDataFolder(); var securityService = new SecurityService(_algorithm.Portfolio.CashBook, marketHoursDatabase, symbolPropertiesDataBase, _algorithm); _algorithm.Securities.SetSecurityService(securityService); _dataManager = new DataManager(feed, new UniverseSelection(_algorithm, securityService), _algorithm, _algorithm.TimeKeeper, marketHoursDatabase, true); _algorithm.SubscriptionManager.SetDataManager(_dataManager); _algorithm.AddSecurities(resolution, equities, forex, crypto); _synchronizer = new TestableLiveSynchronizer(_manualTimeProvider); _synchronizer.Initialize(_algorithm, _dataManager); feed.Initialize(_algorithm, job, resultHandler, mapFileProvider, new LocalDiskFactorFileProvider(mapFileProvider), fileProvider, _dataManager, _synchronizer); _algorithm.PostInitialize(); Thread.Sleep(150); // small handicap for the data to be pumped so TimeSlices have data of all subscriptions return(feed); }
public void RemovalFromUniverseAndDataFeedMakesSecurityNotTradable() { SymbolCache.Clear(); var algorithm = new AlgorithmStub(new MockDataFeedWithSubscription()); var orderProcessorMock = new Mock <IOrderProcessor>(); orderProcessorMock.Setup(m => m.GetOpenOrders(It.IsAny <Func <Order, bool> >())).Returns(new List <Order>()); algorithm.Transactions.SetOrderProcessor(orderProcessorMock.Object); algorithm.SetStartDate(2012, 3, 27); algorithm.SetEndDate(2012, 3, 30); algorithm.AddUniverse("my-custom-universe", dt => dt.Day < 30 ? new List <string> { "CPRT" } : Enumerable.Empty <string>()); // OnEndOfTimeStep will add all pending universe additions algorithm.OnEndOfTimeStep(); var universe = algorithm.UniverseManager.Values.First(); var securityChanges = algorithm.DataManager.UniverseSelection.ApplyUniverseSelection( universe, algorithm.EndDate.ConvertToUtc(algorithm.TimeZone).Subtract(TimeSpan.FromDays(2)), new BaseDataCollection( algorithm.UtcTime, Symbol.Create("CPRT", SecurityType.Equity, Market.USA), new List <BaseData>() ) ); Assert.AreEqual(1, securityChanges.AddedSecurities.Count); Assert.AreEqual(0, securityChanges.RemovedSecurities.Count); var security = securityChanges.AddedSecurities.First(); Assert.IsTrue(security.IsTradable); securityChanges = algorithm.DataManager.UniverseSelection.ApplyUniverseSelection( universe, algorithm.EndDate.ConvertToUtc(algorithm.TimeZone), new BaseDataCollection( algorithm.UtcTime, Symbol.Create("CPRT", SecurityType.Equity, Market.USA), new List <BaseData>() ) ); Assert.AreEqual(0, securityChanges.AddedSecurities.Count); Assert.AreEqual(1, securityChanges.RemovedSecurities.Count); Assert.AreEqual(security.Symbol, securityChanges.RemovedSecurities.First().Symbol); Assert.IsFalse(security.IsTradable); }
public void OptionChainEnumerator(bool fillForward) { var job = new BacktestNodePacket(); var resultHandler = new BacktestingResultHandler(); var feed = new FileSystemDataFeed(); var algorithm = new AlgorithmStub(feed); algorithm.Transactions.SetOrderProcessor(new FakeOrderProcessor()); algorithm.SetStartDate(new DateTime(2014, 06, 06)); algorithm.SetEndDate(new DateTime(2014, 06, 09)); algorithm.SetOptionChainProvider(new BacktestingOptionChainProvider(TestGlobals.DataCacheProvider, TestGlobals.MapFileProvider)); var dataPermissionManager = new DataPermissionManager(); using var synchronizer = new Synchronizer(); synchronizer.Initialize(algorithm, algorithm.DataManager); feed.Initialize(algorithm, job, resultHandler, TestGlobals.MapFileProvider, TestGlobals.FactorFileProvider, TestGlobals.DataProvider, algorithm.DataManager, synchronizer, dataPermissionManager.DataChannelProvider); var option = algorithm.AddOption("AAPL", fillDataForward: fillForward); option.SetFilter(filter => filter.FrontMonth()); algorithm.PostInitialize(); using var cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(30)); var count = 0; var lastMonth = algorithm.StartDate.Month; foreach (var timeSlice in synchronizer.StreamData(cancellationTokenSource.Token)) { if (!timeSlice.IsTimePulse && timeSlice.UniverseData?.Count > 0) { var baseDataCollection = timeSlice.UniverseData.Single().Value; if (baseDataCollection.Symbol.SecurityType == SecurityType.Option) { var nyTime = timeSlice.Time.ConvertFromUtc(algorithm.TimeZone); Assert.AreEqual(new TimeSpan(9, 30, 0).Add(TimeSpan.FromMinutes((count % 390) + 1)), nyTime.TimeOfDay, $"Failed on: {nyTime}"); Assert.IsNotNull(baseDataCollection.Underlying); // make sure the underlying time stamp is getting updated Assert.AreEqual(nyTime.TimeOfDay, baseDataCollection.Underlying.EndTime.TimeOfDay); Assert.AreEqual(nyTime.TimeOfDay, baseDataCollection.EndTime.ConvertFromUtc(algorithm.TimeZone).TimeOfDay); Assert.IsTrue(!baseDataCollection.FilteredContracts.IsNullOrEmpty()); count++; } } } feed.Exit(); algorithm.DataManager.RemoveAllSubscriptions(); // 9:30 to 15:59 -> 6.5 hours * 60 => 390 minutes * 2 days = 780 Assert.AreEqual(780, count); }
public void CoarseFundamentalHasFundamentalDataFalseExcludedInFineUniverseSelection() { var algorithm = new AlgorithmStub(new MockDataFeed()); algorithm.SetEndDate(Time.EndOfTime); algorithm.SetStartDate(DateTime.UtcNow.Subtract(TimeSpan.FromDays(10))); algorithm.AddUniverse( coarse => coarse.Select(c => c.Symbol), fine => fine.Select(f => f.Symbol) ); // OnEndOfTimeStep will add all pending universe additions algorithm.OnEndOfTimeStep(); var universe = algorithm.UniverseManager.Values.First(); var securityChanges = algorithm.DataManager.UniverseSelection.ApplyUniverseSelection( universe, algorithm.EndDate.ConvertToUtc(algorithm.TimeZone).Subtract(TimeSpan.FromDays(1)), new BaseDataCollection( DateTime.UtcNow, Symbols.AAPL, new[] { new CoarseFundamental { Symbol = Symbols.AAPL, HasFundamentalData = true }, new CoarseFundamental { Symbol = Symbols.SPY, HasFundamentalData = false } } ) ); Assert.AreEqual(1, securityChanges.Count); Assert.AreEqual(Symbols.AAPL, securityChanges.AddedSecurities.First().Symbol); }
public void CreatedEquityIsNotAddedToSymbolCache() { SymbolCache.Clear(); var algorithm = new AlgorithmStub(new MockDataFeed()); algorithm.SetEndDate(Time.EndOfTime); algorithm.SetStartDate(DateTime.UtcNow.Subtract(TimeSpan.FromDays(10))); algorithm.AddUniverse(CoarseSelectionFunction, FineSelectionFunction); // OnEndOfTimeStep will add all pending universe additions algorithm.OnEndOfTimeStep(); var universe = algorithm.UniverseManager.Values.First(); var securityChanges = algorithm.DataManager.UniverseSelection.ApplyUniverseSelection( universe, algorithm.EndDate.ConvertToUtc(algorithm.TimeZone).Subtract(TimeSpan.FromDays(1)), new BaseDataCollection( DateTime.UtcNow, Symbols.AAPL, new[] { new CoarseFundamental { Symbol = Symbols.AAPL, HasFundamentalData = true }, new CoarseFundamental { Symbol = Symbols.SPY, HasFundamentalData = false } } ) ); Symbol symbol; Assert.AreEqual(1, securityChanges.AddedSecurities.Count); Assert.AreEqual(Symbols.AAPL, securityChanges.AddedSecurities.First().Symbol); Assert.IsFalse(SymbolCache.TryGetSymbol("AAPL", out symbol)); Assert.IsFalse(SymbolCache.TryGetSymbol("SPY", out symbol)); }
public void FutureChainEnumerator(bool fillForward) { var job = new BacktestNodePacket(); var resultHandler = new BacktestingResultHandler(); var feed = new FileSystemDataFeed(); var algorithm = new AlgorithmStub(feed); algorithm.Transactions.SetOrderProcessor(new FakeOrderProcessor()); algorithm.SetStartDate(new DateTime(2013, 10, 07)); algorithm.SetEndDate(new DateTime(2013, 10, 08)); algorithm.SetFutureChainProvider(new BacktestingFutureChainProvider(TestGlobals.DataCacheProvider)); var dataPermissionManager = new DataPermissionManager(); using var synchronizer = new Synchronizer(); synchronizer.Initialize(algorithm, algorithm.DataManager); feed.Initialize(algorithm, job, resultHandler, TestGlobals.MapFileProvider, TestGlobals.FactorFileProvider, TestGlobals.DataProvider, algorithm.DataManager, synchronizer, dataPermissionManager.DataChannelProvider); var future = algorithm.AddFuture("ES", fillDataForward: fillForward); future.SetFilter(0, 300); algorithm.PostInitialize(); using var cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(30)); var count = 0L; var lastMonth = algorithm.StartDate.Month; foreach (var timeSlice in synchronizer.StreamData(cancellationTokenSource.Token)) { if (!timeSlice.IsTimePulse && timeSlice.UniverseData?.Count > 0) { var nyTime = timeSlice.Time.ConvertFromUtc(algorithm.TimeZone); var currentExpectedTime = new TimeSpan(0, 0, 0).Add(TimeSpan.FromMinutes(count % (24 * 60))); while (!future.Exchange.DateTimeIsOpen(nyTime.Date.Add(currentExpectedTime).AddMinutes(-1))) { // skip closed market times currentExpectedTime = new TimeSpan(0, 0, 0).Add(TimeSpan.FromMinutes(++count % (24 * 60))); } var universeData = timeSlice.UniverseData.OrderBy(kvp => kvp.Key.Configuration.Symbol).ToList(); var chainData = universeData[0].Value; Log.Trace($"{nyTime}. Count: {count}. Universe Data Count {universeData.Count}"); Assert.AreEqual(currentExpectedTime, nyTime.TimeOfDay, $"Failed on: {nyTime}. Count: {count}"); Assert.IsTrue(timeSlice.UniverseData.All(kvp => kvp.Value.EndTime.ConvertFromUtc(algorithm.TimeZone).TimeOfDay == nyTime.TimeOfDay)); if (chainData.FilteredContracts.IsNullOrEmpty()) { Assert.AreEqual(new DateTime(2013, 10, 09), nyTime, $"Unexpected chain FilteredContracts was empty on {nyTime}"); } if (universeData.Count == 1) { // the chain Assert.IsTrue(universeData.Any(kvp => kvp.Key.Configuration.Symbol == future.Symbol)); } else { // we have 2 universe data, the chain and the continuous future Assert.AreEqual(2, universeData.Count); Assert.IsTrue(universeData.All(kvp => kvp.Key.Configuration.Symbol.SecurityType == SecurityType.Future)); Assert.IsTrue(universeData.Any(kvp => kvp.Key.Configuration.Symbol == future.Symbol)); Assert.IsTrue(universeData.Any(kvp => kvp.Key.Configuration.Symbol.ID.Symbol.Contains("CONTINUOUS", StringComparison.InvariantCultureIgnoreCase))); var continuousData = universeData[1].Value; Assert.AreEqual(currentExpectedTime, nyTime.TimeOfDay, $"Failed on: {nyTime}"); Assert.IsTrue(!chainData.FilteredContracts.IsNullOrEmpty()); } count++; } } feed.Exit(); algorithm.DataManager.RemoveAllSubscriptions(); // 2 days worth of minute data Assert.AreEqual(24 * 2 * 60 + 1, count); }