public void CreatedEquityIsNotAddedToSymbolCache()
{
SymbolCache.Clear();
var algorithm = new AlgorithmStub(new MockDataFeed());
algorithm.SetEndDate(Time.EndOfTime);
algorithm.SetStartDate(DateTime.UtcNow.Subtract(TimeSpan.FromDays(10)));
algorithm.AddUniverse(CoarseSelectionFunction, FineSelectionFunction);
var universe = algorithm.UniverseManager.Values.First();
var securityChanges = algorithm.DataManager.UniverseSelection.ApplyUniverseSelection(
universe,
algorithm.EndDate.ConvertToUtc(algorithm.TimeZone).Subtract(TimeSpan.FromDays(1)),
new BaseDataCollection(
DateTime.UtcNow,
Symbol.Create("GOOG", SecurityType.Equity, Market.USA),
new List <BaseData>()
)
);
Symbol symbol;
Assert.AreEqual(1, securityChanges.AddedSecurities.Count);
Assert.AreEqual(Symbol.Create("SPY", SecurityType.Equity, Market.USA),
securityChanges.AddedSecurities.First().Symbol);
Assert.IsFalse(SymbolCache.TryGetSymbol("GOOG", out symbol));
Assert.IsFalse(SymbolCache.TryGetSymbol("SPY", out symbol));
}
private IDataFeed RunDataFeed(out FuncDataQueueHandler dataQueueHandler, Func <FuncDataQueueHandler, IEnumerable <BaseData> > getNextTicksFunction = null,
Resolution resolution = Resolution.Second, List <string> equities = null, List <string> forex = null)
{
_algorithm.SetStartDate(_startDate);
var lastTime = _manualTimeProvider.GetUtcNow();
getNextTicksFunction = getNextTicksFunction ?? (fdqh =>
{
var time = _manualTimeProvider.GetUtcNow();
if (time == lastTime)
{
return(Enumerable.Empty <BaseData>());
}
lastTime = time;
return(fdqh.Subscriptions.Where(symbol => !_algorithm.UniverseManager.ContainsKey(symbol)) // its not a universe
.Select(symbol => new Tick(lastTime.ConvertFromUtc(TimeZones.NewYork), symbol, 1, 2)
{
Quantity = 1,
// Symbol could not be in the Securities collections for the custom Universe tests. AlgorithmManager is in charge of adding them, and we are not executing that code here.
TickType = _algorithm.Securities.ContainsKey(symbol) ? _algorithm.Securities[symbol].SubscriptionDataConfig.TickType : TickType.Trade
}));
});
// job is used to send into DataQueueHandler
var job = new LiveNodePacket();
// result handler is used due to dependency in SubscriptionDataReader
var resultHandler = new BacktestingResultHandler();
dataQueueHandler = new FuncDataQueueHandler(getNextTicksFunction);
var feed = new TestableLiveTradingDataFeed(dataQueueHandler, _manualTimeProvider);
var mapFileProvider = new LocalDiskMapFileProvider();
var fileProvider = new DefaultDataProvider();
var dataManager = new DataManager(feed, new UniverseSelection(feed, _algorithm),
_algorithm.Settings, _algorithm.TimeKeeper);
_algorithm.SubscriptionManager.SetDataManager(dataManager);
_algorithm.AddSecurities(resolution, equities, forex);
feed.Initialize(_algorithm, job, resultHandler, mapFileProvider, new LocalDiskFactorFileProvider(mapFileProvider), fileProvider, dataManager);
_algorithm.PostInitialize();
Thread.Sleep(150); // small handicap for the data to be pumped so TimeSlices have data of all subscriptions
var feedThreadStarted = new ManualResetEvent(false);
Task.Factory.StartNew(() =>
{
feedThreadStarted.Set();
feed.Run();
});
// wait for feed.Run to actually begin
feedThreadStarted.WaitOne();
return(feed);
}
private IDataFeed RunDataFeed(out FuncDataQueueHandler dataQueueHandler, Func <FuncDataQueueHandler, IEnumerable <BaseData> > getNextTicksFunction = null,
Resolution resolution = Resolution.Second, List <string> equities = null, List <string> forex = null, List <string> crypto = null)
{
_algorithm.SetStartDate(_startDate);
var lastTime = _manualTimeProvider.GetUtcNow();
getNextTicksFunction = getNextTicksFunction ?? (fdqh =>
{
var time = _manualTimeProvider.GetUtcNow();
if (time == lastTime)
{
return(Enumerable.Empty <BaseData>());
}
lastTime = time;
var tickTime = lastTime.AddMinutes(-1).ConvertFromUtc(TimeZones.NewYork);
return(fdqh.Subscriptions.Where(symbol => !_algorithm.UniverseManager.ContainsKey(symbol)) // its not a universe
.Select(symbol => new Tick(tickTime, symbol, 1, 2)
{
Quantity = 1,
// Symbol could not be in the Securities collections for the custom Universe tests. AlgorithmManager is in charge of adding them, and we are not executing that code here.
TickType = _algorithm.Securities.ContainsKey(symbol) ? _algorithm.Securities[symbol].SubscriptionDataConfig.TickType : TickType.Trade
}).ToList());
});
// job is used to send into DataQueueHandler
var job = new LiveNodePacket();
// result handler is used due to dependency in SubscriptionDataReader
var resultHandler = new BacktestingResultHandler();
dataQueueHandler = new FuncDataQueueHandler(getNextTicksFunction);
var feed = new TestableLiveTradingDataFeed(dataQueueHandler);
var mapFileProvider = new LocalDiskMapFileProvider();
var fileProvider = new DefaultDataProvider();
var marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
var symbolPropertiesDataBase = SymbolPropertiesDatabase.FromDataFolder();
var securityService = new SecurityService(_algorithm.Portfolio.CashBook, marketHoursDatabase, symbolPropertiesDataBase, _algorithm);
_algorithm.Securities.SetSecurityService(securityService);
_dataManager = new DataManager(feed,
new UniverseSelection(_algorithm, securityService),
_algorithm,
_algorithm.TimeKeeper,
marketHoursDatabase,
true);
_algorithm.SubscriptionManager.SetDataManager(_dataManager);
_algorithm.AddSecurities(resolution, equities, forex, crypto);
_synchronizer = new TestableLiveSynchronizer(_manualTimeProvider);
_synchronizer.Initialize(_algorithm, _dataManager);
feed.Initialize(_algorithm, job, resultHandler, mapFileProvider,
new LocalDiskFactorFileProvider(mapFileProvider), fileProvider, _dataManager, _synchronizer);
_algorithm.PostInitialize();
Thread.Sleep(150); // small handicap for the data to be pumped so TimeSlices have data of all subscriptions
return(feed);
}
public void RemovalFromUniverseAndDataFeedMakesSecurityNotTradable()
{
SymbolCache.Clear();
var algorithm = new AlgorithmStub(new MockDataFeedWithSubscription());
var orderProcessorMock = new Mock <IOrderProcessor>();
orderProcessorMock.Setup(m => m.GetOpenOrders(It.IsAny <Func <Order, bool> >())).Returns(new List <Order>());
algorithm.Transactions.SetOrderProcessor(orderProcessorMock.Object);
algorithm.SetStartDate(2012, 3, 27);
algorithm.SetEndDate(2012, 3, 30);
algorithm.AddUniverse("my-custom-universe", dt => dt.Day < 30 ? new List <string> {
"CPRT"
} : Enumerable.Empty <string>());
// OnEndOfTimeStep will add all pending universe additions
algorithm.OnEndOfTimeStep();
var universe = algorithm.UniverseManager.Values.First();
var securityChanges = algorithm.DataManager.UniverseSelection.ApplyUniverseSelection(
universe,
algorithm.EndDate.ConvertToUtc(algorithm.TimeZone).Subtract(TimeSpan.FromDays(2)),
new BaseDataCollection(
algorithm.UtcTime,
Symbol.Create("CPRT", SecurityType.Equity, Market.USA),
new List <BaseData>()
)
);
Assert.AreEqual(1, securityChanges.AddedSecurities.Count);
Assert.AreEqual(0, securityChanges.RemovedSecurities.Count);
var security = securityChanges.AddedSecurities.First();
Assert.IsTrue(security.IsTradable);
securityChanges = algorithm.DataManager.UniverseSelection.ApplyUniverseSelection(
universe,
algorithm.EndDate.ConvertToUtc(algorithm.TimeZone),
new BaseDataCollection(
algorithm.UtcTime,
Symbol.Create("CPRT", SecurityType.Equity, Market.USA),
new List <BaseData>()
)
);
Assert.AreEqual(0, securityChanges.AddedSecurities.Count);
Assert.AreEqual(1, securityChanges.RemovedSecurities.Count);
Assert.AreEqual(security.Symbol, securityChanges.RemovedSecurities.First().Symbol);
Assert.IsFalse(security.IsTradable);
}
public void OptionChainEnumerator(bool fillForward)
{
var job = new BacktestNodePacket();
var resultHandler = new BacktestingResultHandler();
var feed = new FileSystemDataFeed();
var algorithm = new AlgorithmStub(feed);
algorithm.Transactions.SetOrderProcessor(new FakeOrderProcessor());
algorithm.SetStartDate(new DateTime(2014, 06, 06));
algorithm.SetEndDate(new DateTime(2014, 06, 09));
algorithm.SetOptionChainProvider(new BacktestingOptionChainProvider(TestGlobals.DataCacheProvider, TestGlobals.MapFileProvider));
var dataPermissionManager = new DataPermissionManager();
using var synchronizer = new Synchronizer();
synchronizer.Initialize(algorithm, algorithm.DataManager);
feed.Initialize(algorithm, job, resultHandler, TestGlobals.MapFileProvider, TestGlobals.FactorFileProvider, TestGlobals.DataProvider, algorithm.DataManager, synchronizer, dataPermissionManager.DataChannelProvider);
var option = algorithm.AddOption("AAPL", fillDataForward: fillForward);
option.SetFilter(filter => filter.FrontMonth());
algorithm.PostInitialize();
using var cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(30));
var count = 0;
var lastMonth = algorithm.StartDate.Month;
foreach (var timeSlice in synchronizer.StreamData(cancellationTokenSource.Token))
{
if (!timeSlice.IsTimePulse && timeSlice.UniverseData?.Count > 0)
{
var baseDataCollection = timeSlice.UniverseData.Single().Value;
if (baseDataCollection.Symbol.SecurityType == SecurityType.Option)
{
var nyTime = timeSlice.Time.ConvertFromUtc(algorithm.TimeZone);
Assert.AreEqual(new TimeSpan(9, 30, 0).Add(TimeSpan.FromMinutes((count % 390) + 1)), nyTime.TimeOfDay, $"Failed on: {nyTime}");
Assert.IsNotNull(baseDataCollection.Underlying);
// make sure the underlying time stamp is getting updated
Assert.AreEqual(nyTime.TimeOfDay, baseDataCollection.Underlying.EndTime.TimeOfDay);
Assert.AreEqual(nyTime.TimeOfDay, baseDataCollection.EndTime.ConvertFromUtc(algorithm.TimeZone).TimeOfDay);
Assert.IsTrue(!baseDataCollection.FilteredContracts.IsNullOrEmpty());
count++;
}
}
}
feed.Exit();
algorithm.DataManager.RemoveAllSubscriptions();
// 9:30 to 15:59 -> 6.5 hours * 60 => 390 minutes * 2 days = 780
Assert.AreEqual(780, count);
}
public void CoarseFundamentalHasFundamentalDataFalseExcludedInFineUniverseSelection()
{
var algorithm = new AlgorithmStub(new MockDataFeed());
algorithm.SetEndDate(Time.EndOfTime);
algorithm.SetStartDate(DateTime.UtcNow.Subtract(TimeSpan.FromDays(10)));
algorithm.AddUniverse(
coarse => coarse.Select(c => c.Symbol),
fine => fine.Select(f => f.Symbol)
);
// OnEndOfTimeStep will add all pending universe additions
algorithm.OnEndOfTimeStep();
var universe = algorithm.UniverseManager.Values.First();
var securityChanges = algorithm.DataManager.UniverseSelection.ApplyUniverseSelection(
universe,
algorithm.EndDate.ConvertToUtc(algorithm.TimeZone).Subtract(TimeSpan.FromDays(1)),
new BaseDataCollection(
DateTime.UtcNow,
Symbols.AAPL,
new[]
{
new CoarseFundamental
{
Symbol = Symbols.AAPL,
HasFundamentalData = true
},
new CoarseFundamental
{
Symbol = Symbols.SPY,
HasFundamentalData = false
}
}
)
);
Assert.AreEqual(1, securityChanges.Count);
Assert.AreEqual(Symbols.AAPL, securityChanges.AddedSecurities.First().Symbol);
}
public void CreatedEquityIsNotAddedToSymbolCache()
{
SymbolCache.Clear();
var algorithm = new AlgorithmStub(new MockDataFeed());
algorithm.SetEndDate(Time.EndOfTime);
algorithm.SetStartDate(DateTime.UtcNow.Subtract(TimeSpan.FromDays(10)));
algorithm.AddUniverse(CoarseSelectionFunction, FineSelectionFunction);
// OnEndOfTimeStep will add all pending universe additions
algorithm.OnEndOfTimeStep();
var universe = algorithm.UniverseManager.Values.First();
var securityChanges = algorithm.DataManager.UniverseSelection.ApplyUniverseSelection(
universe,
algorithm.EndDate.ConvertToUtc(algorithm.TimeZone).Subtract(TimeSpan.FromDays(1)),
new BaseDataCollection(
DateTime.UtcNow,
Symbols.AAPL,
new[]
{
new CoarseFundamental
{
Symbol = Symbols.AAPL,
HasFundamentalData = true
},
new CoarseFundamental
{
Symbol = Symbols.SPY,
HasFundamentalData = false
}
}
)
);
Symbol symbol;
Assert.AreEqual(1, securityChanges.AddedSecurities.Count);
Assert.AreEqual(Symbols.AAPL, securityChanges.AddedSecurities.First().Symbol);
Assert.IsFalse(SymbolCache.TryGetSymbol("AAPL", out symbol));
Assert.IsFalse(SymbolCache.TryGetSymbol("SPY", out symbol));
}
public void FutureChainEnumerator(bool fillForward)
{
var job = new BacktestNodePacket();
var resultHandler = new BacktestingResultHandler();
var feed = new FileSystemDataFeed();
var algorithm = new AlgorithmStub(feed);
algorithm.Transactions.SetOrderProcessor(new FakeOrderProcessor());
algorithm.SetStartDate(new DateTime(2013, 10, 07));
algorithm.SetEndDate(new DateTime(2013, 10, 08));
algorithm.SetFutureChainProvider(new BacktestingFutureChainProvider(TestGlobals.DataCacheProvider));
var dataPermissionManager = new DataPermissionManager();
using var synchronizer = new Synchronizer();
synchronizer.Initialize(algorithm, algorithm.DataManager);
feed.Initialize(algorithm, job, resultHandler, TestGlobals.MapFileProvider, TestGlobals.FactorFileProvider, TestGlobals.DataProvider, algorithm.DataManager, synchronizer, dataPermissionManager.DataChannelProvider);
var future = algorithm.AddFuture("ES", fillDataForward: fillForward);
future.SetFilter(0, 300);
algorithm.PostInitialize();
using var cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(30));
var count = 0L;
var lastMonth = algorithm.StartDate.Month;
foreach (var timeSlice in synchronizer.StreamData(cancellationTokenSource.Token))
{
if (!timeSlice.IsTimePulse && timeSlice.UniverseData?.Count > 0)
{
var nyTime = timeSlice.Time.ConvertFromUtc(algorithm.TimeZone);
var currentExpectedTime = new TimeSpan(0, 0, 0).Add(TimeSpan.FromMinutes(count % (24 * 60)));
while (!future.Exchange.DateTimeIsOpen(nyTime.Date.Add(currentExpectedTime).AddMinutes(-1)))
{
// skip closed market times
currentExpectedTime = new TimeSpan(0, 0, 0).Add(TimeSpan.FromMinutes(++count % (24 * 60)));
}
var universeData = timeSlice.UniverseData.OrderBy(kvp => kvp.Key.Configuration.Symbol).ToList();
var chainData = universeData[0].Value;
Log.Trace($"{nyTime}. Count: {count}. Universe Data Count {universeData.Count}");
Assert.AreEqual(currentExpectedTime, nyTime.TimeOfDay, $"Failed on: {nyTime}. Count: {count}");
Assert.IsTrue(timeSlice.UniverseData.All(kvp => kvp.Value.EndTime.ConvertFromUtc(algorithm.TimeZone).TimeOfDay == nyTime.TimeOfDay));
if (chainData.FilteredContracts.IsNullOrEmpty())
{
Assert.AreEqual(new DateTime(2013, 10, 09), nyTime, $"Unexpected chain FilteredContracts was empty on {nyTime}");
}
if (universeData.Count == 1)
{
// the chain
Assert.IsTrue(universeData.Any(kvp => kvp.Key.Configuration.Symbol == future.Symbol));
}
else
{
// we have 2 universe data, the chain and the continuous future
Assert.AreEqual(2, universeData.Count);
Assert.IsTrue(universeData.All(kvp => kvp.Key.Configuration.Symbol.SecurityType == SecurityType.Future));
Assert.IsTrue(universeData.Any(kvp => kvp.Key.Configuration.Symbol == future.Symbol));
Assert.IsTrue(universeData.Any(kvp => kvp.Key.Configuration.Symbol.ID.Symbol.Contains("CONTINUOUS", StringComparison.InvariantCultureIgnoreCase)));
var continuousData = universeData[1].Value;
Assert.AreEqual(currentExpectedTime, nyTime.TimeOfDay, $"Failed on: {nyTime}");
Assert.IsTrue(!chainData.FilteredContracts.IsNullOrEmpty());
}
count++;
}
}
feed.Exit();
algorithm.DataManager.RemoveAllSubscriptions();
// 2 days worth of minute data
Assert.AreEqual(24 * 2 * 60 + 1, count);
}