public void TestsFileSystemDataFeedSpeed()
{
var job = new BacktestNodePacket();
var resultHandler = new BacktestingResultHandler();
var mapFileProvider = new LocalDiskMapFileProvider();
var factorFileProvider = new LocalDiskFactorFileProvider(mapFileProvider);
var dataProvider = new DefaultDataProvider();
var algorithm = PerformanceBenchmarkAlgorithms.SingleSecurity_Second;
var feed = new FileSystemDataFeed();
var marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
var symbolPropertiesDataBase = SymbolPropertiesDatabase.FromDataFolder();
var dataPermissionManager = new DataPermissionManager();
var dataManager = new DataManager(feed,
new UniverseSelection(
algorithm,
new SecurityService(algorithm.Portfolio.CashBook, marketHoursDatabase, symbolPropertiesDataBase, algorithm, RegisteredSecurityDataTypesProvider.Null, new SecurityCacheProvider(algorithm.Portfolio)),
dataPermissionManager,
new DefaultDataProvider()),
algorithm,
algorithm.TimeKeeper,
marketHoursDatabase,
false,
RegisteredSecurityDataTypesProvider.Null,
dataPermissionManager);
algorithm.SubscriptionManager.SetDataManager(dataManager);
var synchronizer = new Synchronizer();
synchronizer.Initialize(algorithm, dataManager);
feed.Initialize(algorithm, job, resultHandler, mapFileProvider, factorFileProvider, dataProvider, dataManager, synchronizer, dataPermissionManager.DataChannelProvider);
algorithm.Initialize();
algorithm.PostInitialize();
var cancellationTokenSource = new CancellationTokenSource();
var count = 0;
var stopwatch = Stopwatch.StartNew();
var lastMonth = algorithm.StartDate.Month;
foreach (var timeSlice in synchronizer.StreamData(cancellationTokenSource.Token))
{
if (timeSlice.Time.Month != lastMonth)
{
var elapsed = stopwatch.Elapsed.TotalSeconds;
var thousands = count / 1000d;
Log.Trace($"{DateTime.Now} - Time: {timeSlice.Time}: KPS: {thousands / elapsed}");
lastMonth = timeSlice.Time.Month;
}
count++;
}
Log.Trace("Count: " + count);
stopwatch.Stop();
feed.Exit();
dataManager.RemoveAllSubscriptions();
Log.Trace($"Elapsed time: {stopwatch.Elapsed} KPS: {count / 1000d / stopwatch.Elapsed.TotalSeconds}");
}
public void TestsFileSystemDataFeedSpeed()
{
var job = new BacktestNodePacket();
var resultHandler = new BacktestingResultHandler();
var mapFileProvider = new LocalDiskMapFileProvider();
var factorFileProvider = new LocalDiskFactorFileProvider(mapFileProvider);
var dataProvider = new DefaultDataProvider();
var algorithm = PerformanceBenchmarkAlgorithms.SingleSecurity_Second;
var feed = new FileSystemDataFeed();
var dataManager = new DataManager(feed, new UniverseSelection(feed, algorithm), algorithm.Settings, algorithm.TimeKeeper);
algorithm.SubscriptionManager.SetDataManager(dataManager);
feed.Initialize(algorithm, job, resultHandler, mapFileProvider, factorFileProvider, dataProvider, dataManager);
algorithm.Initialize();
algorithm.PostInitialize();
var feedThreadStarted = new ManualResetEvent(false);
var dataFeedThread = new Thread(() =>
{
feedThreadStarted.Set();
feed.Run();
})
{
IsBackground = true
};
dataFeedThread.Start();
feedThreadStarted.WaitOne();
var count = 0;
var stopwatch = Stopwatch.StartNew();
var lastMonth = algorithm.StartDate.Month;
foreach (var timeSlice in feed)
{
if (timeSlice.Time.Month != lastMonth)
{
var elapsed = stopwatch.Elapsed.TotalSeconds;
var thousands = count / 1000d;
Console.WriteLine($"{DateTime.Now} - Time: {timeSlice.Time}: KPS: {thousands/elapsed}");
lastMonth = timeSlice.Time.Month;
}
count++;
}
Console.WriteLine("Count: " + count);
stopwatch.Stop();
Console.WriteLine($"Elapsed time: {stopwatch.Elapsed} KPS: {count/1000d/stopwatch.Elapsed.TotalSeconds}");
}
public void OptionChainEnumerator(bool fillForward)
{
var job = new BacktestNodePacket();
var resultHandler = new BacktestingResultHandler();
var feed = new FileSystemDataFeed();
var algorithm = new AlgorithmStub(feed);
algorithm.Transactions.SetOrderProcessor(new FakeOrderProcessor());
algorithm.SetStartDate(new DateTime(2014, 06, 06));
algorithm.SetEndDate(new DateTime(2014, 06, 09));
algorithm.SetOptionChainProvider(new BacktestingOptionChainProvider(TestGlobals.DataCacheProvider, TestGlobals.MapFileProvider));
var dataPermissionManager = new DataPermissionManager();
using var synchronizer = new Synchronizer();
synchronizer.Initialize(algorithm, algorithm.DataManager);
feed.Initialize(algorithm, job, resultHandler, TestGlobals.MapFileProvider, TestGlobals.FactorFileProvider, TestGlobals.DataProvider, algorithm.DataManager, synchronizer, dataPermissionManager.DataChannelProvider);
var option = algorithm.AddOption("AAPL", fillDataForward: fillForward);
option.SetFilter(filter => filter.FrontMonth());
algorithm.PostInitialize();
using var cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(30));
var count = 0;
var lastMonth = algorithm.StartDate.Month;
foreach (var timeSlice in synchronizer.StreamData(cancellationTokenSource.Token))
{
if (!timeSlice.IsTimePulse && timeSlice.UniverseData?.Count > 0)
{
var baseDataCollection = timeSlice.UniverseData.Single().Value;
if (baseDataCollection.Symbol.SecurityType == SecurityType.Option)
{
var nyTime = timeSlice.Time.ConvertFromUtc(algorithm.TimeZone);
Assert.AreEqual(new TimeSpan(9, 30, 0).Add(TimeSpan.FromMinutes((count % 390) + 1)), nyTime.TimeOfDay, $"Failed on: {nyTime}");
Assert.IsNotNull(baseDataCollection.Underlying);
// make sure the underlying time stamp is getting updated
Assert.AreEqual(nyTime.TimeOfDay, baseDataCollection.Underlying.EndTime.TimeOfDay);
Assert.AreEqual(nyTime.TimeOfDay, baseDataCollection.EndTime.ConvertFromUtc(algorithm.TimeZone).TimeOfDay);
Assert.IsTrue(!baseDataCollection.FilteredContracts.IsNullOrEmpty());
count++;
}
}
}
feed.Exit();
algorithm.DataManager.RemoveAllSubscriptions();
// 9:30 to 15:59 -> 6.5 hours * 60 => 390 minutes * 2 days = 780
Assert.AreEqual(780, count);
}
public void TestsFileSystemDataFeedSpeed()
{
var job = new BacktestNodePacket();
var resultHandler = new BacktestingResultHandler();
var mapFileProvider = new LocalDiskMapFileProvider();
var factorFileProvider = new LocalDiskFactorFileProvider(mapFileProvider);
var dataProvider = new DefaultDataProvider();
var algorithm = PerformanceBenchmarkAlgorithms.SingleSecurity_Second;
var feed = new FileSystemDataFeed();
var marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
var symbolPropertiesDataBase = SymbolPropertiesDatabase.FromDataFolder();
var dataManager = new DataManager(feed,
new UniverseSelection(feed,
algorithm,
new SecurityService(algorithm.Portfolio.CashBook, marketHoursDatabase, symbolPropertiesDataBase, algorithm)),
algorithm.Settings,
algorithm.TimeKeeper,
marketHoursDatabase);
algorithm.SubscriptionManager.SetDataManager(dataManager);
feed.Initialize(algorithm, job, resultHandler, mapFileProvider, factorFileProvider, dataProvider, dataManager);
algorithm.Initialize();
algorithm.PostInitialize();
var count = 0;
var stopwatch = Stopwatch.StartNew();
var lastMonth = algorithm.StartDate.Month;
foreach (var timeSlice in feed)
{
if (timeSlice.Time.Month != lastMonth)
{
var elapsed = stopwatch.Elapsed.TotalSeconds;
var thousands = count / 1000d;
Console.WriteLine($"{DateTime.Now} - Time: {timeSlice.Time}: KPS: {thousands/elapsed}");
lastMonth = timeSlice.Time.Month;
}
count++;
}
Console.WriteLine("Count: " + count);
stopwatch.Stop();
feed.Exit();
Console.WriteLine($"Elapsed time: {stopwatch.Elapsed} KPS: {count/1000d/stopwatch.Elapsed.TotalSeconds}");
}
public void TestsFileSystemDataFeedSpeed()
{
var job = new BacktestNodePacket();
var resultHandler = new BacktestingResultHandler();
var mapFileProvider = new LocalDiskMapFileProvider();
var factorFileProvider = new LocalDiskFactorFileProvider(mapFileProvider);
var dataProvider = new DefaultDataProvider();
var algorithm = new BenchmarkTest();
var feed = new FileSystemDataFeed();
feed.Initialize(algorithm, job, resultHandler, mapFileProvider, factorFileProvider, dataProvider);
algorithm.Initialize();
var feedThreadStarted = new ManualResetEvent(false);
Task.Factory.StartNew(() =>
{
feedThreadStarted.Set();
feed.Run();
});
feedThreadStarted.WaitOne();
var stopwatch = Stopwatch.StartNew();
var lastMonth = -1;
var count = 0;
foreach (var timeSlice in feed)
{
if (timeSlice.Time.Month != lastMonth)
{
Console.WriteLine(DateTime.Now + " - Time: " + timeSlice.Time);
lastMonth = timeSlice.Time.Month;
}
count++;
}
Console.WriteLine("Count: " + count);
stopwatch.Stop();
Console.WriteLine("Elapsed time: " + stopwatch.Elapsed);
}
public void FutureChainEnumerator(bool fillForward)
{
var job = new BacktestNodePacket();
var resultHandler = new BacktestingResultHandler();
var feed = new FileSystemDataFeed();
var algorithm = new AlgorithmStub(feed);
algorithm.Transactions.SetOrderProcessor(new FakeOrderProcessor());
algorithm.SetStartDate(new DateTime(2013, 10, 07));
algorithm.SetEndDate(new DateTime(2013, 10, 08));
algorithm.SetFutureChainProvider(new BacktestingFutureChainProvider(TestGlobals.DataCacheProvider));
var dataPermissionManager = new DataPermissionManager();
using var synchronizer = new Synchronizer();
synchronizer.Initialize(algorithm, algorithm.DataManager);
feed.Initialize(algorithm, job, resultHandler, TestGlobals.MapFileProvider, TestGlobals.FactorFileProvider, TestGlobals.DataProvider, algorithm.DataManager, synchronizer, dataPermissionManager.DataChannelProvider);
var future = algorithm.AddFuture("ES", fillDataForward: fillForward);
future.SetFilter(0, 300);
algorithm.PostInitialize();
using var cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(30));
var count = 0L;
var lastMonth = algorithm.StartDate.Month;
foreach (var timeSlice in synchronizer.StreamData(cancellationTokenSource.Token))
{
if (!timeSlice.IsTimePulse && timeSlice.UniverseData?.Count > 0)
{
var nyTime = timeSlice.Time.ConvertFromUtc(algorithm.TimeZone);
var currentExpectedTime = new TimeSpan(0, 0, 0).Add(TimeSpan.FromMinutes(count % (24 * 60)));
while (!future.Exchange.DateTimeIsOpen(nyTime.Date.Add(currentExpectedTime).AddMinutes(-1)))
{
// skip closed market times
currentExpectedTime = new TimeSpan(0, 0, 0).Add(TimeSpan.FromMinutes(++count % (24 * 60)));
}
var universeData = timeSlice.UniverseData.OrderBy(kvp => kvp.Key.Configuration.Symbol).ToList();
var chainData = universeData[0].Value;
Log.Trace($"{nyTime}. Count: {count}. Universe Data Count {universeData.Count}");
Assert.AreEqual(currentExpectedTime, nyTime.TimeOfDay, $"Failed on: {nyTime}. Count: {count}");
Assert.IsTrue(timeSlice.UniverseData.All(kvp => kvp.Value.EndTime.ConvertFromUtc(algorithm.TimeZone).TimeOfDay == nyTime.TimeOfDay));
if (chainData.FilteredContracts.IsNullOrEmpty())
{
Assert.AreEqual(new DateTime(2013, 10, 09), nyTime, $"Unexpected chain FilteredContracts was empty on {nyTime}");
}
if (universeData.Count == 1)
{
// the chain
Assert.IsTrue(universeData.Any(kvp => kvp.Key.Configuration.Symbol == future.Symbol));
}
else
{
// we have 2 universe data, the chain and the continuous future
Assert.AreEqual(2, universeData.Count);
Assert.IsTrue(universeData.All(kvp => kvp.Key.Configuration.Symbol.SecurityType == SecurityType.Future));
Assert.IsTrue(universeData.Any(kvp => kvp.Key.Configuration.Symbol == future.Symbol));
Assert.IsTrue(universeData.Any(kvp => kvp.Key.Configuration.Symbol.ID.Symbol.Contains("CONTINUOUS", StringComparison.InvariantCultureIgnoreCase)));
var continuousData = universeData[1].Value;
Assert.AreEqual(currentExpectedTime, nyTime.TimeOfDay, $"Failed on: {nyTime}");
Assert.IsTrue(!chainData.FilteredContracts.IsNullOrEmpty());
}
count++;
}
}
feed.Exit();
algorithm.DataManager.RemoveAllSubscriptions();
// 2 days worth of minute data
Assert.AreEqual(24 * 2 * 60 + 1, count);
}