public LocalHistoryBrokerage()
{
var mapFileProvider = TestGlobals.MapFileProvider;
var dataProvider = TestGlobals.DataProvider;
_dataCacheProvider = new ZipDataCacheProvider(dataProvider);
var factorFileProvider = TestGlobals.FactorFileProvider;
var dataPermissionManager = new DataPermissionManager();
mapFileProvider.Initialize(dataProvider);
factorFileProvider.Initialize(mapFileProvider, dataProvider);
_historyProvider = new SubscriptionDataReaderHistoryProvider();
_historyProvider.Initialize(
new HistoryProviderInitializeParameters(
null,
null,
dataProvider,
_dataCacheProvider,
mapFileProvider,
factorFileProvider,
null,
true,
dataPermissionManager
)
);
}
public void TestsFileSystemDataFeedSpeed()
{
var job = new BacktestNodePacket();
var resultHandler = new BacktestingResultHandler();
var mapFileProvider = new LocalDiskMapFileProvider();
var factorFileProvider = new LocalDiskFactorFileProvider(mapFileProvider);
var dataProvider = new DefaultDataProvider();
var algorithm = PerformanceBenchmarkAlgorithms.SingleSecurity_Second;
var feed = new FileSystemDataFeed();
var marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
var symbolPropertiesDataBase = SymbolPropertiesDatabase.FromDataFolder();
var dataPermissionManager = new DataPermissionManager();
var dataManager = new DataManager(feed,
new UniverseSelection(
algorithm,
new SecurityService(algorithm.Portfolio.CashBook, marketHoursDatabase, symbolPropertiesDataBase, algorithm, RegisteredSecurityDataTypesProvider.Null, new SecurityCacheProvider(algorithm.Portfolio)),
dataPermissionManager,
new DefaultDataProvider()),
algorithm,
algorithm.TimeKeeper,
marketHoursDatabase,
false,
RegisteredSecurityDataTypesProvider.Null,
dataPermissionManager);
algorithm.SubscriptionManager.SetDataManager(dataManager);
var synchronizer = new Synchronizer();
synchronizer.Initialize(algorithm, dataManager);
feed.Initialize(algorithm, job, resultHandler, mapFileProvider, factorFileProvider, dataProvider, dataManager, synchronizer, dataPermissionManager.DataChannelProvider);
algorithm.Initialize();
algorithm.PostInitialize();
var cancellationTokenSource = new CancellationTokenSource();
var count = 0;
var stopwatch = Stopwatch.StartNew();
var lastMonth = algorithm.StartDate.Month;
foreach (var timeSlice in synchronizer.StreamData(cancellationTokenSource.Token))
{
if (timeSlice.Time.Month != lastMonth)
{
var elapsed = stopwatch.Elapsed.TotalSeconds;
var thousands = count / 1000d;
Log.Trace($"{DateTime.Now} - Time: {timeSlice.Time}: KPS: {thousands / elapsed}");
lastMonth = timeSlice.Time.Month;
}
count++;
}
Log.Trace("Count: " + count);
stopwatch.Stop();
feed.Exit();
dataManager.RemoveAllSubscriptions();
Log.Trace($"Elapsed time: {stopwatch.Elapsed} KPS: {count / 1000d / stopwatch.Elapsed.TotalSeconds}");
}
public void TestAlgorithmManagerSpeed()
{
var algorithm = PerformanceBenchmarkAlgorithms.SingleSecurity_Second;
var algorithmManager = new AlgorithmManager(false);
var job = new BacktestNodePacket(1, 2, "3", null, 9m, $"{nameof(AlgorithmManagerTests)}.{nameof(TestAlgorithmManagerSpeed)}");
var feed = new MockDataFeed();
var marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
var symbolPropertiesDataBase = SymbolPropertiesDatabase.FromDataFolder();
var dataPermissionManager = new DataPermissionManager();
var dataManager = new DataManager(feed,
new UniverseSelection(
algorithm,
new SecurityService(algorithm.Portfolio.CashBook,
marketHoursDatabase,
symbolPropertiesDataBase,
algorithm,
RegisteredSecurityDataTypesProvider.Null,
new SecurityCacheProvider(algorithm.Portfolio)),
dataPermissionManager,
new DefaultDataProvider()),
algorithm,
algorithm.TimeKeeper,
marketHoursDatabase,
false,
RegisteredSecurityDataTypesProvider.Null,
dataPermissionManager);
algorithm.SubscriptionManager.SetDataManager(dataManager);
var transactions = new BacktestingTransactionHandler();
var results = new BacktestingResultHandler();
var realtime = new BacktestingRealTimeHandler();
var leanManager = new NullLeanManager();
var alphas = new NullAlphaHandler();
var token = new CancellationToken();
var nullSynchronizer = new NullSynchronizer(algorithm);
algorithm.Initialize();
algorithm.PostInitialize();
results.Initialize(job, new QuantConnect.Messaging.Messaging(), new Api.Api(), transactions);
results.SetAlgorithm(algorithm, algorithm.Portfolio.TotalPortfolioValue);
transactions.Initialize(algorithm, new BacktestingBrokerage(algorithm), results);
feed.Initialize(algorithm, job, results, null, null, null, dataManager, null, null);
Log.Trace("Starting algorithm manager loop to process " + nullSynchronizer.Count + " time slices");
var sw = Stopwatch.StartNew();
algorithmManager.Run(job, algorithm, nullSynchronizer, transactions, results, realtime, leanManager, alphas, token);
sw.Stop();
realtime.Exit();
results.Exit();
var thousands = nullSynchronizer.Count / 1000d;
var seconds = sw.Elapsed.TotalSeconds;
Log.Trace("COUNT: " + nullSynchronizer.Count + " KPS: " + thousands / seconds);
}
public void RemovesExistingConfig()
{
var dataPermissionManager = new DataPermissionManager();
var dataFeed = new TestDataFeed();
var dataManager = new DataManager(dataFeed,
new UniverseSelection(_algorithm,
_securityService,
dataPermissionManager,
new DefaultDataProvider()),
_algorithm,
_algorithm.TimeKeeper,
MarketHoursDatabase.AlwaysOpen,
false,
new RegisteredSecurityDataTypesProvider(),
dataPermissionManager);
var config = new SubscriptionDataConfig(typeof(TradeBar),
Symbols.SPY,
Resolution.Daily,
TimeZones.NewYork,
TimeZones.NewYork,
false,
false,
false);
Assert.IsTrue(ReferenceEquals(dataManager.SubscriptionManagerGetOrAdd(config), config));
Assert.AreEqual(1, dataManager.GetSubscriptionDataConfigs(config.Symbol).Count);
// we didn't add any subscription yet
Assert.IsFalse(dataManager.RemoveSubscription(config));
var request = new SubscriptionRequest(false,
null,
new Security(Symbols.SPY,
SecurityExchangeHours.AlwaysOpen(DateTimeZone.Utc),
new Cash(Currencies.USD, 1, 1),
SymbolProperties.GetDefault(Currencies.USD),
new IdentityCurrencyConverter(Currencies.USD),
new RegisteredSecurityDataTypesProvider(),
new SecurityCache()),
config,
new DateTime(2019, 1, 1),
new DateTime(2019, 1, 1));
dataFeed.Subscription = new Subscription(request,
new EnqueueableEnumerator <SubscriptionData>(),
null);
Assert.IsTrue(dataManager.AddSubscription(request));
Assert.IsTrue(dataManager.RemoveSubscription(config));
Assert.AreEqual(0, dataManager.GetSubscriptionDataConfigs(config.Symbol).Count);
dataManager.RemoveAllSubscriptions();
}
public void Setup()
{
var sunday = new LocalMarketHours(DayOfWeek.Sunday, new TimeSpan(17, 0, 0), TimeSpan.FromTicks(Time.OneDay.Ticks - 1));
var monday = LocalMarketHours.OpenAllDay(DayOfWeek.Monday);
var tuesday = LocalMarketHours.OpenAllDay(DayOfWeek.Tuesday);
var wednesday = LocalMarketHours.OpenAllDay(DayOfWeek.Wednesday);
var thursday = LocalMarketHours.OpenAllDay(DayOfWeek.Thursday);
var friday = new LocalMarketHours(DayOfWeek.Friday, TimeSpan.Zero, new TimeSpan(17, 0, 0));
var earlyCloses = new Dictionary <DateTime, TimeSpan>();
var lateOpens = new Dictionary <DateTime, TimeSpan>();
_exchangeHours = new SecurityExchangeHours(TimeZones.NewYork, USHoliday.Dates.Select(x => x.Date), new[]
{
sunday, monday, tuesday, wednesday, thursday, friday
}.ToDictionary(x => x.DayOfWeek), earlyCloses, lateOpens);
_liveTradingDataFeed = new TestableLiveTradingDataFeed();
var jobPacket = new LiveNodePacket()
{
DeployId = "",
Brokerage = BrokerageName.OandaBrokerage.ToString(),
DataQueueHandler = "LiveDataQueue"
};
var algo = new TestAlgorithm();
var marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
var symbolPropertiesDataBase = SymbolPropertiesDatabase.FromDataFolder();
var dataPermissionManager = new DataPermissionManager();
var dataProvider = new DefaultDataProvider();
var dataManager = new DataManager(_liveTradingDataFeed,
new UniverseSelection(
algo,
new SecurityService(algo.Portfolio.CashBook, marketHoursDatabase, symbolPropertiesDataBase, algo, RegisteredSecurityDataTypesProvider.Null, new SecurityCacheProvider(algo.Portfolio)),
dataPermissionManager,
dataProvider),
algo,
algo.TimeKeeper,
marketHoursDatabase,
true,
RegisteredSecurityDataTypesProvider.Null,
dataPermissionManager);
algo.SubscriptionManager.SetDataManager(dataManager);
_liveSynchronizer = new LiveSynchronizer();
_liveSynchronizer.Initialize(algo, dataManager);
_liveTradingDataFeed.Initialize(algo, jobPacket, new LiveTradingResultHandler(), new LocalDiskMapFileProvider(),
null, dataProvider, dataManager, _liveSynchronizer, new DataChannelProvider());
algo.Initialize();
_config = SecurityTests.CreateTradeBarConfig();
}
public void OptionChainEnumerator(bool fillForward)
{
var job = new BacktestNodePacket();
var resultHandler = new BacktestingResultHandler();
var feed = new FileSystemDataFeed();
var algorithm = new AlgorithmStub(feed);
algorithm.Transactions.SetOrderProcessor(new FakeOrderProcessor());
algorithm.SetStartDate(new DateTime(2014, 06, 06));
algorithm.SetEndDate(new DateTime(2014, 06, 09));
algorithm.SetOptionChainProvider(new BacktestingOptionChainProvider(TestGlobals.DataCacheProvider, TestGlobals.MapFileProvider));
var dataPermissionManager = new DataPermissionManager();
using var synchronizer = new Synchronizer();
synchronizer.Initialize(algorithm, algorithm.DataManager);
feed.Initialize(algorithm, job, resultHandler, TestGlobals.MapFileProvider, TestGlobals.FactorFileProvider, TestGlobals.DataProvider, algorithm.DataManager, synchronizer, dataPermissionManager.DataChannelProvider);
var option = algorithm.AddOption("AAPL", fillDataForward: fillForward);
option.SetFilter(filter => filter.FrontMonth());
algorithm.PostInitialize();
using var cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(30));
var count = 0;
var lastMonth = algorithm.StartDate.Month;
foreach (var timeSlice in synchronizer.StreamData(cancellationTokenSource.Token))
{
if (!timeSlice.IsTimePulse && timeSlice.UniverseData?.Count > 0)
{
var baseDataCollection = timeSlice.UniverseData.Single().Value;
if (baseDataCollection.Symbol.SecurityType == SecurityType.Option)
{
var nyTime = timeSlice.Time.ConvertFromUtc(algorithm.TimeZone);
Assert.AreEqual(new TimeSpan(9, 30, 0).Add(TimeSpan.FromMinutes((count % 390) + 1)), nyTime.TimeOfDay, $"Failed on: {nyTime}");
Assert.IsNotNull(baseDataCollection.Underlying);
// make sure the underlying time stamp is getting updated
Assert.AreEqual(nyTime.TimeOfDay, baseDataCollection.Underlying.EndTime.TimeOfDay);
Assert.AreEqual(nyTime.TimeOfDay, baseDataCollection.EndTime.ConvertFromUtc(algorithm.TimeZone).TimeOfDay);
Assert.IsTrue(!baseDataCollection.FilteredContracts.IsNullOrEmpty());
count++;
}
}
}
feed.Exit();
algorithm.DataManager.RemoveAllSubscriptions();
// 9:30 to 15:59 -> 6.5 hours * 60 => 390 minutes * 2 days = 780
Assert.AreEqual(780, count);
}
public void ReturnsExistingConfig()
{
var dataPermissionManager = new DataPermissionManager();
var dataManager = new DataManager(new NullDataFeed(),
new UniverseSelection(_algorithm,
_securityService,
dataPermissionManager,
new DefaultDataProvider()),
_algorithm,
_algorithm.TimeKeeper,
MarketHoursDatabase.AlwaysOpen,
false,
new RegisteredSecurityDataTypesProvider(),
dataPermissionManager);
var config = new SubscriptionDataConfig(typeof(TradeBar),
Symbols.SPY,
Resolution.Daily,
TimeZones.NewYork,
TimeZones.NewYork,
false,
false,
false);
var sameConfig = dataManager.SubscriptionManagerGetOrAdd(config);
Assert.IsTrue(ReferenceEquals(sameConfig, config));
Assert.AreEqual(1, dataManager.GetSubscriptionDataConfigs(config.Symbol).Count);
var otherInstance = new SubscriptionDataConfig(config.Type,
config.Symbol,
config.Resolution,
config.DataTimeZone,
config.ExchangeTimeZone,
config.FillDataForward,
config.ExtendedMarketHours,
config.IsInternalFeed);
sameConfig = dataManager.SubscriptionManagerGetOrAdd(otherInstance);
Assert.IsTrue(ReferenceEquals(sameConfig, config));
Assert.AreEqual(1, dataManager.GetSubscriptionDataConfigs(config.Symbol).Count);
dataManager.RemoveAllSubscriptions();
}
private PortfolioLooperAlgorithm CreateAlgorithm(IEnumerable <Order> orders)
{
var algorithm = new PortfolioLooperAlgorithm(100000m, orders);
// Create MHDB and Symbol properties DB instances for the DataManager
var marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
var symbolPropertiesDataBase = SymbolPropertiesDatabase.FromDataFolder();
var dataPermissionManager = new DataPermissionManager();
var dataManager = new DataManager(new QuantConnect.Report.MockDataFeed(),
new UniverseSelection(
algorithm,
new SecurityService(algorithm.Portfolio.CashBook,
marketHoursDatabase,
symbolPropertiesDataBase,
algorithm,
RegisteredSecurityDataTypesProvider.Null,
new SecurityCacheProvider(algorithm.Portfolio)),
dataPermissionManager),
algorithm,
algorithm.TimeKeeper,
marketHoursDatabase,
false,
RegisteredSecurityDataTypesProvider.Null,
dataPermissionManager);
var securityService = new SecurityService(algorithm.Portfolio.CashBook,
marketHoursDatabase,
symbolPropertiesDataBase,
algorithm,
RegisteredSecurityDataTypesProvider.Null,
new SecurityCacheProvider(algorithm.Portfolio));
var transactions = new BacktestingTransactionHandler();
var results = new BacktestingResultHandler();
// Initialize security services and other properties so that we
// don't get null reference exceptions during our re-calculation
algorithm.Securities.SetSecurityService(securityService);
algorithm.SubscriptionManager.SetDataManager(dataManager);
return(algorithm);
}
private Tuple <QCAlgorithm, DataManager> GetAlgorithmAndDataManager()
{
var algorithm = new QCAlgorithm();
var marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
var symbolPropertiesDatabase = SymbolPropertiesDatabase.FromDataFolder();
var dataPermissionManager = new DataPermissionManager();
var dataManager = new DataManager(
new MockDataFeed(),
new UniverseSelection(
algorithm,
new SecurityService(
algorithm.Portfolio.CashBook,
marketHoursDatabase,
symbolPropertiesDatabase,
algorithm,
RegisteredSecurityDataTypesProvider.Null,
new SecurityCacheProvider(algorithm.Portfolio)),
dataPermissionManager,
new DefaultDataProvider()),
algorithm,
algorithm.TimeKeeper,
marketHoursDatabase,
false,
RegisteredSecurityDataTypesProvider.Null,
dataPermissionManager);
var securityService = new SecurityService(
algorithm.Portfolio.CashBook,
marketHoursDatabase,
symbolPropertiesDatabase,
algorithm,
RegisteredSecurityDataTypesProvider.Null,
new SecurityCacheProvider(algorithm.Portfolio));
algorithm.SubscriptionManager.SetDataManager(dataManager);
algorithm.Securities.SetSecurityService(securityService);
return(Tuple.Create(algorithm, dataManager));
}
public void AppliesDividendsOnce()
{
// init algorithm
var algorithm = new AlgorithmStub(new MockDataFeed());
algorithm.SetLiveMode(true);
var dividend = new Dividend(Symbols.SPY, DateTime.UtcNow, 10m, 100m);
var feed = new MockDataFeed();
var marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
var symbolPropertiesDataBase = SymbolPropertiesDatabase.FromDataFolder();
var dataPermissionManager = new DataPermissionManager();
var dataManager = new DataManager(feed,
new UniverseSelection(
algorithm,
new SecurityService(algorithm.Portfolio.CashBook, marketHoursDatabase, symbolPropertiesDataBase, algorithm, RegisteredSecurityDataTypesProvider.Null, new SecurityCacheProvider(algorithm.Portfolio)),
dataPermissionManager,
new DefaultDataProvider()),
algorithm,
algorithm.TimeKeeper,
marketHoursDatabase,
true,
RegisteredSecurityDataTypesProvider.Null,
dataPermissionManager);
var synchronizer = new NullSynchronizer(algorithm, dividend);
algorithm.SubscriptionManager.SetDataManager(dataManager);
algorithm.AddSecurities(equities: new List <string> {
"SPY"
});
algorithm.Securities[Symbols.SPY].Holdings.SetHoldings(100m, 1);
algorithm.PostInitialize();
var initializedCash = algorithm.Portfolio.CashBook[Currencies.USD].Amount;
// init algorithm manager
var manager = new AlgorithmManager(true);
var job = new LiveNodePacket
{
UserId = 1,
ProjectId = 2,
DeployId = $"{nameof(PaperBrokerageTests)}.{nameof(AppliesDividendsOnce)}"
};
var results = new LiveTradingResultHandler();
var transactions = new BacktestingTransactionHandler();
var brokerage = new PaperBrokerage(algorithm, job);
// initialize results and transactions
results.Initialize(job, new EventMessagingHandler(), new Api.Api(), transactions);
results.SetAlgorithm(algorithm, algorithm.Portfolio.TotalPortfolioValue);
transactions.Initialize(algorithm, brokerage, results);
var realTime = new BacktestingRealTimeHandler();
// run algorithm manager
manager.Run(job,
algorithm,
synchronizer,
transactions,
results,
realTime,
new AlgorithmManagerTests.NullLeanManager(),
new AlgorithmManagerTests.NullAlphaHandler(),
new CancellationToken()
);
var postDividendCash = algorithm.Portfolio.CashBook[Currencies.USD].Amount;
realTime.Exit();
results.Exit();
Assert.AreEqual(initializedCash + dividend.Distribution, postDividendCash);
}
/// <summary>
/// Creates an instance of the PortfolioLooper class
/// </summary>
/// <param name="startingCash">Equity curve</param>
/// <param name="orders">Order events</param>
/// <param name="resolution">Optional parameter to override default resolution (Hourly)</param>
private PortfolioLooper(double startingCash, List <Order> orders, Resolution resolution = _resolution)
{
// Initialize the providers that the HistoryProvider requires
var factorFileProvider = Composer.Instance.GetExportedValueByTypeName <IFactorFileProvider>("LocalDiskFactorFileProvider");
var mapFileProvider = Composer.Instance.GetExportedValueByTypeName <IMapFileProvider>("LocalDiskMapFileProvider");
_cacheProvider = new ZipDataCacheProvider(new DefaultDataProvider(), false);
var historyProvider = new SubscriptionDataReaderHistoryProvider();
var dataPermissionManager = new DataPermissionManager();
historyProvider.Initialize(new HistoryProviderInitializeParameters(null, null, null, _cacheProvider, mapFileProvider, factorFileProvider, (_) => { }, false, dataPermissionManager));
Algorithm = new PortfolioLooperAlgorithm((decimal)startingCash, orders);
Algorithm.SetHistoryProvider(historyProvider);
// Dummy LEAN datafeed classes and initializations that essentially do nothing
var job = new BacktestNodePacket(1, 2, "3", null, 9m, $"");
var feed = new MockDataFeed();
// Create MHDB and Symbol properties DB instances for the DataManager
var marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
var symbolPropertiesDataBase = SymbolPropertiesDatabase.FromDataFolder();
_dataManager = new DataManager(feed,
new UniverseSelection(
Algorithm,
new SecurityService(Algorithm.Portfolio.CashBook,
marketHoursDatabase,
symbolPropertiesDataBase,
Algorithm,
RegisteredSecurityDataTypesProvider.Null,
new SecurityCacheProvider(Algorithm.Portfolio)),
dataPermissionManager,
new DefaultDataProvider()),
Algorithm,
Algorithm.TimeKeeper,
marketHoursDatabase,
false,
RegisteredSecurityDataTypesProvider.Null,
dataPermissionManager);
_securityService = new SecurityService(Algorithm.Portfolio.CashBook,
marketHoursDatabase,
symbolPropertiesDataBase,
Algorithm,
RegisteredSecurityDataTypesProvider.Null,
new SecurityCacheProvider(Algorithm.Portfolio));
var transactions = new BacktestingTransactionHandler();
_resultHandler = new BacktestingResultHandler();
// Initialize security services and other properties so that we
// don't get null reference exceptions during our re-calculation
Algorithm.Securities.SetSecurityService(_securityService);
Algorithm.SubscriptionManager.SetDataManager(_dataManager);
// Initializes all the proper Securities from the orders provided by the user
Algorithm.FromOrders(orders);
// Initialize the algorithm
Algorithm.Initialize();
Algorithm.PostInitialize();
// More initialization, this time with Algorithm and other misc. classes
_resultHandler.Initialize(job, new Messaging.Messaging(), new Api.Api(), transactions);
_resultHandler.SetAlgorithm(Algorithm, Algorithm.Portfolio.TotalPortfolioValue);
Algorithm.Transactions.SetOrderProcessor(transactions);
transactions.Initialize(Algorithm, new BacktestingBrokerage(Algorithm), _resultHandler);
feed.Initialize(Algorithm, job, _resultHandler, null, null, null, _dataManager, null, null);
// Begin setting up the currency conversion feed if needed
var coreSecurities = Algorithm.Securities.Values.ToList();
if (coreSecurities.Any(x => x.Symbol.SecurityType == SecurityType.Forex || x.Symbol.SecurityType == SecurityType.Crypto))
{
BaseSetupHandler.SetupCurrencyConversions(Algorithm, _dataManager.UniverseSelection);
var conversionSecurities = Algorithm.Securities.Values.Where(s => !coreSecurities.Contains(s)).ToList();
// Skip the history request if we don't need to convert anything
if (conversionSecurities.Any())
{
// Point-in-time Slices to convert FX and Crypto currencies to the portfolio currency
_conversionSlices = GetHistory(Algorithm, conversionSecurities, resolution);
}
}
}
private void TestSubscriptionSynchronizerSpeed(QCAlgorithm algorithm)
{
var feed = new MockDataFeed();
var marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
var symbolPropertiesDataBase = SymbolPropertiesDatabase.FromDataFolder();
var securityService = new SecurityService(
algorithm.Portfolio.CashBook,
marketHoursDatabase,
symbolPropertiesDataBase,
algorithm,
RegisteredSecurityDataTypesProvider.Null,
new SecurityCacheProvider(algorithm.Portfolio));
algorithm.Securities.SetSecurityService(securityService);
var dataPermissionManager = new DataPermissionManager();
var dataManager = new DataManager(feed,
new UniverseSelection(algorithm, securityService, dataPermissionManager, new DefaultDataProvider()),
algorithm,
algorithm.TimeKeeper,
marketHoursDatabase,
false,
RegisteredSecurityDataTypesProvider.Null,
dataPermissionManager);
algorithm.SubscriptionManager.SetDataManager(dataManager);
algorithm.Initialize();
algorithm.PostInitialize();
// set exchanges to be always open
foreach (var kvp in algorithm.Securities)
{
var security = kvp.Value;
security.Exchange = new SecurityExchange(SecurityExchangeHours.AlwaysOpen(security.Exchange.TimeZone));
}
var endTimeUtc = algorithm.EndDate.ConvertToUtc(TimeZones.NewYork);
var startTimeUtc = algorithm.StartDate.ConvertToUtc(TimeZones.NewYork);
var subscriptionBasedTimeProvider = new SubscriptionFrontierTimeProvider(startTimeUtc, dataManager);
var timeSliceFactory = new TimeSliceFactory(algorithm.TimeZone);
var synchronizer = new SubscriptionSynchronizer(dataManager.UniverseSelection);
synchronizer.SetTimeProvider(subscriptionBasedTimeProvider);
synchronizer.SetTimeSliceFactory(timeSliceFactory);
var totalDataPoints = 0;
var subscriptions = dataManager.DataFeedSubscriptions;
foreach (var kvp in algorithm.Securities)
{
int dataPointCount;
subscriptions.TryAdd(CreateSubscription(algorithm, kvp.Value, startTimeUtc, endTimeUtc, out dataPointCount));
totalDataPoints += dataPointCount;
}
// log what we're doing
Console.WriteLine($"Running {subscriptions.Count()} subscriptions with a total of {totalDataPoints} data points. Start: {algorithm.StartDate:yyyy-MM-dd} End: {algorithm.EndDate:yyyy-MM-dd}");
var count = 0;
DateTime currentTime = DateTime.MaxValue;
DateTime previousValue;
var stopwatch = Stopwatch.StartNew();
var enumerator = synchronizer.Sync(subscriptions, CancellationToken.None).GetEnumerator();
do
{
previousValue = currentTime;
enumerator.MoveNext();
var timeSlice = enumerator.Current;
currentTime = timeSlice.Time;
count += timeSlice.DataPointCount;
}while (currentTime != previousValue);
stopwatch.Stop();
enumerator.DisposeSafely();
var kps = count / 1000d / stopwatch.Elapsed.TotalSeconds;
Console.WriteLine($"Current Time: {currentTime:u} Elapsed time: {(int)stopwatch.Elapsed.TotalSeconds,4}s KPS: {kps,7:.00} COUNT: {count,10}");
Assert.GreaterOrEqual(count, 100); // this assert is for sanity purpose
dataManager.RemoveAllSubscriptions();
}
public StubDataManager(IDataFeed dataFeed, IAlgorithm algorithm, ITimeKeeper timeKeeper, MarketHoursDatabase marketHoursDatabase, SecurityService securityService, DataPermissionManager dataPermissionManager, bool liveMode = false)
: base(dataFeed,
new UniverseSelection(algorithm, securityService, dataPermissionManager, new DefaultDataProvider()),
algorithm,
timeKeeper,
marketHoursDatabase,
liveMode,
RegisteredSecurityDataTypesProvider.Null,
dataPermissionManager)
{
SecurityService = securityService;
algorithm.Securities.SetSecurityService(securityService);
Algorithm = algorithm;
}
public void FutureChainEnumerator(bool fillForward)
{
var job = new BacktestNodePacket();
var resultHandler = new BacktestingResultHandler();
var feed = new FileSystemDataFeed();
var algorithm = new AlgorithmStub(feed);
algorithm.Transactions.SetOrderProcessor(new FakeOrderProcessor());
algorithm.SetStartDate(new DateTime(2013, 10, 07));
algorithm.SetEndDate(new DateTime(2013, 10, 08));
algorithm.SetFutureChainProvider(new BacktestingFutureChainProvider(TestGlobals.DataCacheProvider));
var dataPermissionManager = new DataPermissionManager();
using var synchronizer = new Synchronizer();
synchronizer.Initialize(algorithm, algorithm.DataManager);
feed.Initialize(algorithm, job, resultHandler, TestGlobals.MapFileProvider, TestGlobals.FactorFileProvider, TestGlobals.DataProvider, algorithm.DataManager, synchronizer, dataPermissionManager.DataChannelProvider);
var future = algorithm.AddFuture("ES", fillDataForward: fillForward);
future.SetFilter(0, 300);
algorithm.PostInitialize();
using var cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(30));
var count = 0L;
var lastMonth = algorithm.StartDate.Month;
foreach (var timeSlice in synchronizer.StreamData(cancellationTokenSource.Token))
{
if (!timeSlice.IsTimePulse && timeSlice.UniverseData?.Count > 0)
{
var nyTime = timeSlice.Time.ConvertFromUtc(algorithm.TimeZone);
var currentExpectedTime = new TimeSpan(0, 0, 0).Add(TimeSpan.FromMinutes(count % (24 * 60)));
while (!future.Exchange.DateTimeIsOpen(nyTime.Date.Add(currentExpectedTime).AddMinutes(-1)))
{
// skip closed market times
currentExpectedTime = new TimeSpan(0, 0, 0).Add(TimeSpan.FromMinutes(++count % (24 * 60)));
}
var universeData = timeSlice.UniverseData.OrderBy(kvp => kvp.Key.Configuration.Symbol).ToList();
var chainData = universeData[0].Value;
Log.Trace($"{nyTime}. Count: {count}. Universe Data Count {universeData.Count}");
Assert.AreEqual(currentExpectedTime, nyTime.TimeOfDay, $"Failed on: {nyTime}. Count: {count}");
Assert.IsTrue(timeSlice.UniverseData.All(kvp => kvp.Value.EndTime.ConvertFromUtc(algorithm.TimeZone).TimeOfDay == nyTime.TimeOfDay));
if (chainData.FilteredContracts.IsNullOrEmpty())
{
Assert.AreEqual(new DateTime(2013, 10, 09), nyTime, $"Unexpected chain FilteredContracts was empty on {nyTime}");
}
if (universeData.Count == 1)
{
// the chain
Assert.IsTrue(universeData.Any(kvp => kvp.Key.Configuration.Symbol == future.Symbol));
}
else
{
// we have 2 universe data, the chain and the continuous future
Assert.AreEqual(2, universeData.Count);
Assert.IsTrue(universeData.All(kvp => kvp.Key.Configuration.Symbol.SecurityType == SecurityType.Future));
Assert.IsTrue(universeData.Any(kvp => kvp.Key.Configuration.Symbol == future.Symbol));
Assert.IsTrue(universeData.Any(kvp => kvp.Key.Configuration.Symbol.ID.Symbol.Contains("CONTINUOUS", StringComparison.InvariantCultureIgnoreCase)));
var continuousData = universeData[1].Value;
Assert.AreEqual(currentExpectedTime, nyTime.TimeOfDay, $"Failed on: {nyTime}");
Assert.IsTrue(!chainData.FilteredContracts.IsNullOrEmpty());
}
count++;
}
}
feed.Exit();
algorithm.DataManager.RemoveAllSubscriptions();
// 2 days worth of minute data
Assert.AreEqual(24 * 2 * 60 + 1, count);
}
