/// <summary>
/// Initializes the data feed for the specified job and algorithm
/// </summary>
public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler, IMapFileProvider mapFileProvider, IFactorFileProvider factorFileProvider, IDataProvider dataProvider)
{
_algorithm = algorithm;
_resultHandler = resultHandler;
_mapFileProvider = mapFileProvider;
_factorFileProvider = factorFileProvider;
_dataProvider = dataProvider;
_subscriptions = new SubscriptionCollection();
_universeSelection = new UniverseSelection(this, algorithm);
_cancellationTokenSource = new CancellationTokenSource();
_subscriptionfactory = new SubscriptionDataReaderSubscriptionEnumeratorFactory(_resultHandler, _mapFileProvider, _factorFileProvider, _dataProvider, false, true);
IsActive = true;
var threadCount = Math.Max(1, Math.Min(4, Environment.ProcessorCount - 3));
_controller = new ParallelRunnerController(threadCount);
_controller.Start(_cancellationTokenSource.Token);
var ffres = Time.OneMinute;
_fillForwardResolution = Ref.Create(() => ffres, res => ffres = res);
// wire ourselves up to receive notifications when universes are added/removed
algorithm.UniverseManager.CollectionChanged += (sender, args) =>
{
switch (args.Action)
{
case NotifyCollectionChangedAction.Add:
foreach (var universe in args.NewItems.OfType <Universe>())
{
var config = universe.Configuration;
var start = _algorithm.UtcTime;
var marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
var exchangeHours = marketHoursDatabase.GetExchangeHours(config);
Security security;
if (!_algorithm.Securities.TryGetValue(config.Symbol, out security))
{
// create a canonical security object if it doesn't exist
security = new Security(exchangeHours, config, _algorithm.Portfolio.CashBook[CashBook.AccountCurrency], SymbolProperties.GetDefault(CashBook.AccountCurrency));
}
var end = _algorithm.EndDate.ConvertToUtc(_algorithm.TimeZone);
AddSubscription(new SubscriptionRequest(true, universe, security, config, start, end));
}
break;
case NotifyCollectionChangedAction.Remove:
foreach (var universe in args.OldItems.OfType <Universe>())
{
RemoveSubscription(universe.Configuration);
}
break;
default:
throw new NotImplementedException("The specified action is not implemented: " + args.Action);
}
};
}
public void ChecksMapFileFirstDate()
{
var algorithm = PerformanceBenchmarkAlgorithms.SingleSecurity_Second;
algorithm.Initialize();
algorithm.PostInitialize();
var dataProvider = new DefaultDataProvider();
var resultHandler = new TestResultHandler();
var mapFileProvider = new LocalDiskMapFileProvider();
var factorFileProvider = new LocalDiskFactorFileProvider(mapFileProvider);
var factory = new SubscriptionDataReaderSubscriptionEnumeratorFactory(resultHandler, mapFileProvider, factorFileProvider, dataProvider, true, enablePriceScaling: false);
var universe = algorithm.UniverseManager.Single().Value;
var security = algorithm.AddEquity("AAA", Resolution.Daily);
var securityConfig = security.Subscriptions.First();
// start date is before the first date in the map file
var subscriptionRequest = new SubscriptionRequest(false, universe, security, securityConfig, new DateTime(2001, 12, 1),
new DateTime(2016, 11, 1));
var enumerator = factory.CreateEnumerator(subscriptionRequest, dataProvider);
// should initialize the data source reader
enumerator.MoveNext();
enumerator.Dispose();
factory.DisposeSafely();
resultHandler.Exit();
var message = ((DebugPacket)resultHandler.Messages.Single()).Message;
Assert.IsTrue(message.Equals(
"The starting date for symbol AAA, 2001-11-30, has been adjusted to match map file first date 2002-05-22."));
}
/// <summary>
/// Initializes the data feed for the specified job and algorithm
/// </summary>
public void Initialize(IAlgorithm algorithm,
AlgorithmNodePacket job,
IResultHandler resultHandler,
IMapFileProvider mapFileProvider,
IFactorFileProvider factorFileProvider,
IDataProvider dataProvider,
IDataFeedSubscriptionManager subscriptionManager,
IDataFeedTimeProvider dataFeedTimeProvider)
{
_algorithm = algorithm;
_resultHandler = resultHandler;
_mapFileProvider = mapFileProvider;
_factorFileProvider = factorFileProvider;
_dataProvider = dataProvider;
_subscriptions = subscriptionManager.DataFeedSubscriptions;
_universeSelection = subscriptionManager.UniverseSelection;
_cancellationTokenSource = new CancellationTokenSource();
_subscriptionFactory = new SubscriptionDataReaderSubscriptionEnumeratorFactory(
_resultHandler,
_mapFileProvider,
_factorFileProvider,
_dataProvider,
includeAuxiliaryData: true);
IsActive = true;
var threadCount = Math.Max(1, Math.Min(4, Environment.ProcessorCount - 3));
}
/// <summary>
/// Initializes the data feed for the specified job and algorithm
/// </summary>
public void Initialize(IAlgorithm algorithm,
AlgorithmNodePacket job,
IResultHandler resultHandler,
IMapFileProvider mapFileProvider,
IFactorFileProvider factorFileProvider,
IDataProvider dataProvider,
IDataFeedSubscriptionManager subscriptionManager,
IDataFeedTimeProvider dataFeedTimeProvider,
IDataChannelProvider dataChannelProvider)
{
_algorithm = algorithm;
_resultHandler = resultHandler;
_mapFileProvider = mapFileProvider;
_factorFileProvider = factorFileProvider;
_dataProvider = dataProvider;
_timeProvider = dataFeedTimeProvider.FrontierTimeProvider;
_subscriptions = subscriptionManager.DataFeedSubscriptions;
_cancellationTokenSource = new CancellationTokenSource();
_subscriptionFactory = new SubscriptionDataReaderSubscriptionEnumeratorFactory(
_resultHandler,
_mapFileProvider,
_factorFileProvider,
_dataProvider,
includeAuxiliaryData: true,
enablePriceScaling: false);
IsActive = true;
}
public void ChecksMapFileFirstDate()
{
var algorithm = PerformanceBenchmarkAlgorithms.SingleSecurity_Second;
algorithm.Initialize();
algorithm.PostInitialize();
var resultHandler = new TestResultHandler();
using var cache = new ZipDataCacheProvider(TestGlobals.DataProvider);
var factory = new SubscriptionDataReaderSubscriptionEnumeratorFactory(resultHandler, TestGlobals.MapFileProvider, TestGlobals.FactorFileProvider, cache, enablePriceScaling: false);
var universe = algorithm.UniverseManager.Single().Value;
var security = algorithm.AddEquity("AAA", Resolution.Daily);
var securityConfig = security.Subscriptions.First();
// start date is before the first date in the map file
var subscriptionRequest = new SubscriptionRequest(false, universe, security, securityConfig, new DateTime(2001, 12, 1),
new DateTime(2016, 11, 1));
var enumerator = factory.CreateEnumerator(subscriptionRequest, TestGlobals.DataProvider);
// should initialize the data source reader
enumerator.MoveNext();
enumerator.Dispose();
factory.DisposeSafely();
resultHandler.Exit();
var message = ((DebugPacket)resultHandler.Messages.Single()).Message;
Assert.IsTrue(message.Equals(
"The starting dates for the following symbols have been adjusted to match their map files first date: [AAA, 2020-09-09]"));
}
public void TestDataFeedEnumeratorStackSpeed()
{
var algorithm = PerformanceBenchmarkAlgorithms.SingleSecurity_Second;
algorithm.Initialize();
algorithm.PostInitialize();
var dataProvider = new DefaultDataProvider();
var resultHandler = new BacktestingResultHandler();
var mapFileProvider = new LocalDiskMapFileProvider();
var factorFileProvider = new LocalDiskFactorFileProvider(mapFileProvider);
var factory = new SubscriptionDataReaderSubscriptionEnumeratorFactory(resultHandler, mapFileProvider, factorFileProvider, dataProvider, true, enablePriceScaling: false);
var universe = algorithm.UniverseManager.Single().Value;
var security = algorithm.Securities.Single().Value;
var securityConfig = security.Subscriptions.First();
var subscriptionRequest = new SubscriptionRequest(false, universe, security, securityConfig, algorithm.StartDate, algorithm.EndDate);
var enumerator = factory.CreateEnumerator(subscriptionRequest, dataProvider);
var count = 0;
var stopwatch = Stopwatch.StartNew();
var lastMonth = algorithm.StartDate.Month;
while (enumerator.MoveNext())
{
var current = enumerator.Current;
if (current == null)
{
Log.Trace("ERROR: Current is null");
continue;
}
if (current.Time.Month != lastMonth)
{
var elapsed = stopwatch.Elapsed.TotalSeconds;
var thousands = count / 1000d;
Log.Trace($"{DateTime.Now} - Time: {current.Time}: KPS: {thousands / elapsed}");
lastMonth = current.Time.Month;
}
count++;
}
Log.Trace("Count: " + count);
stopwatch.Stop();
enumerator.Dispose();
factory.DisposeSafely();
Log.Trace($"Elapsed time: {stopwatch.Elapsed} KPS: {count / 1000d / stopwatch.Elapsed.TotalSeconds}");
}