/// <summary>
/// Initializes the data feed for the specified job and algorithm
/// </summary>
public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler, IMapFileProvider mapFileProvider, IFactorFileProvider factorFileProvider, IDataProvider dataProvider)
{
_algorithm = algorithm;
_resultHandler = resultHandler;
_mapFileProvider = mapFileProvider;
_factorFileProvider = factorFileProvider;
_dataProvider = dataProvider;
_subscriptions = new SubscriptionCollection();
_universeSelection = new UniverseSelection(this, algorithm);
_cancellationTokenSource = new CancellationTokenSource();
_subscriptionfactory = new SubscriptionDataReaderSubscriptionEnumeratorFactory(_resultHandler, _mapFileProvider, _factorFileProvider, _dataProvider, false, true);
IsActive = true;
var threadCount = Math.Max(1, Math.Min(4, Environment.ProcessorCount - 3));
_controller = new ParallelRunnerController(threadCount);
_controller.Start(_cancellationTokenSource.Token);
var ffres = Time.OneMinute;
_fillForwardResolution = Ref.Create(() => ffres, res => ffres = res);
// wire ourselves up to receive notifications when universes are added/removed
algorithm.UniverseManager.CollectionChanged += (sender, args) =>
{
switch (args.Action)
{
case NotifyCollectionChangedAction.Add:
foreach (var universe in args.NewItems.OfType <Universe>())
{
var config = universe.Configuration;
var start = _algorithm.UtcTime;
var marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
var exchangeHours = marketHoursDatabase.GetExchangeHours(config);
Security security;
if (!_algorithm.Securities.TryGetValue(config.Symbol, out security))
{
// create a canonical security object if it doesn't exist
security = new Security(exchangeHours, config, _algorithm.Portfolio.CashBook[CashBook.AccountCurrency], SymbolProperties.GetDefault(CashBook.AccountCurrency));
}
var end = _algorithm.EndDate.ConvertToUtc(_algorithm.TimeZone);
AddSubscription(new SubscriptionRequest(true, universe, security, config, start, end));
}
break;
case NotifyCollectionChangedAction.Remove:
foreach (var universe in args.OldItems.OfType <Universe>())
{
RemoveSubscription(universe.Configuration);
}
break;
default:
throw new NotImplementedException("The specified action is not implemented: " + args.Action);
}
};
}
/// <summary>
/// Initializes the data feed for the specified job and algorithm
/// </summary>
public void Initialize(IAlgorithm algorithm,
AlgorithmNodePacket job,
IResultHandler resultHandler,
IMapFileProvider mapFileProvider,
IFactorFileProvider factorFileProvider,
IDataProvider dataProvider,
IDataFeedSubscriptionManager subscriptionManager,
IDataFeedTimeProvider dataFeedTimeProvider)
{
_algorithm = algorithm;
_resultHandler = resultHandler;
_mapFileProvider = mapFileProvider;
_factorFileProvider = factorFileProvider;
_dataProvider = dataProvider;
_subscriptions = subscriptionManager.DataFeedSubscriptions;
_universeSelection = subscriptionManager.UniverseSelection;
_cancellationTokenSource = new CancellationTokenSource();
_subscriptionFactory = new SubscriptionDataReaderSubscriptionEnumeratorFactory(
_resultHandler,
_mapFileProvider,
_factorFileProvider,
_dataProvider,
includeAuxiliaryData: true);
IsActive = true;
var threadCount = Math.Max(1, Math.Min(4, Environment.ProcessorCount - 3));
}
/// <summary>
/// Initializes a new instance of the <see cref="SubscriptionSynchronizer"/> class
/// </summary>
/// <param name="universeSelection">The universe selection instance used to handle universe
/// selection subscription output</param>
/// <param name="sliceTimeZone">The time zone of the created slice object</param>
/// <param name="cashBook">The cash book, used for creating the cash book updates</param>
/// <returns>A time slice for the specified frontier time</returns>
/// <param name="frontierUtc">The initial UTC frontier time to syncronize at</param>
public SubscriptionSynchronizer(UniverseSelection universeSelection, DateTimeZone sliceTimeZone, CashBook cashBook, DateTime frontierUtc)
{
_frontier = frontierUtc;
_universeSelection = universeSelection;
_sliceTimeZone = sliceTimeZone;
_cashBook = cashBook;
}
/// <summary>
/// Creates a new instance of the DataManager
/// </summary>
public DataManager(IDataFeed dataFeed, UniverseSelection universeSelection, IAlgorithmSettings algorithmSettings, ITimeKeeper timeKeeper)
{
_dataFeed = dataFeed;
UniverseSelection = universeSelection;
_algorithmSettings = algorithmSettings;
AvailableDataTypes = SubscriptionManager.DefaultDataTypes();
_timeKeeper = timeKeeper;
}
/// <summary>
/// Initializes a new instance of the <see cref="SubscriptionSynchronizer"/> class
/// </summary>
/// <param name="universeSelection">The universe selection instance used to handle universe
/// selection subscription output</param>
/// <param name="sliceTimeZone">The time zone of the created slice object</param>
/// <param name="cashBook">The cash book, used for creating the cash book updates</param>
/// <returns>A time slice for the specified frontier time</returns>
/// <param name="timeProvider">The time provider, used to obtain the current frontier UTC value.</param>
public SubscriptionSynchronizer(UniverseSelection universeSelection, DateTimeZone sliceTimeZone,
CashBook cashBook, ITimeProvider timeProvider)
{
_timeProvider = timeProvider;
_universeSelection = universeSelection;
_sliceTimeZone = sliceTimeZone;
_cashBook = cashBook;
}
/// <summary>
/// Initializes the data feed for the specified job and algorithm
/// </summary>
public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler, IMapFileProvider mapFileProvider, IFactorFileProvider factorFileProvider)
{
_algorithm = algorithm;
_resultHandler = resultHandler;
_mapFileProvider = mapFileProvider;
_factorFileProvider = factorFileProvider;
_subscriptions = new SubscriptionCollection();
_universeSelection = new UniverseSelection(this, algorithm, job.Controls);
_cancellationTokenSource = new CancellationTokenSource();
IsActive = true;
var threadCount = Math.Max(1, Math.Min(4, Environment.ProcessorCount - 3));
_controller = new ParallelRunnerController(threadCount);
_controller.Start(_cancellationTokenSource.Token);
var ffres = Time.OneMinute;
_fillForwardResolution = Ref.Create(() => ffres, res => ffres = res);
// wire ourselves up to receive notifications when universes are added/removed
algorithm.UniverseManager.CollectionChanged += (sender, args) =>
{
switch (args.Action)
{
case NotifyCollectionChangedAction.Add:
foreach (var universe in args.NewItems.OfType<Universe>())
{
var config = universe.Configuration;
var start = _frontierUtc != DateTime.MinValue ? _frontierUtc : _algorithm.StartDate.ConvertToUtc(_algorithm.TimeZone);
var marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
var exchangeHours = marketHoursDatabase.GetExchangeHours(config);
Security security;
if (!_algorithm.Securities.TryGetValue(config.Symbol, out security))
{
// create a canonical security object if it doesn't exist
security = new Security(exchangeHours, config, _algorithm.Portfolio.CashBook[CashBook.AccountCurrency], SymbolProperties.GetDefault(CashBook.AccountCurrency));
}
var end = _algorithm.EndDate.ConvertToUtc(_algorithm.TimeZone);
AddSubscription(new SubscriptionRequest(true, universe, security, config, start, end));
}
break;
case NotifyCollectionChangedAction.Remove:
foreach (var universe in args.OldItems.OfType<Universe>())
{
RemoveSubscription(universe.Configuration);
}
break;
default:
throw new NotImplementedException("The specified action is not implemented: " + args.Action);
}
};
}
/// <summary>
/// Initializes the data feed for the specified job and algorithm
/// </summary>
public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler, IMapFileProvider mapFileProvider, IFactorFileProvider factorFileProvider)
{
_algorithm = algorithm;
_resultHandler = resultHandler;
_mapFileProvider = mapFileProvider;
_factorFileProvider = factorFileProvider;
_subscriptions = new ConcurrentDictionary <Symbol, Subscription>();
_universeSelection = new UniverseSelection(this, algorithm, job.Controls);
_cancellationTokenSource = new CancellationTokenSource();
IsActive = true;
var threadCount = Math.Max(1, Math.Min(4, Environment.ProcessorCount - 3));
_controller = new ParallelRunnerController(threadCount);
_controller.Start(_cancellationTokenSource.Token);
var ffres = Time.OneSecond;
_fillForwardResolution = Ref.Create(() => ffres, res => ffres = res);
// find the minimum resolution, ignoring ticks
ffres = ResolveFillForwardResolution(algorithm);
// wire ourselves up to receive notifications when universes are added/removed
algorithm.UniverseManager.CollectionChanged += (sender, args) =>
{
switch (args.Action)
{
case NotifyCollectionChangedAction.Add:
foreach (var universe in args.NewItems.OfType <Universe>())
{
var start = _frontierUtc != DateTime.MinValue ? _frontierUtc : _algorithm.StartDate.ConvertToUtc(_algorithm.TimeZone);
AddUniverseSubscription(universe, start, _algorithm.EndDate.ConvertToUtc(_algorithm.TimeZone));
}
break;
case NotifyCollectionChangedAction.Remove:
foreach (var universe in args.OldItems.OfType <Universe>())
{
Subscription subscription;
if (_subscriptions.TryGetValue(universe.Configuration.Symbol, out subscription))
{
RemoveSubscription(subscription);
}
}
break;
default:
throw new NotImplementedException("The specified action is not implemented: " + args.Action);
}
};
}
/// <summary>
/// Initializes the data feed for the specified job and algorithm
/// </summary>
public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler, IMapFileProvider mapFileProvider, IFactorFileProvider factorFileProvider)
{
_algorithm = algorithm;
_resultHandler = resultHandler;
_mapFileProvider = mapFileProvider;
_factorFileProvider = factorFileProvider;
_subscriptions = new ConcurrentDictionary<Symbol, Subscription>();
_universeSelection = new UniverseSelection(this, algorithm, job.Controls);
_cancellationTokenSource = new CancellationTokenSource();
IsActive = true;
var threadCount = Math.Max(1, Math.Min(4, Environment.ProcessorCount - 3));
_controller = new ParallelRunnerController(threadCount);
_controller.Start(_cancellationTokenSource.Token);
var ffres = Time.OneSecond;
_fillForwardResolution = Ref.Create(() => ffres, res => ffres = res);
// find the minimum resolution, ignoring ticks
ffres = ResolveFillForwardResolution(algorithm);
// wire ourselves up to receive notifications when universes are added/removed
algorithm.UniverseManager.CollectionChanged += (sender, args) =>
{
switch (args.Action)
{
case NotifyCollectionChangedAction.Add:
foreach (var universe in args.NewItems.OfType<Universe>())
{
var start = _frontierUtc != DateTime.MinValue ? _frontierUtc : _algorithm.StartDate.ConvertToUtc(_algorithm.TimeZone);
AddUniverseSubscription(universe, start, _algorithm.EndDate.ConvertToUtc(_algorithm.TimeZone));
}
break;
case NotifyCollectionChangedAction.Remove:
foreach (var universe in args.OldItems.OfType<Universe>())
{
Subscription subscription;
if (_subscriptions.TryGetValue(universe.Configuration.Symbol, out subscription))
{
RemoveSubscription(subscription);
}
}
break;
default:
throw new NotImplementedException("The specified action is not implemented: " + args.Action);
}
};
}
/// <summary>
/// Initializes the data feed for the specified job and algorithm
/// </summary>
public void Initialize(IAlgorithm algorithm,
AlgorithmNodePacket job,
IResultHandler resultHandler,
IMapFileProvider mapFileProvider,
IFactorFileProvider factorFileProvider,
IDataProvider dataProvider,
IDataFeedSubscriptionManager subscriptionManager,
IDataFeedTimeProvider dataFeedTimeProvider)
{
if (!(job is LiveNodePacket))
{
throw new ArgumentException("The LiveTradingDataFeed requires a LiveNodePacket.");
}
_cancellationTokenSource = new CancellationTokenSource();
_algorithm = algorithm;
_job = (LiveNodePacket)job;
_timeProvider = dataFeedTimeProvider.TimeProvider;
_dataQueueHandler = GetDataQueueHandler();
_dataProvider = dataProvider;
_channelProvider = GetDataChannelProvider();
_frontierTimeProvider = dataFeedTimeProvider.FrontierTimeProvider;
_customExchange = new BaseDataExchange("CustomDataExchange")
{
SleepInterval = 10
};
// sleep is controlled on this exchange via the GetNextTicksEnumerator
_exchange = new BaseDataExchange("DataQueueExchange")
{
SleepInterval = 0
};
_exchange.AddEnumerator(DataQueueHandlerSymbol, GetNextTicksEnumerator());
_subscriptions = subscriptionManager.DataFeedSubscriptions;
_universeSelection = subscriptionManager.UniverseSelection;
// run the exchanges
Task.Run(() => _exchange.Start(_cancellationTokenSource.Token));
Task.Run(() => _customExchange.Start(_cancellationTokenSource.Token));
IsActive = true;
}
/// <summary>
/// Live trading datafeed handler provides a base implementation of a live trading datafeed. Derived types
/// need only implement the GetNextTicks() function to return unprocessed ticks from a data source.
/// This creates a new data feed with a DataFeedEndpoint of LiveTrading.
/// </summary>
public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler)
{
if (!(job is LiveNodePacket))
{
throw new ArgumentException("The LiveTradingDataFeed requires a LiveNodePacket.");
}
_job = (LiveNodePacket)job;
_isActive = true;
_algorithm = algorithm;
_resultHandler = resultHandler;
_cancellationTokenSource = new CancellationTokenSource();
_universeSelection = new UniverseSelection(this, algorithm, true);
_dataQueue = Composer.Instance.GetExportedValueByTypeName <IDataQueueHandler>(Configuration.Config.Get("data-queue-handler", "LiveDataQueue"));
Bridge = new BusyBlockingCollection <TimeSlice>();
_subscriptions = new ConcurrentDictionary <SymbolSecurityType, LiveSubscription>();
var periodStart = DateTime.UtcNow.ConvertFromUtc(algorithm.TimeZone).AddDays(-7);
var periodEnd = Time.EndOfTime;
foreach (var security in algorithm.Securities.Values)
{
var subscription = CreateSubscription(algorithm, resultHandler, security, periodStart, periodEnd);
_subscriptions.AddOrUpdate(new SymbolSecurityType(subscription), subscription);
}
// request for data from these symbols
var symbols = BuildTypeSymbolList(algorithm.Securities.Values);
if (symbols.Any())
{
// don't subscribe if there's nothing there, this allows custom data to
// work without an IDataQueueHandler implementation by specifying LiveDataQueue
// in the configuration, that implementation throws on every method, but we actually
// don't need it if we're only doing custom data
_dataQueue.Subscribe(_job, symbols);
}
}
public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler)
{
if (algorithm.SubscriptionManager.Subscriptions.Count == 0 && algorithm.Universe == null)
{
throw new Exception("No subscriptions registered and no universe defined.");
}
_algorithm = algorithm;
_resultHandler = resultHandler;
_subscriptions = new ConcurrentDictionary <SymbolSecurityType, Subscription>();
_cancellationTokenSource = new CancellationTokenSource();
_universeSelection = new UniverseSelection(this, algorithm, false);
IsActive = true;
Bridge = new BusyBlockingCollection <TimeSlice>(100);
// find the minimum resolution, ignoring ticks
_fillForwardResolution = algorithm.SubscriptionManager.Subscriptions
.Where(x => x.Resolution != Resolution.Tick)
.Select(x => x.Resolution)
.DefaultIfEmpty(algorithm.UniverseSettings.Resolution)
.Min();
// initialize the original user defined securities
foreach (var security in _algorithm.Securities.Values)
{
var subscription = CreateSubscription(resultHandler, security, algorithm.StartDate, algorithm.EndDate, _fillForwardResolution, true);
if (subscription != null)
{
_subscriptions.AddOrUpdate(new SymbolSecurityType(security), subscription);
// prime the pump, run method checks current before move next calls
PrimeSubscriptionPump(subscription, true);
}
}
}
/// <summary>
/// Execute the primary thread for retrieving stock data.
/// 1. Subscribe to the streams requested.
/// 2. Build bars or tick data requested, primary loop increment smallest possible.
/// </summary>
public void Run()
{
//Initialize:
// Set up separate thread to handle stream and building packets:
var streamThread = new Thread(StreamStoreConsumer);
streamThread.Start();
Thread.Sleep(5); // Wait a little for the other thread to init.
// This thread converts data into bars "on" the second - assuring the bars are close as
// possible to a second unit tradebar (starting at 0 milliseconds).
var realtime = new RealTimeSynchronizedTimer(TimeSpan.FromSeconds(1), utcTriggerTime =>
{
// determine if we're on even time boundaries for data emit
var onMinute = utcTriggerTime.Second == 0;
var onHour = onMinute && utcTriggerTime.Minute == 0;
// Determine if this subscription needs to be archived:
var items = new List <KeyValuePair <Security, List <BaseData> > >();
var changes = SecurityChanges.None;
var performedUniverseSelection = new HashSet <string>();
foreach (var kvp in _subscriptions)
{
var subscription = kvp.Value;
if (subscription.Configuration.Resolution == Resolution.Tick)
{
continue;
}
var localTime = new DateTime(utcTriggerTime.Ticks - subscription.OffsetProvider.GetOffsetTicks(utcTriggerTime));
var onDay = onHour && localTime.Hour == 0;
// perform universe selection if requested on day changes (don't perform multiple times per market)
if (onDay && _algorithm.Universe != null && performedUniverseSelection.Add(subscription.Configuration.Market))
{
var coarse = UniverseSelection.GetCoarseFundamentals(subscription.Configuration.Market, subscription.TimeZone, localTime.Date, true);
OnFundamental(FundamentalType.Coarse, utcTriggerTime, subscription.Configuration, coarse.ToList());
}
var triggerArchive = false;
switch (subscription.Configuration.Resolution)
{
case Resolution.Second:
triggerArchive = true;
break;
case Resolution.Minute:
triggerArchive = onMinute;
break;
case Resolution.Hour:
triggerArchive = onHour;
break;
case Resolution.Daily:
triggerArchive = onDay;
break;
}
if (triggerArchive)
{
var data = subscription.StreamStore.TriggerArchive(utcTriggerTime);
if (data != null)
{
items.Add(new KeyValuePair <Security, List <BaseData> >(subscription.Security, new List <BaseData> {
data
}));
}
}
}
// don't try to add if we're already cancelling
if (_cancellationTokenSource.IsCancellationRequested)
{
return;
}
Bridge.Add(TimeSlice.Create(utcTriggerTime, _algorithm.TimeZone, _algorithm.Portfolio.CashBook, items, changes));
});
//Start the realtime sampler above
realtime.Start();
while (!_cancellationTokenSource.IsCancellationRequested && !_endOfBridges)
{
// main work of this class is done in the realtime and stream store consumer threads
Thread.Sleep(1000);
}
//Dispose of the realtime clock.
realtime.Stop();
//Stop thread
_isActive = false;
//Exit Live DataStream Feed:
Log.Trace("LiveTradingDataFeed.Run(): Exiting LiveTradingDataFeed Run Method");
}
/// <summary>
/// Creates a new instance of the DataManager
/// </summary>
public DataManager(
IDataFeed dataFeed,
UniverseSelection universeSelection,
IAlgorithm algorithm,
ITimeKeeper timeKeeper,
MarketHoursDatabase marketHoursDatabase,
bool liveMode,
IRegisteredSecurityDataTypesProvider registeredTypesProvider,
IDataPermissionManager dataPermissionManager)
{
_dataFeed = dataFeed;
UniverseSelection = universeSelection;
UniverseSelection.SetDataManager(this);
_algorithmSettings = algorithm.Settings;
AvailableDataTypes = SubscriptionManager.DefaultDataTypes();
_timeKeeper = timeKeeper;
_marketHoursDatabase = marketHoursDatabase;
_liveMode = liveMode;
_registeredTypesProvider = registeredTypesProvider;
_dataPermissionManager = dataPermissionManager;
// wire ourselves up to receive notifications when universes are added/removed
algorithm.UniverseManager.CollectionChanged += (sender, args) =>
{
switch (args.Action)
{
case NotifyCollectionChangedAction.Add:
foreach (var universe in args.NewItems.OfType <Universe>())
{
var config = universe.Configuration;
var start = algorithm.UtcTime;
var end = algorithm.LiveMode ? Time.EndOfTime
: algorithm.EndDate.ConvertToUtc(algorithm.TimeZone);
Security security;
if (!algorithm.Securities.TryGetValue(config.Symbol, out security))
{
// create a canonical security object if it doesn't exist
security = new Security(
_marketHoursDatabase.GetExchangeHours(config),
config,
algorithm.Portfolio.CashBook[algorithm.AccountCurrency],
SymbolProperties.GetDefault(algorithm.AccountCurrency),
algorithm.Portfolio.CashBook,
RegisteredSecurityDataTypesProvider.Null,
new SecurityCache()
);
}
AddSubscription(
new SubscriptionRequest(true,
universe,
security,
config,
start,
end));
}
break;
case NotifyCollectionChangedAction.Remove:
foreach (var universe in args.OldItems.OfType <Universe>())
{
// removing the subscription will be handled by the SubscriptionSynchronizer
// in the next loop as well as executing a UniverseSelection one last time.
if (!universe.DisposeRequested)
{
universe.Dispose();
}
}
break;
default:
throw new NotImplementedException("The specified action is not implemented: " + args.Action);
}
};
}
/// <summary>
/// Creates a new instance of the DataManager
/// </summary>
public DataManager(IDataFeed dataFeed, UniverseSelection universeSelection, IAlgorithmSettings algorithmSettings)
{
_dataFeed = dataFeed;
UniverseSelection = universeSelection;
_algorithmSettings = algorithmSettings;
}
/// <summary>
/// Runs a single backtest/live job from the job queue
/// </summary>
/// <param name="job">The algorithm job to be processed</param>
/// <param name="assemblyPath">The path to the algorithm's assembly</param>
public void Run(AlgorithmNodePacket job, string assemblyPath)
{
var algorithm = default(IAlgorithm);
var algorithmManager = new AlgorithmManager(_liveMode);
//Start monitoring the backtest active status:
var statusPing = new StateCheck.Ping(algorithmManager, _systemHandlers.Api, _algorithmHandlers.Results, _systemHandlers.Notify, job);
var statusPingThread = new Thread(statusPing.Run);
statusPingThread.Start();
try
{
//Reset thread holders.
var initializeComplete = false;
Thread threadFeed = null;
Thread threadTransactions = null;
Thread threadResults = null;
Thread threadRealTime = null;
//-> Initialize messaging system
_systemHandlers.Notify.SetChannel(job.Channel);
//-> Set the result handler type for this algorithm job, and launch the associated result thread.
_algorithmHandlers.Results.Initialize(job, _systemHandlers.Notify, _systemHandlers.Api, _algorithmHandlers.DataFeed, _algorithmHandlers.Setup, _algorithmHandlers.Transactions);
threadResults = new Thread(_algorithmHandlers.Results.Run, 0) {Name = "Result Thread"};
threadResults.Start();
IBrokerage brokerage = null;
try
{
// Save algorithm to cache, load algorithm instance:
algorithm = _algorithmHandlers.Setup.CreateAlgorithmInstance(assemblyPath, job.Language);
// set the history provider before setting up the algorithm
_algorithmHandlers.HistoryProvider.Initialize(job, progress =>
{
// send progress updates to the result handler only during initialization
if (!algorithm.GetLocked() || algorithm.IsWarmingUp)
{
_algorithmHandlers.Results.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.History,
string.Format("Processing history {0}%...", progress));
}
});
algorithm.HistoryProvider = _algorithmHandlers.HistoryProvider;
// initialize the default brokerage message handler
algorithm.BrokerageMessageHandler = new DefaultBrokerageMessageHandler(algorithm, job, _algorithmHandlers.Results, _systemHandlers.Api);
//Initialize the internal state of algorithm and job: executes the algorithm.Initialize() method.
initializeComplete = _algorithmHandlers.Setup.Setup(algorithm, out brokerage, job, _algorithmHandlers.Results, _algorithmHandlers.Transactions, _algorithmHandlers.RealTime);
// set this again now that we've actually added securities
_algorithmHandlers.Results.SetAlgorithm(algorithm);
//If there are any reasons it failed, pass these back to the IDE.
if (!initializeComplete || algorithm.ErrorMessages.Count > 0 || _algorithmHandlers.Setup.Errors.Count > 0)
{
initializeComplete = false;
//Get all the error messages: internal in algorithm and external in setup handler.
var errorMessage = String.Join(",", algorithm.ErrorMessages);
errorMessage += String.Join(",", _algorithmHandlers.Setup.Errors);
Log.Error("Engine.Run(): " + errorMessage);
_algorithmHandlers.Results.RuntimeError(errorMessage);
_systemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, errorMessage);
}
}
catch (Exception err)
{
var runtimeMessage = "Algorithm.Initialize() Error: " + err.Message + " Stack Trace: " + err.StackTrace;
_algorithmHandlers.Results.RuntimeError(runtimeMessage, err.StackTrace);
_systemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, runtimeMessage);
}
//-> Using the job + initialization: load the designated handlers:
if (initializeComplete)
{
//-> Reset the backtest stopwatch; we're now running the algorithm.
var startTime = DateTime.Now;
//Set algorithm as locked; set it to live mode if we're trading live, and set it to locked for no further updates.
algorithm.SetAlgorithmId(job.AlgorithmId);
algorithm.SetLocked();
//Wire up the universe selection event handler before kicking off the data feed
var universeSelection = new UniverseSelection(_algorithmHandlers.DataFeed, algorithm, _liveMode);
_algorithmHandlers.DataFeed.UniverseSelection += (sender, args) => universeSelection.ApplyUniverseSelection(args);
//Load the associated handlers for data, transaction and realtime events:
_algorithmHandlers.DataFeed.Initialize(algorithm, job, _algorithmHandlers.Results);
_algorithmHandlers.Transactions.Initialize(algorithm, brokerage, _algorithmHandlers.Results);
_algorithmHandlers.RealTime.Setup(algorithm, job, _algorithmHandlers.Results, _systemHandlers.Api);
// wire up the brokerage message handler
brokerage.Message += (sender, message) => algorithm.BrokerageMessageHandler.Handle(message);
//Send status to user the algorithm is now executing.
_algorithmHandlers.Results.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.Running);
//Launch the data, transaction and realtime handlers into dedicated threads
threadFeed = new Thread(_algorithmHandlers.DataFeed.Run) {Name = "DataFeed Thread"};
threadTransactions = new Thread(_algorithmHandlers.Transactions.Run) {Name = "Transaction Thread"};
threadRealTime = new Thread(_algorithmHandlers.RealTime.Run) {Name = "RealTime Thread"};
//Launch the data feed, result sending, and transaction models/handlers in separate threads.
threadFeed.Start(); // Data feed pushing data packets into thread bridge;
threadTransactions.Start(); // Transaction modeller scanning new order requests
threadRealTime.Start(); // RealTime scan time for time based events:
// Result manager scanning message queue: (started earlier)
_algorithmHandlers.Results.DebugMessage(string.Format("Launching analysis for {0} with LEAN Engine v{1}", job.AlgorithmId, Constants.Version));
try
{
//Create a new engine isolator class
var isolator = new Isolator();
// Execute the Algorithm Code:
var complete = isolator.ExecuteWithTimeLimit(_algorithmHandlers.Setup.MaximumRuntime, algorithmManager.TimeLoopWithinLimits, () =>
{
try
{
//Run Algorithm Job:
// -> Using this Data Feed,
// -> Send Orders to this TransactionHandler,
// -> Send Results to ResultHandler.
algorithmManager.Run(job, algorithm, _algorithmHandlers.DataFeed, _algorithmHandlers.Transactions, _algorithmHandlers.Results, _algorithmHandlers.RealTime, isolator.CancellationToken);
}
catch (Exception err)
{
//Debugging at this level is difficult, stack trace needed.
Log.Error(err);
algorithm.RunTimeError = err;
algorithmManager.SetStatus(AlgorithmStatus.RuntimeError);
return;
}
Log.Trace("Engine.Run(): Exiting Algorithm Manager");
}, job.RamAllocation);
if (!complete)
{
Log.Error("Engine.Main(): Failed to complete in time: " + _algorithmHandlers.Setup.MaximumRuntime.ToString("F"));
throw new Exception("Failed to complete algorithm within " + _algorithmHandlers.Setup.MaximumRuntime.ToString("F")
+ " seconds. Please make it run faster.");
}
// Algorithm runtime error:
if (algorithm.RunTimeError != null)
{
throw algorithm.RunTimeError;
}
}
catch (Exception err)
{
//Error running the user algorithm: purge datafeed, send error messages, set algorithm status to failed.
Log.Error("Engine.Run(): Breaking out of parent try-catch: " + err.Message + " " + err.StackTrace);
if (_algorithmHandlers.DataFeed != null) _algorithmHandlers.DataFeed.Exit();
if (_algorithmHandlers.Results != null)
{
var message = "Runtime Error: " + err.Message;
Log.Trace("Engine.Run(): Sending runtime error to user...");
_algorithmHandlers.Results.LogMessage(message);
_algorithmHandlers.Results.RuntimeError(message, err.StackTrace);
_systemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, message + " Stack Trace: " + err.StackTrace);
}
}
//Send result data back: this entire code block could be rewritten.
// todo: - Split up statistics class, its enormous.
// todo: - Make a dedicated Statistics.Benchmark class.
// todo: - Move all creation and transmission of statistics out of primary engine loop.
// todo: - Statistics.Generate(algorithm, resulthandler, transactionhandler);
try
{
var trades = algorithm.TradeBuilder.ClosedTrades;
var charts = new Dictionary<string, Chart>(_algorithmHandlers.Results.Charts);
var orders = new Dictionary<int, Order>(_algorithmHandlers.Transactions.Orders);
var holdings = new Dictionary<string, Holding>();
var banner = new Dictionary<string, string>();
var statisticsResults = new StatisticsResults();
try
{
//Generates error when things don't exist (no charting logged, runtime errors in main algo execution)
const string strategyEquityKey = "Strategy Equity";
const string equityKey = "Equity";
const string dailyPerformanceKey = "Daily Performance";
const string benchmarkKey = "Benchmark";
// make sure we've taken samples for these series before just blindly requesting them
if (charts.ContainsKey(strategyEquityKey) &&
charts[strategyEquityKey].Series.ContainsKey(equityKey) &&
charts[strategyEquityKey].Series.ContainsKey(dailyPerformanceKey))
{
var equity = charts[strategyEquityKey].Series[equityKey].Values;
var performance = charts[strategyEquityKey].Series[dailyPerformanceKey].Values;
var profitLoss = new SortedDictionary<DateTime, decimal>(algorithm.Transactions.TransactionRecord);
var totalTransactions = algorithm.Transactions.GetOrders(x => x.Status.IsFill()).Count();
var benchmark = charts[benchmarkKey].Series[benchmarkKey].Values;
statisticsResults = StatisticsBuilder.Generate(trades, profitLoss, equity, performance, benchmark,
_algorithmHandlers.Setup.StartingPortfolioValue, algorithm.Portfolio.TotalFees, totalTransactions);
}
}
catch (Exception err)
{
Log.Error("Algorithm.Node.Engine(): Error generating statistics packet: " + err.Message);
}
//Diagnostics Completed, Send Result Packet:
var totalSeconds = (DateTime.Now - startTime).TotalSeconds;
var dataPoints = algorithmManager.DataPoints + _algorithmHandlers.HistoryProvider.DataPointCount;
_algorithmHandlers.Results.DebugMessage(
string.Format("Algorithm Id:({0}) completed in {1} seconds at {2}k data points per second. Processing total of {3} data points.",
job.AlgorithmId, totalSeconds.ToString("F2"), ((dataPoints/(double) 1000)/totalSeconds).ToString("F0"),
dataPoints.ToString("N0")));
_algorithmHandlers.Results.SendFinalResult(job, orders, algorithm.Transactions.TransactionRecord, holdings, statisticsResults, banner);
}
catch (Exception err)
{
Log.Error("Engine.Main(): Error sending analysis result: " + err.Message + " ST >> " + err.StackTrace);
}
//Before we return, send terminate commands to close up the threads
_algorithmHandlers.Transactions.Exit();
_algorithmHandlers.DataFeed.Exit();
_algorithmHandlers.RealTime.Exit();
}
//Close result handler:
_algorithmHandlers.Results.Exit();
statusPing.Exit();
//Wait for the threads to complete:
var ts = Stopwatch.StartNew();
while ((_algorithmHandlers.Results.IsActive
|| (_algorithmHandlers.Transactions != null && _algorithmHandlers.Transactions.IsActive)
|| (_algorithmHandlers.DataFeed != null && _algorithmHandlers.DataFeed.IsActive)
|| (_algorithmHandlers.RealTime != null && _algorithmHandlers.RealTime.IsActive))
&& ts.ElapsedMilliseconds < 30*1000)
{
Thread.Sleep(100);
Log.Trace("Waiting for threads to exit...");
}
//Terminate threads still in active state.
if (threadFeed != null && threadFeed.IsAlive) threadFeed.Abort();
if (threadTransactions != null && threadTransactions.IsAlive) threadTransactions.Abort();
if (threadResults != null && threadResults.IsAlive) threadResults.Abort();
if (statusPingThread != null && statusPingThread.IsAlive) statusPingThread.Abort();
if (brokerage != null)
{
brokerage.Disconnect();
}
if (_algorithmHandlers.Setup != null)
{
_algorithmHandlers.Setup.Dispose();
}
Log.Trace("Engine.Main(): Analysis Completed and Results Posted.");
}
catch (Exception err)
{
Log.Error("Engine.Main(): Error running algorithm: " + err.Message + " >> " + err.StackTrace);
}
finally
{
//No matter what for live mode; make sure we've set algorithm status in the API for "not running" conditions:
if (_liveMode && algorithmManager.State != AlgorithmStatus.Running && algorithmManager.State != AlgorithmStatus.RuntimeError)
_systemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, algorithmManager.State);
_algorithmHandlers.Results.Exit();
_algorithmHandlers.DataFeed.Exit();
_algorithmHandlers.Transactions.Exit();
_algorithmHandlers.RealTime.Exit();
}
}
/// <summary>
/// Initializes the data feed for the specified job and algorithm
/// </summary>
public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler, IMapFileProvider mapFileProvider, IFactorFileProvider factorFileProvider)
{
if (!(job is LiveNodePacket))
{
throw new ArgumentException("The LiveTradingDataFeed requires a LiveNodePacket.");
}
_cancellationTokenSource = new CancellationTokenSource();
_algorithm = algorithm;
_job = (LiveNodePacket)job;
_resultHandler = resultHandler;
_timeProvider = GetTimeProvider();
_dataQueueHandler = GetDataQueueHandler();
_frontierTimeProvider = new ManualTimeProvider(_timeProvider.GetUtcNow());
_customExchange = new BaseDataExchange("CustomDataExchange")
{
SleepInterval = 10
};
// sleep is controlled on this exchange via the GetNextTicksEnumerator
_exchange = new BaseDataExchange("DataQueueExchange")
{
SleepInterval = 0
};
_exchange.AddEnumerator(DataQueueHandlerSymbol, GetNextTicksEnumerator());
_subscriptions = new ConcurrentDictionary <Symbol, Subscription>();
_bridge = new BusyBlockingCollection <TimeSlice>();
_universeSelection = new UniverseSelection(this, algorithm, job.Controls);
// run the exchanges
Task.Run(() => _exchange.Start(_cancellationTokenSource.Token));
Task.Run(() => _customExchange.Start(_cancellationTokenSource.Token));
// this value will be modified via calls to AddSubscription/RemoveSubscription
var ffres = Time.OneMinute;
_fillForwardResolution = Ref.Create(() => ffres, v => ffres = v);
// wire ourselves up to receive notifications when universes are added/removed
var start = _timeProvider.GetUtcNow();
algorithm.UniverseManager.CollectionChanged += (sender, args) =>
{
switch (args.Action)
{
case NotifyCollectionChangedAction.Add:
foreach (var universe in args.NewItems.OfType <Universe>())
{
_subscriptions[universe.Configuration.Symbol] = CreateUniverseSubscription(universe, start, Time.EndOfTime);
UpdateFillForwardResolution();
}
break;
case NotifyCollectionChangedAction.Remove:
foreach (var universe in args.OldItems.OfType <Universe>())
{
Subscription subscription;
if (_subscriptions.TryGetValue(universe.Configuration.Symbol, out subscription))
{
RemoveSubscription(subscription.Configuration.Symbol);
}
}
break;
default:
throw new NotImplementedException("The specified action is not implemented: " + args.Action);
}
};
}
/// <summary>
/// Initializes a new instance of the <see cref="SubscriptionSynchronizer"/> class
/// </summary>
/// <param name="universeSelection">The universe selection instance used to handle universe
/// selection subscription output</param>
public SubscriptionSynchronizer(UniverseSelection universeSelection)
{
_universeSelection = universeSelection;
}
/// <summary>
/// Initializes the data feed for the specified job and algorithm
/// </summary>
public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler,
IMapFileProvider mapFileProvider, IFactorFileProvider factorFileProvider,
IDataProvider dataProvider, IDataFeedSubscriptionManager subscriptionManager)
{
if (!(job is LiveNodePacket))
{
throw new ArgumentException("The LiveTradingDataFeed requires a LiveNodePacket.");
}
_cancellationTokenSource = new CancellationTokenSource();
_algorithm = algorithm;
_job = (LiveNodePacket)job;
_resultHandler = resultHandler;
_timeProvider = GetTimeProvider();
_dataQueueHandler = GetDataQueueHandler();
_dataProvider = dataProvider;
_dataCacheProvider = new SingleEntryDataCacheProvider(dataProvider);
_frontierTimeProvider = new ManualTimeProvider(_timeProvider.GetUtcNow());
_customExchange = new BaseDataExchange("CustomDataExchange")
{
SleepInterval = 10
};
// sleep is controlled on this exchange via the GetNextTicksEnumerator
_exchange = new BaseDataExchange("DataQueueExchange")
{
SleepInterval = 0
};
_exchange.AddEnumerator(DataQueueHandlerSymbol, GetNextTicksEnumerator());
_subscriptions = subscriptionManager.DataFeedSubscriptions;
_bridge = new BusyBlockingCollection <TimeSlice>();
_universeSelection = subscriptionManager.UniverseSelection;
// run the exchanges
Task.Run(() => _exchange.Start(_cancellationTokenSource.Token));
Task.Run(() => _customExchange.Start(_cancellationTokenSource.Token));
// wire ourselves up to receive notifications when universes are added/removed
var start = _timeProvider.GetUtcNow();
algorithm.UniverseManager.CollectionChanged += (sender, args) =>
{
switch (args.Action)
{
case NotifyCollectionChangedAction.Add:
foreach (var universe in args.NewItems.OfType <Universe>())
{
var config = universe.Configuration;
var marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
var exchangeHours = marketHoursDatabase.GetExchangeHours(config);
Security security;
if (!_algorithm.Securities.TryGetValue(config.Symbol, out security))
{
// create a canonical security object
security = new Security(exchangeHours, config, _algorithm.Portfolio.CashBook[CashBook.AccountCurrency], SymbolProperties.GetDefault(CashBook.AccountCurrency));
}
AddSubscription(new SubscriptionRequest(true, universe, security, config, start, Time.EndOfTime));
}
break;
case NotifyCollectionChangedAction.Remove:
foreach (var universe in args.OldItems.OfType <Universe>())
{
RemoveSubscription(universe.Configuration);
}
break;
default:
throw new NotImplementedException("The specified action is not implemented: " + args.Action);
}
};
}
/// <summary>
/// Creates a new instance of the DataManager
/// </summary>
public DataManager(IDataFeed dataFeed, IAlgorithm algorithm)
{
_dataFeed = dataFeed;
UniverseSelection = new UniverseSelection(dataFeed, algorithm);
}
/// <summary>
/// Live trading datafeed handler provides a base implementation of a live trading datafeed. Derived types
/// need only implement the GetNextTicks() function to return unprocessed ticks from a data source.
/// This creates a new data feed with a DataFeedEndpoint of LiveTrading.
/// </summary>
public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler)
{
if (!(job is LiveNodePacket))
{
throw new ArgumentException("The LiveTradingDataFeed requires a LiveNodePacket.");
}
_job = (LiveNodePacket)job;
_isActive = true;
_algorithm = algorithm;
_resultHandler = resultHandler;
_cancellationTokenSource = new CancellationTokenSource();
_universeSelection = new UniverseSelection(this, algorithm, true);
_dataQueue = Composer.Instance.GetExportedValueByTypeName<IDataQueueHandler>(Configuration.Config.Get("data-queue-handler", "LiveDataQueue"));
Bridge = new BusyBlockingCollection<TimeSlice>();
_subscriptions = new ConcurrentDictionary<SymbolSecurityType, LiveSubscription>();
var periodStart = DateTime.UtcNow.ConvertFromUtc(algorithm.TimeZone).AddDays(-7);
var periodEnd = Time.EndOfTime;
foreach (var security in algorithm.Securities.Values)
{
var subscription = CreateSubscription(algorithm, resultHandler, security, periodStart, periodEnd);
_subscriptions.AddOrUpdate(new SymbolSecurityType(subscription), subscription);
}
// request for data from these symbols
var symbols = BuildTypeSymbolList(algorithm.Securities.Values);
if (symbols.Any())
{
// don't subscribe if there's nothing there, this allows custom data to
// work without an IDataQueueHandler implementation by specifying LiveDataQueue
// in the configuration, that implementation throws on every method, but we actually
// don't need it if we're only doing custom data
_dataQueue.Subscribe(_job, symbols);
}
}
/// <summary>
/// Initializes the data feed for the specified job and algorithm
/// </summary>
public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler, IMapFileProvider mapFileProvider, IFactorFileProvider factorFileProvider)
{
if (!(job is LiveNodePacket))
{
throw new ArgumentException("The LiveTradingDataFeed requires a LiveNodePacket.");
}
_cancellationTokenSource = new CancellationTokenSource();
_algorithm = algorithm;
_job = (LiveNodePacket) job;
_resultHandler = resultHandler;
_timeProvider = GetTimeProvider();
_dataQueueHandler = GetDataQueueHandler();
_frontierTimeProvider = new ManualTimeProvider(_timeProvider.GetUtcNow());
_customExchange = new BaseDataExchange("CustomDataExchange") {SleepInterval = 10};
// sleep is controlled on this exchange via the GetNextTicksEnumerator
_exchange = new BaseDataExchange("DataQueueExchange"){SleepInterval = 0};
_exchange.AddEnumerator(DataQueueHandlerSymbol, GetNextTicksEnumerator());
_subscriptions = new ConcurrentDictionary<Symbol, List<Subscription>>();
_bridge = new BusyBlockingCollection<TimeSlice>();
_universeSelection = new UniverseSelection(this, algorithm, job.Controls);
// run the exchanges
Task.Run(() => _exchange.Start(_cancellationTokenSource.Token));
Task.Run(() => _customExchange.Start(_cancellationTokenSource.Token));
// this value will be modified via calls to AddSubscription/RemoveSubscription
var ffres = Time.OneMinute;
_fillForwardResolution = Ref.Create(() => ffres, v => ffres = v);
// wire ourselves up to receive notifications when universes are added/removed
var start = _timeProvider.GetUtcNow();
algorithm.UniverseManager.CollectionChanged += (sender, args) =>
{
switch (args.Action)
{
case NotifyCollectionChangedAction.Add:
foreach (var universe in args.NewItems.OfType<Universe>())
{
_subscriptions.Add(universe.Configuration.Symbol, CreateUniverseSubscription(universe, start, Time.EndOfTime));
UpdateFillForwardResolution();
}
break;
case NotifyCollectionChangedAction.Remove:
foreach (var universe in args.OldItems.OfType<Universe>())
{
RemoveSubscription(universe.Configuration.Symbol);
}
break;
default:
throw new NotImplementedException("The specified action is not implemented: " + args.Action);
}
};
}
/// <summary>
/// Initializes a new instance of the <see cref="SubscriptionSynchronizer"/> class
/// </summary>
/// <param name="universeSelection">The universe selection instance used to handle universe
/// selection subscription output</param>
/// <returns>A time slice for the specified frontier time</returns>
public SubscriptionSynchronizer(UniverseSelection universeSelection)
{
_universeSelection = universeSelection;
}
public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler)
{
if (algorithm.SubscriptionManager.Subscriptions.Count == 0 && algorithm.Universe == null)
{
throw new Exception("No subscriptions registered and no universe defined.");
}
_algorithm = algorithm;
_resultHandler = resultHandler;
_subscriptions = new ConcurrentDictionary<SymbolSecurityType, Subscription>();
_cancellationTokenSource = new CancellationTokenSource();
_universeSelection = new UniverseSelection(this, algorithm, false);
IsActive = true;
Bridge = new BusyBlockingCollection<TimeSlice>(100);
// find the minimum resolution, ignoring ticks
_fillForwardResolution = algorithm.SubscriptionManager.Subscriptions
.Where(x => x.Resolution != Resolution.Tick)
.Select(x => x.Resolution)
.DefaultIfEmpty(algorithm.UniverseSettings.Resolution)
.Min();
// initialize the original user defined securities
foreach (var security in _algorithm.Securities.Values)
{
var subscription = CreateSubscription(resultHandler, security, algorithm.StartDate, algorithm.EndDate, _fillForwardResolution, true);
if (subscription != null)
{
_subscriptions.AddOrUpdate(new SymbolSecurityType(security), subscription);
// prime the pump, run method checks current before move next calls
PrimeSubscriptionPump(subscription, true);
}
}
}
/// <summary>
/// Initializes the data feed for the specified job and algorithm
/// </summary>
public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler, IMapFileProvider mapFileProvider, IFactorFileProvider factorFileProvider)
{
if (!(job is LiveNodePacket))
{
throw new ArgumentException("The LiveTradingDataFeed requires a LiveNodePacket.");
}
_cancellationTokenSource = new CancellationTokenSource();
_algorithm = algorithm;
_job = (LiveNodePacket) job;
_resultHandler = resultHandler;
_timeProvider = GetTimeProvider();
_dataQueueHandler = GetDataQueueHandler();
_frontierTimeProvider = new ManualTimeProvider(_timeProvider.GetUtcNow());
_customExchange = new BaseDataExchange("CustomDataExchange") {SleepInterval = 10};
// sleep is controlled on this exchange via the GetNextTicksEnumerator
_exchange = new BaseDataExchange("DataQueueExchange"){SleepInterval = 0};
_exchange.AddEnumerator(DataQueueHandlerSymbol, GetNextTicksEnumerator());
_subscriptions = new SubscriptionCollection();
_bridge = new BusyBlockingCollection<TimeSlice>();
_universeSelection = new UniverseSelection(this, algorithm, job.Controls);
// run the exchanges
Task.Run(() => _exchange.Start(_cancellationTokenSource.Token));
Task.Run(() => _customExchange.Start(_cancellationTokenSource.Token));
// this value will be modified via calls to AddSubscription/RemoveSubscription
var ffres = Time.OneMinute;
_fillForwardResolution = Ref.Create(() => ffres, v => ffres = v);
// wire ourselves up to receive notifications when universes are added/removed
var start = _timeProvider.GetUtcNow();
algorithm.UniverseManager.CollectionChanged += (sender, args) =>
{
switch (args.Action)
{
case NotifyCollectionChangedAction.Add:
foreach (var universe in args.NewItems.OfType<Universe>())
{
var config = universe.Configuration;
var marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
var exchangeHours = marketHoursDatabase.GetExchangeHours(config);
Security security;
if (!_algorithm.Securities.TryGetValue(config.Symbol, out security))
{
// create a canonical security object
security = new Security(exchangeHours, config, _algorithm.Portfolio.CashBook[CashBook.AccountCurrency], SymbolProperties.GetDefault(CashBook.AccountCurrency));
}
AddSubscription(new SubscriptionRequest(true, universe, security, config, start, Time.EndOfTime));
// Not sure if this is needed but left here because of this:
// https://github.com/QuantConnect/Lean/commit/029d70bde6ca83a1eb0c667bb5cc4444bea05678
UpdateFillForwardResolution();
}
break;
case NotifyCollectionChangedAction.Remove:
foreach (var universe in args.OldItems.OfType<Universe>())
{
RemoveSubscription(universe.Configuration);
}
break;
default:
throw new NotImplementedException("The specified action is not implemented: " + args.Action);
}
};
}
/// <summary>
/// Creates a new instance of the DataManager
/// </summary>
public DataManager(
IDataFeed dataFeed,
UniverseSelection universeSelection,
IAlgorithm algorithm,
ITimeKeeper timeKeeper,
MarketHoursDatabase marketHoursDatabase)
{
_dataFeed = dataFeed;
UniverseSelection = universeSelection;
UniverseSelection.SetDataManager(this);
_algorithmSettings = algorithm.Settings;
AvailableDataTypes = SubscriptionManager.DefaultDataTypes();
_timeKeeper = timeKeeper;
_marketHoursDatabase = marketHoursDatabase;
var liveStart = DateTime.UtcNow;
// wire ourselves up to receive notifications when universes are added/removed
algorithm.UniverseManager.CollectionChanged += (sender, args) =>
{
switch (args.Action)
{
case NotifyCollectionChangedAction.Add:
foreach (var universe in args.NewItems.OfType <Universe>())
{
var config = universe.Configuration;
var start = algorithm.LiveMode ? liveStart : algorithm.UtcTime;
var end = algorithm.LiveMode ? Time.EndOfTime
: algorithm.EndDate.ConvertToUtc(algorithm.TimeZone);
Security security;
if (!algorithm.Securities.TryGetValue(config.Symbol, out security))
{
// create a canonical security object if it doesn't exist
security = new Security(
_marketHoursDatabase.GetExchangeHours(config),
config,
algorithm.Portfolio.CashBook[CashBook.AccountCurrency],
SymbolProperties.GetDefault(CashBook.AccountCurrency),
algorithm.Portfolio.CashBook
);
}
AddSubscription(
new SubscriptionRequest(true,
universe,
security,
config,
start,
end));
}
break;
case NotifyCollectionChangedAction.Remove:
foreach (var universe in args.OldItems.OfType <Universe>())
{
RemoveSubscription(universe.Configuration);
}
break;
default:
throw new NotImplementedException("The specified action is not implemented: " + args.Action);
}
};
}
/// <summary>
/// Initializes the data feed for the specified job and algorithm
/// </summary>
public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler, IMapFileProvider mapFileProvider, IFactorFileProvider factorFileProvider, IDataFileProvider dataFileProvider)
{
if (!(job is LiveNodePacket))
{
throw new ArgumentException("The LiveTradingDataFeed requires a LiveNodePacket.");
}
_cancellationTokenSource = new CancellationTokenSource();
_algorithm = algorithm;
_job = (LiveNodePacket)job;
_resultHandler = resultHandler;
_timeProvider = GetTimeProvider();
_dataQueueHandler = GetDataQueueHandler();
_dataFileProvider = dataFileProvider;
_frontierTimeProvider = new ManualTimeProvider(_timeProvider.GetUtcNow());
_customExchange = new BaseDataExchange("CustomDataExchange")
{
SleepInterval = 10
};
// sleep is controlled on this exchange via the GetNextTicksEnumerator
_exchange = new BaseDataExchange("DataQueueExchange")
{
SleepInterval = 0
};
_exchange.AddEnumerator(DataQueueHandlerSymbol, GetNextTicksEnumerator());
_subscriptions = new SubscriptionCollection();
_bridge = new BusyBlockingCollection <TimeSlice>();
_universeSelection = new UniverseSelection(this, algorithm, job.Controls);
// run the exchanges
Task.Run(() => _exchange.Start(_cancellationTokenSource.Token));
Task.Run(() => _customExchange.Start(_cancellationTokenSource.Token));
// this value will be modified via calls to AddSubscription/RemoveSubscription
var ffres = Time.OneMinute;
_fillForwardResolution = Ref.Create(() => ffres, v => ffres = v);
// wire ourselves up to receive notifications when universes are added/removed
var start = _timeProvider.GetUtcNow();
algorithm.UniverseManager.CollectionChanged += (sender, args) =>
{
switch (args.Action)
{
case NotifyCollectionChangedAction.Add:
foreach (var universe in args.NewItems.OfType <Universe>())
{
var config = universe.Configuration;
var marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
var exchangeHours = marketHoursDatabase.GetExchangeHours(config);
Security security;
if (!_algorithm.Securities.TryGetValue(config.Symbol, out security))
{
// create a canonical security object
security = new Security(exchangeHours, config, _algorithm.Portfolio.CashBook[CashBook.AccountCurrency], SymbolProperties.GetDefault(CashBook.AccountCurrency));
}
AddSubscription(new SubscriptionRequest(true, universe, security, config, start, Time.EndOfTime));
// Not sure if this is needed but left here because of this:
// https://github.com/QuantConnect/Lean/commit/029d70bde6ca83a1eb0c667bb5cc4444bea05678
UpdateFillForwardResolution();
}
break;
case NotifyCollectionChangedAction.Remove:
foreach (var universe in args.OldItems.OfType <Universe>())
{
RemoveSubscription(universe.Configuration);
}
break;
default:
throw new NotImplementedException("The specified action is not implemented: " + args.Action);
}
};
}
/// <summary>
/// Initializes a new instance of the <see cref="SubscriptionSynchronizer"/> class
/// </summary>
/// <param name="universeSelection">The universe selection instance used to handle universe
/// selection subscription output</param>
/// <param name="cashBook">The cash book, used for creating the cash book updates</param>
/// <returns>A time slice for the specified frontier time</returns>
public SubscriptionSynchronizer(UniverseSelection universeSelection,
CashBook cashBook)
{
_universeSelection = universeSelection;
_cashBook = cashBook;
}
