/// <summary>
/// Initializes a new instance of the <see cref="LeanEngineSystemHandlers"/> class with the specified handles
/// </summary>
/// <param name="jobQueue">The job queue used to acquire algorithm jobs</param>
/// <param name="api">The api instance used for communicating limits and status</param>
/// <param name="notify">The messaging handler user for passing messages from the algorithm to listeners</param>
/// <param name="leanManagement"></param>
public LeanEngineSystemHandlers(IJobQueueHandler jobQueue, IApi api, IMessagingHandler notify, ILeanManagement leanManagement)
{
if (jobQueue == null)
{
throw new ArgumentNullException("jobQueue");
}
if (api == null)
{
throw new ArgumentNullException("api");
}
if (notify == null)
{
throw new ArgumentNullException("notify");
}
if (leanManagement == null)
{
throw new ArgumentNullException("leanManagement");
}
_api = api;
_jobQueue = jobQueue;
_notify = notify;
_leanManagement = leanManagement;
}
/// <summary>
/// Launch the algorithm manager to run this strategy
/// </summary>
/// <param name="job">Algorithm job</param>
/// <param name="algorithm">Algorithm instance</param>
/// <param name="feed">Datafeed object</param>
/// <param name="transactions">Transaction manager object</param>
/// <param name="results">Result handler object</param>
/// <param name="realtime">Realtime processing object</param>
/// <param name="commands">The command queue for relaying extenal commands to the algorithm</param>
/// <param name="systemHandlersServer"></param>
/// <param name="leanManagement">ILeanManagement implementation that is updated periodically with the IAlgorithm instance</param>
/// <param name="token">Cancellation token</param>
/// <remarks>Modify with caution</remarks>
public void Run(AlgorithmNodePacket job, IAlgorithm algorithm, IDataFeed feed, ITransactionHandler transactions, IResultHandler results, IRealTimeHandler realtime, ILeanManagement leanManagement, CancellationToken token)
{
//Initialize:
_dataPointCount = 0;
_algorithm = algorithm;
var portfolioValue = algorithm.Portfolio.TotalPortfolioValue;
var backtestMode = (job.Type == PacketType.BacktestNode);
var methodInvokers = new Dictionary <Type, MethodInvoker>();
var marginCallFrequency = TimeSpan.FromMinutes(5);
var nextMarginCallTime = DateTime.MinValue;
var settlementScanFrequency = TimeSpan.FromMinutes(30);
var nextSettlementScanTime = DateTime.MinValue;
var delistings = new List <Delisting>();
//Initialize Properties:
_algorithmId = job.AlgorithmId;
_algorithm.Status = AlgorithmStatus.Running;
_previousTime = algorithm.StartDate.Date;
//Create the method accessors to push generic types into algorithm: Find all OnData events:
// Algorithm 2.0 data accessors
var hasOnDataTradeBars = AddMethodInvoker <TradeBars>(algorithm, methodInvokers);
var hasOnDataQuoteBars = AddMethodInvoker <QuoteBars>(algorithm, methodInvokers);
var hasOnDataOptionChains = AddMethodInvoker <OptionChains>(algorithm, methodInvokers);
var hasOnDataTicks = AddMethodInvoker <Ticks>(algorithm, methodInvokers);
// dividend and split events
var hasOnDataDividends = AddMethodInvoker <Dividends>(algorithm, methodInvokers);
var hasOnDataSplits = AddMethodInvoker <Splits>(algorithm, methodInvokers);
var hasOnDataDelistings = AddMethodInvoker <Delistings>(algorithm, methodInvokers);
var hasOnDataSymbolChangedEvents = AddMethodInvoker <SymbolChangedEvents>(algorithm, methodInvokers);
// Algorithm 3.0 data accessors
var hasOnDataSlice = algorithm.GetType().GetMethods()
.Where(x => x.Name == "OnData" && x.GetParameters().Length == 1 && x.GetParameters()[0].ParameterType == typeof(Slice))
.FirstOrDefault(x => x.DeclaringType == algorithm.GetType()) != null;
//Go through the subscription types and create invokers to trigger the event handlers for each custom type:
foreach (var config in algorithm.SubscriptionManager.Subscriptions)
{
//If type is a custom feed, check for a dedicated event handler
if (config.IsCustomData)
{
//Get the matching method for this event handler - e.g. public void OnData(Quandl data) { .. }
var genericMethod = (algorithm.GetType()).GetMethod("OnData", new[] { config.Type });
//If we already have this Type-handler then don't add it to invokers again.
if (methodInvokers.ContainsKey(config.Type))
{
continue;
}
//If we couldnt find the event handler, let the user know we can't fire that event.
if (genericMethod == null && !hasOnDataSlice)
{
algorithm.RunTimeError = new Exception("Data event handler not found, please create a function matching this template: public void OnData(" + config.Type.Name + " data) { }");
_algorithm.Status = AlgorithmStatus.RuntimeError;
return;
}
if (genericMethod != null)
{
methodInvokers.Add(config.Type, genericMethod.DelegateForCallMethod());
}
}
}
//Loop over the queues: get a data collection, then pass them all into relevent methods in the algorithm.
Log.Trace("AlgorithmManager.Run(): Begin DataStream - Start: " + algorithm.StartDate + " Stop: " + algorithm.EndDate);
foreach (var timeSlice in Stream(job, algorithm, feed, results, token))
{
// reset our timer on each loop
_currentTimeStepTime = DateTime.UtcNow;
//Check this backtest is still running:
if (_algorithm.Status != AlgorithmStatus.Running)
{
Log.Error(string.Format("AlgorithmManager.Run(): Algorithm state changed to {0} at {1}", _algorithm.Status, timeSlice.Time));
break;
}
//Execute with TimeLimit Monitor:
if (token.IsCancellationRequested)
{
Log.Error("AlgorithmManager.Run(): CancellationRequestion at " + timeSlice.Time);
return;
}
// Update the ILeanManagement
leanManagement.Update();
var time = timeSlice.Time;
_dataPointCount += timeSlice.DataPointCount;
//If we're in backtest mode we need to capture the daily performance. We do this here directly
//before updating the algorithm state with the new data from this time step, otherwise we'll
//produce incorrect samples (they'll take into account this time step's new price values)
if (backtestMode)
{
//On day-change sample equity and daily performance for statistics calculations
if (_previousTime.Date != time.Date)
{
SampleBenchmark(algorithm, results, _previousTime.Date);
//Sample the portfolio value over time for chart.
results.SampleEquity(_previousTime, Math.Round(algorithm.Portfolio.TotalPortfolioValue, 4));
//Check for divide by zero
if (portfolioValue == 0m)
{
results.SamplePerformance(_previousTime.Date, 0);
}
else
{
results.SamplePerformance(_previousTime.Date, Math.Round((algorithm.Portfolio.TotalPortfolioValue - portfolioValue) * 100 / portfolioValue, 10));
}
portfolioValue = algorithm.Portfolio.TotalPortfolioValue;
}
if (portfolioValue <= 0)
{
string logMessage = "AlgorithmManager.Run(): Portfolio value is less than or equal to zero";
Log.Trace(logMessage);
results.SystemDebugMessage(logMessage);
break;
}
}
else
{
// live mode continously sample the benchmark
SampleBenchmark(algorithm, results, time);
}
//Update algorithm state after capturing performance from previous day
// If backtesting, we need to check if there are realtime events in the past
// which didn't fire because at the scheduled times there was no data (i.e. markets closed)
// and fire them with the correct date/time.
if (backtestMode)
{
realtime.ScanPastEvents(time);
}
//Set the algorithm and real time handler's time
algorithm.SetDateTime(time);
if (timeSlice.Slice.SymbolChangedEvents.Count != 0)
{
if (hasOnDataSymbolChangedEvents)
{
methodInvokers[typeof(SymbolChangedEvents)](algorithm, timeSlice.Slice.SymbolChangedEvents);
}
foreach (var symbol in timeSlice.Slice.SymbolChangedEvents.Keys)
{
// cancel all orders for the old symbol
foreach (var ticket in transactions.GetOrderTickets(x => x.Status.IsOpen() && x.Symbol == symbol))
{
ticket.Cancel("Open order cancelled on symbol changed event");
}
}
}
if (timeSlice.SecurityChanges != SecurityChanges.None)
{
foreach (var security in timeSlice.SecurityChanges.AddedSecurities)
{
if (!algorithm.Securities.ContainsKey(security.Symbol))
{
// add the new security
algorithm.Securities.Add(security);
}
}
}
//On each time step push the real time prices to the cashbook so we can have updated conversion rates
foreach (var update in timeSlice.CashBookUpdateData)
{
var cash = update.Target;
foreach (var data in update.Data)
{
cash.Update(data);
}
}
//Update the securities properties: first before calling user code to avoid issues with data
foreach (var update in timeSlice.SecuritiesUpdateData)
{
var security = update.Target;
foreach (var data in update.Data)
{
security.SetMarketPrice(data);
}
// Send market price updates to the TradeBuilder
algorithm.TradeBuilder.SetMarketPrice(security.Symbol, security.Price);
}
// fire real time events after we've updated based on the new data
realtime.SetTime(timeSlice.Time);
// process fill models on the updated data before entering algorithm, applies to all non-market orders
transactions.ProcessSynchronousEvents();
// process end of day delistings
ProcessDelistedSymbols(algorithm, delistings);
//Check if the user's signalled Quit: loop over data until day changes.
if (algorithm.Status == AlgorithmStatus.Stopped)
{
Log.Trace("AlgorithmManager.Run(): Algorithm quit requested.");
break;
}
if (algorithm.RunTimeError != null)
{
_algorithm.Status = AlgorithmStatus.RuntimeError;
Log.Trace(string.Format("AlgorithmManager.Run(): Algorithm encountered a runtime error at {0}. Error: {1}", timeSlice.Time, algorithm.RunTimeError));
break;
}
// perform margin calls, in live mode we can also use realtime to emit these
if (time >= nextMarginCallTime || (_liveMode && nextMarginCallTime > DateTime.UtcNow))
{
// determine if there are possible margin call orders to be executed
bool issueMarginCallWarning;
var marginCallOrders = algorithm.Portfolio.ScanForMarginCall(out issueMarginCallWarning);
if (marginCallOrders.Count != 0)
{
var executingMarginCall = false;
try
{
// tell the algorithm we're about to issue the margin call
algorithm.OnMarginCall(marginCallOrders);
executingMarginCall = true;
// execute the margin call orders
var executedTickets = algorithm.Portfolio.MarginCallModel.ExecuteMarginCall(marginCallOrders);
foreach (var ticket in executedTickets)
{
algorithm.Error(string.Format("{0} - Executed MarginCallOrder: {1} - Quantity: {2} @ {3}", algorithm.Time, ticket.Symbol, ticket.Quantity, ticket.AverageFillPrice));
}
}
catch (Exception err)
{
algorithm.RunTimeError = err;
_algorithm.Status = AlgorithmStatus.RuntimeError;
var locator = executingMarginCall ? "Portfolio.MarginCallModel.ExecuteMarginCall" : "OnMarginCall";
Log.Error(string.Format("AlgorithmManager.Run(): RuntimeError: {0}: ", locator) + err);
return;
}
}
// we didn't perform a margin call, but got the warning flag back, so issue the warning to the algorithm
else if (issueMarginCallWarning)
{
try
{
algorithm.OnMarginCallWarning();
}
catch (Exception err)
{
algorithm.RunTimeError = err;
_algorithm.Status = AlgorithmStatus.RuntimeError;
Log.Error("AlgorithmManager.Run(): RuntimeError: OnMarginCallWarning: " + err);
return;
}
}
nextMarginCallTime = time + marginCallFrequency;
}
// perform check for settlement of unsettled funds
if (time >= nextSettlementScanTime || (_liveMode && nextSettlementScanTime > DateTime.UtcNow))
{
algorithm.Portfolio.ScanForCashSettlement(algorithm.UtcTime);
nextSettlementScanTime = time + settlementScanFrequency;
}
// before we call any events, let the algorithm know about universe changes
if (timeSlice.SecurityChanges != SecurityChanges.None)
{
try
{
algorithm.OnSecuritiesChanged(timeSlice.SecurityChanges);
}
catch (Exception err)
{
algorithm.RunTimeError = err;
_algorithm.Status = AlgorithmStatus.RuntimeError;
Log.Error("AlgorithmManager.Run(): RuntimeError: OnSecuritiesChanged event: " + err);
return;
}
}
// apply dividends
foreach (var dividend in timeSlice.Slice.Dividends.Values)
{
Log.Trace("AlgorithmManager.Run(): {0}: Applying Dividend for {1}", algorithm.Time, dividend.Symbol.ToString());
algorithm.Portfolio.ApplyDividend(dividend);
}
// apply splits
foreach (var split in timeSlice.Slice.Splits.Values)
{
try
{
Log.Trace("AlgorithmManager.Run(): {0}: Applying Split for {1}", algorithm.Time, split.Symbol.ToString());
algorithm.Portfolio.ApplySplit(split);
// apply the split to open orders as well in raw mode, all other modes are split adjusted
if (_liveMode || algorithm.Securities[split.Symbol].DataNormalizationMode == DataNormalizationMode.Raw)
{
// in live mode we always want to have our order match the order at the brokerage, so apply the split to the orders
var openOrders = transactions.GetOrderTickets(ticket => ticket.Status.IsOpen() && ticket.Symbol == split.Symbol);
algorithm.BrokerageModel.ApplySplit(openOrders.ToList(), split);
}
}
catch (Exception err)
{
algorithm.RunTimeError = err;
_algorithm.Status = AlgorithmStatus.RuntimeError;
Log.Error("AlgorithmManager.Run(): RuntimeError: Split event: " + err);
return;
}
}
//Update registered consolidators for this symbol index
try
{
foreach (var update in timeSlice.ConsolidatorUpdateData)
{
var resolutionTimeSpan = update.Target.Resolution.ToTimeSpan();
var consolidators = update.Target.Consolidators;
foreach (var consolidator in consolidators)
{
foreach (var dataPoint in update.Data)
{
// Filter out data with resolution higher than the data subscription resolution.
// This is needed to avoid feeding in higher resolution data, typically fill-forward bars.
// It also prevents volume-based indicators or consolidators summing up volume to generate
// invalid values.
var algorithmTimeSpan = resolutionTimeSpan == TimeSpan.FromTicks(0)
? TimeSpan.FromTicks(0)
: TimeSpan.FromSeconds(1);
if (update.Target.Resolution == Resolution.Tick ||
algorithm.UtcTime.RoundDown(algorithmTimeSpan) == dataPoint.EndTime.RoundUp(resolutionTimeSpan).ConvertToUtc(update.Target.ExchangeTimeZone))
{
consolidator.Update(dataPoint);
}
}
// scan for time after we've pumped all the data through for this consolidator
var localTime = time.ConvertFromUtc(update.Target.ExchangeTimeZone);
consolidator.Scan(localTime);
}
}
}
catch (Exception err)
{
algorithm.RunTimeError = err;
_algorithm.Status = AlgorithmStatus.RuntimeError;
Log.Error("AlgorithmManager.Run(): RuntimeError: Consolidators update: " + err);
return;
}
// fire custom event handlers
foreach (var update in timeSlice.CustomData)
{
MethodInvoker methodInvoker;
if (!methodInvokers.TryGetValue(update.DataType, out methodInvoker))
{
continue;
}
try
{
foreach (var dataPoint in update.Data)
{
if (update.DataType.IsInstanceOfType(dataPoint))
{
methodInvoker(algorithm, dataPoint);
}
}
}
catch (Exception err)
{
algorithm.RunTimeError = err;
_algorithm.Status = AlgorithmStatus.RuntimeError;
Log.Error("AlgorithmManager.Run(): RuntimeError: Custom Data: " + err);
return;
}
}
try
{
// fire off the dividend and split events before pricing events
if (hasOnDataDividends && timeSlice.Slice.Dividends.Count != 0)
{
methodInvokers[typeof(Dividends)](algorithm, timeSlice.Slice.Dividends);
}
if (hasOnDataSplits && timeSlice.Slice.Splits.Count != 0)
{
methodInvokers[typeof(Splits)](algorithm, timeSlice.Slice.Splits);
}
if (hasOnDataDelistings && timeSlice.Slice.Delistings.Count != 0)
{
methodInvokers[typeof(Delistings)](algorithm, timeSlice.Slice.Delistings);
}
}
catch (Exception err)
{
algorithm.RunTimeError = err;
_algorithm.Status = AlgorithmStatus.RuntimeError;
Log.Error("AlgorithmManager.Run(): RuntimeError: Dividends/Splits/Delistings: " + err);
return;
}
// run the delisting logic after firing delisting events
HandleDelistedSymbols(algorithm, timeSlice.Slice.Delistings, delistings);
//After we've fired all other events in this second, fire the pricing events:
try
{
// TODO: For backwards compatibility only. Remove in 2017
// For compatibility with Forex Trade data, moving
if (timeSlice.Slice.QuoteBars.Count > 0)
{
foreach (var tradeBar in timeSlice.Slice.QuoteBars.Where(x => x.Key.ID.SecurityType == SecurityType.Forex))
{
timeSlice.Slice.Bars.Add(tradeBar.Value.Collapse());
}
}
if (hasOnDataTradeBars && timeSlice.Slice.Bars.Count > 0)
{
methodInvokers[typeof(TradeBars)](algorithm, timeSlice.Slice.Bars);
}
if (hasOnDataQuoteBars && timeSlice.Slice.QuoteBars.Count > 0)
{
methodInvokers[typeof(QuoteBars)](algorithm, timeSlice.Slice.QuoteBars);
}
if (hasOnDataOptionChains && timeSlice.Slice.OptionChains.Count > 0)
{
methodInvokers[typeof(OptionChains)](algorithm, timeSlice.Slice.OptionChains);
}
if (hasOnDataTicks && timeSlice.Slice.Ticks.Count > 0)
{
methodInvokers[typeof(Ticks)](algorithm, timeSlice.Slice.Ticks);
}
}
catch (Exception err)
{
algorithm.RunTimeError = err;
_algorithm.Status = AlgorithmStatus.RuntimeError;
Log.Error("AlgorithmManager.Run(): RuntimeError: New Style Mode: " + err);
return;
}
try
{
if (timeSlice.Slice.HasData)
{
// EVENT HANDLER v3.0 -- all data in a single event
algorithm.OnData(timeSlice.Slice);
}
}
catch (Exception err)
{
algorithm.RunTimeError = err;
_algorithm.Status = AlgorithmStatus.RuntimeError;
Log.Error("AlgorithmManager.Run(): RuntimeError: Slice: " + err);
return;
}
//If its the historical/paper trading models, wait until market orders have been "filled"
// Manually trigger the event handler to prevent thread switch.
transactions.ProcessSynchronousEvents();
//Save the previous time for the sample calculations
_previousTime = time;
// Process any required events of the results handler such as sampling assets, equity, or stock prices.
results.ProcessSynchronousEvents();
} // End of ForEach feed.Bridge.GetConsumingEnumerable
// stop timing the loops
_currentTimeStepTime = DateTime.MinValue;
//Stream over:: Send the final packet and fire final events:
Log.Trace("AlgorithmManager.Run(): Firing On End Of Algorithm...");
try
{
algorithm.OnEndOfAlgorithm();
}
catch (Exception err)
{
_algorithm.Status = AlgorithmStatus.RuntimeError;
algorithm.RunTimeError = new Exception("Error running OnEndOfAlgorithm(): " + err.Message, err.InnerException);
Log.Error("AlgorithmManager.OnEndOfAlgorithm(): " + err);
return;
}
// Process any required events of the results handler such as sampling assets, equity, or stock prices.
results.ProcessSynchronousEvents(forceProcess: true);
//Liquidate Holdings for Calculations:
if (_algorithm.Status == AlgorithmStatus.Liquidated && _liveMode)
{
Log.Trace("AlgorithmManager.Run(): Liquidating algorithm holdings...");
algorithm.Liquidate();
results.LogMessage("Algorithm Liquidated");
results.SendStatusUpdate(AlgorithmStatus.Liquidated);
}
//Manually stopped the algorithm
if (_algorithm.Status == AlgorithmStatus.Stopped)
{
Log.Trace("AlgorithmManager.Run(): Stopping algorithm...");
results.LogMessage("Algorithm Stopped");
results.SendStatusUpdate(AlgorithmStatus.Stopped);
}
//Backtest deleted.
if (_algorithm.Status == AlgorithmStatus.Deleted)
{
Log.Trace("AlgorithmManager.Run(): Deleting algorithm...");
results.DebugMessage("Algorithm Id:(" + job.AlgorithmId + ") Deleted by request.");
results.SendStatusUpdate(AlgorithmStatus.Deleted);
}
//Algorithm finished, send regardless of commands:
results.SendStatusUpdate(AlgorithmStatus.Completed);
//Take final samples:
results.SampleRange(algorithm.GetChartUpdates());
results.SampleEquity(_previousTime, Math.Round(algorithm.Portfolio.TotalPortfolioValue, 4));
SampleBenchmark(algorithm, results, backtestMode ? _previousTime.Date : _previousTime);
//Check for divide by zero
if (portfolioValue == 0m)
{
results.SamplePerformance(backtestMode ? _previousTime.Date : _previousTime, 0m);
}
else
{
results.SamplePerformance(backtestMode ? _previousTime.Date : _previousTime,
Math.Round((algorithm.Portfolio.TotalPortfolioValue - portfolioValue) * 100 / portfolioValue, 10));
}
} // End of Run();
