/// <summary> /// Launch the algorithm manager to run this strategy /// </summary> /// <param name="job">Algorithm job</param> /// <param name="algorithm">Algorithm instance</param> /// <param name="synchronizer">Instance which implements <see cref="ISynchronizer"/>. Used to stream the data</param> /// <param name="transactions">Transaction manager object</param> /// <param name="results">Result handler object</param> /// <param name="realtime">Realtime processing object</param> /// <param name="leanManager">ILeanManager implementation that is updated periodically with the IAlgorithm instance</param> /// <param name="alphas">Alpha handler used to process algorithm generated insights</param> /// <param name="token">Cancellation token</param> /// <remarks>Modify with caution</remarks> public void Run(AlgorithmNodePacket job, IAlgorithm algorithm, ISynchronizer synchronizer, ITransactionHandler transactions, IResultHandler results, IRealTimeHandler realtime, ILeanManager leanManager, IAlphaHandler alphas, CancellationToken token) { //Initialize: DataPoints = 0; _algorithm = algorithm; var backtestMode = (job.Type == PacketType.BacktestNode); var methodInvokers = new Dictionary <Type, MethodInvoker>(); var marginCallFrequency = TimeSpan.FromMinutes(5); var nextMarginCallTime = DateTime.MinValue; var settlementScanFrequency = TimeSpan.FromMinutes(30); var nextSettlementScanTime = DateTime.MinValue; var time = algorithm.StartDate.Date; var pendingDelistings = new List <Delisting>(); var splitWarnings = new List <Split>(); //Initialize Properties: AlgorithmId = job.AlgorithmId; //Create the method accessors to push generic types into algorithm: Find all OnData events: // Algorithm 2.0 data accessors var hasOnDataTradeBars = AddMethodInvoker <TradeBars>(algorithm, methodInvokers); var hasOnDataQuoteBars = AddMethodInvoker <QuoteBars>(algorithm, methodInvokers); var hasOnDataOptionChains = AddMethodInvoker <OptionChains>(algorithm, methodInvokers); var hasOnDataTicks = AddMethodInvoker <Ticks>(algorithm, methodInvokers); // dividend and split events var hasOnDataDividends = AddMethodInvoker <Dividends>(algorithm, methodInvokers); var hasOnDataSplits = AddMethodInvoker <Splits>(algorithm, methodInvokers); var hasOnDataDelistings = AddMethodInvoker <Delistings>(algorithm, methodInvokers); var hasOnDataSymbolChangedEvents = AddMethodInvoker <SymbolChangedEvents>(algorithm, methodInvokers); //Go through the subscription types and create invokers to trigger the event handlers for each custom type: foreach (var config in algorithm.SubscriptionManager.Subscriptions) { //If type is a custom feed, check for a dedicated event handler if (config.IsCustomData) { //Get the matching method for this event handler - e.g. public void OnData(Quandl data) { .. } var genericMethod = (algorithm.GetType()).GetMethod("OnData", new[] { config.Type }); //If we already have this Type-handler then don't add it to invokers again. if (methodInvokers.ContainsKey(config.Type)) { continue; } if (genericMethod != null) { methodInvokers.Add(config.Type, genericMethod.DelegateForCallMethod()); } } } // Schedule a daily event for sampling at midnight every night algorithm.Schedule.On("Daily Sampling", algorithm.Schedule.DateRules.EveryDay(), algorithm.Schedule.TimeRules.Midnight, () => { results.Sample(algorithm.UtcTime); }); //Loop over the queues: get a data collection, then pass them all into relevent methods in the algorithm. Log.Trace($"AlgorithmManager.Run(): Begin DataStream - Start: {algorithm.StartDate} Stop: {algorithm.EndDate} Time: {algorithm.Time} Warmup: {algorithm.IsWarmingUp}"); foreach (var timeSlice in Stream(algorithm, synchronizer, results, token)) { // reset our timer on each loop TimeLimit.StartNewTimeStep(); //Check this backtest is still running: if (_algorithm.Status != AlgorithmStatus.Running && _algorithm.RunTimeError == null) { Log.Error($"AlgorithmManager.Run(): Algorithm state changed to {_algorithm.Status} at {timeSlice.Time.ToStringInvariant()}"); break; } //Execute with TimeLimit Monitor: if (token.IsCancellationRequested) { Log.Error($"AlgorithmManager.Run(): CancellationRequestion at {timeSlice.Time.ToStringInvariant()}"); return; } // Update the ILeanManager leanManager.Update(); time = timeSlice.Time; DataPoints += timeSlice.DataPointCount; if (backtestMode && algorithm.Portfolio.TotalPortfolioValue <= 0) { var logMessage = "AlgorithmManager.Run(): Portfolio value is less than or equal to zero, stopping algorithm."; Log.Error(logMessage); results.SystemDebugMessage(logMessage); break; } // If backtesting/warmup, we need to check if there are realtime events in the past // which didn't fire because at the scheduled times there was no data (i.e. markets closed) // and fire them with the correct date/time. realtime.ScanPastEvents(time); //Set the algorithm and real time handler's time algorithm.SetDateTime(time); // the time pulse are just to advance algorithm time, lets shortcut the loop here if (timeSlice.IsTimePulse) { continue; } // Update the current slice before firing scheduled events or any other task algorithm.SetCurrentSlice(timeSlice.Slice); if (timeSlice.Slice.SymbolChangedEvents.Count != 0) { if (hasOnDataSymbolChangedEvents) { methodInvokers[typeof(SymbolChangedEvents)](algorithm, timeSlice.Slice.SymbolChangedEvents); } foreach (var symbol in timeSlice.Slice.SymbolChangedEvents.Keys) { // cancel all orders for the old symbol foreach (var ticket in transactions.GetOpenOrderTickets(x => x.Symbol == symbol)) { ticket.Cancel("Open order cancelled on symbol changed event"); } } } if (timeSlice.SecurityChanges != SecurityChanges.None) { foreach (var security in timeSlice.SecurityChanges.AddedSecurities) { security.IsTradable = true; // uses TryAdd, so don't need to worry about duplicates here algorithm.Securities.Add(security); } var activeSecurities = algorithm.UniverseManager.ActiveSecurities; foreach (var security in timeSlice.SecurityChanges.RemovedSecurities) { if (!activeSecurities.ContainsKey(security.Symbol)) { security.IsTradable = false; } } leanManager.OnSecuritiesChanged(timeSlice.SecurityChanges); realtime.OnSecuritiesChanged(timeSlice.SecurityChanges); results.OnSecuritiesChanged(timeSlice.SecurityChanges); } //Update the securities properties: first before calling user code to avoid issues with data foreach (var update in timeSlice.SecuritiesUpdateData) { var security = update.Target; security.Update(update.Data, update.DataType, update.ContainsFillForwardData); if (!update.IsInternalConfig) { // Send market price updates to the TradeBuilder algorithm.TradeBuilder.SetMarketPrice(security.Symbol, security.Price); } } //Update the securities properties with any universe data if (timeSlice.UniverseData.Count > 0) { foreach (var kvp in timeSlice.UniverseData) { foreach (var data in kvp.Value.Data) { Security security; if (algorithm.Securities.TryGetValue(data.Symbol, out security)) { security.Cache.StoreData(new[] { data }, data.GetType()); } } } } // poke each cash object to update from the recent security data foreach (var cash in algorithm.Portfolio.CashBook.Values.Where(x => x.CurrencyConversion != null)) { cash.Update(); } // security prices got updated algorithm.Portfolio.InvalidateTotalPortfolioValue(); // process fill models on the updated data before entering algorithm, applies to all non-market orders transactions.ProcessSynchronousEvents(); // fire real time events after we've updated based on the new data realtime.SetTime(timeSlice.Time); // process split warnings for options ProcessSplitSymbols(algorithm, splitWarnings, pendingDelistings); //Check if the user's signalled Quit: loop over data until day changes. if (_algorithm.Status != AlgorithmStatus.Running && _algorithm.RunTimeError == null) { Log.Error($"AlgorithmManager.Run(): Algorithm state changed to {_algorithm.Status} at {timeSlice.Time.ToStringInvariant()}"); break; } if (algorithm.RunTimeError != null) { Log.Error($"AlgorithmManager.Run(): Stopping, encountered a runtime error at {algorithm.UtcTime} UTC."); return; } // perform margin calls, in live mode we can also use realtime to emit these if (time >= nextMarginCallTime || (_liveMode && nextMarginCallTime > DateTime.UtcNow)) { // determine if there are possible margin call orders to be executed bool issueMarginCallWarning; var marginCallOrders = algorithm.Portfolio.MarginCallModel.GetMarginCallOrders(out issueMarginCallWarning); if (marginCallOrders.Count != 0) { var executingMarginCall = false; try { // tell the algorithm we're about to issue the margin call algorithm.OnMarginCall(marginCallOrders); executingMarginCall = true; // execute the margin call orders var executedTickets = algorithm.Portfolio.MarginCallModel.ExecuteMarginCall(marginCallOrders); foreach (var ticket in executedTickets) { algorithm.Error($"{algorithm.Time.ToStringInvariant()} - Executed MarginCallOrder: {ticket.Symbol} - " + $"Quantity: {ticket.Quantity.ToStringInvariant()} @ {ticket.AverageFillPrice.ToStringInvariant()}" ); } } catch (Exception err) { algorithm.SetRuntimeError(err, executingMarginCall ? "Portfolio.MarginCallModel.ExecuteMarginCall" : "OnMarginCall"); return; } } // we didn't perform a margin call, but got the warning flag back, so issue the warning to the algorithm else if (issueMarginCallWarning) { try { algorithm.OnMarginCallWarning(); } catch (Exception err) { algorithm.SetRuntimeError(err, "OnMarginCallWarning"); return; } } nextMarginCallTime = time + marginCallFrequency; } // perform check for settlement of unsettled funds if (time >= nextSettlementScanTime || (_liveMode && nextSettlementScanTime > DateTime.UtcNow)) { algorithm.Portfolio.ScanForCashSettlement(algorithm.UtcTime); nextSettlementScanTime = time + settlementScanFrequency; } // before we call any events, let the algorithm know about universe changes if (timeSlice.SecurityChanges != SecurityChanges.None) { try { var algorithmSecurityChanges = new SecurityChanges(timeSlice.SecurityChanges) { // by default for user code we want to filter out custom securities FilterCustomSecurities = true, // by default for user code we want to filter out internal securities FilterInternalSecurities = true }; algorithm.OnSecuritiesChanged(algorithmSecurityChanges); algorithm.OnFrameworkSecuritiesChanged(algorithmSecurityChanges); } catch (Exception err) { algorithm.SetRuntimeError(err, "OnSecuritiesChanged"); return; } } // apply dividends foreach (var dividend in timeSlice.Slice.Dividends.Values) { Log.Debug($"AlgorithmManager.Run(): {algorithm.Time}: Applying Dividend: {dividend}"); Security security = null; if (_liveMode && algorithm.Securities.TryGetValue(dividend.Symbol, out security)) { Log.Trace($"AlgorithmManager.Run(): {algorithm.Time}: Pre-Dividend: {dividend}. " + $"Security Holdings: {security.Holdings.Quantity} Account Currency Holdings: " + $"{algorithm.Portfolio.CashBook[algorithm.AccountCurrency].Amount}"); } var mode = algorithm.SubscriptionManager.SubscriptionDataConfigService .GetSubscriptionDataConfigs(dividend.Symbol) .DataNormalizationMode(); // apply the dividend event to the portfolio algorithm.Portfolio.ApplyDividend(dividend, _liveMode, mode); if (_liveMode && security != null) { Log.Trace($"AlgorithmManager.Run(): {algorithm.Time}: Post-Dividend: {dividend}. Security " + $"Holdings: {security.Holdings.Quantity} Account Currency Holdings: " + $"{algorithm.Portfolio.CashBook[algorithm.AccountCurrency].Amount}"); } } // apply splits foreach (var split in timeSlice.Slice.Splits.Values) { try { // only process split occurred events (ignore warnings) if (split.Type != SplitType.SplitOccurred) { continue; } Log.Debug($"AlgorithmManager.Run(): {algorithm.Time}: Applying Split for {split.Symbol}"); Security security = null; if (_liveMode && algorithm.Securities.TryGetValue(split.Symbol, out security)) { Log.Trace($"AlgorithmManager.Run(): {algorithm.Time}: Pre-Split for {split}. Security Price: {security.Price} Holdings: {security.Holdings.Quantity}"); } var mode = algorithm.SubscriptionManager.SubscriptionDataConfigService .GetSubscriptionDataConfigs(split.Symbol) .DataNormalizationMode(); // apply the split event to the portfolio algorithm.Portfolio.ApplySplit(split, _liveMode, mode); if (_liveMode && security != null) { Log.Trace($"AlgorithmManager.Run(): {algorithm.Time}: Post-Split for {split}. Security Price: {security.Price} Holdings: {security.Holdings.Quantity}"); } // apply the split to open orders as well in raw mode, all other modes are split adjusted if (_liveMode || mode == DataNormalizationMode.Raw) { // in live mode we always want to have our order match the order at the brokerage, so apply the split to the orders var openOrders = transactions.GetOpenOrderTickets(ticket => ticket.Symbol == split.Symbol); algorithm.BrokerageModel.ApplySplit(openOrders.ToList(), split); } } catch (Exception err) { algorithm.SetRuntimeError(err, "Split event"); return; } } //Update registered consolidators for this symbol index try { if (timeSlice.ConsolidatorUpdateData.Count > 0) { var timeKeeper = algorithm.TimeKeeper; foreach (var update in timeSlice.ConsolidatorUpdateData) { var localTime = timeKeeper.GetLocalTimeKeeper(update.Target.ExchangeTimeZone).LocalTime; var consolidators = update.Target.Consolidators; foreach (var consolidator in consolidators) { foreach (var dataPoint in update.Data) { // only push data into consolidators on the native, subscribed to resolution if (EndTimeIsInNativeResolution(update.Target, dataPoint.EndTime)) { consolidator.Update(dataPoint); } } // scan for time after we've pumped all the data through for this consolidator consolidator.Scan(localTime); } } } } catch (Exception err) { algorithm.SetRuntimeError(err, "Consolidators update"); return; } // fire custom event handlers foreach (var update in timeSlice.CustomData) { MethodInvoker methodInvoker; if (!methodInvokers.TryGetValue(update.DataType, out methodInvoker)) { continue; } try { foreach (var dataPoint in update.Data) { if (update.DataType.IsInstanceOfType(dataPoint)) { methodInvoker(algorithm, dataPoint); } } } catch (Exception err) { algorithm.SetRuntimeError(err, "Custom Data"); return; } } try { // fire off the dividend and split events before pricing events if (hasOnDataDividends && timeSlice.Slice.Dividends.Count != 0) { methodInvokers[typeof(Dividends)](algorithm, timeSlice.Slice.Dividends); } if (hasOnDataSplits && timeSlice.Slice.Splits.Count != 0) { methodInvokers[typeof(Splits)](algorithm, timeSlice.Slice.Splits); } if (hasOnDataDelistings && timeSlice.Slice.Delistings.Count != 0) { methodInvokers[typeof(Delistings)](algorithm, timeSlice.Slice.Delistings); } } catch (Exception err) { algorithm.SetRuntimeError(err, "Dividends/Splits/Delistings"); return; } // Only track pending delistings in non-live mode. if (!algorithm.LiveMode) { // Keep this up to date even though we don't process delistings here anymore foreach (var delisting in timeSlice.Slice.Delistings.Values) { if (delisting.Type == DelistingType.Warning) { // Store our delistings warnings because they are still used by ProcessSplitSymbols above pendingDelistings.Add(delisting); } else { // If we have an actual delisting event, remove it from pending delistings var index = pendingDelistings.FindIndex(x => x.Symbol == delisting.Symbol); if (index != -1) { pendingDelistings.RemoveAt(index); } } } } // run split logic after firing split events HandleSplitSymbols(timeSlice.Slice.Splits, splitWarnings); //After we've fired all other events in this second, fire the pricing events: try { if (hasOnDataTradeBars && timeSlice.Slice.Bars.Count > 0) { methodInvokers[typeof(TradeBars)](algorithm, timeSlice.Slice.Bars); } if (hasOnDataQuoteBars && timeSlice.Slice.QuoteBars.Count > 0) { methodInvokers[typeof(QuoteBars)](algorithm, timeSlice.Slice.QuoteBars); } if (hasOnDataOptionChains && timeSlice.Slice.OptionChains.Count > 0) { methodInvokers[typeof(OptionChains)](algorithm, timeSlice.Slice.OptionChains); } if (hasOnDataTicks && timeSlice.Slice.Ticks.Count > 0) { methodInvokers[typeof(Ticks)](algorithm, timeSlice.Slice.Ticks); } } catch (Exception err) { algorithm.SetRuntimeError(err, "methodInvokers"); return; } try { if (timeSlice.Slice.HasData) { // EVENT HANDLER v3.0 -- all data in a single event algorithm.OnData(timeSlice.Slice); } // always turn the crank on this method to ensure universe selection models function properly on day changes w/out data algorithm.OnFrameworkData(timeSlice.Slice); } catch (Exception err) { algorithm.SetRuntimeError(err, "OnData"); return; } //If its the historical/paper trading models, wait until market orders have been "filled" // Manually trigger the event handler to prevent thread switch. transactions.ProcessSynchronousEvents(); // sample alpha charts now that we've updated time/price information and after transactions // are processed so that insights closed because of new order based insights get updated alphas.ProcessSynchronousEvents(); // send the alpha statistics to the result handler for storage/transmit with the result packets results.SetAlphaRuntimeStatistics(alphas.RuntimeStatistics); // Process any required events of the results handler such as sampling assets, equity, or stock prices. results.ProcessSynchronousEvents(); // poke the algorithm at the end of each time step algorithm.OnEndOfTimeStep(); } // End of ForEach feed.Bridge.GetConsumingEnumerable // stop timing the loops TimeLimit.StopEnforcingTimeLimit(); //Stream over:: Send the final packet and fire final events: Log.Trace("AlgorithmManager.Run(): Firing On End Of Algorithm..."); try { algorithm.OnEndOfAlgorithm(); } catch (Exception err) { algorithm.SetRuntimeError(err, "OnEndOfAlgorithm"); return; } // final processing now that the algorithm has completed alphas.ProcessSynchronousEvents(); // send the final alpha statistics to the result handler for storage/transmit with the result packets results.SetAlphaRuntimeStatistics(alphas.RuntimeStatistics); // Process any required events of the results handler such as sampling assets, equity, or stock prices. results.ProcessSynchronousEvents(forceProcess: true); //Liquidate Holdings for Calculations: if (_algorithm.Status == AlgorithmStatus.Liquidated && _liveMode) { Log.Trace("AlgorithmManager.Run(): Liquidating algorithm holdings..."); algorithm.Liquidate(); results.LogMessage("Algorithm Liquidated"); results.SendStatusUpdate(AlgorithmStatus.Liquidated); } //Manually stopped the algorithm if (_algorithm.Status == AlgorithmStatus.Stopped) { Log.Trace("AlgorithmManager.Run(): Stopping algorithm..."); results.LogMessage("Algorithm Stopped"); results.SendStatusUpdate(AlgorithmStatus.Stopped); } //Backtest deleted. if (_algorithm.Status == AlgorithmStatus.Deleted) { Log.Trace("AlgorithmManager.Run(): Deleting algorithm..."); results.DebugMessage("Algorithm Id:(" + job.AlgorithmId + ") Deleted by request."); results.SendStatusUpdate(AlgorithmStatus.Deleted); } //Algorithm finished, send regardless of commands: results.SendStatusUpdate(AlgorithmStatus.Completed); SetStatus(AlgorithmStatus.Completed); //Take final samples: results.Sample(time); } // End of Run();