public void Initialize(AlgorithmNodePacket job,
IMessagingHandler messagingHandler,
IApi api,
IDataFeed dataFeed,
ISetupHandler setupHandler,
ITransactionHandler transactionHandler)
{
_job = job;
}
/// <summary>
/// Initializes a new instance of the <see cref="LeanEngineAlgorithmHandlers"/> class from the specified handlers
/// </summary>
/// <param name="results">The result handler for communicating results from the algorithm</param>
/// <param name="setup">The setup handler used to initialize algorithm state</param>
/// <param name="dataFeed">The data feed handler used to pump data to the algorithm</param>
/// <param name="transactions">The transaction handler used to process orders from the algorithm</param>
/// <param name="realTime">The real time handler used to process real time events</param>
/// <param name="historyProvider">The history provider used to process historical data requests</param>
/// <param name="commandQueue">The command queue handler used to receive external commands for the algorithm</param>
/// <param name="mapFileProvider">The map file provider used to retrieve map files for the data feed</param>
public LeanEngineAlgorithmHandlers(IResultHandler results,
ISetupHandler setup,
IDataFeed dataFeed,
ITransactionHandler transactions,
IRealTimeHandler realTime,
IHistoryProvider historyProvider,
ICommandQueueHandler commandQueue,
IMapFileProvider mapFileProvider
)
{
if (results == null)
{
throw new ArgumentNullException("results");
}
if (setup == null)
{
throw new ArgumentNullException("setup");
}
if (dataFeed == null)
{
throw new ArgumentNullException("dataFeed");
}
if (transactions == null)
{
throw new ArgumentNullException("transactions");
}
if (realTime == null)
{
throw new ArgumentNullException("realTime");
}
if (historyProvider == null)
{
throw new ArgumentNullException("realTime");
}
if (commandQueue == null)
{
throw new ArgumentNullException("commandQueue");
}
if (mapFileProvider == null)
{
throw new ArgumentNullException("mapFileProvider");
}
_results = results;
_setup = setup;
_dataFeed = dataFeed;
_transactions = transactions;
_realTime = realTime;
_historyProvider = historyProvider;
_commandQueue = commandQueue;
_mapFileProvider = mapFileProvider;
}
/// <summary>
/// Initialize the result handler with this result packet.
/// </summary>
/// <param name="job">Algorithm job packet for this result handler</param>
/// <param name="messagingHandler"></param>
/// <param name="api"></param>
/// <param name="dataFeed"></param>
/// <param name="setupHandler"></param>
/// <param name="transactionHandler"></param>
public void Initialize(AlgorithmNodePacket job, IMessagingHandler messagingHandler, IApi api, IDataFeed dataFeed, ISetupHandler setupHandler, ITransactionHandler transactionHandler)
{
//Redirect the log messages here:
_job = job;
var desktopLogging = new FunctionalLogHandler(DebugMessage, DebugMessage, ErrorMessage);
Log.LogHandler = new CompositeLogHandler(new[] { desktopLogging, Log.LogHandler });
}
/// <summary>
/// Initialize the result handler with this result packet.
/// </summary>
/// <param name="job">Algorithm job packet for this result handler</param>
/// <param name="messagingHandler"></param>
/// <param name="api"></param>
/// <param name="dataFeed"></param>
/// <param name="setupHandler"></param>
/// <param name="transactionHandler"></param>
public void Initialize(AlgorithmNodePacket job, IMessagingHandler messagingHandler, IApi api, IDataFeed dataFeed, ISetupHandler setupHandler, ITransactionHandler transactionHandler)
{
_api = api;
_dataFeed = dataFeed;
_messagingHandler = messagingHandler;
_setupHandler = setupHandler;
_transactionHandler = transactionHandler;
_job = (LiveNodePacket)job;
if (_job == null) throw new Exception("LiveResultHandler.Constructor(): Submitted Job type invalid.");
_deployId = _job.DeployId;
_compileId = _job.CompileId;
}
/// <summary>
/// Initialize the result handler with this result packet.
/// </summary>
/// <param name="job">Algorithm job packet for this result handler</param>
/// <param name="messagingHandler">The handler responsible for communicating messages to listeners</param>
/// <param name="api">The api instance used for handling logs</param>
/// <param name="dataFeed"></param>
/// <param name="setupHandler"></param>
/// <param name="transactionHandler"></param>
public void Initialize(AlgorithmNodePacket job, IMessagingHandler messagingHandler, IApi api, IDataFeed dataFeed, ISetupHandler setupHandler, ITransactionHandler transactionHandler)
{
_api = api;
_messagingHandler = messagingHandler;
_transactionHandler = transactionHandler;
_setupHandler = setupHandler;
_job = (BacktestNodePacket)job;
if (_job == null)
{
throw new Exception("BacktestingResultHandler.Constructor(): Submitted Job type invalid.");
}
_compileId = _job.CompileId;
_backtestId = _job.BacktestId;
}
/// <summary>
/// Initialize the result handler with this result packet.
/// </summary>
/// <param name="job">Algorithm job packet for this result handler</param>
/// <param name="messagingHandler"></param>
/// <param name="api"></param>
/// <param name="dataFeed"></param>
/// <param name="setupHandler"></param>
/// <param name="transactionHandler"></param>
public void Initialize(AlgorithmNodePacket job, IMessagingHandler messagingHandler, IApi api, IDataFeed dataFeed, ISetupHandler setupHandler, ITransactionHandler transactionHandler)
{
//Redirect the log messages here:
_job = job;
var desktopLogging = new FunctionalLogHandler(DebugMessage, DebugMessage, ErrorMessage);
Log.LogHandler = new CompositeLogHandler(new[] { desktopLogging, Log.LogHandler });
}
/********************************************************
* CLASS METHODS
*********************************************************/
/// <summary>
/// Launch the algorithm manager to run this strategy
/// </summary>
/// <param name="job">Algorithm job</param>
/// <param name="algorithm">Algorithm instance</param>
/// <param name="feed">Datafeed object</param>
/// <param name="transactions">Transaction manager object</param>
/// <param name="results">Result handler object</param>
/// <param name="setup">Setup handler object</param>
/// <param name="realtime">Realtime processing object</param>
/// <remarks>Modify with caution</remarks>
public static void Run(AlgorithmNodePacket job, IAlgorithm algorithm, IDataFeed feed, ITransactionHandler transactions, IResultHandler results, ISetupHandler setup, IRealTimeHandler realtime)
{
//Initialize:
_dataPointCount = 0;
var backwardsCompatibilityMode = false;
var tradebarsType = typeof(TradeBars);
var ticksType = typeof(Ticks);
var startingPortfolioValue = setup.StartingCapital;
var backtestMode = (job.Type == PacketType.BacktestNode);
var methodInvokers = new Dictionary <Type, MethodInvoker>();
var marginCallFrequency = TimeSpan.FromMinutes(5);
var nextMarginCallTime = DateTime.MinValue;
//Initialize Properties:
_frontier = setup.StartingDate;
_algorithmId = job.AlgorithmId;
_algorithmState = AlgorithmStatus.Running;
_previousTime = setup.StartingDate.Date;
//Create the method accessors to push generic types into algorithm: Find all OnData events:
//Algorithm 1.0 Data Accessors.
//If the users defined these methods, add them in manually. This allows keeping backwards compatibility to algorithm 1.0.
var oldTradeBarsMethodInfo = (algorithm.GetType()).GetMethod("OnTradeBar", new[] { typeof(Dictionary <string, TradeBar>) });
var oldTicksMethodInfo = (algorithm.GetType()).GetMethod("OnTick", new[] { typeof(Dictionary <string, List <Tick> >) });
//Algorithm 2.0 Data Generics Accessors.
//New hidden access to tradebars with custom type.
var newTradeBarsMethodInfo = (algorithm.GetType()).GetMethod("OnData", new[] { tradebarsType });
var newTicksMethodInfo = (algorithm.GetType()).GetMethod("OnData", new[] { ticksType });
if (newTradeBarsMethodInfo == null && newTicksMethodInfo == null)
{
backwardsCompatibilityMode = true;
if (oldTradeBarsMethodInfo != null)
{
methodInvokers.Add(tradebarsType, oldTradeBarsMethodInfo.DelegateForCallMethod());
}
if (oldTradeBarsMethodInfo != null)
{
methodInvokers.Add(ticksType, oldTicksMethodInfo.DelegateForCallMethod());
}
}
else
{
backwardsCompatibilityMode = false;
if (newTradeBarsMethodInfo != null)
{
methodInvokers.Add(tradebarsType, newTradeBarsMethodInfo.DelegateForCallMethod());
}
if (newTicksMethodInfo != null)
{
methodInvokers.Add(ticksType, newTicksMethodInfo.DelegateForCallMethod());
}
}
//Go through the subscription types and create invokers to trigger the event handlers for each custom type:
foreach (var config in feed.Subscriptions)
{
//If type is a tradebar, combine tradebars and ticks into unified array:
if (config.Type.Name != "TradeBar" && config.Type.Name != "Tick")
{
//Get the matching method for this event handler - e.g. public void OnData(Quandl data) { .. }
var genericMethod = (algorithm.GetType()).GetMethod("OnData", new[] { config.Type });
//If we already have this Type-handler then don't add it to invokers again.
if (methodInvokers.ContainsKey(config.Type))
{
continue;
}
//If we couldnt find the event handler, let the user know we can't fire that event.
if (genericMethod == null)
{
algorithm.RunTimeError = new Exception("Data event handler not found, please create a function matching this template: public void OnData(" + config.Type.Name + " data) { }");
_algorithmState = AlgorithmStatus.RuntimeError;
return;
}
methodInvokers.Add(config.Type, genericMethod.DelegateForCallMethod());
}
}
//Loop over the queues: get a data collection, then pass them all into relevent methods in the algorithm.
Log.Debug("AlgorithmManager.Run(): Algorithm initialized, launching time loop.");
foreach (var newData in DataStream.GetData(feed, setup.StartingDate))
{
//Check this backtest is still running:
if (_algorithmState != AlgorithmStatus.Running)
{
break;
}
//Go over each time stamp we've collected, pass it into the algorithm in order:
foreach (var time in newData.Keys)
{
//Set the time frontier:
_frontier = time;
//Execute with TimeLimit Monitor:
if (Isolator.IsCancellationRequested)
{
return;
}
//If we're in backtest mode we need to capture the daily performance. We do this here directly
//before updating the algorithm state with the new data from this time step, otherwise we'll
//produce incorrect samples (they'll take into account this time step's new price values)
if (backtestMode)
{
//Refresh the realtime event monitor:
//in backtest mode use the algorithms clock as realtime.
realtime.SetTime(time);
//On day-change sample equity and daily performance for statistics calculations
if (_previousTime.Date != time.Date)
{
//Sample the portfolio value over time for chart.
results.SampleEquity(_previousTime, Math.Round(algorithm.Portfolio.TotalPortfolioValue, 4));
//Check for divide by zero
if (startingPortfolioValue == 0m)
{
results.SamplePerformance(_previousTime.Date, 0);
}
else
{
results.SamplePerformance(_previousTime.Date, Math.Round((algorithm.Portfolio.TotalPortfolioValue - startingPortfolioValue) * 100 / startingPortfolioValue, 10));
}
startingPortfolioValue = algorithm.Portfolio.TotalPortfolioValue;
}
}
//Update algorithm state after capturing performance from previous day
//On each time step push the real time prices to the cashbook so we can have updated conversion rates
algorithm.Portfolio.CashBook.Update(newData[time]);
//Update the securities properties: first before calling user code to avoid issues with data
algorithm.Securities.Update(time, newData[time]);
// perform margin calls
if (time >= nextMarginCallTime)
{
// determine if there are possible margin call orders to be executed
var marginCallOrders = algorithm.Portfolio.ScanForMarginCall();
if (marginCallOrders.Count != 0)
{
// execute the margin call orders
var executedOrders = algorithm.Portfolio.MarginCallModel.ExecuteMarginCall(marginCallOrders);
foreach (var order in executedOrders)
{
algorithm.Error(string.Format("Executed MarginCallOrder: {0} - Quantity: {1} @ {2}", order.Symbol, order.Quantity, order.Price));
}
}
nextMarginCallTime = time + marginCallFrequency;
}
//Check if the user's signalled Quit: loop over data until day changes.
if (algorithm.GetQuit())
{
_algorithmState = AlgorithmStatus.Quit;
break;
}
//Pass in the new time first:
algorithm.SetDateTime(time);
//Trigger the data events: Invoke the types we have data for:
var oldBars = new Dictionary <string, TradeBar>();
var oldTicks = new Dictionary <string, List <Tick> >();
var newBars = new TradeBars(time);
var newTicks = new Ticks(time);
//Invoke all non-tradebars, non-ticks methods and build up the TradeBars and Ticks dictionaries
// --> i == Subscription Configuration Index, so we don't need to compare types.
foreach (var i in newData[time].Keys)
{
//Data point and config of this point:
var dataPoints = newData[time][i];
var config = feed.Subscriptions[i];
//Keep track of how many data points we've processed
_dataPointCount += dataPoints.Count;
//We don't want to pump data that we added just for currency conversions
if (config.IsInternalFeed)
{
continue;
}
//Create TradeBars Unified Data --> OR --> invoke generic data event. One loop.
foreach (var dataPoint in dataPoints)
{
//Update registered consolidators for this symbol index
try
{
for (var j = 0; j < config.Consolidators.Count; j++)
{
config.Consolidators[j].Update(dataPoint);
}
}
catch (Exception err)
{
algorithm.RunTimeError = err;
_algorithmState = AlgorithmStatus.RuntimeError;
Log.Error("AlgorithmManager.Run(): RuntimeError: Consolidators update: " + err.Message);
return;
}
switch (config.Type.Name)
{
case "TradeBar":
var bar = dataPoint as TradeBar;
try
{
if (bar != null)
{
if (backwardsCompatibilityMode)
{
if (!oldBars.ContainsKey(bar.Symbol))
{
oldBars.Add(bar.Symbol, bar);
}
}
else
{
if (!newBars.ContainsKey(bar.Symbol))
{
newBars.Add(bar.Symbol, bar);
}
}
}
}
catch (Exception err)
{
Log.Error(time.ToLongTimeString() + " >> " + bar.Time.ToLongTimeString() + " >> " + bar.Symbol + " >> " + bar.Value.ToString("C"));
Log.Error("AlgorithmManager.Run(): Failed to add TradeBar (" + bar.Symbol + ") Time: (" + time.ToLongTimeString() + ") Count:(" + newBars.Count + ") " + err.Message);
}
break;
case "Tick":
var tick = dataPoint as Tick;
if (tick != null)
{
if (backwardsCompatibilityMode)
{
if (!oldTicks.ContainsKey(tick.Symbol))
{
oldTicks.Add(tick.Symbol, new List <Tick>());
}
oldTicks[tick.Symbol].Add(tick);
}
else
{
if (!newTicks.ContainsKey(tick.Symbol))
{
newTicks.Add(tick.Symbol, new List <Tick>());
}
newTicks[tick.Symbol].Add(tick);
}
}
break;
default:
//Send data into the generic algorithm event handlers
try
{
methodInvokers[config.Type](algorithm, dataPoint);
}
catch (Exception err)
{
algorithm.RunTimeError = err;
_algorithmState = AlgorithmStatus.RuntimeError;
Log.Debug("AlgorithmManager.Run(): RuntimeError: Custom Data: " + err.Message + " STACK >>> " + err.StackTrace);
return;
}
break;
}
}
}
//After we've fired all other events in this second, fire the pricing events:
if (backwardsCompatibilityMode)
{
//Log.Debug("AlgorithmManager.Run(): Invoking v1.0 Event Handlers...");
try
{
if (oldTradeBarsMethodInfo != null && oldBars.Count > 0)
{
methodInvokers[tradebarsType](algorithm, oldBars);
}
if (oldTicksMethodInfo != null && oldTicks.Count > 0)
{
methodInvokers[ticksType](algorithm, oldTicks);
}
}
catch (Exception err)
{
algorithm.RunTimeError = err;
_algorithmState = AlgorithmStatus.RuntimeError;
Log.Debug("AlgorithmManager.Run(): RuntimeError: Backwards Compatibility Mode: " + err.Message + " STACK >>> " + err.StackTrace);
return;
}
}
else
{
//Log.Debug("AlgorithmManager.Run(): Invoking v2.0 Event Handlers...");
try
{
if (newTradeBarsMethodInfo != null && newBars.Count > 0)
{
methodInvokers[tradebarsType](algorithm, newBars);
}
if (newTicksMethodInfo != null && newTicks.Count > 0)
{
methodInvokers[ticksType](algorithm, newTicks);
}
}
catch (Exception err)
{
algorithm.RunTimeError = err;
_algorithmState = AlgorithmStatus.RuntimeError;
Log.Debug("AlgorithmManager.Run(): RuntimeError: New Style Mode: " + err.Message + " STACK >>> " + err.StackTrace);
return;
}
}
//If its the historical/paper trading models, wait until market orders have been "filled"
// Manually trigger the event handler to prevent thread switch.
transactions.ProcessSynchronousEvents();
//Save the previous time for the sample calculations
_previousTime = time;
// Process any required events of the results handler such as sampling assets, equity, or stock prices.
results.ProcessSynchronousEvents();
} // End of Time Loop
} // End of ForEach DataStream
//Stream over:: Send the final packet and fire final events:
Log.Trace("AlgorithmManager.Run(): Firing On End Of Algorithm...");
try
{
algorithm.OnEndOfAlgorithm();
}
catch (Exception err)
{
_algorithmState = AlgorithmStatus.RuntimeError;
algorithm.RunTimeError = new Exception("Error running OnEndOfAlgorithm(): " + err.Message, err.InnerException);
Log.Debug("AlgorithmManager.OnEndOfAlgorithm(): " + err.Message + " STACK >>> " + err.StackTrace);
return;
}
// Process any required events of the results handler such as sampling assets, equity, or stock prices.
results.ProcessSynchronousEvents(forceProcess: true);
//Liquidate Holdings for Calculations:
if (_algorithmState == AlgorithmStatus.Liquidated || !Engine.LiveMode)
{
Log.Trace("AlgorithmManager.Run(): Liquidating algorithm holdings...");
algorithm.Liquidate();
results.LogMessage("Algorithm Liquidated");
results.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.Liquidated);
}
//Manually stopped the algorithm
if (_algorithmState == AlgorithmStatus.Stopped)
{
Log.Trace("AlgorithmManager.Run(): Stopping algorithm...");
results.LogMessage("Algorithm Stopped");
results.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.Stopped);
}
//Backtest deleted.
if (_algorithmState == AlgorithmStatus.Deleted)
{
Log.Trace("AlgorithmManager.Run(): Deleting algorithm...");
results.DebugMessage("Algorithm Id:(" + job.AlgorithmId + ") Deleted by request.");
results.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.Deleted);
}
//Algorithm finished, send regardless of commands:
results.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.Completed);
//Take final samples:
results.SampleRange(algorithm.GetChartUpdates());
results.SampleEquity(_frontier, Math.Round(algorithm.Portfolio.TotalPortfolioValue, 4));
results.SamplePerformance(_frontier, Math.Round((algorithm.Portfolio.TotalPortfolioValue - startingPortfolioValue) * 100 / startingPortfolioValue, 10));
} // End of Run();
/// <summary>
/// Initializes a new instance of the <see cref="LeanEngineAlgorithmHandlers"/> class from the specified handlers
/// </summary>
/// <param name="results">The result handler for communicating results from the algorithm</param>
/// <param name="setup">The setup handler used to initialize algorithm state</param>
/// <param name="dataFeed">The data feed handler used to pump data to the algorithm</param>
/// <param name="transactions">The transaction handler used to process orders from the algorithm</param>
/// <param name="realTime">The real time handler used to process real time events</param>
/// <param name="historyProvider">The history provider used to process historical data requests</param>
/// <param name="commandQueue">The command queue handler used to receive external commands for the algorithm</param>
/// <param name="mapFileProvider">The map file provider used to retrieve map files for the data feed</param>
/// <param name="factorFileProvider">Map file provider used as a map file source for the data feed</param>
/// <param name="dataFileProvider">file provider used to retrieve security data if it is not on the file system</param>
public LeanEngineAlgorithmHandlers(IResultHandler results,
ISetupHandler setup,
IDataFeed dataFeed,
ITransactionHandler transactions,
IRealTimeHandler realTime,
IHistoryProvider historyProvider,
ICommandQueueHandler commandQueue,
IMapFileProvider mapFileProvider,
IFactorFileProvider factorFileProvider,
IDataFileProvider dataFileProvider
)
{
if (results == null)
{
throw new ArgumentNullException("results");
}
if (setup == null)
{
throw new ArgumentNullException("setup");
}
if (dataFeed == null)
{
throw new ArgumentNullException("dataFeed");
}
if (transactions == null)
{
throw new ArgumentNullException("transactions");
}
if (realTime == null)
{
throw new ArgumentNullException("realTime");
}
if (historyProvider == null)
{
throw new ArgumentNullException("realTime");
}
if (commandQueue == null)
{
throw new ArgumentNullException("commandQueue");
}
if (mapFileProvider == null)
{
throw new ArgumentNullException("mapFileProvider");
}
if (factorFileProvider == null)
{
throw new ArgumentNullException("factorFileProvider");
}
if (dataFileProvider == null)
{
throw new ArgumentNullException("dataFileProvider");
}
_results = results;
_setup = setup;
_dataFeed = dataFeed;
_transactions = transactions;
_realTime = realTime;
_historyProvider = historyProvider;
_commandQueue = commandQueue;
_mapFileProvider = mapFileProvider;
_factorFileProvider = factorFileProvider;
_dataFileProvider = dataFileProvider;
}
/********************************************************
* CLASS METHODS
*********************************************************/
/// <summary>
/// Primary Analysis Thread:
/// </summary>
public static void Main(string[] args)
{
//Initialize:
var algorithmPath = "";
string mode = "RELEASE";
AlgorithmNodePacket job = null;
var algorithm = default(IAlgorithm);
var startTime = DateTime.Now;
Log.LogHandler = Composer.Instance.GetExportedValueByTypeName <ILogHandler>(Config.Get("log-handler", "CompositeLogHandler"));
_version = DateTime.ParseExact(Config.Get("version", DateTime.Now.ToString(DateFormat.UI)), DateFormat.UI, CultureInfo.InvariantCulture);
#if DEBUG
mode = "DEBUG";
#endif
//Name thread for the profiler:
Thread.CurrentThread.Name = "Algorithm Analysis Thread";
Log.Trace("Engine.Main(): LEAN ALGORITHMIC TRADING ENGINE v" + _version + " Mode: " + mode);
Log.Trace("Engine.Main(): Started " + DateTime.Now.ToShortTimeString());
Log.Trace("Engine.Main(): Memory " + OS.ApplicationMemoryUsed + "Mb-App " + +OS.TotalPhysicalMemoryUsed + "Mb-Used " + OS.TotalPhysicalMemory + "Mb-Total");
//Import external libraries specific to physical server location (cloud/local)
try
{
// grab the right export based on configuration
Api = Composer.Instance.GetExportedValueByTypeName <IApi>(Config.Get("api-handler"));
Notify = Composer.Instance.GetExportedValueByTypeName <IMessagingHandler>(Config.Get("messaging-handler"));
JobQueue = Composer.Instance.GetExportedValueByTypeName <IJobQueueHandler>(Config.Get("job-queue-handler"));
}
catch (CompositionException compositionException)
{ Log.Error("Engine.Main(): Failed to load library: " + compositionException); }
//Setup packeting, queue and controls system: These don't do much locally.
Api.Initialize();
Notify.Initialize();
JobQueue.Initialize();
//Start monitoring the backtest active status:
var statusPingThread = new Thread(StateCheck.Ping.Run);
statusPingThread.Start();
do
{
try
{
//Reset algo manager internal variables preparing for a new algorithm.
AlgorithmManager.ResetManager();
//Reset thread holders.
var initializeComplete = false;
Thread threadFeed = null;
Thread threadTransactions = null;
Thread threadResults = null;
Thread threadRealTime = null;
do
{
//-> Pull job from QuantConnect job queue, or, pull local build:
job = JobQueue.NextJob(out algorithmPath); // Blocking.
if (!IsLocal && LiveMode && (job.Version < Version || (job.Version == Version && job.Redelivered)))
{
//Tiny chance there was an uncontrolled collapse of a server, resulting in an old user task circulating.
//In this event kill the old algorithm and leave a message so the user can later review.
JobQueue.AcknowledgeJob(job);
Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, _collapseMessage);
Notify.SetChannel(job.Channel);
Notify.RuntimeError(job.AlgorithmId, _collapseMessage);
job = null;
}
} while (job == null);
//-> Initialize messaging system
Notify.SetChannel(job.Channel);
//-> Create SetupHandler to configure internal algorithm state:
SetupHandler = GetSetupHandler(job.SetupEndpoint);
//-> Set the result handler type for this algorithm job, and launch the associated result thread.
ResultHandler = GetResultHandler(job);
threadResults = new Thread(ResultHandler.Run, 0)
{
Name = "Result Thread"
};
threadResults.Start();
try
{
// Save algorithm to cache, load algorithm instance:
algorithm = SetupHandler.CreateAlgorithmInstance(algorithmPath);
//Initialize the internal state of algorithm and job: executes the algorithm.Initialize() method.
initializeComplete = SetupHandler.Setup(algorithm, out _brokerage, job);
//If there are any reasons it failed, pass these back to the IDE.
if (!initializeComplete || algorithm.ErrorMessages.Count > 0 || SetupHandler.Errors.Count > 0)
{
initializeComplete = false;
//Get all the error messages: internal in algorithm and external in setup handler.
var errorMessage = String.Join(",", algorithm.ErrorMessages);
errorMessage += String.Join(",", SetupHandler.Errors);
throw new Exception(errorMessage);
}
}
catch (Exception err)
{
var runtimeMessage = "Algorithm.Initialize() Error: " + err.Message + " Stack Trace: " + err.StackTrace;
ResultHandler.RuntimeError(runtimeMessage, err.StackTrace);
Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, runtimeMessage);
}
//-> Using the job + initialization: load the designated handlers:
if (initializeComplete)
{
//-> Reset the backtest stopwatch; we're now running the algorithm.
startTime = DateTime.Now;
//Set algorithm as locked; set it to live mode if we're trading live, and set it to locked for no further updates.
algorithm.SetAlgorithmId(job.AlgorithmId);
algorithm.SetLiveMode(LiveMode);
algorithm.SetLocked();
//Load the associated handlers for data, transaction and realtime events:
ResultHandler.SetAlgorithm(algorithm);
DataFeed = GetDataFeedHandler(algorithm, job);
TransactionHandler = GetTransactionHandler(algorithm, _brokerage, ResultHandler, job);
RealTimeHandler = GetRealTimeHandler(algorithm, _brokerage, DataFeed, ResultHandler, job);
//Set the error handlers for the brokerage asynchronous errors.
SetupHandler.SetupErrorHandler(ResultHandler, _brokerage);
//Send status to user the algorithm is now executing.
ResultHandler.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.Running);
//Launch the data, transaction and realtime handlers into dedicated threads
threadFeed = new Thread(DataFeed.Run, 0)
{
Name = "DataFeed Thread"
};
threadTransactions = new Thread(TransactionHandler.Run, 0)
{
Name = "Transaction Thread"
};
threadRealTime = new Thread(RealTimeHandler.Run, 0)
{
Name = "RealTime Thread"
};
//Launch the data feed, result sending, and transaction models/handlers in separate threads.
threadFeed.Start(); // Data feed pushing data packets into thread bridge;
threadTransactions.Start(); // Transaction modeller scanning new order requests
threadRealTime.Start(); // RealTime scan time for time based events:
// Result manager scanning message queue: (started earlier)
try
{
// Execute the Algorithm Code:
var complete = Isolator.ExecuteWithTimeLimit(SetupHandler.MaximumRuntime, () =>
{
try
{
//Run Algorithm Job:
// -> Using this Data Feed,
// -> Send Orders to this TransactionHandler,
// -> Send Results to ResultHandler.
AlgorithmManager.Run(job, algorithm, DataFeed, TransactionHandler, ResultHandler, SetupHandler, RealTimeHandler);
}
catch (Exception err)
{
//Debugging at this level is difficult, stack trace needed.
Log.Error("Engine.Run(): Error in Algo Manager: " + err.Message + " ST >> " + err.StackTrace);
}
Log.Trace("Engine.Run(): Exiting Algorithm Manager");
}, MaximumRamAllocation);
if (!complete)
{
Log.Error("Engine.Main(): Failed to complete in time: " + SetupHandler.MaximumRuntime.ToString("F"));
throw new Exception("Failed to complete algorithm within " + SetupHandler.MaximumRuntime.ToString("F") + " seconds. Please make it run faster.");
}
// Algorithm runtime error:
if (algorithm.RunTimeError != null)
{
throw algorithm.RunTimeError;
}
}
catch (Exception err)
{
//Error running the user algorithm: purge datafeed, send error messages, set algorithm status to failed.
Log.Error("Engine.Run(): Breaking out of parent try-catch: " + err.Message + " " + err.StackTrace);
if (DataFeed != null)
{
DataFeed.Exit();
}
if (ResultHandler != null)
{
var message = "Runtime Error: " + err.Message;
Log.Trace("Engine.Run(): Sending runtime error to user...");
ResultHandler.LogMessage(message);
ResultHandler.RuntimeError(message, err.StackTrace);
Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, message + " Stack Trace: " + err.StackTrace);
}
}
//Send result data back: this entire code block could be rewritten.
// todo: - Split up statistics class, its enormous.
// todo: - Make a dedicated Statistics.Benchmark class.
// todo: - Move all creation and transmission of statistics out of primary engine loop.
// todo: - Statistics.Generate(algorithm, resulthandler, transactionhandler);
try
{
var charts = new Dictionary <string, Chart>(ResultHandler.Charts);
var orders = new Dictionary <int, Order>(algorithm.Transactions.Orders);
var holdings = new Dictionary <string, Holding>();
var statistics = new Dictionary <string, string>();
var banner = new Dictionary <string, string>();
try
{
//Generates error when things don't exist (no charting logged, runtime errors in main algo execution)
const string strategyEquityKey = "Strategy Equity";
const string equityKey = "Equity";
const string dailyPerformanceKey = "Daily Performance";
// make sure we've taken samples for these series before just blindly requesting them
if (charts.ContainsKey(strategyEquityKey) &&
charts[strategyEquityKey].Series.ContainsKey(equityKey) &&
charts[strategyEquityKey].Series.ContainsKey(dailyPerformanceKey))
{
var equity = charts[strategyEquityKey].Series[equityKey].Values;
var performance = charts[strategyEquityKey].Series[dailyPerformanceKey].Values;
var profitLoss =
new SortedDictionary <DateTime, decimal>(algorithm.Transactions.TransactionRecord);
statistics = Statistics.Statistics.Generate(equity, profitLoss, performance,
SetupHandler.StartingCapital, 252);
}
}
catch (Exception err)
{
Log.Error("Algorithm.Node.Engine(): Error generating statistics packet: " + err.Message);
}
//Diagnostics Completed, Send Result Packet:
var totalSeconds = (DateTime.Now - startTime).TotalSeconds;
ResultHandler.DebugMessage(string.Format("Algorithm Id:({0}) completed in {1} seconds at {2}k data points per second. Processing total of {3} data points.",
job.AlgorithmId, totalSeconds.ToString("F2"), ((AlgorithmManager.DataPoints / (double)1000) / totalSeconds).ToString("F0"), AlgorithmManager.DataPoints.ToString("N0")));
ResultHandler.SendFinalResult(job, orders, algorithm.Transactions.TransactionRecord, holdings, statistics, banner);
}
catch (Exception err)
{
Log.Error("Engine.Main(): Error sending analysis result: " + err.Message + " ST >> " + err.StackTrace);
}
//Before we return, send terminate commands to close up the threads
TransactionHandler.Exit();
DataFeed.Exit();
RealTimeHandler.Exit();
}
//Close result handler:
ResultHandler.Exit();
//Wait for the threads to complete:
var ts = Stopwatch.StartNew();
while ((ResultHandler.IsActive || (TransactionHandler != null && TransactionHandler.IsActive) || (DataFeed != null && DataFeed.IsActive)) && ts.ElapsedMilliseconds < 30 * 1000)
{
Thread.Sleep(100); Log.Trace("Waiting for threads to exit...");
}
if (threadFeed != null && threadFeed.IsAlive)
{
threadFeed.Abort();
}
if (threadTransactions != null && threadTransactions.IsAlive)
{
threadTransactions.Abort();
}
if (threadResults != null && threadResults.IsAlive)
{
threadResults.Abort();
}
Log.Trace("Engine.Main(): Analysis Completed and Results Posted.");
}
catch (Exception err)
{
Log.Error("Engine.Main(): Error running algorithm: " + err.Message + " >> " + err.StackTrace);
}
finally
{
//Delete the message from the job queue:
JobQueue.AcknowledgeJob(job);
Log.Trace("Engine.Main(): Packet removed from queue: " + job.AlgorithmId);
//No matter what for live mode; make sure we've set algorithm status in the API for "not running" conditions:
if (LiveMode && AlgorithmManager.State != AlgorithmStatus.Running && AlgorithmManager.State != AlgorithmStatus.RuntimeError)
{
Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmManager.State);
}
//Attempt to clean up ram usage:
GC.Collect();
}
//If we're running locally will execute just once.
} while (!IsLocal);
// Send the exit signal and then kill the thread
StateCheck.Ping.Exit();
// Make the console window pause so we can read log output before exiting and killing the application completely
Console.Read();
//Finally if ping thread still not complete, kill.
if (statusPingThread != null && statusPingThread.IsAlive)
{
statusPingThread.Abort();
}
if (Log.LogHandler != null)
{
Log.LogHandler.Dispose();
}
}
/// <summary>
/// Initialize the result handler with this result packet.
/// </summary>
/// <param name="job">Algorithm job packet for this result handler</param>
/// <param name="messagingHandler">The handler responsible for communicating messages to listeners</param>
/// <param name="api">The api instance used for handling logs</param>
/// <param name="dataFeed"></param>
/// <param name="setupHandler"></param>
/// <param name="transactionHandler"></param>
public void Initialize(AlgorithmNodePacket job, IMessagingHandler messagingHandler, IApi api, IDataFeed dataFeed, ISetupHandler setupHandler, ITransactionHandler transactionHandler)
{
_api = api;
_messagingHandler = messagingHandler;
_transactionHandler = transactionHandler;
_job = (BacktestNodePacket)job;
if (_job == null) throw new Exception("BacktestingResultHandler.Constructor(): Submitted Job type invalid.");
_compileId = _job.CompileId;
_backtestId = _job.BacktestId;
//Get the resample period:
var totalMinutes = (_job.PeriodFinish - _job.PeriodStart).TotalMinutes;
var resampleMinutes = (totalMinutes < (_minimumSamplePeriod * _samples)) ? _minimumSamplePeriod : (totalMinutes / _samples); // Space out the sampling every
_resamplePeriod = TimeSpan.FromMinutes(resampleMinutes);
Log.Trace("BacktestingResultHandler(): Sample Period Set: " + resampleMinutes.ToString("00.00"));
}
/// <summary>
/// Initialize the result handler with this result packet.
/// </summary>
/// <param name="job">Algorithm job packet for this result handler</param>
/// <param name="messagingHandler">The handler responsible for communicating messages to listeners</param>
/// <param name="api">The api instance used for handling logs</param>
/// <param name="dataFeed"></param>
/// <param name="setupHandler"></param>
/// <param name="transactionHandler"></param>
public void Initialize(AlgorithmNodePacket job, IMessagingHandler messagingHandler, IApi api, IDataFeed dataFeed, ISetupHandler setupHandler, ITransactionHandler transactionHandler)
{
_api = api;
_messagingHandler = messagingHandler;
_transactionHandler = transactionHandler;
_job = (BacktestNodePacket)job;
if (_job == null)
{
throw new Exception("BacktestingResultHandler.Constructor(): Submitted Job type invalid.");
}
_compileId = _job.CompileId;
_backtestId = _job.BacktestId;
//Get the resample period:
var totalMinutes = (_job.PeriodFinish - _job.PeriodStart).TotalMinutes;
var resampleMinutes = (totalMinutes < (_minimumSamplePeriod * _samples)) ? _minimumSamplePeriod : (totalMinutes / _samples); // Space out the sampling every
_resamplePeriod = TimeSpan.FromMinutes(resampleMinutes);
Log.Trace("BacktestingResultHandler(): Sample Period Set: " + resampleMinutes.ToString("00.00"));
}
/// <summary>
/// Initializes a new instance of the <see cref="LeanEngineAlgorithmHandlers"/> class from the specified handlers
/// </summary>
/// <param name="results">The result handler for communicating results from the algorithm</param>
/// <param name="setup">The setup handler used to initialize algorithm state</param>
/// <param name="dataFeed">The data feed handler used to pump data to the algorithm</param>
/// <param name="transactions">The transaction handler used to process orders from the algorithm</param>
/// <param name="realTime">The real time handler used to process real time events</param>
/// <param name="mapFileProvider">The map file provider used to retrieve map files for the data feed</param>
/// <param name="factorFileProvider">Map file provider used as a map file source for the data feed</param>
/// <param name="dataProvider">file provider used to retrieve security data if it is not on the file system</param>
/// <param name="alphas">The alpha handler used to process generated insights</param>
/// <param name="objectStore">The object store used for persistence</param>
/// <param name="dataPermissionsManager">The data permission manager to use</param>
public LeanEngineAlgorithmHandlers(IResultHandler results,
ISetupHandler setup,
IDataFeed dataFeed,
ITransactionHandler transactions,
IRealTimeHandler realTime,
IMapFileProvider mapFileProvider,
IFactorFileProvider factorFileProvider,
IDataProvider dataProvider,
IAlphaHandler alphas,
IObjectStore objectStore,
IDataPermissionManager dataPermissionsManager
)
{
if (results == null)
{
throw new ArgumentNullException(nameof(results));
}
if (setup == null)
{
throw new ArgumentNullException(nameof(setup));
}
if (dataFeed == null)
{
throw new ArgumentNullException(nameof(dataFeed));
}
if (transactions == null)
{
throw new ArgumentNullException(nameof(transactions));
}
if (realTime == null)
{
throw new ArgumentNullException(nameof(realTime));
}
if (mapFileProvider == null)
{
throw new ArgumentNullException(nameof(mapFileProvider));
}
if (factorFileProvider == null)
{
throw new ArgumentNullException(nameof(factorFileProvider));
}
if (dataProvider == null)
{
throw new ArgumentNullException(nameof(dataProvider));
}
if (alphas == null)
{
throw new ArgumentNullException(nameof(alphas));
}
if (objectStore == null)
{
throw new ArgumentNullException(nameof(objectStore));
}
if (dataPermissionsManager == null)
{
throw new ArgumentNullException(nameof(dataPermissionsManager));
}
Results = results;
Setup = setup;
DataFeed = dataFeed;
Transactions = transactions;
RealTime = realTime;
MapFileProvider = mapFileProvider;
FactorFileProvider = factorFileProvider;
DataProvider = dataProvider;
Alphas = alphas;
ObjectStore = objectStore;
DataPermissionsManager = dataPermissionsManager;
}
/********************************************************
* CLASS METHODS
*********************************************************/
/// <summary>
/// Primary Analysis Thread:
/// </summary>
public static void Main(string[] args)
{
//Initialize:
AlgorithmNodePacket job = null;
var timer = Stopwatch.StartNew();
var algorithm = default(IAlgorithm);
//Name thread for the profiler:
Thread.CurrentThread.Name = "Algorithm Analysis Thread";
Log.Trace("Engine.Main(): Started " + DateTime.Now.ToShortTimeString());
Log.Trace("Engine.Main(): Memory " + OS.ApplicationMemoryUsed + "Mb-App " + +OS.TotalPhysicalMemoryUsed + "Mb-Used " + OS.TotalPhysicalMemory + "Mb-Total");
//Import external libraries specific to physical server location (cloud/local)
var catalog = new AggregateCatalog();
catalog.Catalogs.Add(new DirectoryCatalog(@"../../Extensions"));
var container = new CompositionContainer(catalog);
try
{
// grab the right export based on configuration
Notify = container.GetExportedValueByTypeName <IMessagingHandler>(Config.Get("messaging-handler"));
Queue = container.GetExportedValueByTypeName <IQueueHandler>(Config.Get("queue-handler"));
Api = container.GetExportedValueByTypeName <IApi>(Config.Get("api-handler"));
}
catch (CompositionException compositionException)
{ Log.Error("Engine.Main(): Failed to load library: " + compositionException); }
//Setup packeting, queue and controls system: These don't do much locally.
Api.Initialize();
Notify.Initialize();
Queue.Initialize(_liveMode);
//Start monitoring the backtest active status:
var statusPingThread = new Thread(StateCheck.Ping.Run);
statusPingThread.Start();
do
{
try
{
//Clean up cache directories:
//Reset algo manager internal variables preparing for a new algorithm.
AlgorithmManager.ResetManager();
//Reset thread holders.
var initializeComplete = false;
Thread threadFeed = null;
Thread threadTransactions = null;
Thread threadResults = null;
Thread threadRealTime = null;
//-> Pull job from QuantConnect job queue, or, pull local build:
var algorithmPath = "";
job = Queue.NextJob(out algorithmPath); // Blocking.
//-> Initialize messaging system
Notify.SetChannel(job.Channel);
//-> Reset the backtest stopwatch; we're now running the algorithm.
timer.Restart();
//-> Create SetupHandler to configure internal algorithm state:
SetupHandler = GetSetupHandler(job.SetupEndpoint);
//-> Set the result handler type for this algorithm job, and launch the associated result thread.
ResultHandler = GetResultHandler(job);
threadResults = new Thread(ResultHandler.Run, 0)
{
Name = "Result Thread"
};
threadResults.Start();
try
{
// Save algorithm to cache, load algorithm instance:
algorithm = SetupHandler.CreateAlgorithmInstance(algorithmPath);
//Initialize the internal state of algorithm and job: executes the algorithm.Initialize() method.
initializeComplete = SetupHandler.Setup(algorithm, out _brokerage, job);
//If there are any reasons it failed, pass these back to the IDE.
if (!initializeComplete || algorithm.ErrorMessages.Count > 0 || SetupHandler.Errors.Count > 0)
{
initializeComplete = false;
//Get all the error messages: internal in algorithm and external in setup handler.
var errorMessage = String.Join(",", algorithm.ErrorMessages);
errorMessage += String.Join(",", SetupHandler.Errors);
throw new Exception(errorMessage);
}
}
catch (Exception err)
{
ResultHandler.RuntimeError("Algorithm.Initialize() Error: " + err.Message, err.StackTrace);
}
//-> Using the job + initialization: load the designated handlers:
if (initializeComplete)
{
//Set algorithm as locked; set it to live mode if we're trading live, and set it to locked for no further updates.
algorithm.SetAlgorithmId(job.AlgorithmId);
algorithm.SetLiveMode(LiveMode);
algorithm.SetLocked();
//Load the associated handlers for data, transaction and realtime events:
ResultHandler.SetAlgorithm(algorithm);
DataFeed = GetDataFeedHandler(algorithm, job);
TransactionHandler = GetTransactionHandler(algorithm, _brokerage, ResultHandler, job);
RealTimeHandler = GetRealTimeHandler(algorithm, _brokerage, DataFeed, ResultHandler, job);
//Set the error handlers for the brokerage asynchronous errors.
SetupHandler.SetupErrorHandler(ResultHandler, _brokerage);
//Send status to user the algorithm is now executing.
ResultHandler.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.Running);
//Launch the data, transaction and realtime handlers into dedicated threads
threadFeed = new Thread(DataFeed.Run, 0)
{
Name = "DataFeed Thread"
};
threadTransactions = new Thread(TransactionHandler.Run, 0)
{
Name = "Transaction Thread"
};
threadRealTime = new Thread(RealTimeHandler.Run, 0)
{
Name = "RealTime Thread"
};
//Launch the data feed, result sending, and transaction models/handlers in separate threads.
threadFeed.Start(); // Data feed pushing data packets into thread bridge;
threadTransactions.Start(); // Transaction modeller scanning new order requests
threadRealTime.Start(); // RealTime scan time for time based events:
// Result manager scanning message queue: (started earlier)
try
{
// Execute the Algorithm Code:
var complete = Isolator.ExecuteWithTimeLimit(SetupHandler.MaximumRuntime, () =>
{
try
{
//Run Algorithm Job:
// -> Using this Data Feed,
// -> Send Orders to this TransactionHandler,
// -> Send Results to ResultHandler.
AlgorithmManager.Run(job, algorithm, DataFeed, TransactionHandler, ResultHandler, SetupHandler, RealTimeHandler);
}
catch (Exception err)
{
//Debugging at this level is difficult, stack trace needed.
Log.Error("Engine.Run(): Error in Algo Manager: " + err.Message + " ST >> " + err.StackTrace);
}
Log.Trace("Engine.Run(): Exiting Algorithm Manager");
}, MaximumRamAllocation);
if (!complete)
{
Log.Error("Engine.Main(): Failed to complete in time: " + SetupHandler.MaximumRuntime.ToString("F"));
throw new Exception("Failed to complete algorithm within " + SetupHandler.MaximumRuntime.ToString("F") + " seconds. Please make it run faster.");
}
// Algorithm runtime error:
if (AlgorithmManager.RunTimeError != null)
{
throw AlgorithmManager.RunTimeError;
}
}
catch (Exception err)
{
//Error running the user algorithm: purge datafeed, send error messages, set algorithm status to failed.
Log.Error("Engine.Run(): Breaking out of parent try-catch: " + err.Message + " " + err.StackTrace);
if (DataFeed != null)
{
DataFeed.Exit();
}
if (ResultHandler != null)
{
var runtimeMessage = "Runtime Error: " + err.Message + " Stack Trace: " + err.StackTrace;
ResultHandler.RuntimeError(runtimeMessage, err.StackTrace);
Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, runtimeMessage);
}
}
//Send result data back: this entire code block could be rewritten.
// todo: - Split up statistics class, its enormous.
// todo: - Make a dedicated Statistics.Benchmark class.
// todo: - Elegently manage failure scenarios where no equity present.
// todo: - Move all creation and transmission of statistics out of primary engine loop.
// todo: - Statistics.Generate(algorithm, resulthandler, transactionhandler);
try
{
var charts = new Dictionary <string, Chart>(ResultHandler.Charts);
var orders = new Dictionary <int, Order>(algorithm.Transactions.Orders);
var holdings = new Dictionary <string, Holding>();
var statistics = new Dictionary <string, string>();
var banner = new Dictionary <string, string>();
try
{
//Generates error when things don't exist (no charting logged, runtime errors in main algo execution)
const string strategyEquityKey = "Strategy Equity";
const string equityKey = "Equity";
const string dailyPerformanceKey = "Daily Performance";
// make sure we've taken samples for these series before just blindly requesting them
if (charts.ContainsKey(strategyEquityKey) && charts[strategyEquityKey].Series.ContainsKey(equityKey) && charts[strategyEquityKey].Series.ContainsKey(dailyPerformanceKey))
{
var equity = charts[strategyEquityKey].Series[equityKey].Values;
var performance = charts[strategyEquityKey].Series[dailyPerformanceKey].Values;
var profitLoss = new SortedDictionary <DateTime, decimal>(algorithm.Transactions.TransactionRecord);
statistics = Statistics.Statistics.Generate(equity, profitLoss, performance, SetupHandler.StartingCapital, 252);
}
}
catch (Exception err) {
Log.Error("Algorithm.Node.Engine(): Error generating result packet: " + err.Message);
}
//Diagnostics Completed:
ResultHandler.DebugMessage("Algorithm Id:(" + job.AlgorithmId + ") completed analysis in " + timer.Elapsed.TotalSeconds.ToString("F2") + " seconds");
//Send the result packet:
ResultHandler.SendFinalResult(job, orders, algorithm.Transactions.TransactionRecord, holdings, statistics, banner);
}
catch (Exception err)
{
Log.Error("Engine.Main(): Error sending analysis result: " + err.Message + " ST >> " + err.StackTrace);
}
//Before we return, send terminate commands to close up the threads
timer.Stop(); //Algorithm finished running.
TransactionHandler.Exit();
DataFeed.Exit();
RealTimeHandler.Exit();
AlgorithmManager.ResetManager();
}
//Close result handler:
ResultHandler.Exit();
//Wait for the threads to complete:
Log.Trace("Engine.Main(): Waiting for threads to deactivate...");
var ts = Stopwatch.StartNew();
while ((ResultHandler.IsActive || (TransactionHandler != null && TransactionHandler.IsActive) || (DataFeed != null && DataFeed.IsActive)) && ts.ElapsedMilliseconds < 60 * 1000)
{
Thread.Sleep(100);
DataFeed.Exit();
Log.Trace("WAITING >> Result: " + ResultHandler.IsActive + " Transaction: " + TransactionHandler.IsActive + " DataFeed: " + DataFeed.IsActive + " RealTime: " + RealTimeHandler.IsActive);
}
Log.Trace("Engine.Main(): Closing Threads...");
if (threadFeed != null && threadFeed.IsAlive)
{
threadFeed.Abort();
}
if (threadTransactions != null && threadTransactions.IsAlive)
{
threadTransactions.Abort();
}
if (threadResults != null && threadResults.IsAlive)
{
threadResults.Abort();
}
Log.Trace("Engine.Main(): Analysis Completed and Results Posted.");
}
catch (Exception err)
{
Log.Error("Engine.Main(): Error running algorithm: " + err.Message + " >> " + err.StackTrace);
}
finally
{
//Delete the message from the queue before another worker picks it up:
Queue.AcknowledgeJob(job);
Log.Trace("Engine.Main(): Packet removed from queue: " + job.AlgorithmId);
GC.Collect();
}
//If we're running locally will execute just once.
} while (!IsLocal);
// Send the exit signal and then kill the thread
StateCheck.Ping.Exit();
// Make the console window pause so we can read log output before exiting and killing the application completely
Console.ReadKey();
//Finally if ping thread still not complete, kill.
if (statusPingThread != null && statusPingThread.IsAlive)
{
statusPingThread.Abort();
}
}
/********************************************************
* CLASS METHODS
*********************************************************/
/// <summary>
/// Launch the algorithm manager to run this strategy
/// </summary>
/// <param name="job">Algorithm job</param>
/// <param name="algorithm">Algorithm instance</param>
/// <param name="feed">Datafeed object</param>
/// <param name="transactions">Transaction manager object</param>
/// <param name="results">Result handler object</param>
/// <param name="setup">Setup handler object</param>
/// <param name="realtime">Realtime processing object</param>
/// <remarks>Modify with caution</remarks>
public static void Run(AlgorithmNodePacket job, IAlgorithm algorithm, IDataFeed feed, ITransactionHandler transactions, IResultHandler results, ISetupHandler setup, IRealTimeHandler realtime)
{
//Initialize:
var backwardsCompatibilityMode = false;
var tradebarsType = typeof(TradeBars);
var ticksType = typeof(Ticks);
var startingPerformance = setup.StartingCapital;
var backtestMode = (job.Type == PacketType.BacktestNode);
var methodInvokers = new Dictionary <Type, MethodInvoker>();
//Initialize Properties:
_nextSample = new DateTime();
_frontier = setup.StartingDate;
_runtimeError = null;
_algorithmId = job.AlgorithmId;
_algorithmState = AlgorithmStatus.Running;
_previousTime = setup.StartingDate.Date;
//Create the method accessors to push generic types into algorithm: Find all OnData events:
//Algorithm 1.0 Data Accessors.
//If the users defined these methods, add them in manually. This allows keeping backwards compatibility to algorithm 1.0.
var oldTradeBarsMethodInfo = (algorithm.GetType()).GetMethod("OnTradeBar", new[] { typeof(Dictionary <string, TradeBar>) });
var oldTicksMethodInfo = (algorithm.GetType()).GetMethod("OnTick", new[] { typeof(Dictionary <string, List <Tick> >) });
//Algorithm 2.0 Data Generics Accessors.
//New hidden access to tradebars with custom type.
var newTradeBarsMethodInfo = (algorithm.GetType()).GetMethod("OnData", new[] { tradebarsType });
var newTicksMethodInfo = (algorithm.GetType()).GetMethod("OnData", new[] { ticksType });
if (newTradeBarsMethodInfo == null && newTicksMethodInfo == null)
{
backwardsCompatibilityMode = true;
if (oldTradeBarsMethodInfo != null)
{
methodInvokers.Add(tradebarsType, oldTradeBarsMethodInfo.DelegateForCallMethod());
}
if (oldTradeBarsMethodInfo != null)
{
methodInvokers.Add(ticksType, oldTicksMethodInfo.DelegateForCallMethod());
}
}
else
{
backwardsCompatibilityMode = false;
if (newTradeBarsMethodInfo != null)
{
methodInvokers.Add(tradebarsType, newTradeBarsMethodInfo.DelegateForCallMethod());
}
if (newTicksMethodInfo != null)
{
methodInvokers.Add(ticksType, newTicksMethodInfo.DelegateForCallMethod());
}
}
//Go through the subscription types and create invokers to trigger the event handlers for each custom type:
foreach (var config in feed.Subscriptions)
{
//If type is a tradebar, combine tradebars and ticks into unified array:
if (config.Type.Name != "TradeBar" && config.Type.Name != "Tick")
{
//Get the matching method for this event handler - e.g. public void OnData(Quandl data) { .. }
var genericMethod = (algorithm.GetType()).GetMethod("OnData", new[] { config.Type });
//Is we already have this Type-handler then don't add it to invokers again.
if (methodInvokers.ContainsKey(config.Type))
{
continue;
}
//If we couldnt find the event handler, let the user know we can't fire that event.
if (genericMethod == null)
{
_runtimeError = new Exception("Data event handler not found, please create a function matching this template: public void OnData(" + config.Type.Name + " data) { }");
return;
}
methodInvokers.Add(config.Type, genericMethod.DelegateForCallMethod());
}
}
//Loop over the queues: get a data collection, then pass them all into relevent methods in the algorithm.
Log.Debug("AlgorithmManager.Run(): Algorithm initialized, launching time loop.");
foreach (var newData in DataStream.GetData(feed, setup.StartingDate))
{
//Check this backtest is still running:
if (_algorithmState != AlgorithmStatus.Running)
{
break;
}
//Go over each time stamp we've collected, pass it into the algorithm in order:
foreach (var time in newData.Keys)
{
//Set the time frontier:
_frontier = time;
//Execute with TimeLimit Monitor:
if (Isolator.IsCancellationRequested)
{
return;
}
//Refresh the realtime event monitor:
realtime.SetTime(time);
//Fire EOD if the time packet we just processed is greater
if (backtestMode && _previousTime.Date != time.Date)
{
//Sample the portfolio value over time for chart.
results.SampleEquity(_previousTime, Math.Round(algorithm.Portfolio.TotalPortfolioValue, 4));
if (startingPerformance == 0)
{
results.SamplePerformance(_previousTime.Date, 0);
}
else
{
results.SamplePerformance(_previousTime.Date, Math.Round((algorithm.Portfolio.TotalPortfolioValue - startingPerformance) * 100 / startingPerformance, 10));
}
startingPerformance = algorithm.Portfolio.TotalPortfolioValue;
}
//Check if the user's signalled Quit: loop over data until day changes.
if (algorithm.GetQuit())
{
_algorithmState = AlgorithmStatus.Quit;
break;
}
//Pass in the new time first:
algorithm.SetDateTime(time);
//Trigger the data events: Invoke the types we have data for:
var oldBars = new Dictionary <string, TradeBar>();
var oldTicks = new Dictionary <string, List <Tick> >();
var newBars = new TradeBars(time);
var newTicks = new Ticks(time);
//Invoke all non-tradebars, non-ticks methods:
// --> i == Subscription Configuration Index, so we don't need to compare types.
foreach (var i in newData[time].Keys)
{
//Data point and config of this point:
var dataPoints = newData[time][i];
var config = feed.Subscriptions[i];
//Create TradeBars Unified Data --> OR --> invoke generic data event. One loop.
foreach (var dataPoint in dataPoints)
{
//Update the securities properties: first before calling user code to avoid issues with data
algorithm.Securities.Update(time, dataPoint);
//Update registered consolidators for this symbol index
for (var j = 0; j < config.Consolidators.Count; j++)
{
config.Consolidators[j].Update(dataPoint);
}
switch (config.Type.Name)
{
case "TradeBar":
var bar = dataPoint as TradeBar;
try
{
if (bar != null)
{
if (backwardsCompatibilityMode)
{
if (!oldBars.ContainsKey(bar.Symbol))
{
oldBars.Add(bar.Symbol, bar);
}
}
else
{
if (!newBars.ContainsKey(bar.Symbol))
{
newBars.Add(bar.Symbol, bar);
}
}
}
}
catch (Exception err)
{
Log.Error(time.ToLongTimeString() + " >> " + bar.Time.ToLongTimeString() + " >> " + bar.Symbol + " >> " + bar.Value.ToString("C"));
Log.Error("AlgorithmManager.Run(): Failed to add TradeBar (" + bar.Symbol + ") Time: (" + time.ToLongTimeString() + ") Count:(" + newBars.Count + ") " + err.Message);
}
break;
case "Tick":
var tick = dataPoint as Tick;
if (tick != null)
{
if (backwardsCompatibilityMode)
{
if (!oldTicks.ContainsKey(tick.Symbol))
{
oldTicks.Add(tick.Symbol, new List <Tick>());
}
oldTicks[tick.Symbol].Add(tick);
}
else
{
if (!newTicks.ContainsKey(tick.Symbol))
{
newTicks.Add(tick.Symbol, new List <Tick>());
}
newTicks[tick.Symbol].Add(tick);
}
}
break;
default:
//Send data into the generic algorithm event handlers
try
{
methodInvokers[config.Type](algorithm, dataPoint);
}
catch (Exception err)
{
_runtimeError = err;
Log.Error("AlgorithmManager.Run(): RuntimeError: Custom Data: " + err.Message + " STACK >>> " + err.StackTrace);
return;
}
break;
}
}
}
//After we've fired all other events in this second, fire the pricing events:
if (backwardsCompatibilityMode)
{
//Log.Debug("AlgorithmManager.Run(): Invoking v1.0 Event Handlers...");
try
{
if (oldTradeBarsMethodInfo != null && oldBars.Count > 0)
{
methodInvokers[tradebarsType](algorithm, oldBars);
}
if (oldTicksMethodInfo != null && oldTicks.Count > 0)
{
methodInvokers[ticksType](algorithm, oldTicks);
}
}
catch (Exception err)
{
_runtimeError = err;
Log.Error("AlgorithmManager.Run(): RuntimeError: Backwards Compatibility Mode: " + err.Message + " STACK >>> " + err.StackTrace);
return;
}
}
else
{
//Log.Debug("AlgorithmManager.Run(): Invoking v2.0 Event Handlers...");
try
{
if (newTradeBarsMethodInfo != null && newBars.Count > 0)
{
methodInvokers[tradebarsType](algorithm, newBars);
}
if (newTicksMethodInfo != null && newTicks.Count > 0)
{
methodInvokers[ticksType](algorithm, newTicks);
}
}
catch (Exception err)
{
_runtimeError = err;
Log.Error("AlgorithmManager.Run(): RuntimeError: New Style Mode: " + err.Message + " STACK >>> " + err.StackTrace);
return;
}
}
//If its the historical/paper trading models, wait until its flagged as "ready"
if (job.TransactionEndpoint == TransactionHandlerEndpoint.Backtesting)
{
while (!transactions.Ready)
{
Thread.Yield();
}
}
if (time > _nextSample)
{
//Set next sample time: 4000 samples per backtest
_nextSample = time.Add(results.ResamplePeriod);
//Sample the portfolio value over time for chart.
results.SampleEquity(time, Math.Round(algorithm.Portfolio.TotalPortfolioValue, 4));
//Also add the user samples / plots to the result handler tracking:
results.SampleRange(algorithm.GetChartUpdates());
//Sample the asset pricing:
foreach (var security in algorithm.Securities.Values)
{
results.SampleAssetPrices(security.Symbol, time, security.Price);
}
}
ProcessMessages(results, algorithm);
//Save the previous time for the sample calculations
_previousTime = time;
} // End of Time Loop
} // End of ForEach DataStream
//Stream over:: Send the final packet and fire final events:
Log.Trace("AlgorithmManager.Run(): Firing On End Of Algorithm...");
try
{
algorithm.OnEndOfAlgorithm();
}
catch (Exception err)
{
_runtimeError = new Exception("Error running OnEndOfAlgorithm(): " + err.Message, err.InnerException);
return;
}
//Process the final messages from the algorithm
ProcessMessages(results, algorithm);
//Liquidate Holdings for Calculations:
if (_algorithmState == AlgorithmStatus.Liquidated || Engine.IsLocal || job.TransactionEndpoint == TransactionHandlerEndpoint.Backtesting)
{
Log.Trace("AlgorithmManager.Run(): Liquidating algorithm holdings...");
algorithm.Liquidate();
results.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.Liquidated);
}
//Manually stopped the algorithm
if (_algorithmState == AlgorithmStatus.Stopped)
{
Log.Trace("AlgorithmManager.Run(): Stopping algorithm...");
results.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.Stopped);
}
//Backtest deleted.
if (_algorithmState == AlgorithmStatus.Deleted)
{
Log.Trace("AlgorithmManager.Run(): Deleting algorithm...");
results.DebugMessage("Algorithm Id:(" + job.AlgorithmId + ") Deleted by request.");
results.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.Deleted);
}
//Algorithm finished, send regardless of commands:
results.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.Completed);
//Take final samples:
results.SampleRange(algorithm.GetChartUpdates());
results.SampleEquity(_frontier, Math.Round(algorithm.Portfolio.TotalPortfolioValue, 4));
results.SamplePerformance(_frontier, Math.Round((algorithm.Portfolio.TotalPortfolioValue - startingPerformance) * 100 / startingPerformance, 10));
} // End of Run();
/********************************************************
* CLASS METHODS
*********************************************************/
/// <summary>
/// Primary Analysis Thread:
/// </summary>
public static void Main(string[] args)
{
//Initialize:
var algorithmPath = "";
string mode = "RELEASE";
AlgorithmNodePacket job = null;
var algorithm = default(IAlgorithm);
var startTime = DateTime.Now;
Log.LogHandler = Composer.Instance.GetExportedValueByTypeName<ILogHandler>(Config.Get("log-handler", "CompositeLogHandler"));
#if DEBUG
mode = "DEBUG";
#endif
//Name thread for the profiler:
Thread.CurrentThread.Name = "Algorithm Analysis Thread";
Log.Trace("Engine.Main(): LEAN ALGORITHMIC TRADING ENGINE v" + Constants.Version + " Mode: " + mode);
Log.Trace("Engine.Main(): Started " + DateTime.Now.ToShortTimeString());
Log.Trace("Engine.Main(): Memory " + OS.ApplicationMemoryUsed + "Mb-App " + +OS.TotalPhysicalMemoryUsed + "Mb-Used " + OS.TotalPhysicalMemory + "Mb-Total");
//Import external libraries specific to physical server location (cloud/local)
try
{
// grab the right export based on configuration
Api = Composer.Instance.GetExportedValueByTypeName<IApi>(Config.Get("api-handler"));
Notify = Composer.Instance.GetExportedValueByTypeName<IMessagingHandler>(Config.Get("messaging-handler"));
JobQueue = Composer.Instance.GetExportedValueByTypeName<IJobQueueHandler>(Config.Get("job-queue-handler"));
}
catch (CompositionException compositionException)
{ Log.Error("Engine.Main(): Failed to load library: " + compositionException);
}
//Setup packeting, queue and controls system: These don't do much locally.
Api.Initialize();
Notify.Initialize();
JobQueue.Initialize();
//Start monitoring the backtest active status:
var statusPingThread = new Thread(StateCheck.Ping.Run);
statusPingThread.Start();
try
{
//Reset algo manager internal variables preparing for a new algorithm.
AlgorithmManager.ResetManager();
//Reset thread holders.
var initializeComplete = false;
Thread threadFeed = null;
Thread threadTransactions = null;
Thread threadResults = null;
Thread threadRealTime = null;
do
{
//-> Pull job from QuantConnect job queue, or, pull local build:
job = JobQueue.NextJob(out algorithmPath); // Blocking.
// if the job version doesn't match this instance version then we can't process it
// we also don't want to reprocess redelivered live jobs
if (job.Version != Constants.Version || (LiveMode && job.Redelivered))
{
Log.Error("Engine.Run(): Job Version: " + job.Version + " Deployed Version: " + Constants.Version);
//Tiny chance there was an uncontrolled collapse of a server, resulting in an old user task circulating.
//In this event kill the old algorithm and leave a message so the user can later review.
JobQueue.AcknowledgeJob(job);
Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, _collapseMessage);
Notify.SetChannel(job.Channel);
Notify.RuntimeError(job.AlgorithmId, _collapseMessage);
job = null;
}
} while (job == null);
//-> Initialize messaging system
Notify.SetChannel(job.Channel);
//-> Create SetupHandler to configure internal algorithm state:
SetupHandler = GetSetupHandler(job.SetupEndpoint);
//-> Set the result handler type for this algorithm job, and launch the associated result thread.
ResultHandler = GetResultHandler(job);
threadResults = new Thread(ResultHandler.Run, 0) {Name = "Result Thread"};
threadResults.Start();
try
{
// Save algorithm to cache, load algorithm instance:
algorithm = SetupHandler.CreateAlgorithmInstance(algorithmPath);
//Initialize the internal state of algorithm and job: executes the algorithm.Initialize() method.
initializeComplete = SetupHandler.Setup(algorithm, out _brokerage, job);
//If there are any reasons it failed, pass these back to the IDE.
if (!initializeComplete || algorithm.ErrorMessages.Count > 0 || SetupHandler.Errors.Count > 0)
{
initializeComplete = false;
//Get all the error messages: internal in algorithm and external in setup handler.
var errorMessage = String.Join(",", algorithm.ErrorMessages);
errorMessage += String.Join(",", SetupHandler.Errors);
ResultHandler.RuntimeError(errorMessage);
Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError);
}
}
catch (Exception err)
{
var runtimeMessage = "Algorithm.Initialize() Error: " + err.Message + " Stack Trace: " + err.StackTrace;
ResultHandler.RuntimeError(runtimeMessage, err.StackTrace);
Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, runtimeMessage);
}
//-> Using the job + initialization: load the designated handlers:
if (initializeComplete)
{
//-> Reset the backtest stopwatch; we're now running the algorithm.
startTime = DateTime.Now;
//Set algorithm as locked; set it to live mode if we're trading live, and set it to locked for no further updates.
algorithm.SetAlgorithmId(job.AlgorithmId);
algorithm.SetLiveMode(LiveMode);
algorithm.SetLocked();
//Load the associated handlers for data, transaction and realtime events:
ResultHandler.SetAlgorithm(algorithm);
DataFeed = GetDataFeedHandler(algorithm, job);
TransactionHandler = GetTransactionHandler(algorithm, _brokerage, ResultHandler, job);
RealTimeHandler = GetRealTimeHandler(algorithm, _brokerage, DataFeed, ResultHandler, job);
//Set the error handlers for the brokerage asynchronous errors.
SetupHandler.SetupErrorHandler(ResultHandler, _brokerage);
//Send status to user the algorithm is now executing.
ResultHandler.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.Running);
//Launch the data, transaction and realtime handlers into dedicated threads
threadFeed = new Thread(DataFeed.Run) {Name = "DataFeed Thread"};
threadTransactions = new Thread(TransactionHandler.Run) {Name = "Transaction Thread"};
threadRealTime = new Thread(RealTimeHandler.Run) {Name = "RealTime Thread"};
//Launch the data feed, result sending, and transaction models/handlers in separate threads.
threadFeed.Start(); // Data feed pushing data packets into thread bridge;
threadTransactions.Start(); // Transaction modeller scanning new order requests
threadRealTime.Start(); // RealTime scan time for time based events:
// Result manager scanning message queue: (started earlier)
ResultHandler.DebugMessage(string.Format("Launching analysis for {0} with LEAN Engine v{1}", job.AlgorithmId, Constants.Version));
try
{
// Execute the Algorithm Code:
var complete = Isolator.ExecuteWithTimeLimit(SetupHandler.MaximumRuntime, AlgorithmManager.TimeLoopWithinLimits, () =>
{
try
{
//Run Algorithm Job:
// -> Using this Data Feed,
// -> Send Orders to this TransactionHandler,
// -> Send Results to ResultHandler.
AlgorithmManager.Run(job, algorithm, DataFeed, TransactionHandler, ResultHandler, SetupHandler, RealTimeHandler);
}
catch (Exception err)
{
//Debugging at this level is difficult, stack trace needed.
Log.Error("Engine.Run", err);
}
Log.Trace("Engine.Run(): Exiting Algorithm Manager");
}, job.UserPlan == UserPlan.Free ? 1024 : MaximumRamAllocation);
if (!complete)
{
Log.Error("Engine.Main(): Failed to complete in time: " + SetupHandler.MaximumRuntime.ToString("F"));
throw new Exception("Failed to complete algorithm within " + SetupHandler.MaximumRuntime.ToString("F") + " seconds. Please make it run faster.");
}
// Algorithm runtime error:
if (algorithm.RunTimeError != null)
{
throw algorithm.RunTimeError;
}
}
catch (Exception err)
{
//Error running the user algorithm: purge datafeed, send error messages, set algorithm status to failed.
Log.Error("Engine.Run(): Breaking out of parent try-catch: " + err.Message + " " + err.StackTrace);
if (DataFeed != null) DataFeed.Exit();
if (ResultHandler != null)
{
var message = "Runtime Error: " + err.Message;
Log.Trace("Engine.Run(): Sending runtime error to user...");
ResultHandler.LogMessage(message);
ResultHandler.RuntimeError(message, err.StackTrace);
Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, message + " Stack Trace: " + err.StackTrace);
}
}
//Send result data back: this entire code block could be rewritten.
// todo: - Split up statistics class, its enormous.
// todo: - Make a dedicated Statistics.Benchmark class.
// todo: - Move all creation and transmission of statistics out of primary engine loop.
// todo: - Statistics.Generate(algorithm, resulthandler, transactionhandler);
try
{
var charts = new Dictionary<string, Chart>(ResultHandler.Charts);
var orders = new Dictionary<int, Order>(algorithm.Transactions.Orders);
var holdings = new Dictionary<string, Holding>();
var statistics = new Dictionary<string, string>();
var banner = new Dictionary<string, string>();
try
{
//Generates error when things don't exist (no charting logged, runtime errors in main algo execution)
const string strategyEquityKey = "Strategy Equity";
const string equityKey = "Equity";
const string dailyPerformanceKey = "Daily Performance";
// make sure we've taken samples for these series before just blindly requesting them
if (charts.ContainsKey(strategyEquityKey) &&
charts[strategyEquityKey].Series.ContainsKey(equityKey) &&
charts[strategyEquityKey].Series.ContainsKey(dailyPerformanceKey))
{
var equity = charts[strategyEquityKey].Series[equityKey].Values;
var performance = charts[strategyEquityKey].Series[dailyPerformanceKey].Values;
var profitLoss =
new SortedDictionary<DateTime, decimal>(algorithm.Transactions.TransactionRecord);
statistics = Statistics.Statistics.Generate(equity, profitLoss, performance,
SetupHandler.StartingPortfolioValue, algorithm.Portfolio.TotalFees, 252);
}
}
catch (Exception err)
{
Log.Error("Algorithm.Node.Engine(): Error generating statistics packet: " + err.Message);
}
//Diagnostics Completed, Send Result Packet:
var totalSeconds = (DateTime.Now - startTime).TotalSeconds;
ResultHandler.DebugMessage(string.Format("Algorithm Id:({0}) completed in {1} seconds at {2}k data points per second. Processing total of {3} data points.",
job.AlgorithmId, totalSeconds.ToString("F2"), ((AlgorithmManager.DataPoints / (double)1000) / totalSeconds).ToString("F0"), AlgorithmManager.DataPoints.ToString("N0")));
ResultHandler.SendFinalResult(job, orders, algorithm.Transactions.TransactionRecord, holdings, statistics, banner);
}
catch (Exception err)
{
Log.Error("Engine.Main(): Error sending analysis result: " + err.Message + " ST >> " + err.StackTrace);
}
//Before we return, send terminate commands to close up the threads
TransactionHandler.Exit();
DataFeed.Exit();
RealTimeHandler.Exit();
}
//Close result handler:
ResultHandler.Exit();
StateCheck.Ping.Exit();
//Wait for the threads to complete:
var ts = Stopwatch.StartNew();
while ((ResultHandler.IsActive || (TransactionHandler != null && TransactionHandler.IsActive) || (DataFeed != null && DataFeed.IsActive)) && ts.ElapsedMilliseconds < 30 * 1000)
{
Thread.Sleep(100); Log.Trace("Waiting for threads to exit...");
}
//Terminate threads still in active state.
if (threadFeed != null && threadFeed.IsAlive) threadFeed.Abort();
if (threadTransactions != null && threadTransactions.IsAlive) threadTransactions.Abort();
if (threadResults != null && threadResults.IsAlive) threadResults.Abort();
if (statusPingThread != null && statusPingThread.IsAlive) statusPingThread.Abort();
if (_brokerage != null)
{
_brokerage.Disconnect();
}
if (SetupHandler != null)
{
SetupHandler.Dispose();
}
Log.Trace("Engine.Main(): Analysis Completed and Results Posted.");
}
catch (Exception err)
{
Log.Error("Engine.Main(): Error running algorithm: " + err.Message + " >> " + err.StackTrace);
}
finally
{
//No matter what for live mode; make sure we've set algorithm status in the API for "not running" conditions:
if (LiveMode && AlgorithmManager.State != AlgorithmStatus.Running && AlgorithmManager.State != AlgorithmStatus.RuntimeError)
Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmManager.State);
//Delete the message from the job queue:
JobQueue.AcknowledgeJob(job);
Log.Trace("Engine.Main(): Packet removed from queue: " + job.AlgorithmId);
//Attempt to clean up ram usage:
GC.Collect();
}
//Final disposals.
Api.Dispose();
// Make the console window pause so we can read log output before exiting and killing the application completely
if (IsLocal)
{
Log.Trace("Engine.Main(): Analysis Complete. Press any key to continue.");
Console.Read();
}
Log.LogHandler.Dispose();
}
public void Initialize(AlgorithmNodePacket job, IMessagingHandler messagingHandler, IApi api, IDataFeed dataFeed, ISetupHandler setupHandler, ITransactionHandler transactionHandler)
{
_shadow.Initialize(job, messagingHandler, api, dataFeed, setupHandler, transactionHandler);
}
/// <summary>
/// Initialize the result handler with this result packet.
/// </summary>
/// <param name="packet">Algorithm job packet for this result handler</param>
/// <param name="messagingHandler"></param>
/// <param name="api"></param>
/// <param name="dataFeed"></param>
/// <param name="setupHandler"></param>
/// <param name="transactionHandler"></param>
public void Initialize(AlgorithmNodePacket packet, IMessagingHandler messagingHandler, IApi api, IDataFeed dataFeed, ISetupHandler setupHandler, ITransactionHandler transactionHandler)
{
// we expect one of two types here, the backtest node packet or the live node packet
var job = packet as BacktestNodePacket;
if (job != null)
{
_algorithmNode = new BacktestConsoleStatusHandler(job);
}
else
{
var live = packet as LiveNodePacket;
if (live == null)
{
throw new ArgumentException("Unexpected AlgorithmNodeType: " + packet.GetType().Name);
}
_algorithmNode = new LiveConsoleStatusHandler(live);
}
_resamplePeriod = _algorithmNode.ComputeSampleEquityPeriod();
var time = DateTime.Now.ToString("yyyy-MM-dd-HH-mm");
_chartDirectory = Path.Combine("../../../Charts/", packet.AlgorithmId, time);
if (Directory.Exists(_chartDirectory))
{
foreach (var file in Directory.EnumerateFiles(_chartDirectory, "*.csv", SearchOption.AllDirectories))
{
File.Delete(file);
}
Directory.Delete(_chartDirectory, true);
}
Directory.CreateDirectory(_chartDirectory);
_messagingHandler = messagingHandler;
}
/// <summary>
/// Initialize the result handler with this result packet.
/// </summary>
/// <param name="packet">Algorithm job packet for this result handler</param>
/// <param name="messagingHandler"></param>
/// <param name="api"></param>
/// <param name="dataFeed"></param>
/// <param name="setupHandler"></param>
/// <param name="transactionHandler"></param>
public void Initialize(AlgorithmNodePacket packet, IMessagingHandler messagingHandler, IApi api, IDataFeed dataFeed, ISetupHandler setupHandler, ITransactionHandler transactionHandler)
{
// we expect one of two types here, the backtest node packet or the live node packet
var job = packet as BacktestNodePacket;
if (job != null)
{
_algorithmNode = new BacktestConsoleStatusHandler(job);
}
else
{
var live = packet as LiveNodePacket;
if (live == null)
{
throw new ArgumentException("Unexpected AlgorithmNodeType: " + packet.GetType().Name);
}
_algorithmNode = new LiveConsoleStatusHandler(live);
}
_resamplePeriod = _algorithmNode.ComputeSampleEquityPeriod();
var time = DateTime.Now.ToString("yyyy-MM-dd-HH-mm");
_chartDirectory = Path.Combine("../../../Charts/", packet.AlgorithmId, time);
if (Directory.Exists(_chartDirectory))
{
foreach (var file in Directory.EnumerateFiles(_chartDirectory, "*.csv", SearchOption.AllDirectories))
{
File.Delete(file);
}
Directory.Delete(_chartDirectory, true);
}
Directory.CreateDirectory(_chartDirectory);
_messagingHandler = messagingHandler;
}
/********************************************************
* CLASS METHODS
*********************************************************/
/// <summary>
/// Launch the algorithm manager to run this strategy
/// </summary>
/// <param name="job">Algorithm job</param>
/// <param name="algorithm">Algorithm instance</param>
/// <param name="feed">Datafeed object</param>
/// <param name="transactions">Transaction manager object</param>
/// <param name="results">Result handler object</param>
/// <param name="setup">Setup handler object</param>
/// <param name="realtime">Realtime processing object</param>
/// <remarks>Modify with caution</remarks>
public static void Run(AlgorithmNodePacket job, IAlgorithm algorithm, IDataFeed feed, ITransactionHandler transactions, IResultHandler results, ISetupHandler setup, IRealTimeHandler realtime)
{
//Initialize:
var backwardsCompatibilityMode = false;
var tradebarsType = typeof (TradeBars);
var ticksType = typeof(Ticks);
var startingPerformance = setup.StartingCapital;
var backtestMode = (job.Type == PacketType.BacktestNode);
var methodInvokers = new Dictionary<Type, MethodInvoker>();
//Initialize Properties:
_frontier = setup.StartingDate;
_runtimeError = null;
_algorithmId = job.AlgorithmId;
_algorithmState = AlgorithmStatus.Running;
_previousTime = setup.StartingDate.Date;
//Create the method accessors to push generic types into algorithm: Find all OnData events:
//Algorithm 1.0 Data Accessors.
//If the users defined these methods, add them in manually. This allows keeping backwards compatibility to algorithm 1.0.
var oldTradeBarsMethodInfo = (algorithm.GetType()).GetMethod("OnTradeBar", new[] { typeof(Dictionary<string, TradeBar>) });
var oldTicksMethodInfo = (algorithm.GetType()).GetMethod("OnTick", new[] { typeof(Dictionary<string, List<Tick>>) });
//Algorithm 2.0 Data Generics Accessors.
//New hidden access to tradebars with custom type.
var newTradeBarsMethodInfo = (algorithm.GetType()).GetMethod("OnData", new[] { tradebarsType });
var newTicksMethodInfo = (algorithm.GetType()).GetMethod("OnData", new[] { ticksType });
if (newTradeBarsMethodInfo == null && newTicksMethodInfo == null)
{
backwardsCompatibilityMode = true;
if (oldTradeBarsMethodInfo != null) methodInvokers.Add(tradebarsType, oldTradeBarsMethodInfo.DelegateForCallMethod());
if (oldTradeBarsMethodInfo != null) methodInvokers.Add(ticksType, oldTicksMethodInfo.DelegateForCallMethod());
}
else
{
backwardsCompatibilityMode = false;
if (newTradeBarsMethodInfo != null) methodInvokers.Add(tradebarsType, newTradeBarsMethodInfo.DelegateForCallMethod());
if (newTicksMethodInfo != null) methodInvokers.Add(ticksType, newTicksMethodInfo.DelegateForCallMethod());
}
//Go through the subscription types and create invokers to trigger the event handlers for each custom type:
foreach (var config in feed.Subscriptions)
{
//If type is a tradebar, combine tradebars and ticks into unified array:
if (config.Type.Name != "TradeBar" && config.Type.Name != "Tick")
{
//Get the matching method for this event handler - e.g. public void OnData(Quandl data) { .. }
var genericMethod = (algorithm.GetType()).GetMethod("OnData", new[] { config.Type });
//Is we already have this Type-handler then don't add it to invokers again.
if (methodInvokers.ContainsKey(config.Type)) continue;
//If we couldnt find the event handler, let the user know we can't fire that event.
if (genericMethod == null)
{
_runtimeError = new Exception("Data event handler not found, please create a function matching this template: public void OnData(" + config.Type.Name + " data) { }");
_algorithmState = AlgorithmStatus.RuntimeError;
return;
}
methodInvokers.Add(config.Type, genericMethod.DelegateForCallMethod());
}
}
//Loop over the queues: get a data collection, then pass them all into relevent methods in the algorithm.
Log.Debug("AlgorithmManager.Run(): Algorithm initialized, launching time loop.");
foreach (var newData in DataStream.GetData(feed, setup.StartingDate))
{
//Check this backtest is still running:
if (_algorithmState != AlgorithmStatus.Running) break;
//Go over each time stamp we've collected, pass it into the algorithm in order:
foreach (var time in newData.Keys)
{
//Set the time frontier:
_frontier = time;
//Execute with TimeLimit Monitor:
if (Isolator.IsCancellationRequested) return;
//Refresh the realtime event monitor:
realtime.SetTime(time);
//Fire EOD if the time packet we just processed is greater
if (backtestMode && _previousTime.Date != time.Date)
{
//Sample the portfolio value over time for chart.
results.SampleEquity(_previousTime, Math.Round(algorithm.Portfolio.TotalPortfolioValue, 4));
if (startingPerformance == 0)
{
results.SamplePerformance(_previousTime.Date, 0);
}
else
{
results.SamplePerformance(_previousTime.Date, Math.Round((algorithm.Portfolio.TotalPortfolioValue - startingPerformance) * 100 / startingPerformance, 10));
}
startingPerformance = algorithm.Portfolio.TotalPortfolioValue;
}
//Check if the user's signalled Quit: loop over data until day changes.
if (algorithm.GetQuit())
{
_algorithmState = AlgorithmStatus.Quit;
break;
}
//Pass in the new time first:
algorithm.SetDateTime(time);
//Trigger the data events: Invoke the types we have data for:
var oldBars = new Dictionary<string, TradeBar>();
var oldTicks = new Dictionary<string, List<Tick>>();
var newBars = new TradeBars(time);
var newTicks = new Ticks(time);
//Invoke all non-tradebars, non-ticks methods:
// --> i == Subscription Configuration Index, so we don't need to compare types.
foreach (var i in newData[time].Keys)
{
//Data point and config of this point:
var dataPoints = newData[time][i];
var config = feed.Subscriptions[i];
//Create TradeBars Unified Data --> OR --> invoke generic data event. One loop.
foreach (var dataPoint in dataPoints)
{
//Update the securities properties: first before calling user code to avoid issues with data
algorithm.Securities.Update(time, dataPoint);
//Update registered consolidators for this symbol index
for (var j = 0; j < config.Consolidators.Count; j++)
{
config.Consolidators[j].Update(dataPoint);
}
switch (config.Type.Name)
{
case "TradeBar":
var bar = dataPoint as TradeBar;
try
{
if (bar != null)
{
if (backwardsCompatibilityMode)
{
if (!oldBars.ContainsKey(bar.Symbol)) oldBars.Add(bar.Symbol, bar);
}
else
{
if (!newBars.ContainsKey(bar.Symbol)) newBars.Add(bar.Symbol, bar);
}
}
}
catch (Exception err)
{
Log.Error(time.ToLongTimeString() + " >> " + bar.Time.ToLongTimeString() + " >> " + bar.Symbol + " >> " + bar.Value.ToString("C"));
Log.Error("AlgorithmManager.Run(): Failed to add TradeBar (" + bar.Symbol + ") Time: (" + time.ToLongTimeString() + ") Count:(" + newBars.Count + ") " + err.Message);
}
break;
case "Tick":
var tick = dataPoint as Tick;
if (tick != null)
{
if (backwardsCompatibilityMode) {
if (!oldTicks.ContainsKey(tick.Symbol)) { oldTicks.Add(tick.Symbol, new List<Tick>()); }
oldTicks[tick.Symbol].Add(tick);
}
else
{
if (!newTicks.ContainsKey(tick.Symbol)) { newTicks.Add(tick.Symbol, new List<Tick>()); }
newTicks[tick.Symbol].Add(tick);
}
}
break;
default:
//Send data into the generic algorithm event handlers
try
{
methodInvokers[config.Type](algorithm, dataPoint);
}
catch (Exception err)
{
_runtimeError = err;
_algorithmState = AlgorithmStatus.RuntimeError;
Log.Debug("AlgorithmManager.Run(): RuntimeError: Custom Data: " + err.Message + " STACK >>> " + err.StackTrace);
return;
}
break;
}
}
}
//After we've fired all other events in this second, fire the pricing events:
if (backwardsCompatibilityMode)
{
//Log.Debug("AlgorithmManager.Run(): Invoking v1.0 Event Handlers...");
try
{
if (oldTradeBarsMethodInfo != null && oldBars.Count > 0) methodInvokers[tradebarsType](algorithm, oldBars);
if (oldTicksMethodInfo != null && oldTicks.Count > 0) methodInvokers[ticksType](algorithm, oldTicks);
}
catch (Exception err)
{
_runtimeError = err;
_algorithmState = AlgorithmStatus.RuntimeError;
Log.Debug("AlgorithmManager.Run(): RuntimeError: Backwards Compatibility Mode: " + err.Message + " STACK >>> " + err.StackTrace);
return;
}
}
else
{
//Log.Debug("AlgorithmManager.Run(): Invoking v2.0 Event Handlers...");
try
{
if (newTradeBarsMethodInfo != null && newBars.Count > 0) methodInvokers[tradebarsType](algorithm, newBars);
if (newTicksMethodInfo != null && newTicks.Count > 0) methodInvokers[ticksType](algorithm, newTicks);
}
catch (Exception err)
{
_runtimeError = err;
_algorithmState = AlgorithmStatus.RuntimeError;
Log.Debug("AlgorithmManager.Run(): RuntimeError: New Style Mode: " + err.Message + " STACK >>> " + err.StackTrace);
return;
}
}
//If its the historical/paper trading models, wait until market orders have been "filled"
// Manually trigger the event handler to prevent thread switch.
transactions.ProcessSynchronousEvents();
//Save the previous time for the sample calculations
_previousTime = time;
} // End of Time Loop
// Process any required events of the results handler such as sampling assets, equity, or stock prices.
results.ProcessSynchronousEvents();
} // End of ForEach DataStream
//Stream over:: Send the final packet and fire final events:
Log.Trace("AlgorithmManager.Run(): Firing On End Of Algorithm...");
try
{
algorithm.OnEndOfAlgorithm();
}
catch (Exception err)
{
_algorithmState = AlgorithmStatus.RuntimeError;
_runtimeError = new Exception("Error running OnEndOfAlgorithm(): " + err.Message, err.InnerException);
Log.Debug("AlgorithmManager.OnEndOfAlgorithm(): " + err.Message + " STACK >>> " + err.StackTrace);
return;
}
// Process any required events of the results handler such as sampling assets, equity, or stock prices.
results.ProcessSynchronousEvents(forceProcess: true);
//Liquidate Holdings for Calculations:
if (_algorithmState == AlgorithmStatus.Liquidated || !Engine.LiveMode)
{
Log.Trace("AlgorithmManager.Run(): Liquidating algorithm holdings...");
algorithm.Liquidate();
results.LogMessage("Algorithm Liquidated");
results.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.Liquidated);
}
//Manually stopped the algorithm
if (_algorithmState == AlgorithmStatus.Stopped)
{
Log.Trace("AlgorithmManager.Run(): Stopping algorithm...");
results.LogMessage("Algorithm Stopped");
results.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.Stopped);
}
//Backtest deleted.
if (_algorithmState == AlgorithmStatus.Deleted)
{
Log.Trace("AlgorithmManager.Run(): Deleting algorithm...");
results.DebugMessage("Algorithm Id:(" + job.AlgorithmId + ") Deleted by request.");
results.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.Deleted);
}
//Algorithm finished, send regardless of commands:
results.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.Completed);
//Take final samples:
results.SampleRange(algorithm.GetChartUpdates());
results.SampleEquity(_frontier, Math.Round(algorithm.Portfolio.TotalPortfolioValue, 4));
results.SamplePerformance(_frontier, Math.Round((algorithm.Portfolio.TotalPortfolioValue - startingPerformance) * 100 / startingPerformance, 10));
}
/********************************************************
* CLASS METHODS
*********************************************************/
/// <summary>
/// Launch the algorithm manager to run this strategy
/// </summary>
/// <param name="job">Algorithm job</param>
/// <param name="algorithm">Algorithm instance</param>
/// <param name="feed">Datafeed object</param>
/// <param name="transactions">Transaction manager object</param>
/// <param name="results">Result handler object</param>
/// <param name="setup">Setup handler object</param>
/// <param name="realtime">Realtime processing object</param>
/// <remarks>Modify with caution</remarks>
public static void Run(AlgorithmNodePacket job, IAlgorithm algorithm, IDataFeed feed, ITransactionHandler transactions, IResultHandler results, ISetupHandler setup, IRealTimeHandler realtime)
{
//Initialize:
_dataPointCount = 0;
var startingPortfolioValue = setup.StartingPortfolioValue;
var backtestMode = (job.Type == PacketType.BacktestNode);
var methodInvokers = new Dictionary <Type, MethodInvoker>();
var marginCallFrequency = TimeSpan.FromMinutes(5);
var nextMarginCallTime = DateTime.MinValue;
//Initialize Properties:
_algorithmId = job.AlgorithmId;
_algorithmState = AlgorithmStatus.Running;
_previousTime = setup.StartingDate.Date;
//Create the method accessors to push generic types into algorithm: Find all OnData events:
// Algorithm 1.0 data accessors
var hasOnTradeBar = AddMethodInvoker <Dictionary <string, TradeBar> >(algorithm, methodInvokers, "OnTradeBar");
var hasOnTick = AddMethodInvoker <Dictionary <string, List <Tick> > >(algorithm, methodInvokers, "OnTick");
// Algorithm 2.0 data accessors
var hasOnDataTradeBars = AddMethodInvoker <TradeBars>(algorithm, methodInvokers);
var hasOnDataTicks = AddMethodInvoker <Ticks>(algorithm, methodInvokers);
// determine what mode we're in
var backwardsCompatibilityMode = !hasOnDataTradeBars && !hasOnDataTicks;
// dividend and split events
var hasOnDataDividends = AddMethodInvoker <Dividends>(algorithm, methodInvokers);
var hasOnDataSplits = AddMethodInvoker <Splits>(algorithm, methodInvokers);
//Go through the subscription types and create invokers to trigger the event handlers for each custom type:
foreach (var config in feed.Subscriptions)
{
//If type is a tradebar, combine tradebars and ticks into unified array:
if (config.Type.Name != "TradeBar" && config.Type.Name != "Tick")
{
//Get the matching method for this event handler - e.g. public void OnData(Quandl data) { .. }
var genericMethod = (algorithm.GetType()).GetMethod("OnData", new[] { config.Type });
//If we already have this Type-handler then don't add it to invokers again.
if (methodInvokers.ContainsKey(config.Type))
{
continue;
}
//If we couldnt find the event handler, let the user know we can't fire that event.
if (genericMethod == null)
{
algorithm.RunTimeError = new Exception("Data event handler not found, please create a function matching this template: public void OnData(" + config.Type.Name + " data) { }");
_algorithmState = AlgorithmStatus.RuntimeError;
return;
}
methodInvokers.Add(config.Type, genericMethod.DelegateForCallMethod());
}
}
//Loop over the queues: get a data collection, then pass them all into relevent methods in the algorithm.
Log.Debug("AlgorithmManager.Run(): Algorithm initialized, launching time loop.");
foreach (var newData in DataStream.GetData(feed, setup.StartingDate))
{
// reset our timer on each loop
_currentTimeStepStopwatch = Stopwatch.StartNew();
//Check this backtest is still running:
if (_algorithmState != AlgorithmStatus.Running)
{
break;
}
//Execute with TimeLimit Monitor:
if (Isolator.IsCancellationRequested)
{
return;
}
var time = DataStream.AlorithmTime;
//If we're in backtest mode we need to capture the daily performance. We do this here directly
//before updating the algorithm state with the new data from this time step, otherwise we'll
//produce incorrect samples (they'll take into account this time step's new price values)
if (backtestMode)
{
//Refresh the realtime event monitor:
//in backtest mode use the algorithms clock as realtime.
realtime.SetTime(time);
//On day-change sample equity and daily performance for statistics calculations
if (_previousTime.Date != time.Date)
{
//Sample the portfolio value over time for chart.
results.SampleEquity(_previousTime, Math.Round(algorithm.Portfolio.TotalPortfolioValue, 4));
//Check for divide by zero
if (startingPortfolioValue == 0m)
{
results.SamplePerformance(_previousTime.Date, 0);
}
else
{
results.SamplePerformance(_previousTime.Date, Math.Round((algorithm.Portfolio.TotalPortfolioValue - startingPortfolioValue) * 100 / startingPortfolioValue, 10));
}
startingPortfolioValue = algorithm.Portfolio.TotalPortfolioValue;
}
}
//Update algorithm state after capturing performance from previous day
//On each time step push the real time prices to the cashbook so we can have updated conversion rates
algorithm.Portfolio.CashBook.Update(newData);
//Update the securities properties: first before calling user code to avoid issues with data
algorithm.Securities.Update(time, newData);
// perform margin calls, in live mode we can also use realtime to emit these
if (time >= nextMarginCallTime || (Engine.LiveMode && nextMarginCallTime > DateTime.Now))
{
// determine if there are possible margin call orders to be executed
bool issueMarginCallWarning;
var marginCallOrders = algorithm.Portfolio.ScanForMarginCall(out issueMarginCallWarning);
if (marginCallOrders.Count != 0)
{
try
{
// tell the algorithm we're about to issue the margin call
algorithm.OnMarginCall(marginCallOrders);
}
catch (Exception err)
{
algorithm.RunTimeError = err;
_algorithmState = AlgorithmStatus.RuntimeError;
Log.Debug("AlgorithmManager.Run(): RuntimeError: OnMarginCall: " + err.Message + " STACK >>> " + err.StackTrace);
return;
}
// execute the margin call orders
var executedOrders = algorithm.Portfolio.MarginCallModel.ExecuteMarginCall(marginCallOrders);
foreach (var order in executedOrders)
{
algorithm.Error(string.Format("{0} - Executed MarginCallOrder: {1} - Quantity: {2} @ {3}", algorithm.Time, order.Symbol, order.Quantity, order.Price));
}
}
// we didn't perform a margin call, but got the warning flag back, so issue the warning to the algorithm
else if (issueMarginCallWarning)
{
try
{
algorithm.OnMarginCallWarning();
}
catch (Exception err)
{
algorithm.RunTimeError = err;
_algorithmState = AlgorithmStatus.RuntimeError;
Log.Debug("AlgorithmManager.Run(): RuntimeError: OnMarginCallWarning: " + err.Message + " STACK >>> " + err.StackTrace);
}
}
nextMarginCallTime = time + marginCallFrequency;
}
//Check if the user's signalled Quit: loop over data until day changes.
if (algorithm.GetQuit())
{
_algorithmState = AlgorithmStatus.Quit;
break;
}
if (algorithm.RunTimeError != null)
{
_algorithmState = AlgorithmStatus.RuntimeError;
break;
}
//Pass in the new time first:
algorithm.SetDateTime(time);
//Trigger the data events: Invoke the types we have data for:
var oldBars = new Dictionary <string, TradeBar>();
var oldTicks = new Dictionary <string, List <Tick> >();
var newBars = new TradeBars(time);
var newTicks = new Ticks(time);
var newDividends = new Dividends(time);
var newSplits = new Splits(time);
//Invoke all non-tradebars, non-ticks methods and build up the TradeBars and Ticks dictionaries
// --> i == Subscription Configuration Index, so we don't need to compare types.
foreach (var i in newData.Keys)
{
//Data point and config of this point:
var dataPoints = newData[i];
var config = feed.Subscriptions[i];
//Keep track of how many data points we've processed
_dataPointCount += dataPoints.Count;
//We don't want to pump data that we added just for currency conversions
if (config.IsInternalFeed)
{
continue;
}
//Create TradeBars Unified Data --> OR --> invoke generic data event. One loop.
// Aggregate Dividends and Splits -- invoke portfolio application methods
foreach (var dataPoint in dataPoints)
{
var dividend = dataPoint as Dividend;
if (dividend != null)
{
Log.Trace("AlgorithmManager.Run(): Applying Dividend for " + dividend.Symbol);
// if this is a dividend apply to portfolio
algorithm.Portfolio.ApplyDividend(dividend);
if (hasOnDataDividends)
{
// and add to our data dictionary to pump into OnData(Dividends data)
newDividends.Add(dividend);
}
continue;
}
var split = dataPoint as Split;
if (split != null)
{
Log.Trace("AlgorithmManager.Run(): Applying Split for " + split.Symbol);
// if this is a split apply to portfolio
algorithm.Portfolio.ApplySplit(split);
if (hasOnDataSplits)
{
// and add to our data dictionary to pump into OnData(Splits data)
newSplits.Add(split);
}
continue;
}
//Update registered consolidators for this symbol index
try
{
for (var j = 0; j < config.Consolidators.Count; j++)
{
config.Consolidators[j].Update(dataPoint);
}
}
catch (Exception err)
{
algorithm.RunTimeError = err;
_algorithmState = AlgorithmStatus.RuntimeError;
Log.Error("AlgorithmManager.Run(): RuntimeError: Consolidators update: " + err.Message);
return;
}
// TRADEBAR -- add to our dictionary
var bar = dataPoint as TradeBar;
if (bar != null)
{
try
{
if (backwardsCompatibilityMode)
{
oldBars[bar.Symbol] = bar;
}
else
{
newBars[bar.Symbol] = bar;
}
}
catch (Exception err)
{
Log.Error(time.ToLongTimeString() + " >> " + bar.Time.ToLongTimeString() + " >> " + bar.Symbol + " >> "
+ bar.Value.ToString("C"));
Log.Error("AlgorithmManager.Run(): Failed to add TradeBar (" + bar.Symbol + ") Time: (" + time.ToLongTimeString()
+ ") Count:(" + newBars.Count + ") " + err.Message);
}
continue;
}
// TICK -- add to our dictionary
var tick = dataPoint as Tick;
if (tick != null)
{
if (backwardsCompatibilityMode)
{
List <Tick> ticks;
if (!oldTicks.TryGetValue(tick.Symbol, out ticks))
{
ticks = new List <Tick>(3);
oldTicks.Add(tick.Symbol, ticks);
}
ticks.Add(tick);
}
else
{
List <Tick> ticks;
if (!newTicks.TryGetValue(tick.Symbol, out ticks))
{
ticks = new List <Tick>(3);
newTicks.Add(tick.Symbol, ticks);
}
ticks.Add(tick);
}
continue;
}
// if it was nothing else then it must be custom data
// CUSTOM DATA -- invoke on data method
//Send data into the generic algorithm event handlers
try
{
methodInvokers[config.Type](algorithm, dataPoint);
}
catch (Exception err)
{
algorithm.RunTimeError = err;
_algorithmState = AlgorithmStatus.RuntimeError;
Log.Debug("AlgorithmManager.Run(): RuntimeError: Custom Data: " + err.Message + " STACK >>> " + err.StackTrace);
return;
}
}
}
try
{
// fire off the dividend and split events before pricing events
if (hasOnDataDividends && newDividends.Count != 0)
{
methodInvokers[typeof(Dividends)](algorithm, newDividends);
}
if (hasOnDataSplits && newSplits.Count != 0)
{
methodInvokers[typeof(Splits)](algorithm, newSplits);
}
}
catch (Exception err)
{
algorithm.RunTimeError = err;
_algorithmState = AlgorithmStatus.RuntimeError;
Log.Debug("AlgorithmManager.Run(): RuntimeError: Dividends/Splits: " + err.Message + " STACK >>> " + err.StackTrace);
return;
}
//After we've fired all other events in this second, fire the pricing events:
if (backwardsCompatibilityMode)
{
//Log.Debug("AlgorithmManager.Run(): Invoking v1.0 Event Handlers...");
try
{
if (hasOnTradeBar && oldBars.Count > 0)
{
methodInvokers[typeof(Dictionary <string, TradeBar>)](algorithm, oldBars);
}
if (hasOnTick && oldTicks.Count > 0)
{
methodInvokers[typeof(Dictionary <string, List <Tick> >)](algorithm, oldTicks);
}
}
catch (Exception err)
{
algorithm.RunTimeError = err;
_algorithmState = AlgorithmStatus.RuntimeError;
Log.Debug("AlgorithmManager.Run(): RuntimeError: Backwards Compatibility Mode: " + err.Message + " STACK >>> " + err.StackTrace);
return;
}
}
else
{
//Log.Debug("AlgorithmManager.Run(): Invoking v2.0 Event Handlers...");
try
{
if (hasOnDataTradeBars && newBars.Count > 0)
{
methodInvokers[typeof(TradeBars)](algorithm, newBars);
}
if (hasOnDataTicks && newTicks.Count > 0)
{
methodInvokers[typeof(Ticks)](algorithm, newTicks);
}
}
catch (Exception err)
{
algorithm.RunTimeError = err;
_algorithmState = AlgorithmStatus.RuntimeError;
Log.Debug("AlgorithmManager.Run(): RuntimeError: New Style Mode: " + err.Message + " STACK >>> " + err.StackTrace);
return;
}
}
//If its the historical/paper trading models, wait until market orders have been "filled"
// Manually trigger the event handler to prevent thread switch.
transactions.ProcessSynchronousEvents();
//Save the previous time for the sample calculations
_previousTime = time;
// Process any required events of the results handler such as sampling assets, equity, or stock prices.
results.ProcessSynchronousEvents();
} // End of ForEach DataStream
//Stream over:: Send the final packet and fire final events:
Log.Trace("AlgorithmManager.Run(): Firing On End Of Algorithm...");
try
{
algorithm.OnEndOfAlgorithm();
}
catch (Exception err)
{
_algorithmState = AlgorithmStatus.RuntimeError;
algorithm.RunTimeError = new Exception("Error running OnEndOfAlgorithm(): " + err.Message, err.InnerException);
Log.Debug("AlgorithmManager.OnEndOfAlgorithm(): " + err.Message + " STACK >>> " + err.StackTrace);
return;
}
// Process any required events of the results handler such as sampling assets, equity, or stock prices.
results.ProcessSynchronousEvents(forceProcess: true);
//Liquidate Holdings for Calculations:
if (_algorithmState == AlgorithmStatus.Liquidated || !Engine.LiveMode)
{
// without this we can't liquidate equities since the exchange is 'technically' closed
var hackedFrontier = algorithm.Time == DateTime.MinValue ? DateTime.MinValue : algorithm.Time.AddMilliseconds(-1);
algorithm.SetDateTime(hackedFrontier);
foreach (var security in algorithm.Securities)
{
security.Value.SetMarketPrice(hackedFrontier, null);
}
Log.Trace("AlgorithmManager.Run(): Liquidating algorithm holdings...");
algorithm.Liquidate();
results.LogMessage("Algorithm Liquidated");
results.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.Liquidated);
}
//Manually stopped the algorithm
if (_algorithmState == AlgorithmStatus.Stopped)
{
Log.Trace("AlgorithmManager.Run(): Stopping algorithm...");
results.LogMessage("Algorithm Stopped");
results.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.Stopped);
}
//Backtest deleted.
if (_algorithmState == AlgorithmStatus.Deleted)
{
Log.Trace("AlgorithmManager.Run(): Deleting algorithm...");
results.DebugMessage("Algorithm Id:(" + job.AlgorithmId + ") Deleted by request.");
results.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.Deleted);
}
//Algorithm finished, send regardless of commands:
results.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.Completed);
//Take final samples:
results.SampleRange(algorithm.GetChartUpdates());
results.SampleEquity(DataStream.AlorithmTime, Math.Round(algorithm.Portfolio.TotalPortfolioValue, 4));
results.SamplePerformance(DataStream.AlorithmTime, Math.Round((algorithm.Portfolio.TotalPortfolioValue - startingPortfolioValue) * 100 / startingPortfolioValue, 10));
} // End of Run();