/// <summary>
/// Initializes a new instance of the <see cref="LeanEngineAlgorithmHandlers"/> class from the specified handlers
/// </summary>
/// <param name="results">The result handler for communicating results from the algorithm</param>
/// <param name="setup">The setup handler used to initialize algorithm state</param>
/// <param name="dataFeed">The data feed handler used to pump data to the algorithm</param>
/// <param name="transactions">The transaction handler used to process orders from the algorithm</param>
/// <param name="realTime">The real time handler used to process real time events</param>
/// <param name="historyProvider">The history provider used to process historical data requests</param>
/// <param name="commandQueue">The command queue handler used to receive external commands for the algorithm</param>
/// <param name="mapFileProvider">The map file provider used to retrieve map files for the data feed</param>
public LeanEngineAlgorithmHandlers(IResultHandler results,
ISetupHandler setup,
IDataFeed dataFeed,
ITransactionHandler transactions,
IRealTimeHandler realTime,
IHistoryProvider historyProvider,
ICommandQueueHandler commandQueue,
IMapFileProvider mapFileProvider
)
{
if (results == null)
{
throw new ArgumentNullException("results");
}
if (setup == null)
{
throw new ArgumentNullException("setup");
}
if (dataFeed == null)
{
throw new ArgumentNullException("dataFeed");
}
if (transactions == null)
{
throw new ArgumentNullException("transactions");
}
if (realTime == null)
{
throw new ArgumentNullException("realTime");
}
if (historyProvider == null)
{
throw new ArgumentNullException("realTime");
}
if (commandQueue == null)
{
throw new ArgumentNullException("commandQueue");
}
if (mapFileProvider == null)
{
throw new ArgumentNullException("mapFileProvider");
}
_results = results;
_setup = setup;
_dataFeed = dataFeed;
_transactions = transactions;
_realTime = realTime;
_historyProvider = historyProvider;
_commandQueue = commandQueue;
_mapFileProvider = mapFileProvider;
}
/// <summary>
/// Primary entry point to setup a new algorithm
/// </summary>
/// <param name="algorithm">Algorithm instance</param>
/// <param name="brokerage">New brokerage output instance</param>
/// <param name="job">Algorithm job task</param>
/// <param name="resultHandler">The configured result handler</param>
/// <param name="transactionHandler">The configurated transaction handler</param>
/// <param name="realTimeHandler">The configured real time handler</param>
/// <returns>True on successfully setting up the algorithm state, or false on error.</returns>
public bool Setup(IAlgorithm algorithm, out IBrokerage brokerage, AlgorithmNodePacket job, IResultHandler resultHandler, ITransactionHandler transactionHandler, IRealTimeHandler realTimeHandler)
{
_algorithm = algorithm;
brokerage = default(IBrokerage);
// verify we were given the correct job packet type
var liveJob = job as LiveNodePacket;
if (liveJob == null)
{
AddInitializationError("BrokerageSetupHandler requires a LiveNodePacket");
return false;
}
// verify the brokerage was specified
if (string.IsNullOrWhiteSpace(liveJob.Brokerage))
{
AddInitializationError("A brokerage must be specified");
return false;
}
// attach to the message event to relay brokerage specific initialization messages
EventHandler<BrokerageMessageEvent> brokerageOnMessage = (sender, args) =>
{
if (args.Type == BrokerageMessageType.Error)
{
AddInitializationError(string.Format("Brokerage Error Code: {0} - {1}", args.Code, args.Message));
}
};
try
{
Log.Trace("BrokerageSetupHandler.Setup(): Initializing algorithm...");
//Execute the initialize code:
var isolator = new Isolator();
var initializeComplete = isolator.ExecuteWithTimeLimit(TimeSpan.FromSeconds(10), () =>
{
try
{
//Set the live trading level asset/ram allocation limits.
//Protects algorithm from linux killing the job by excess memory:
switch (job.ServerType)
{
case ServerType.Server1024:
algorithm.SetAssetLimits(100, 20, 10);
break;
case ServerType.Server2048:
algorithm.SetAssetLimits(400, 50, 30);
break;
default: //512
algorithm.SetAssetLimits(50, 25, 15);
break;
}
//Algorithm is live, not backtesting:
algorithm.SetLiveMode(true);
//Initialize the algorithm's starting date
algorithm.SetDateTime(DateTime.UtcNow);
//Set the source impl for the event scheduling
algorithm.Schedule.SetEventSchedule(realTimeHandler);
//Initialise the algorithm, get the required data:
algorithm.Initialize();
}
catch (Exception err)
{
AddInitializationError(err.Message);
}
});
if (!initializeComplete)
{
AddInitializationError("Initialization timed out.");
return false;
}
try
{
// find the correct brokerage factory based on the specified brokerage in the live job packet
_factory = Composer.Instance.Single<IBrokerageFactory>(factory => factory.BrokerageType.MatchesTypeName(liveJob.Brokerage));
}
catch (Exception err)
{
Log.Error("BrokerageSetupHandler.Setup(): Error resolving brokerage factory for " + liveJob.Brokerage + ". " + err.Message);
AddInitializationError("Unable to locate factory for brokerage: " + liveJob.Brokerage);
}
// let the world know what we're doing since logging in can take a minute
resultHandler.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.LoggingIn, "Logging into brokerage...");
// initialize the correct brokerage using the resolved factory
brokerage = _factory.CreateBrokerage(liveJob, algorithm);
if (brokerage == null)
{
AddInitializationError("Failed to create instance of brokerage: " + liveJob.Brokerage);
return false;
}
brokerage.Message += brokerageOnMessage;
// set the transaction models base on the brokerage properties
SetupHandler.UpdateTransactionModels(algorithm, algorithm.BrokerageModel);
algorithm.Transactions.SetOrderProcessor(transactionHandler);
algorithm.PostInitialize();
try
{
// this can fail for various reasons, such as already being logged in somewhere else
brokerage.Connect();
}
catch (Exception err)
{
Log.Error(err);
AddInitializationError("Error connecting to brokerage: " + err.Message);
return false;
}
if (!brokerage.IsConnected)
{
// if we're reporting that we're not connected, bail
AddInitializationError("Unable to connect to brokerage.");
return false;
}
try
{
// set the algorithm's cash balance for each currency
var cashBalance = brokerage.GetCashBalance();
foreach (var cash in cashBalance)
{
Log.Trace("BrokerageSetupHandler.Setup(): Setting " + cash.Symbol + " cash to " + cash.Quantity);
algorithm.SetCash(cash.Symbol, cash.Quantity, cash.ConversionRate);
}
}
catch (Exception err)
{
Log.Error(err);
AddInitializationError("Error getting cash balance from brokerage: " + err.Message);
return false;
}
try
{
// populate the algorithm with the account's outstanding orders
var openOrders = brokerage.GetOpenOrders();
foreach (var order in openOrders)
{
// be sure to assign order IDs such that we increment from the SecurityTransactionManager to avoid ID collisions
Log.Trace("BrokerageSetupHandler.Setup(): Has open order: " + order.Symbol + " - " + order.Quantity);
order.Id = algorithm.Transactions.GetIncrementOrderId();
transactionHandler.Orders.AddOrUpdate(order.Id, order, (i, o) => order);
}
}
catch (Exception err)
{
Log.Error(err);
AddInitializationError("Error getting open orders from brokerage: " + err.Message);
return false;
}
try
{
// populate the algorithm with the account's current holdings
var holdings = brokerage.GetAccountHoldings();
var minResolution = new Lazy<Resolution>(() => algorithm.Securities.Min(x => x.Value.Resolution));
foreach (var holding in holdings)
{
Log.Trace("BrokerageSetupHandler.Setup(): Has existing holding: " + holding);
if (!algorithm.Portfolio.ContainsKey(holding.Symbol))
{
Log.Trace("BrokerageSetupHandler.Setup(): Adding unrequested security: " + holding.Symbol);
// for items not directly requested set leverage to 1 and at the min resolution
algorithm.AddSecurity(holding.Type, holding.Symbol, minResolution.Value, null, true, 1.0m, false);
}
algorithm.Portfolio[holding.Symbol].SetHoldings(holding.AveragePrice, (int) holding.Quantity);
algorithm.Securities[holding.Symbol].SetMarketPrice(new TradeBar
{
Time = DateTime.Now,
Open = holding.MarketPrice,
High = holding.MarketPrice,
Low = holding.MarketPrice,
Close = holding.MarketPrice,
Volume = 0,
Symbol = holding.Symbol,
DataType = MarketDataType.TradeBar
});
}
}
catch (Exception err)
{
Log.Error(err);
AddInitializationError("Error getting account holdings from brokerage: " + err.Message);
return false;
}
// call this after we've initialized everything from the brokerage since we may have added some holdings/currencies
algorithm.Portfolio.CashBook.EnsureCurrencyDataFeeds(algorithm.Securities, algorithm.SubscriptionManager, SecurityExchangeHoursProvider.FromDataFolder());
//Set the starting portfolio value for the strategy to calculate performance:
StartingPortfolioValue = algorithm.Portfolio.TotalPortfolioValue;
StartingDate = DateTime.Now;
}
catch (Exception err)
{
AddInitializationError(err.Message);
}
finally
{
if (brokerage != null)
{
brokerage.Message -= brokerageOnMessage;
}
}
return Errors.Count == 0;
}
/********************************************************
* CLASS METHODS
*********************************************************/
/// <summary>
/// Primary Analysis Thread:
/// </summary>
public static void Main(string[] args)
{
//Initialize:
var algorithmPath = "";
string mode = "RELEASE";
AlgorithmNodePacket job = null;
var algorithm = default(IAlgorithm);
var startTime = DateTime.Now;
Log.LogHandler = Composer.Instance.GetExportedValueByTypeName<ILogHandler>(Config.Get("log-handler", "CompositeLogHandler"));
#if DEBUG
mode = "DEBUG";
#endif
//Name thread for the profiler:
Thread.CurrentThread.Name = "Algorithm Analysis Thread";
Log.Trace("Engine.Main(): LEAN ALGORITHMIC TRADING ENGINE v" + Constants.Version + " Mode: " + mode);
Log.Trace("Engine.Main(): Started " + DateTime.Now.ToShortTimeString());
Log.Trace("Engine.Main(): Memory " + OS.ApplicationMemoryUsed + "Mb-App " + +OS.TotalPhysicalMemoryUsed + "Mb-Used " + OS.TotalPhysicalMemory + "Mb-Total");
//Import external libraries specific to physical server location (cloud/local)
try
{
// grab the right export based on configuration
Api = Composer.Instance.GetExportedValueByTypeName<IApi>(Config.Get("api-handler"));
Notify = Composer.Instance.GetExportedValueByTypeName<IMessagingHandler>(Config.Get("messaging-handler"));
JobQueue = Composer.Instance.GetExportedValueByTypeName<IJobQueueHandler>(Config.Get("job-queue-handler"));
}
catch (CompositionException compositionException)
{ Log.Error("Engine.Main(): Failed to load library: " + compositionException);
}
//Setup packeting, queue and controls system: These don't do much locally.
Api.Initialize();
Notify.Initialize();
JobQueue.Initialize();
//Start monitoring the backtest active status:
var statusPingThread = new Thread(StateCheck.Ping.Run);
statusPingThread.Start();
try
{
//Reset algo manager internal variables preparing for a new algorithm.
AlgorithmManager.ResetManager();
//Reset thread holders.
var initializeComplete = false;
Thread threadFeed = null;
Thread threadTransactions = null;
Thread threadResults = null;
Thread threadRealTime = null;
do
{
//-> Pull job from QuantConnect job queue, or, pull local build:
job = JobQueue.NextJob(out algorithmPath); // Blocking.
// if the job version doesn't match this instance version then we can't process it
// we also don't want to reprocess redelivered live jobs
if (job.Version != Constants.Version || (LiveMode && job.Redelivered))
{
Log.Error("Engine.Run(): Job Version: " + job.Version + " Deployed Version: " + Constants.Version);
//Tiny chance there was an uncontrolled collapse of a server, resulting in an old user task circulating.
//In this event kill the old algorithm and leave a message so the user can later review.
JobQueue.AcknowledgeJob(job);
Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, _collapseMessage);
Notify.SetChannel(job.Channel);
Notify.RuntimeError(job.AlgorithmId, _collapseMessage);
job = null;
}
} while (job == null);
//-> Initialize messaging system
Notify.SetChannel(job.Channel);
//-> Create SetupHandler to configure internal algorithm state:
SetupHandler = GetSetupHandler(job.SetupEndpoint);
//-> Set the result handler type for this algorithm job, and launch the associated result thread.
ResultHandler = GetResultHandler(job);
threadResults = new Thread(ResultHandler.Run, 0) {Name = "Result Thread"};
threadResults.Start();
try
{
// Save algorithm to cache, load algorithm instance:
algorithm = SetupHandler.CreateAlgorithmInstance(algorithmPath);
//Initialize the internal state of algorithm and job: executes the algorithm.Initialize() method.
initializeComplete = SetupHandler.Setup(algorithm, out _brokerage, job);
//If there are any reasons it failed, pass these back to the IDE.
if (!initializeComplete || algorithm.ErrorMessages.Count > 0 || SetupHandler.Errors.Count > 0)
{
initializeComplete = false;
//Get all the error messages: internal in algorithm and external in setup handler.
var errorMessage = String.Join(",", algorithm.ErrorMessages);
errorMessage += String.Join(",", SetupHandler.Errors);
ResultHandler.RuntimeError(errorMessage);
Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError);
}
}
catch (Exception err)
{
var runtimeMessage = "Algorithm.Initialize() Error: " + err.Message + " Stack Trace: " + err.StackTrace;
ResultHandler.RuntimeError(runtimeMessage, err.StackTrace);
Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, runtimeMessage);
}
//-> Using the job + initialization: load the designated handlers:
if (initializeComplete)
{
//-> Reset the backtest stopwatch; we're now running the algorithm.
startTime = DateTime.Now;
//Set algorithm as locked; set it to live mode if we're trading live, and set it to locked for no further updates.
algorithm.SetAlgorithmId(job.AlgorithmId);
algorithm.SetLiveMode(LiveMode);
algorithm.SetLocked();
//Load the associated handlers for data, transaction and realtime events:
ResultHandler.SetAlgorithm(algorithm);
DataFeed = GetDataFeedHandler(algorithm, job);
TransactionHandler = GetTransactionHandler(algorithm, _brokerage, ResultHandler, job);
RealTimeHandler = GetRealTimeHandler(algorithm, _brokerage, DataFeed, ResultHandler, job);
//Set the error handlers for the brokerage asynchronous errors.
SetupHandler.SetupErrorHandler(ResultHandler, _brokerage);
//Send status to user the algorithm is now executing.
ResultHandler.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.Running);
//Launch the data, transaction and realtime handlers into dedicated threads
threadFeed = new Thread(DataFeed.Run) {Name = "DataFeed Thread"};
threadTransactions = new Thread(TransactionHandler.Run) {Name = "Transaction Thread"};
threadRealTime = new Thread(RealTimeHandler.Run) {Name = "RealTime Thread"};
//Launch the data feed, result sending, and transaction models/handlers in separate threads.
threadFeed.Start(); // Data feed pushing data packets into thread bridge;
threadTransactions.Start(); // Transaction modeller scanning new order requests
threadRealTime.Start(); // RealTime scan time for time based events:
// Result manager scanning message queue: (started earlier)
ResultHandler.DebugMessage(string.Format("Launching analysis for {0} with LEAN Engine v{1}", job.AlgorithmId, Constants.Version));
try
{
// Execute the Algorithm Code:
var complete = Isolator.ExecuteWithTimeLimit(SetupHandler.MaximumRuntime, AlgorithmManager.TimeLoopWithinLimits, () =>
{
try
{
//Run Algorithm Job:
// -> Using this Data Feed,
// -> Send Orders to this TransactionHandler,
// -> Send Results to ResultHandler.
AlgorithmManager.Run(job, algorithm, DataFeed, TransactionHandler, ResultHandler, SetupHandler, RealTimeHandler);
}
catch (Exception err)
{
//Debugging at this level is difficult, stack trace needed.
Log.Error("Engine.Run", err);
}
Log.Trace("Engine.Run(): Exiting Algorithm Manager");
}, job.UserPlan == UserPlan.Free ? 1024 : MaximumRamAllocation);
if (!complete)
{
Log.Error("Engine.Main(): Failed to complete in time: " + SetupHandler.MaximumRuntime.ToString("F"));
throw new Exception("Failed to complete algorithm within " + SetupHandler.MaximumRuntime.ToString("F") + " seconds. Please make it run faster.");
}
// Algorithm runtime error:
if (algorithm.RunTimeError != null)
{
throw algorithm.RunTimeError;
}
}
catch (Exception err)
{
//Error running the user algorithm: purge datafeed, send error messages, set algorithm status to failed.
Log.Error("Engine.Run(): Breaking out of parent try-catch: " + err.Message + " " + err.StackTrace);
if (DataFeed != null) DataFeed.Exit();
if (ResultHandler != null)
{
var message = "Runtime Error: " + err.Message;
Log.Trace("Engine.Run(): Sending runtime error to user...");
ResultHandler.LogMessage(message);
ResultHandler.RuntimeError(message, err.StackTrace);
Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, message + " Stack Trace: " + err.StackTrace);
}
}
//Send result data back: this entire code block could be rewritten.
// todo: - Split up statistics class, its enormous.
// todo: - Make a dedicated Statistics.Benchmark class.
// todo: - Move all creation and transmission of statistics out of primary engine loop.
// todo: - Statistics.Generate(algorithm, resulthandler, transactionhandler);
try
{
var charts = new Dictionary<string, Chart>(ResultHandler.Charts);
var orders = new Dictionary<int, Order>(algorithm.Transactions.Orders);
var holdings = new Dictionary<string, Holding>();
var statistics = new Dictionary<string, string>();
var banner = new Dictionary<string, string>();
try
{
//Generates error when things don't exist (no charting logged, runtime errors in main algo execution)
const string strategyEquityKey = "Strategy Equity";
const string equityKey = "Equity";
const string dailyPerformanceKey = "Daily Performance";
// make sure we've taken samples for these series before just blindly requesting them
if (charts.ContainsKey(strategyEquityKey) &&
charts[strategyEquityKey].Series.ContainsKey(equityKey) &&
charts[strategyEquityKey].Series.ContainsKey(dailyPerformanceKey))
{
var equity = charts[strategyEquityKey].Series[equityKey].Values;
var performance = charts[strategyEquityKey].Series[dailyPerformanceKey].Values;
var profitLoss =
new SortedDictionary<DateTime, decimal>(algorithm.Transactions.TransactionRecord);
statistics = Statistics.Statistics.Generate(equity, profitLoss, performance,
SetupHandler.StartingPortfolioValue, algorithm.Portfolio.TotalFees, 252);
}
}
catch (Exception err)
{
Log.Error("Algorithm.Node.Engine(): Error generating statistics packet: " + err.Message);
}
//Diagnostics Completed, Send Result Packet:
var totalSeconds = (DateTime.Now - startTime).TotalSeconds;
ResultHandler.DebugMessage(string.Format("Algorithm Id:({0}) completed in {1} seconds at {2}k data points per second. Processing total of {3} data points.",
job.AlgorithmId, totalSeconds.ToString("F2"), ((AlgorithmManager.DataPoints / (double)1000) / totalSeconds).ToString("F0"), AlgorithmManager.DataPoints.ToString("N0")));
ResultHandler.SendFinalResult(job, orders, algorithm.Transactions.TransactionRecord, holdings, statistics, banner);
}
catch (Exception err)
{
Log.Error("Engine.Main(): Error sending analysis result: " + err.Message + " ST >> " + err.StackTrace);
}
//Before we return, send terminate commands to close up the threads
TransactionHandler.Exit();
DataFeed.Exit();
RealTimeHandler.Exit();
}
//Close result handler:
ResultHandler.Exit();
StateCheck.Ping.Exit();
//Wait for the threads to complete:
var ts = Stopwatch.StartNew();
while ((ResultHandler.IsActive || (TransactionHandler != null && TransactionHandler.IsActive) || (DataFeed != null && DataFeed.IsActive)) && ts.ElapsedMilliseconds < 30 * 1000)
{
Thread.Sleep(100); Log.Trace("Waiting for threads to exit...");
}
//Terminate threads still in active state.
if (threadFeed != null && threadFeed.IsAlive) threadFeed.Abort();
if (threadTransactions != null && threadTransactions.IsAlive) threadTransactions.Abort();
if (threadResults != null && threadResults.IsAlive) threadResults.Abort();
if (statusPingThread != null && statusPingThread.IsAlive) statusPingThread.Abort();
if (_brokerage != null)
{
_brokerage.Disconnect();
}
if (SetupHandler != null)
{
SetupHandler.Dispose();
}
Log.Trace("Engine.Main(): Analysis Completed and Results Posted.");
}
catch (Exception err)
{
Log.Error("Engine.Main(): Error running algorithm: " + err.Message + " >> " + err.StackTrace);
}
finally
{
//No matter what for live mode; make sure we've set algorithm status in the API for "not running" conditions:
if (LiveMode && AlgorithmManager.State != AlgorithmStatus.Running && AlgorithmManager.State != AlgorithmStatus.RuntimeError)
Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmManager.State);
//Delete the message from the job queue:
JobQueue.AcknowledgeJob(job);
Log.Trace("Engine.Main(): Packet removed from queue: " + job.AlgorithmId);
//Attempt to clean up ram usage:
GC.Collect();
}
//Final disposals.
Api.Dispose();
// Make the console window pause so we can read log output before exiting and killing the application completely
if (IsLocal)
{
Log.Trace("Engine.Main(): Analysis Complete. Press any key to continue.");
Console.Read();
}
Log.LogHandler.Dispose();
}
/// <summary>
/// Setup the algorithm cash, dates and portfolio as desired.
/// </summary>
/// <param name="algorithm">Existing algorithm instance</param>
/// <param name="brokerage">New brokerage instance</param>
/// <param name="baseJob">Backtesting job</param>
/// <param name="resultHandler">The configured result handler</param>
/// <param name="transactionHandler">The configuration transaction handler</param>
/// <param name="realTimeHandler">The configured real time handler</param>
/// <returns>Boolean true on successfully setting up the console.</returns>
public bool Setup(IAlgorithm algorithm, IBrokerage brokerage, AlgorithmNodePacket baseJob, IResultHandler resultHandler, ITransactionHandler transactionHandler, IRealTimeHandler realTimeHandler)
{
var initializeComplete = false;
try
{
//Set common variables for console programs:
if (baseJob.Type == PacketType.BacktestNode)
{
var backtestJob = baseJob as BacktestNodePacket;
//Set our default markets
algorithm.SetDefaultMarkets(BacktestingBrokerageFactory.DefaultMarketMap.ToDictionary());
algorithm.SetMaximumOrders(int.MaxValue);
// set our parameters
algorithm.SetParameters(baseJob.Parameters);
algorithm.SetLiveMode(false);
//Set the source impl for the event scheduling
algorithm.Schedule.SetEventSchedule(realTimeHandler);
//Setup Base Algorithm:
algorithm.Initialize();
//Set the time frontier of the algorithm
algorithm.SetDateTime(algorithm.StartDate.ConvertToUtc(algorithm.TimeZone));
//Construct the backtest job packet:
backtestJob.PeriodStart = algorithm.StartDate;
backtestJob.PeriodFinish = algorithm.EndDate;
backtestJob.BacktestId = "LOCALHOST";
backtestJob.UserId = 1001;
backtestJob.Type = PacketType.BacktestNode;
//Backtest Specific Parameters:
StartingDate = backtestJob.PeriodStart;
StartingPortfolioValue = algorithm.Portfolio.Cash;
}
else
{
throw new Exception("The ConsoleSetupHandler is for backtests only. Use the BrokerageSetupHandler.");
}
}
catch (Exception err)
{
Log.Error(err);
Errors.Add("Failed to initialize algorithm: Initialize(): " + err.Message);
}
if (Errors.Count == 0)
{
initializeComplete = true;
}
// set the transaction and settlement models based on the brokerage properties
algorithm.UpdateModels(algorithm.BrokerageModel);
algorithm.Transactions.SetOrderProcessor(transactionHandler);
algorithm.PostInitialize();
return initializeComplete;
}
/// <summary>
/// Setup the algorithm cash, dates and portfolio as desired.
/// </summary>
/// <param name="algorithm">Existing algorithm instance</param>
/// <param name="brokerage">New brokerage instance</param>
/// <param name="baseJob">Backtesting job</param>
/// <param name="resultHandler">The configured result handler</param>
/// <param name="transactionHandler">The configuration transaction handler</param>
/// <param name="realTimeHandler">The configured real time handler</param>
/// <returns>Boolean true on successfully setting up the console.</returns>
public bool Setup(IAlgorithm algorithm, out IBrokerage brokerage, AlgorithmNodePacket baseJob, IResultHandler resultHandler, ITransactionHandler transactionHandler, IRealTimeHandler realTimeHandler)
{
var initializeComplete = false;
try
{
//Set common variables for console programs:
if (baseJob.Type == PacketType.BacktestNode)
{
var backtestJob = baseJob as BacktestNodePacket;
//Set the limits on the algorithm assets (for local no limits)
algorithm.SetAssetLimits(999, 999, 999);
algorithm.SetMaximumOrders(int.MaxValue);
algorithm.SetLiveMode(false);
//Set the source impl for the event scheduling
algorithm.Schedule.SetEventSchedule(realTimeHandler);
//Setup Base Algorithm:
algorithm.Initialize();
//Set the time frontier of the algorithm
algorithm.SetDateTime(algorithm.StartDate.ConvertToUtc(algorithm.TimeZone));
//Add currency data feeds that weren't explicity added in Initialize
algorithm.Portfolio.CashBook.EnsureCurrencyDataFeeds(algorithm.Securities, algorithm.SubscriptionManager, MarketHoursDatabase.FromDataFolder());
//Construct the backtest job packet:
backtestJob.PeriodStart = algorithm.StartDate;
backtestJob.PeriodFinish = algorithm.EndDate;
backtestJob.BacktestId = "LOCALHOST";
backtestJob.UserId = 1001;
backtestJob.Type = PacketType.BacktestNode;
//Backtest Specific Parameters:
StartingDate = backtestJob.PeriodStart;
StartingPortfolioValue = algorithm.Portfolio.Cash;
}
else
{
throw new Exception("The ConsoleSetupHandler is for backtests only. Use the BrokerageSetupHandler.");
}
}
catch (Exception err)
{
Log.Error("ConsoleSetupHandler().Setup(): " + err.Message);
Errors.Add("Failed to initialize algorithm: Initialize(): " + err.Message);
}
if (Errors.Count == 0)
{
initializeComplete = true;
}
// we need to do this after algorithm initialization
brokerage = new BacktestingBrokerage(algorithm);
// set the transaction and settlement models based on the brokerage properties
SetupHandler.UpdateModels(algorithm, algorithm.BrokerageModel);
algorithm.Transactions.SetOrderProcessor(transactionHandler);
algorithm.PostInitialize();
return initializeComplete;
}
/// <summary>
/// Primary entry point to setup a new algorithm
/// </summary>
/// <param name="algorithm">Algorithm instance</param>
/// <param name="brokerage">New brokerage output instance</param>
/// <param name="job">Algorithm job task</param>
/// <param name="resultHandler">The configured result handler</param>
/// <param name="transactionHandler">The configurated transaction handler</param>
/// <param name="realTimeHandler">The configured real time handler</param>
/// <returns>True on successfully setting up the algorithm state, or false on error.</returns>
public bool Setup(IAlgorithm algorithm, IBrokerage brokerage, AlgorithmNodePacket job, IResultHandler resultHandler, ITransactionHandler transactionHandler, IRealTimeHandler realTimeHandler)
{
_algorithm = algorithm;
// verify we were given the correct job packet type
var liveJob = job as LiveNodePacket;
if (liveJob == null)
{
AddInitializationError("BrokerageSetupHandler requires a LiveNodePacket");
return false;
}
// verify the brokerage was specified
if (string.IsNullOrWhiteSpace(liveJob.Brokerage))
{
AddInitializationError("A brokerage must be specified");
return false;
}
// attach to the message event to relay brokerage specific initialization messages
EventHandler<BrokerageMessageEvent> brokerageOnMessage = (sender, args) =>
{
if (args.Type == BrokerageMessageType.Error)
{
AddInitializationError(string.Format("Brokerage Error Code: {0} - {1}", args.Code, args.Message));
}
};
try
{
Log.Trace("BrokerageSetupHandler.Setup(): Initializing algorithm...");
resultHandler.SendStatusUpdate(AlgorithmStatus.Initializing, "Initializing algorithm...");
//Execute the initialize code:
var controls = job.Controls;
var isolator = new Isolator();
var initializeComplete = isolator.ExecuteWithTimeLimit(TimeSpan.FromSeconds(300), () =>
{
try
{
//Set the default brokerage model before initialize
algorithm.SetBrokerageModel(_factory.BrokerageModel);
//Set our parameters
algorithm.SetParameters(job.Parameters);
//Algorithm is live, not backtesting:
algorithm.SetLiveMode(true);
//Initialize the algorithm's starting date
algorithm.SetDateTime(DateTime.UtcNow);
//Set the source impl for the event scheduling
algorithm.Schedule.SetEventSchedule(realTimeHandler);
//Initialise the algorithm, get the required data:
algorithm.Initialize();
if (liveJob.Brokerage != "PaperBrokerage")
{
//Zero the CashBook - we'll populate directly from brokerage
foreach (var kvp in algorithm.Portfolio.CashBook)
{
kvp.Value.SetAmount(0);
}
}
}
catch (Exception err)
{
AddInitializationError(err.Message);
}
});
if (!initializeComplete)
{
AddInitializationError("Initialization timed out.");
return false;
}
// let the world know what we're doing since logging in can take a minute
resultHandler.SendStatusUpdate(AlgorithmStatus.LoggingIn, "Logging into brokerage...");
brokerage.Message += brokerageOnMessage;
algorithm.Transactions.SetOrderProcessor(transactionHandler);
Log.Trace("BrokerageSetupHandler.Setup(): Connecting to brokerage...");
try
{
// this can fail for various reasons, such as already being logged in somewhere else
brokerage.Connect();
}
catch (Exception err)
{
Log.Error(err);
AddInitializationError(string.Format("Error connecting to brokerage: {0}. " +
"This may be caused by incorrect login credentials or an unsupported account type.", err.Message));
return false;
}
if (!brokerage.IsConnected)
{
// if we're reporting that we're not connected, bail
AddInitializationError("Unable to connect to brokerage.");
return false;
}
Log.Trace("BrokerageSetupHandler.Setup(): Fetching cash balance from brokerage...");
try
{
// set the algorithm's cash balance for each currency
var cashBalance = brokerage.GetCashBalance();
foreach (var cash in cashBalance)
{
Log.Trace("BrokerageSetupHandler.Setup(): Setting " + cash.Symbol + " cash to " + cash.Amount);
algorithm.Portfolio.SetCash(cash.Symbol, cash.Amount, cash.ConversionRate);
}
}
catch (Exception err)
{
Log.Error(err);
AddInitializationError("Error getting cash balance from brokerage: " + err.Message);
return false;
}
Log.Trace("BrokerageSetupHandler.Setup(): Fetching open orders from brokerage...");
try
{
// populate the algorithm with the account's outstanding orders
var openOrders = brokerage.GetOpenOrders();
foreach (var order in openOrders)
{
// be sure to assign order IDs such that we increment from the SecurityTransactionManager to avoid ID collisions
Log.Trace("BrokerageSetupHandler.Setup(): Has open order: " + order.Symbol.ToString() + " - " + order.Quantity);
order.Id = algorithm.Transactions.GetIncrementOrderId();
transactionHandler.Orders.AddOrUpdate(order.Id, order, (i, o) => order);
}
}
catch (Exception err)
{
Log.Error(err);
AddInitializationError("Error getting open orders from brokerage: " + err.Message);
return false;
}
Log.Trace("BrokerageSetupHandler.Setup(): Fetching holdings from brokerage...");
try
{
// populate the algorithm with the account's current holdings
var holdings = brokerage.GetAccountHoldings();
var supportedSecurityTypes = new HashSet<SecurityType> { SecurityType.Equity, SecurityType.Forex, SecurityType.Cfd };
var minResolution = new Lazy<Resolution>(() => algorithm.Securities.Select(x => x.Value.Resolution).DefaultIfEmpty(Resolution.Second).Min());
foreach (var holding in holdings)
{
Log.Trace("BrokerageSetupHandler.Setup(): Has existing holding: " + holding);
// verify existing holding security type
if (!supportedSecurityTypes.Contains(holding.Type))
{
Log.Error("BrokerageSetupHandler.Setup(): Unsupported security type: " + holding.Type + "-" + holding.Symbol.Value);
AddInitializationError("Found unsupported security type in existing brokerage holdings: " + holding.Type + ". " +
"QuantConnect currently supports the following security types: " + string.Join(",", supportedSecurityTypes));
// keep aggregating these errors
continue;
}
if (!algorithm.Portfolio.ContainsKey(holding.Symbol))
{
Log.Trace("BrokerageSetupHandler.Setup(): Adding unrequested security: " + holding.Symbol.ToString());
// for items not directly requested set leverage to 1 and at the min resolution
algorithm.AddSecurity(holding.Type, holding.Symbol.Value, minResolution.Value, null, true, 1.0m, false);
}
algorithm.Portfolio[holding.Symbol].SetHoldings(holding.AveragePrice, (int) holding.Quantity);
algorithm.Securities[holding.Symbol].SetMarketPrice(new TradeBar
{
Time = DateTime.Now,
Open = holding.MarketPrice,
High = holding.MarketPrice,
Low = holding.MarketPrice,
Close = holding.MarketPrice,
Volume = 0,
Symbol = holding.Symbol,
DataType = MarketDataType.TradeBar
});
}
}
catch (Exception err)
{
Log.Error(err);
AddInitializationError("Error getting account holdings from brokerage: " + err.Message);
return false;
}
algorithm.PostInitialize();
//Set the starting portfolio value for the strategy to calculate performance:
StartingPortfolioValue = algorithm.Portfolio.TotalPortfolioValue;
StartingDate = DateTime.Now;
}
catch (Exception err)
{
AddInitializationError(err.Message);
}
finally
{
if (brokerage != null)
{
brokerage.Message -= brokerageOnMessage;
}
}
return Errors.Count == 0;
}
/// <summary>
/// Setup the algorithm cash, dates and data subscriptions as desired.
/// </summary>
/// <param name="algorithm">Algorithm instance</param>
/// <param name="brokerage">Brokerage instance</param>
/// <param name="baseJob">Algorithm job</param>
/// <param name="resultHandler">The configured result handler</param>
/// <param name="transactionHandler">The configurated transaction handler</param>
/// <param name="realTimeHandler">The configured real time handler</param>
/// <returns>Boolean true on successfully initializing the algorithm</returns>
public bool Setup(IAlgorithm algorithm, IBrokerage brokerage, AlgorithmNodePacket baseJob, IResultHandler resultHandler, ITransactionHandler transactionHandler, IRealTimeHandler realTimeHandler)
{
var job = baseJob as BacktestNodePacket;
if (job == null)
{
throw new ArgumentException("Expected BacktestNodePacket but received " + baseJob.GetType().Name);
}
Log.Trace(string.Format("BacktestingSetupHandler.Setup(): Setting up job: Plan: {0}, UID: {1}, PID: {2}, Version: {3}, Source: {4}", job.UserPlan, job.UserId, job.ProjectId, job.Version, job.RequestSource));
if (algorithm == null)
{
Errors.Add("Could not create instance of algorithm");
return(false);
}
//Make sure the algorithm start date ok.
if (job.PeriodStart == default(DateTime))
{
Errors.Add("Algorithm start date was never set");
return(false);
}
var controls = job.Controls;
var isolator = new Isolator();
var initializeComplete = isolator.ExecuteWithTimeLimit(TimeSpan.FromMinutes(5), () =>
{
try
{
//Set our parameters
algorithm.SetParameters(job.Parameters);
//Algorithm is backtesting, not live:
algorithm.SetLiveMode(false);
//Set the algorithm time before we even initialize:
algorithm.SetDateTime(job.PeriodStart.ConvertToUtc(algorithm.TimeZone));
//Set the source impl for the event scheduling
algorithm.Schedule.SetEventSchedule(realTimeHandler);
//Initialise the algorithm, get the required data:
algorithm.Initialize();
}
catch (Exception err)
{
Log.Error(err);
Errors.Add("Failed to initialize algorithm: Initialize(): " + err);
}
}, controls.RamAllocation);
//Before continuing, detect if this is ready:
if (!initializeComplete)
{
return(false);
}
algorithm.PostInitialize();
//Calculate the max runtime for the strategy
_maxRuntime = GetMaximumRuntime(job.PeriodStart, job.PeriodFinish, algorithm.SubscriptionManager, baseJob.Controls);
// Python takes forever; lets give it 10x longer to finish.
if (job.Language == Language.Python)
{
_maxRuntime = _maxRuntime.Add(TimeSpan.FromSeconds(_maxRuntime.TotalSeconds * 9));
}
//Get starting capital:
_startingCaptial = algorithm.Portfolio.Cash;
//Max Orders: 10k per backtest:
if (job.UserPlan == UserPlan.Free)
{
_maxOrders = 10000;
}
else
{
_maxOrders = int.MaxValue;
_maxRuntime += _maxRuntime;
}
//Set back to the algorithm,
algorithm.SetMaximumOrders(_maxOrders);
//Starting date of the algorithm:
_startingDate = job.PeriodStart;
//Put into log for debugging:
Log.Trace("SetUp Backtesting: User: "******" ProjectId: " + job.ProjectId + " AlgoId: " + job.AlgorithmId);
Log.Trace("Dates: Start: " + job.PeriodStart.ToShortDateString() + " End: " + job.PeriodFinish.ToShortDateString() + " Cash: " + _startingCaptial.ToString("C"));
if (Errors.Count > 0)
{
initializeComplete = false;
}
return(initializeComplete);
}
/********************************************************
* CLASS METHODS
*********************************************************/
/// <summary>
/// Primary Analysis Thread:
/// </summary>
public static void Main(string[] args)
{
//Initialize:
var algorithmPath = "";
AlgorithmNodePacket job = null;
var timer = Stopwatch.StartNew();
var algorithm = default(IAlgorithm);
_version = DateTime.ParseExact(Config.Get("version", DateTime.Now.ToString(DateFormat.UI)), DateFormat.UI, CultureInfo.InvariantCulture);
//Name thread for the profiler:
Thread.CurrentThread.Name = "Algorithm Analysis Thread";
Log.Trace("Engine.Main(): LEAN ALGORITHMIC TRADING ENGINE v" + _version);
Log.Trace("Engine.Main(): Started " + DateTime.Now.ToShortTimeString());
Log.Trace("Engine.Main(): Memory " + OS.ApplicationMemoryUsed + "Mb-App " + +OS.TotalPhysicalMemoryUsed + "Mb-Used " + OS.TotalPhysicalMemory + "Mb-Total");
//Import external libraries specific to physical server location (cloud/local)
var catalog = new AggregateCatalog();
catalog.Catalogs.Add(new DirectoryCatalog(@"../../Extensions"));
var container = new CompositionContainer(catalog);
try
{
// grab the right export based on configuration
Notify = container.GetExportedValueByTypeName <IMessagingHandler>(Config.Get("messaging-handler"));
Queue = container.GetExportedValueByTypeName <IQueueHandler>(Config.Get("queue-handler"));
Api = container.GetExportedValueByTypeName <IApi>(Config.Get("api-handler"));
}
catch (CompositionException compositionException)
{ Log.Error("Engine.Main(): Failed to load library: " + compositionException); }
//Setup packeting, queue and controls system: These don't do much locally.
Api.Initialize();
Notify.Initialize();
Queue.Initialize(_liveMode);
//Start monitoring the backtest active status:
var statusPingThread = new Thread(StateCheck.Ping.Run);
statusPingThread.Start();
do
{
try
{
//Reset algo manager internal variables preparing for a new algorithm.
AlgorithmManager.ResetManager();
//Reset thread holders.
var initializeComplete = false;
Thread threadFeed = null;
Thread threadTransactions = null;
Thread threadResults = null;
Thread threadRealTime = null;
do
{
//-> Pull job from QuantConnect job queue, or, pull local build:
job = Queue.NextJob(out algorithmPath); // Blocking.
if (!IsLocal && LiveMode && (job.Version < Version || (job.Version == Version && job.Redelivered)))
{
//Tiny chance there was an uncontrolled collapse of a server, resulting in an old user task circulating.
//In this event kill the old algorithm and leave a message so the user can later review.
Queue.AcknowledgeJob(job);
Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, _collapseMessage);
Notify.SetChannel(job.Channel);
Notify.RuntimeError(job.AlgorithmId, _collapseMessage);
job = null;
}
} while (job == null);
//-> Initialize messaging system
Notify.SetChannel(job.Channel);
//-> Reset the backtest stopwatch; we're now running the algorithm.
timer.Restart();
//-> Create SetupHandler to configure internal algorithm state:
SetupHandler = GetSetupHandler(job.SetupEndpoint);
//-> Set the result handler type for this algorithm job, and launch the associated result thread.
ResultHandler = GetResultHandler(job);
threadResults = new Thread(ResultHandler.Run, 0)
{
Name = "Result Thread"
};
threadResults.Start();
try
{
// Save algorithm to cache, load algorithm instance:
algorithm = SetupHandler.CreateAlgorithmInstance(algorithmPath);
//Initialize the internal state of algorithm and job: executes the algorithm.Initialize() method.
initializeComplete = SetupHandler.Setup(algorithm, out _brokerage, job);
//If there are any reasons it failed, pass these back to the IDE.
if (!initializeComplete || algorithm.ErrorMessages.Count > 0 || SetupHandler.Errors.Count > 0)
{
initializeComplete = false;
//Get all the error messages: internal in algorithm and external in setup handler.
var errorMessage = String.Join(",", algorithm.ErrorMessages);
errorMessage += String.Join(",", SetupHandler.Errors);
throw new Exception(errorMessage);
}
}
catch (Exception err)
{
var runtimeMessage = "Algorithm.Initialize() Error: " + err.Message + " Stack Trace: " + err.StackTrace;
ResultHandler.RuntimeError(runtimeMessage, err.StackTrace);
Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, runtimeMessage);
}
//-> Using the job + initialization: load the designated handlers:
if (initializeComplete)
{
//Set algorithm as locked; set it to live mode if we're trading live, and set it to locked for no further updates.
algorithm.SetAlgorithmId(job.AlgorithmId);
algorithm.SetLiveMode(LiveMode);
algorithm.SetLocked();
//Load the associated handlers for data, transaction and realtime events:
ResultHandler.SetAlgorithm(algorithm);
DataFeed = GetDataFeedHandler(algorithm, job);
TransactionHandler = GetTransactionHandler(algorithm, _brokerage, ResultHandler, job);
RealTimeHandler = GetRealTimeHandler(algorithm, _brokerage, DataFeed, ResultHandler, job);
//Set the error handlers for the brokerage asynchronous errors.
SetupHandler.SetupErrorHandler(ResultHandler, _brokerage);
//Send status to user the algorithm is now executing.
ResultHandler.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.Running);
//Launch the data, transaction and realtime handlers into dedicated threads
threadFeed = new Thread(DataFeed.Run, 0)
{
Name = "DataFeed Thread"
};
threadTransactions = new Thread(TransactionHandler.Run, 0)
{
Name = "Transaction Thread"
};
threadRealTime = new Thread(RealTimeHandler.Run, 0)
{
Name = "RealTime Thread"
};
//Launch the data feed, result sending, and transaction models/handlers in separate threads.
threadFeed.Start(); // Data feed pushing data packets into thread bridge;
threadTransactions.Start(); // Transaction modeller scanning new order requests
threadRealTime.Start(); // RealTime scan time for time based events:
// Result manager scanning message queue: (started earlier)
try
{
// Execute the Algorithm Code:
var complete = Isolator.ExecuteWithTimeLimit(SetupHandler.MaximumRuntime, () =>
{
try
{
//Run Algorithm Job:
// -> Using this Data Feed,
// -> Send Orders to this TransactionHandler,
// -> Send Results to ResultHandler.
AlgorithmManager.Run(job, algorithm, DataFeed, TransactionHandler, ResultHandler, SetupHandler, RealTimeHandler);
}
catch (Exception err)
{
//Debugging at this level is difficult, stack trace needed.
Log.Error("Engine.Run(): Error in Algo Manager: " + err.Message + " ST >> " + err.StackTrace);
}
Log.Trace("Engine.Run(): Exiting Algorithm Manager");
}, MaximumRamAllocation);
if (!complete)
{
Log.Error("Engine.Main(): Failed to complete in time: " + SetupHandler.MaximumRuntime.ToString("F"));
throw new Exception("Failed to complete algorithm within " + SetupHandler.MaximumRuntime.ToString("F") + " seconds. Please make it run faster.");
}
// Algorithm runtime error:
if (AlgorithmManager.RunTimeError != null)
{
throw AlgorithmManager.RunTimeError;
}
}
catch (Exception err)
{
//Error running the user algorithm: purge datafeed, send error messages, set algorithm status to failed.
Log.Error("Engine.Run(): Breaking out of parent try-catch: " + err.Message + " " + err.StackTrace);
if (DataFeed != null)
{
DataFeed.Exit();
}
if (ResultHandler != null)
{
var message = "Runtime Error: " + err.Message;
Log.Trace("Engine.Run(): Sending runtime error to user...");
ResultHandler.LogMessage(message);
ResultHandler.RuntimeError(message, err.StackTrace);
Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, message + " Stack Trace: " + err.StackTrace);
}
}
//Send result data back: this entire code block could be rewritten.
// todo: - Split up statistics class, its enormous.
// todo: - Make a dedicated Statistics.Benchmark class.
// todo: - Move all creation and transmission of statistics out of primary engine loop.
// todo: - Statistics.Generate(algorithm, resulthandler, transactionhandler);
try
{
var charts = new Dictionary <string, Chart>(ResultHandler.Charts);
var orders = new Dictionary <int, Order>(algorithm.Transactions.Orders);
var holdings = new Dictionary <string, Holding>();
var statistics = new Dictionary <string, string>();
var banner = new Dictionary <string, string>();
try
{
//Generates error when things don't exist (no charting logged, runtime errors in main algo execution)
const string strategyEquityKey = "Strategy Equity";
const string equityKey = "Equity";
const string dailyPerformanceKey = "Daily Performance";
// make sure we've taken samples for these series before just blindly requesting them
if (charts.ContainsKey(strategyEquityKey) &&
charts[strategyEquityKey].Series.ContainsKey(equityKey) &&
charts[strategyEquityKey].Series.ContainsKey(dailyPerformanceKey))
{
var equity = charts[strategyEquityKey].Series[equityKey].Values;
var performance = charts[strategyEquityKey].Series[dailyPerformanceKey].Values;
var profitLoss =
new SortedDictionary <DateTime, decimal>(algorithm.Transactions.TransactionRecord);
statistics = Statistics.Statistics.Generate(equity, profitLoss, performance,
SetupHandler.StartingCapital, 252);
}
}
catch (Exception err)
{
Log.Error("Algorithm.Node.Engine(): Error generating statistics packet: " + err.Message);
}
//Diagnostics Completed, Send Result Packet:
ResultHandler.DebugMessage("Algorithm Id:(" + job.AlgorithmId + ") completed analysis in " + timer.Elapsed.TotalSeconds.ToString("F2") + " seconds");
ResultHandler.SendFinalResult(job, orders, algorithm.Transactions.TransactionRecord, holdings, statistics, banner);
}
catch (Exception err)
{
Log.Error("Engine.Main(): Error sending analysis result: " + err.Message + " ST >> " + err.StackTrace);
}
//Before we return, send terminate commands to close up the threads
timer.Stop(); //Algorithm finished running.
TransactionHandler.Exit();
DataFeed.Exit();
RealTimeHandler.Exit();
}
//Close result handler:
ResultHandler.Exit();
//Wait for the threads to complete:
var ts = Stopwatch.StartNew();
while ((ResultHandler.IsActive || (TransactionHandler != null && TransactionHandler.IsActive) || (DataFeed != null && DataFeed.IsActive)) && ts.ElapsedMilliseconds < 30 * 1000)
{
Thread.Sleep(100); Log.Trace("Waiting for threads to exit...");
}
if (threadFeed != null && threadFeed.IsAlive)
{
threadFeed.Abort();
}
if (threadTransactions != null && threadTransactions.IsAlive)
{
threadTransactions.Abort();
}
if (threadResults != null && threadResults.IsAlive)
{
threadResults.Abort();
}
Log.Trace("Engine.Main(): Analysis Completed and Results Posted.");
}
catch (Exception err)
{
Log.Error("Engine.Main(): Error running algorithm: " + err.Message + " >> " + err.StackTrace);
}
finally
{
//Delete the message from the job queue:
Queue.AcknowledgeJob(job);
Log.Trace("Engine.Main(): Packet removed from queue: " + job.AlgorithmId);
//No matter what for live mode; make sure we've set algorithm status in the API for "not running" conditions:
if (LiveMode && AlgorithmManager.State != AlgorithmStatus.Running && AlgorithmManager.State != AlgorithmStatus.RuntimeError)
{
Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmManager.State);
}
//Attempt to clean up ram usage:
GC.Collect();
}
//If we're running locally will execute just once.
} while (!IsLocal);
// Send the exit signal and then kill the thread
StateCheck.Ping.Exit();
// Make the console window pause so we can read log output before exiting and killing the application completely
Console.ReadKey();
//Finally if ping thread still not complete, kill.
if (statusPingThread != null && statusPingThread.IsAlive)
{
statusPingThread.Abort();
}
}
/// <summary>
/// Launch the algorithm manager to run this strategy
/// </summary>
/// <param name="job">Algorithm job</param>
/// <param name="algorithm">Algorithm instance</param>
/// <param name="feed">Datafeed object</param>
/// <param name="transactions">Transaction manager object</param>
/// <param name="results">Result handler object</param>
/// <param name="realtime">Realtime processing object</param>
/// <param name="token">Cancellation token</param>
/// <remarks>Modify with caution</remarks>
public void Run(AlgorithmNodePacket job, IAlgorithm algorithm, IDataFeed feed, ITransactionHandler transactions, IResultHandler results, IRealTimeHandler realtime, CancellationToken token)
{
//Initialize:
_dataPointCount = 0;
var startingPortfolioValue = algorithm.Portfolio.TotalPortfolioValue;
var backtestMode = (job.Type == PacketType.BacktestNode);
var methodInvokers = new Dictionary <Type, MethodInvoker>();
var marginCallFrequency = TimeSpan.FromMinutes(5);
var nextMarginCallTime = DateTime.MinValue;
//Initialize Properties:
_algorithmId = job.AlgorithmId;
_algorithmState = AlgorithmStatus.Running;
_previousTime = algorithm.StartDate.Date;
//Create the method accessors to push generic types into algorithm: Find all OnData events:
// Algorithm 2.0 data accessors
var hasOnDataTradeBars = AddMethodInvoker <TradeBars>(algorithm, methodInvokers);
var hasOnDataTicks = AddMethodInvoker <Ticks>(algorithm, methodInvokers);
// dividend and split events
var hasOnDataDividends = AddMethodInvoker <Dividends>(algorithm, methodInvokers);
var hasOnDataSplits = AddMethodInvoker <Splits>(algorithm, methodInvokers);
// Algorithm 3.0 data accessors
var hasOnDataSlice = algorithm.GetType().GetMethods()
.Where(x => x.Name == "OnData" && x.GetParameters().Length == 1 && x.GetParameters()[0].ParameterType == typeof(Slice))
.FirstOrDefault(x => x.DeclaringType == algorithm.GetType()) != null;
//Go through the subscription types and create invokers to trigger the event handlers for each custom type:
foreach (var config in feed.Subscriptions)
{
//If type is a tradebar, combine tradebars and ticks into unified array:
if (config.Type.Name != "TradeBar" && config.Type.Name != "Tick")
{
//Get the matching method for this event handler - e.g. public void OnData(Quandl data) { .. }
var genericMethod = (algorithm.GetType()).GetMethod("OnData", new[] { config.Type });
//If we already have this Type-handler then don't add it to invokers again.
if (methodInvokers.ContainsKey(config.Type))
{
continue;
}
//If we couldnt find the event handler, let the user know we can't fire that event.
if (genericMethod == null && !hasOnDataSlice)
{
algorithm.RunTimeError = new Exception("Data event handler not found, please create a function matching this template: public void OnData(" + config.Type.Name + " data) { }");
_algorithmState = AlgorithmStatus.RuntimeError;
return;
}
if (genericMethod != null)
{
methodInvokers.Add(config.Type, genericMethod.DelegateForCallMethod());
}
}
}
//Loop over the queues: get a data collection, then pass them all into relevent methods in the algorithm.
Log.Trace("AlgorithmManager.Run(): Begin DataStream - Start: " + algorithm.StartDate + " Stop: " + algorithm.EndDate);
foreach (var timeSlice in feed.Bridge.GetConsumingEnumerable(token))
{
// reset our timer on each loop
_currentTimeStepTime = DateTime.UtcNow;
//Check this backtest is still running:
if (_algorithmState != AlgorithmStatus.Running)
{
Log.Error(string.Format("AlgorithmManager.Run(): Algorthm state changed to {0} at {1}", _algorithmState, timeSlice.Time));
break;
}
//Execute with TimeLimit Monitor:
if (token.IsCancellationRequested)
{
Log.Error("AlgorithmManager.Run(): CancellationRequestion at " + timeSlice.Time);
return;
}
var time = timeSlice.Time;
var newData = timeSlice.Data;
//If we're in backtest mode we need to capture the daily performance. We do this here directly
//before updating the algorithm state with the new data from this time step, otherwise we'll
//produce incorrect samples (they'll take into account this time step's new price values)
if (backtestMode)
{
//On day-change sample equity and daily performance for statistics calculations
if (_previousTime.Date != time.Date)
{
//Sample the portfolio value over time for chart.
results.SampleEquity(_previousTime, Math.Round(algorithm.Portfolio.TotalPortfolioValue, 4));
//Check for divide by zero
if (startingPortfolioValue == 0m)
{
results.SamplePerformance(_previousTime.Date, 0);
}
else
{
results.SamplePerformance(_previousTime.Date, Math.Round((algorithm.Portfolio.TotalPortfolioValue - startingPortfolioValue) * 100 / startingPortfolioValue, 10));
}
startingPortfolioValue = algorithm.Portfolio.TotalPortfolioValue;
}
}
//Update algorithm state after capturing performance from previous day
//On each time step push the real time prices to the cashbook so we can have updated conversion rates
algorithm.Portfolio.CashBook.Update(newData);
//Update the securities properties: first before calling user code to avoid issues with data
algorithm.Securities.Update(time, newData);
//Set the algorithm and real time handler's time
algorithm.SetDateTime(time);
realtime.SetTime(time);
// process fill models on the updated data before entering algorithm, applies to all non-market orders
transactions.ProcessSynchronousEvents();
//Check if the user's signalled Quit: loop over data until day changes.
if (algorithm.GetQuit())
{
_algorithmState = AlgorithmStatus.Quit;
Log.Trace("AlgorithmManager.Run(): Algorithm quit requested.");
break;
}
if (algorithm.RunTimeError != null)
{
_algorithmState = AlgorithmStatus.RuntimeError;
Log.Trace(string.Format("AlgorithmManager.Run(): Algorithm encountered a runtime error at {0}. Error: {1}", timeSlice.Time,
algorithm.RunTimeError));
break;
}
// perform margin calls, in live mode we can also use realtime to emit these
if (time >= nextMarginCallTime || (_liveMode && nextMarginCallTime > DateTime.Now))
{
// determine if there are possible margin call orders to be executed
bool issueMarginCallWarning;
var marginCallOrders = algorithm.Portfolio.ScanForMarginCall(out issueMarginCallWarning);
if (marginCallOrders.Count != 0)
{
try
{
// tell the algorithm we're about to issue the margin call
algorithm.OnMarginCall(marginCallOrders);
}
catch (Exception err)
{
algorithm.RunTimeError = err;
_algorithmState = AlgorithmStatus.RuntimeError;
Log.Error("AlgorithmManager.Run(): RuntimeError: OnMarginCall: " + err.Message + " STACK >>> " + err.StackTrace);
return;
}
// execute the margin call orders
var executedOrders = algorithm.Portfolio.MarginCallModel.ExecuteMarginCall(marginCallOrders);
foreach (var order in executedOrders)
{
algorithm.Error(string.Format("{0} - Executed MarginCallOrder: {1} - Quantity: {2} @ {3}", algorithm.Time, order.Symbol, order.Quantity, order.Price));
}
}
// we didn't perform a margin call, but got the warning flag back, so issue the warning to the algorithm
else if (issueMarginCallWarning)
{
try
{
algorithm.OnMarginCallWarning();
}
catch (Exception err)
{
algorithm.RunTimeError = err;
_algorithmState = AlgorithmStatus.RuntimeError;
Log.Error("AlgorithmManager.Run(): RuntimeError: OnMarginCallWarning: " + err.Message + " STACK >>> " + err.StackTrace);
}
}
nextMarginCallTime = time + marginCallFrequency;
}
//Trigger the data events: Invoke the types we have data for:
var newBars = new TradeBars(time);
var newTicks = new Ticks(time);
var newDividends = new Dividends(time);
var newSplits = new Splits(time);
//Invoke all non-tradebars, non-ticks methods and build up the TradeBars and Ticks dictionaries
// --> i == Subscription Configuration Index, so we don't need to compare types.
foreach (var i in newData.Keys)
{
//Data point and config of this point:
var dataPoints = newData[i];
var config = feed.Subscriptions[i];
//Keep track of how many data points we've processed
_dataPointCount += dataPoints.Count;
//We don't want to pump data that we added just for currency conversions
if (config.IsInternalFeed)
{
continue;
}
//Create TradeBars Unified Data --> OR --> invoke generic data event. One loop.
// Aggregate Dividends and Splits -- invoke portfolio application methods
foreach (var dataPoint in dataPoints)
{
var dividend = dataPoint as Dividend;
if (dividend != null)
{
Log.Trace("AlgorithmManager.Run(): Applying Dividend for " + dividend.Symbol);
// if this is a dividend apply to portfolio
algorithm.Portfolio.ApplyDividend(dividend);
if (hasOnDataDividends)
{
// and add to our data dictionary to pump into OnData(Dividends data)
newDividends.Add(dividend);
}
continue;
}
var split = dataPoint as Split;
if (split != null)
{
Log.Trace("AlgorithmManager.Run(): Applying Split for " + split.Symbol);
// if this is a split apply to portfolio
algorithm.Portfolio.ApplySplit(split);
if (hasOnDataSplits)
{
// and add to our data dictionary to pump into OnData(Splits data)
newSplits.Add(split);
}
continue;
}
//Update registered consolidators for this symbol index
try
{
for (var j = 0; j < config.Consolidators.Count; j++)
{
config.Consolidators[j].Update(dataPoint);
}
}
catch (Exception err)
{
algorithm.RunTimeError = err;
_algorithmState = AlgorithmStatus.RuntimeError;
Log.Error("AlgorithmManager.Run(): RuntimeError: Consolidators update: " + err.Message);
return;
}
// TRADEBAR -- add to our dictionary
if (dataPoint.DataType == MarketDataType.TradeBar)
{
var bar = dataPoint as TradeBar;
if (bar != null)
{
newBars[bar.Symbol] = bar;
continue;
}
}
// TICK -- add to our dictionary
if (dataPoint.DataType == MarketDataType.Tick)
{
var tick = dataPoint as Tick;
if (tick != null)
{
List <Tick> ticks;
if (!newTicks.TryGetValue(tick.Symbol, out ticks))
{
ticks = new List <Tick>(3);
newTicks.Add(tick.Symbol, ticks);
}
ticks.Add(tick);
continue;
}
}
// if it was nothing else then it must be custom data
// CUSTOM DATA -- invoke on data method
//Send data into the generic algorithm event handlers
try
{
MethodInvoker methodInvoker;
if (methodInvokers.TryGetValue(config.Type, out methodInvoker))
{
methodInvoker(algorithm, dataPoint);
}
}
catch (Exception err)
{
algorithm.RunTimeError = err;
_algorithmState = AlgorithmStatus.RuntimeError;
Log.Error("AlgorithmManager.Run(): RuntimeError: Custom Data: " + err.Message + " STACK >>> " + err.StackTrace);
return;
}
}
}
try
{
// fire off the dividend and split events before pricing events
if (hasOnDataDividends && newDividends.Count != 0)
{
methodInvokers[typeof(Dividends)](algorithm, newDividends);
}
if (hasOnDataSplits && newSplits.Count != 0)
{
methodInvokers[typeof(Splits)](algorithm, newSplits);
}
}
catch (Exception err)
{
algorithm.RunTimeError = err;
_algorithmState = AlgorithmStatus.RuntimeError;
Log.Error("AlgorithmManager.Run(): RuntimeError: Dividends/Splits: " + err.Message + " STACK >>> " + err.StackTrace);
return;
}
//After we've fired all other events in this second, fire the pricing events:
try
{
if (hasOnDataTradeBars && newBars.Count > 0)
{
methodInvokers[typeof(TradeBars)](algorithm, newBars);
}
if (hasOnDataTicks && newTicks.Count > 0)
{
methodInvokers[typeof(Ticks)](algorithm, newTicks);
}
}
catch (Exception err)
{
algorithm.RunTimeError = err;
_algorithmState = AlgorithmStatus.RuntimeError;
Log.Error("AlgorithmManager.Run(): RuntimeError: New Style Mode: " + err.Message + " STACK >>> " + err.StackTrace);
return;
}
// EVENT HANDLER v3.0 -- all data in a single event
var slice = new Slice(time, newData.Values.SelectMany(x => x),
newBars.Count == 0 ? null : newBars,
newTicks.Count == 0 ? null : newTicks,
newSplits.Count == 0 ? null : newSplits,
newDividends.Count == 0 ? null : newDividends
);
algorithm.OnData(slice);
//If its the historical/paper trading models, wait until market orders have been "filled"
// Manually trigger the event handler to prevent thread switch.
transactions.ProcessSynchronousEvents();
//Save the previous time for the sample calculations
_previousTime = time;
// Process any required events of the results handler such as sampling assets, equity, or stock prices.
results.ProcessSynchronousEvents();
} // End of ForEach feed.Bridge.GetConsumingEnumerable
// stop timing the loops
_currentTimeStepTime = DateTime.MinValue;
//Stream over:: Send the final packet and fire final events:
Log.Trace("AlgorithmManager.Run(): Firing On End Of Algorithm...");
try
{
algorithm.OnEndOfAlgorithm();
}
catch (Exception err)
{
_algorithmState = AlgorithmStatus.RuntimeError;
algorithm.RunTimeError = new Exception("Error running OnEndOfAlgorithm(): " + err.Message, err.InnerException);
Log.Error("AlgorithmManager.OnEndOfAlgorithm(): " + err.Message + " STACK >>> " + err.StackTrace);
return;
}
// Process any required events of the results handler such as sampling assets, equity, or stock prices.
results.ProcessSynchronousEvents(forceProcess: true);
//Liquidate Holdings for Calculations:
if (_algorithmState == AlgorithmStatus.Liquidated || !_liveMode)
{
// without this we can't liquidate equities since the exchange is 'technically' closed
var hackedFrontier = algorithm.Time == DateTime.MinValue ? DateTime.MinValue : algorithm.Time.AddMilliseconds(-1);
algorithm.SetDateTime(hackedFrontier);
foreach (var security in algorithm.Securities)
{
security.Value.SetMarketPrice(hackedFrontier, null);
}
Log.Trace("AlgorithmManager.Run(): Liquidating algorithm holdings...");
algorithm.Liquidate();
results.LogMessage("Algorithm Liquidated");
results.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.Liquidated);
}
//Manually stopped the algorithm
if (_algorithmState == AlgorithmStatus.Stopped)
{
Log.Trace("AlgorithmManager.Run(): Stopping algorithm...");
results.LogMessage("Algorithm Stopped");
results.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.Stopped);
}
//Backtest deleted.
if (_algorithmState == AlgorithmStatus.Deleted)
{
Log.Trace("AlgorithmManager.Run(): Deleting algorithm...");
results.DebugMessage("Algorithm Id:(" + job.AlgorithmId + ") Deleted by request.");
results.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.Deleted);
}
//Algorithm finished, send regardless of commands:
results.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.Completed);
//Take final samples:
results.SampleRange(algorithm.GetChartUpdates());
results.SampleEquity(_previousTime, Math.Round(algorithm.Portfolio.TotalPortfolioValue, 4));
results.SamplePerformance(_previousTime, Math.Round((algorithm.Portfolio.TotalPortfolioValue - startingPortfolioValue) * 100 / startingPortfolioValue, 10));
} // End of Run();
/// <summary>
/// Setup the algorithm cash, dates and portfolio as desired.
/// </summary>
/// <param name="algorithm">Existing algorithm instance</param>
/// <param name="brokerage">New brokerage instance</param>
/// <param name="baseJob">Backtesting job</param>
/// <param name="resultHandler">The configured result handler</param>
/// <param name="transactionHandler">The configuration transaction handler</param>
/// <param name="realTimeHandler">The configured real time handler</param>
/// <returns>Boolean true on successfully setting up the console.</returns>
public bool Setup(IAlgorithm algorithm, IBrokerage brokerage, AlgorithmNodePacket baseJob, IResultHandler resultHandler, ITransactionHandler transactionHandler, IRealTimeHandler realTimeHandler)
{
var initializeComplete = false;
try
{
//Set common variables for console programs:
if (baseJob.Type == PacketType.BacktestNode)
{
var backtestJob = baseJob as BacktestNodePacket;
algorithm.SetMaximumOrders(int.MaxValue);
// set our parameters
algorithm.SetParameters(baseJob.Parameters);
algorithm.SetLiveMode(false);
//Set the source impl for the event scheduling
algorithm.Schedule.SetEventSchedule(realTimeHandler);
//Setup Base Algorithm:
algorithm.Initialize();
//Set the time frontier of the algorithm
algorithm.SetDateTime(algorithm.StartDate.ConvertToUtc(algorithm.TimeZone));
//Construct the backtest job packet:
backtestJob.PeriodStart = algorithm.StartDate;
backtestJob.PeriodFinish = algorithm.EndDate;
backtestJob.BacktestId = "LOCALHOST";
backtestJob.Type = PacketType.BacktestNode;
backtestJob.UserId = !string.IsNullOrWhiteSpace(Config.Get("qc-user-id")) ? Convert.ToInt32(Config.Get("qc-user-id")) : 1001;
backtestJob.Channel = Config.Get("api-access-token");
//Backtest Specific Parameters:
StartingDate = backtestJob.PeriodStart;
StartingPortfolioValue = algorithm.Portfolio.Cash;
}
else
{
throw new Exception("The ConsoleSetupHandler is for backtests only. Use the BrokerageSetupHandler.");
}
}
catch (Exception err)
{
Log.Error(err);
Errors.Add("Failed to initialize algorithm: Initialize(): " + err.Message);
}
if (Errors.Count == 0)
{
initializeComplete = true;
}
algorithm.Transactions.SetOrderProcessor(transactionHandler);
algorithm.PostInitialize();
return initializeComplete;
}
/// <summary>
/// Launch the algorithm manager to run this strategy
/// </summary>
/// <param name="job">Algorithm job</param>
/// <param name="algorithm">Algorithm instance</param>
/// <param name="feed">Datafeed object</param>
/// <param name="transactions">Transaction manager object</param>
/// <param name="results">Result handler object</param>
/// <param name="realtime">Realtime processing object</param>
/// <param name="token">Cancellation token</param>
/// <remarks>Modify with caution</remarks>
public void Run(AlgorithmNodePacket job, IAlgorithm algorithm, IDataFeed feed, ITransactionHandler transactions, IResultHandler results, IRealTimeHandler realtime, CancellationToken token)
{
//Initialize:
_dataPointCount = 0;
var portfolioValue = algorithm.Portfolio.TotalPortfolioValue;
var backtestMode = (job.Type == PacketType.BacktestNode);
var methodInvokers = new Dictionary <Type, MethodInvoker>();
var marginCallFrequency = TimeSpan.FromMinutes(5);
var nextMarginCallTime = DateTime.MinValue;
var delistingTickets = new List <OrderTicket>();
//Initialize Properties:
_algorithmId = job.AlgorithmId;
_algorithmState = AlgorithmStatus.Running;
_previousTime = algorithm.StartDate.Date;
//Create the method accessors to push generic types into algorithm: Find all OnData events:
// Algorithm 2.0 data accessors
var hasOnDataTradeBars = AddMethodInvoker <TradeBars>(algorithm, methodInvokers);
var hasOnDataTicks = AddMethodInvoker <Ticks>(algorithm, methodInvokers);
// dividend and split events
var hasOnDataDividends = AddMethodInvoker <Dividends>(algorithm, methodInvokers);
var hasOnDataSplits = AddMethodInvoker <Splits>(algorithm, methodInvokers);
var hasOnDataDelistings = AddMethodInvoker <Delistings>(algorithm, methodInvokers);
var hasOnDataSymbolChangedEvents = AddMethodInvoker <SymbolChangedEvents>(algorithm, methodInvokers);
// Algorithm 3.0 data accessors
var hasOnDataSlice = algorithm.GetType().GetMethods()
.Where(x => x.Name == "OnData" && x.GetParameters().Length == 1 && x.GetParameters()[0].ParameterType == typeof(Slice))
.FirstOrDefault(x => x.DeclaringType == algorithm.GetType()) != null;
//Go through the subscription types and create invokers to trigger the event handlers for each custom type:
foreach (var config in feed.Subscriptions.Select(x => x.Configuration))
{
//If type is a tradebar, combine tradebars and ticks into unified array:
if (config.Type.Name != "TradeBar" && config.Type.Name != "Tick" && !config.IsInternalFeed)
{
//Get the matching method for this event handler - e.g. public void OnData(Quandl data) { .. }
var genericMethod = (algorithm.GetType()).GetMethod("OnData", new[] { config.Type });
//If we already have this Type-handler then don't add it to invokers again.
if (methodInvokers.ContainsKey(config.Type))
{
continue;
}
//If we couldnt find the event handler, let the user know we can't fire that event.
if (genericMethod == null && !hasOnDataSlice)
{
algorithm.RunTimeError = new Exception("Data event handler not found, please create a function matching this template: public void OnData(" + config.Type.Name + " data) { }");
_algorithmState = AlgorithmStatus.RuntimeError;
return;
}
if (genericMethod != null)
{
methodInvokers.Add(config.Type, genericMethod.DelegateForCallMethod());
}
}
}
// wire up universe selection. it is assumed that the data feed will perform the
// required thread synchronization.
var universeSelection = new UniverseSelection(feed, algorithm, _liveMode);
feed.Fundamental += (sender, args) =>
{
var market = args.Configuration.Market;
var localTime = args.DateTimeUtc.ConvertFromUtc(args.Configuration.TimeZone);
universeSelection.ApplyUniverseSelection(localTime, market, args.Data.OfType <CoarseFundamental>());
};
//Loop over the queues: get a data collection, then pass them all into relevent methods in the algorithm.
Log.Trace("AlgorithmManager.Run(): Begin DataStream - Start: " + algorithm.StartDate + " Stop: " + algorithm.EndDate);
foreach (var timeSlice in Stream(job, algorithm, feed, results, token))
{
// reset our timer on each loop
_currentTimeStepTime = DateTime.UtcNow;
//Check this backtest is still running:
if (_algorithmState != AlgorithmStatus.Running)
{
Log.Error(string.Format("AlgorithmManager.Run(): Algorthm state changed to {0} at {1}", _algorithmState, timeSlice.Time));
break;
}
//Execute with TimeLimit Monitor:
if (token.IsCancellationRequested)
{
Log.Error("AlgorithmManager.Run(): CancellationRequestion at " + timeSlice.Time);
return;
}
var time = timeSlice.Time;
_dataPointCount += timeSlice.DataPointCount;
//If we're in backtest mode we need to capture the daily performance. We do this here directly
//before updating the algorithm state with the new data from this time step, otherwise we'll
//produce incorrect samples (they'll take into account this time step's new price values)
if (backtestMode)
{
//On day-change sample equity and daily performance for statistics calculations
if (_previousTime.Date != time.Date)
{
SampleBenchmark(algorithm, results, _previousTime.Date);
//Sample the portfolio value over time for chart.
results.SampleEquity(_previousTime, Math.Round(algorithm.Portfolio.TotalPortfolioValue, 4));
//Check for divide by zero
if (portfolioValue == 0m)
{
results.SamplePerformance(_previousTime.Date, 0);
}
else
{
results.SamplePerformance(_previousTime.Date, Math.Round((algorithm.Portfolio.TotalPortfolioValue - portfolioValue) * 100 / portfolioValue, 10));
}
portfolioValue = algorithm.Portfolio.TotalPortfolioValue;
}
}
else
{
// live mode continously sample the benchmark
SampleBenchmark(algorithm, results, time);
}
//Update algorithm state after capturing performance from previous day
//Set the algorithm and real time handler's time
algorithm.SetDateTime(time);
if (timeSlice.Slice.SymbolChangedEvents.Count != 0)
{
if (hasOnDataSymbolChangedEvents)
{
methodInvokers[typeof(SymbolChangedEvents)](algorithm, timeSlice.Slice.SymbolChangedEvents);
}
foreach (var symbol in timeSlice.Slice.SymbolChangedEvents.Keys)
{
// cancel all orders for the old symbol
foreach (var ticket in transactions.GetOrderTickets(x => x.Status.IsOpen() && x.Symbol == symbol))
{
ticket.Cancel("Open order cancelled on symbol changed event");
}
}
}
if (timeSlice.SecurityChanges != SecurityChanges.None)
{
foreach (var security in timeSlice.SecurityChanges.AddedSecurities)
{
if (!algorithm.Securities.ContainsKey(security.Symbol))
{
// add the new security
algorithm.Securities.Add(security);
}
}
}
//On each time step push the real time prices to the cashbook so we can have updated conversion rates
foreach (var kvp in timeSlice.CashBookUpdateData)
{
kvp.Key.Update(kvp.Value);
}
//Update the securities properties: first before calling user code to avoid issues with data
foreach (var kvp in timeSlice.SecuritiesUpdateData)
{
kvp.Key.SetMarketPrice(kvp.Value);
// Send market price updates to the TradeBuilder
if (kvp.Value != null)
{
algorithm.TradeBuilder.SetMarketPrice(kvp.Key.Symbol, kvp.Value.Price);
}
}
// fire real time events after we've updated based on the new data
realtime.SetTime(timeSlice.Time);
// process fill models on the updated data before entering algorithm, applies to all non-market orders
transactions.ProcessSynchronousEvents();
if (delistingTickets.Count != 0)
{
for (int i = 0; i < delistingTickets.Count; i++)
{
var ticket = delistingTickets[i];
if (ticket.Status == OrderStatus.Filled)
{
algorithm.Securities.Remove(ticket.Symbol);
delistingTickets.RemoveAt(i--);
Log.Trace("AlgorithmManager.Run(): Delisted Security removed: " + ticket.Symbol.Permtick);
}
}
}
//Check if the user's signalled Quit: loop over data until day changes.
if (algorithm.GetQuit())
{
_algorithmState = AlgorithmStatus.Quit;
Log.Trace("AlgorithmManager.Run(): Algorithm quit requested.");
break;
}
if (algorithm.RunTimeError != null)
{
_algorithmState = AlgorithmStatus.RuntimeError;
Log.Trace(string.Format("AlgorithmManager.Run(): Algorithm encountered a runtime error at {0}. Error: {1}", timeSlice.Time, algorithm.RunTimeError));
break;
}
// perform margin calls, in live mode we can also use realtime to emit these
if (time >= nextMarginCallTime || (_liveMode && nextMarginCallTime > DateTime.Now))
{
// determine if there are possible margin call orders to be executed
bool issueMarginCallWarning;
var marginCallOrders = algorithm.Portfolio.ScanForMarginCall(out issueMarginCallWarning);
if (marginCallOrders.Count != 0)
{
var executingMarginCall = false;
try
{
// tell the algorithm we're about to issue the margin call
algorithm.OnMarginCall(marginCallOrders);
executingMarginCall = true;
// execute the margin call orders
var executedTickets = algorithm.Portfolio.MarginCallModel.ExecuteMarginCall(marginCallOrders);
foreach (var ticket in executedTickets)
{
algorithm.Error(string.Format("{0} - Executed MarginCallOrder: {1} - Quantity: {2} @ {3}", algorithm.Time, ticket.Symbol, ticket.Quantity, ticket.AverageFillPrice));
}
}
catch (Exception err)
{
algorithm.RunTimeError = err;
_algorithmState = AlgorithmStatus.RuntimeError;
var locator = executingMarginCall ? "Portfolio.MarginCallModel.ExecuteMarginCall" : "OnMarginCall";
Log.Error(string.Format("AlgorithmManager.Run(): RuntimeError: {0}: ", locator) + err.Message + " STACK >>> " + err.StackTrace);
return;
}
}
// we didn't perform a margin call, but got the warning flag back, so issue the warning to the algorithm
else if (issueMarginCallWarning)
{
try
{
algorithm.OnMarginCallWarning();
}
catch (Exception err)
{
algorithm.RunTimeError = err;
_algorithmState = AlgorithmStatus.RuntimeError;
Log.Error("AlgorithmManager.Run(): RuntimeError: OnMarginCallWarning: " + err.Message + " STACK >>> " + err.StackTrace);
return;
}
}
nextMarginCallTime = time + marginCallFrequency;
}
// before we call any events, let the algorithm know about universe changes
if (timeSlice.SecurityChanges != SecurityChanges.None)
{
try
{
algorithm.OnSecuritiesChanged(timeSlice.SecurityChanges);
}
catch (Exception err)
{
algorithm.RunTimeError = err;
_algorithmState = AlgorithmStatus.RuntimeError;
Log.Error("AlgorithmManager.Run(): RuntimeError: OnSecuritiesChanged event: " + err.Message);
return;
}
}
// apply dividends
foreach (var dividend in timeSlice.Slice.Dividends.Values)
{
Log.Trace("AlgorithmManager.Run(): Applying Dividend for " + dividend.Symbol, true);
algorithm.Portfolio.ApplyDividend(dividend);
}
// apply splits
foreach (var split in timeSlice.Slice.Splits.Values)
{
try
{
Log.Trace("AlgorithmManager.Run(): Applying Split for " + split.Symbol, true);
algorithm.Portfolio.ApplySplit(split);
// apply the split to open orders as well in raw mode, all other modes are split adjusted
if (_liveMode || algorithm.Securities[split.Symbol].SubscriptionDataConfig.DataNormalizationMode == DataNormalizationMode.Raw)
{
// in live mode we always want to have our order match the order at the brokerage, so apply the split to the orders
var openOrders = transactions.GetOrderTickets(ticket => ticket.Status.IsOpen() && ticket.Symbol == split.Symbol);
algorithm.BrokerageModel.ApplySplit(openOrders.ToList(), split);
}
}
catch (Exception err)
{
algorithm.RunTimeError = err;
_algorithmState = AlgorithmStatus.RuntimeError;
Log.Error("AlgorithmManager.Run(): RuntimeError: Split event: " + err.Message);
return;
}
}
//Update registered consolidators for this symbol index
try
{
foreach (var kvp in timeSlice.ConsolidatorUpdateData)
{
var consolidators = kvp.Key.Consolidators;
foreach (var dataPoint in kvp.Value)
{
foreach (var consolidator in consolidators)
{
consolidator.Update(dataPoint);
}
}
}
}
catch (Exception err)
{
algorithm.RunTimeError = err;
_algorithmState = AlgorithmStatus.RuntimeError;
Log.Error("AlgorithmManager.Run(): RuntimeError: Consolidators update: " + err.Message);
return;
}
// fire custom event handlers
foreach (var kvp in timeSlice.CustomData)
{
MethodInvoker methodInvoker;
if (!methodInvokers.TryGetValue(kvp.Key.SubscriptionDataConfig.Type, out methodInvoker))
{
continue;
}
try
{
foreach (var dataPoint in kvp.Value)
{
methodInvoker(algorithm, dataPoint);
}
}
catch (Exception err)
{
algorithm.RunTimeError = err;
_algorithmState = AlgorithmStatus.RuntimeError;
Log.Error("AlgorithmManager.Run(): RuntimeError: Custom Data: " + err.Message + " STACK >>> " + err.StackTrace);
return;
}
}
try
{
// fire off the dividend and split events before pricing events
if (hasOnDataDividends && timeSlice.Slice.Dividends.Count != 0)
{
methodInvokers[typeof(Dividends)](algorithm, timeSlice.Slice.Dividends);
}
if (hasOnDataSplits && timeSlice.Slice.Splits.Count != 0)
{
methodInvokers[typeof(Splits)](algorithm, timeSlice.Slice.Splits);
}
if (hasOnDataDelistings && timeSlice.Slice.Delistings.Count != 0)
{
methodInvokers[typeof(Delistings)](algorithm, timeSlice.Slice.Delistings);
}
}
catch (Exception err)
{
algorithm.RunTimeError = err;
_algorithmState = AlgorithmStatus.RuntimeError;
Log.Error("AlgorithmManager.Run(): RuntimeError: Dividends/Splits/Delistings: " + err.Message + " STACK >>> " + err.StackTrace);
return;
}
// run the delisting logic after firing delisting events
HandleDelistedSymbols(algorithm, timeSlice.Slice.Delistings, delistingTickets);
//After we've fired all other events in this second, fire the pricing events:
try
{
if (hasOnDataTradeBars && timeSlice.Slice.Bars.Count > 0)
{
methodInvokers[typeof(TradeBars)](algorithm, timeSlice.Slice.Bars);
}
if (hasOnDataTicks && timeSlice.Slice.Ticks.Count > 0)
{
methodInvokers[typeof(Ticks)](algorithm, timeSlice.Slice.Ticks);
}
}
catch (Exception err)
{
algorithm.RunTimeError = err;
_algorithmState = AlgorithmStatus.RuntimeError;
Log.Error("AlgorithmManager.Run(): RuntimeError: New Style Mode: " + err.Message + " STACK >>> " + err.StackTrace);
return;
}
try
{
if (timeSlice.Slice.Count != 0)
{
// EVENT HANDLER v3.0 -- all data in a single event
algorithm.OnData(timeSlice.Slice);
}
}
catch (Exception err)
{
algorithm.RunTimeError = err;
_algorithmState = AlgorithmStatus.RuntimeError;
Log.Error("AlgorithmManager.Run(): RuntimeError: Slice: " + err.Message + " STACK >>> " + err.StackTrace);
return;
}
//If its the historical/paper trading models, wait until market orders have been "filled"
// Manually trigger the event handler to prevent thread switch.
transactions.ProcessSynchronousEvents();
//Save the previous time for the sample calculations
_previousTime = time;
// Process any required events of the results handler such as sampling assets, equity, or stock prices.
results.ProcessSynchronousEvents();
} // End of ForEach feed.Bridge.GetConsumingEnumerable
// stop timing the loops
_currentTimeStepTime = DateTime.MinValue;
//Stream over:: Send the final packet and fire final events:
Log.Trace("AlgorithmManager.Run(): Firing On End Of Algorithm...");
try
{
algorithm.OnEndOfAlgorithm();
}
catch (Exception err)
{
_algorithmState = AlgorithmStatus.RuntimeError;
algorithm.RunTimeError = new Exception("Error running OnEndOfAlgorithm(): " + err.Message, err.InnerException);
Log.Error("AlgorithmManager.OnEndOfAlgorithm(): " + err.Message + " STACK >>> " + err.StackTrace);
return;
}
// Process any required events of the results handler such as sampling assets, equity, or stock prices.
results.ProcessSynchronousEvents(forceProcess: true);
//Liquidate Holdings for Calculations:
if (_algorithmState == AlgorithmStatus.Liquidated && _liveMode)
{
Log.Trace("AlgorithmManager.Run(): Liquidating algorithm holdings...");
algorithm.Liquidate();
results.LogMessage("Algorithm Liquidated");
results.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.Liquidated);
}
//Manually stopped the algorithm
if (_algorithmState == AlgorithmStatus.Stopped)
{
Log.Trace("AlgorithmManager.Run(): Stopping algorithm...");
results.LogMessage("Algorithm Stopped");
results.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.Stopped);
}
//Backtest deleted.
if (_algorithmState == AlgorithmStatus.Deleted)
{
Log.Trace("AlgorithmManager.Run(): Deleting algorithm...");
results.DebugMessage("Algorithm Id:(" + job.AlgorithmId + ") Deleted by request.");
results.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.Deleted);
}
//Algorithm finished, send regardless of commands:
results.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.Completed);
//Take final samples:
results.SampleRange(algorithm.GetChartUpdates());
results.SampleEquity(_previousTime, Math.Round(algorithm.Portfolio.TotalPortfolioValue, 4));
SampleBenchmark(algorithm, results, _previousTime);
results.SamplePerformance(_previousTime, Math.Round((algorithm.Portfolio.TotalPortfolioValue - portfolioValue) * 100 / portfolioValue, 10));
} // End of Run();
/********************************************************
* CLASS METHODS
*********************************************************/
/// <summary>
/// Launch the algorithm manager to run this strategy
/// </summary>
/// <param name="job">Algorithm job</param>
/// <param name="algorithm">Algorithm instance</param>
/// <param name="feed">Datafeed object</param>
/// <param name="transactions">Transaction manager object</param>
/// <param name="results">Result handler object</param>
/// <param name="setup">Setup handler object</param>
/// <param name="realtime">Realtime processing object</param>
/// <remarks>Modify with caution</remarks>
public static void Run(AlgorithmNodePacket job, IAlgorithm algorithm, IDataFeed feed, ITransactionHandler transactions, IResultHandler results, ISetupHandler setup, IRealTimeHandler realtime)
{
//Initialize:
_dataPointCount = 0;
var backwardsCompatibilityMode = false;
var tradebarsType = typeof(TradeBars);
var ticksType = typeof(Ticks);
var startingPortfolioValue = setup.StartingCapital;
var backtestMode = (job.Type == PacketType.BacktestNode);
var methodInvokers = new Dictionary <Type, MethodInvoker>();
//Initialize Properties:
_frontier = setup.StartingDate;
_algorithmId = job.AlgorithmId;
_algorithmState = AlgorithmStatus.Running;
_previousTime = setup.StartingDate.Date;
//Create the method accessors to push generic types into algorithm: Find all OnData events:
//Algorithm 1.0 Data Accessors.
//If the users defined these methods, add them in manually. This allows keeping backwards compatibility to algorithm 1.0.
var oldTradeBarsMethodInfo = (algorithm.GetType()).GetMethod("OnTradeBar", new[] { typeof(Dictionary <string, TradeBar>) });
var oldTicksMethodInfo = (algorithm.GetType()).GetMethod("OnTick", new[] { typeof(Dictionary <string, List <Tick> >) });
//Algorithm 2.0 Data Generics Accessors.
//New hidden access to tradebars with custom type.
var newTradeBarsMethodInfo = (algorithm.GetType()).GetMethod("OnData", new[] { tradebarsType });
var newTicksMethodInfo = (algorithm.GetType()).GetMethod("OnData", new[] { ticksType });
if (newTradeBarsMethodInfo == null && newTicksMethodInfo == null)
{
backwardsCompatibilityMode = true;
if (oldTradeBarsMethodInfo != null)
{
methodInvokers.Add(tradebarsType, oldTradeBarsMethodInfo.DelegateForCallMethod());
}
if (oldTradeBarsMethodInfo != null)
{
methodInvokers.Add(ticksType, oldTicksMethodInfo.DelegateForCallMethod());
}
}
else
{
backwardsCompatibilityMode = false;
if (newTradeBarsMethodInfo != null)
{
methodInvokers.Add(tradebarsType, newTradeBarsMethodInfo.DelegateForCallMethod());
}
if (newTicksMethodInfo != null)
{
methodInvokers.Add(ticksType, newTicksMethodInfo.DelegateForCallMethod());
}
}
//Go through the subscription types and create invokers to trigger the event handlers for each custom type:
foreach (var config in feed.Subscriptions)
{
//If type is a tradebar, combine tradebars and ticks into unified array:
if (config.Type.Name != "TradeBar" && config.Type.Name != "Tick")
{
//Get the matching method for this event handler - e.g. public void OnData(Quandl data) { .. }
var genericMethod = (algorithm.GetType()).GetMethod("OnData", new[] { config.Type });
//If we already have this Type-handler then don't add it to invokers again.
if (methodInvokers.ContainsKey(config.Type))
{
continue;
}
//If we couldnt find the event handler, let the user know we can't fire that event.
if (genericMethod == null)
{
algorithm.RunTimeError = new Exception("Data event handler not found, please create a function matching this template: public void OnData(" + config.Type.Name + " data) { }");
_algorithmState = AlgorithmStatus.RuntimeError;
return;
}
methodInvokers.Add(config.Type, genericMethod.DelegateForCallMethod());
}
}
//Loop over the queues: get a data collection, then pass them all into relevent methods in the algorithm.
Log.Debug("AlgorithmManager.Run(): Algorithm initialized, launching time loop.");
foreach (var newData in DataStream.GetData(feed, setup.StartingDate))
{
//Check this backtest is still running:
if (_algorithmState != AlgorithmStatus.Running)
{
break;
}
//Go over each time stamp we've collected, pass it into the algorithm in order:
foreach (var time in newData.Keys)
{
//Set the time frontier:
_frontier = time;
//Execute with TimeLimit Monitor:
if (Isolator.IsCancellationRequested)
{
return;
}
//If we're in backtest mode we need to capture the daily performance. We do this here directly
//before updating the algorithm state with the new data from this time step, otherwise we'll
//produce incorrect samples (they'll take into account this time step's new price values)
if (backtestMode)
{
//Refresh the realtime event monitor:
//in backtest mode use the algorithms clock as realtime.
realtime.SetTime(time);
//On day-change sample equity and daily performance for statistics calculations
if (_previousTime.Date != time.Date)
{
//Sample the portfolio value over time for chart.
results.SampleEquity(_previousTime, Math.Round(algorithm.Portfolio.TotalPortfolioValue, 4));
//Check for divide by zero
if (startingPortfolioValue == 0m)
{
results.SamplePerformance(_previousTime.Date, 0);
}
else
{
results.SamplePerformance(_previousTime.Date, Math.Round((algorithm.Portfolio.TotalPortfolioValue - startingPortfolioValue) * 100 / startingPortfolioValue, 10));
}
startingPortfolioValue = algorithm.Portfolio.TotalPortfolioValue;
}
}
//Update algorithm state after capturing performance from previous day
//On each time step push the real time prices to the cashbook so we can have updated conversion rates
algorithm.Portfolio.CashBook.Update(newData[time]);
//Update the securities properties: first before calling user code to avoid issues with data
algorithm.Securities.Update(time, newData[time]);
//Check if the user's signalled Quit: loop over data until day changes.
if (algorithm.GetQuit())
{
_algorithmState = AlgorithmStatus.Quit;
break;
}
//Pass in the new time first:
algorithm.SetDateTime(time);
//Trigger the data events: Invoke the types we have data for:
var oldBars = new Dictionary <string, TradeBar>();
var oldTicks = new Dictionary <string, List <Tick> >();
var newBars = new TradeBars(time);
var newTicks = new Ticks(time);
//Invoke all non-tradebars, non-ticks methods and build up the TradeBars and Ticks dictionaries
// --> i == Subscription Configuration Index, so we don't need to compare types.
foreach (var i in newData[time].Keys)
{
//Data point and config of this point:
var dataPoints = newData[time][i];
var config = feed.Subscriptions[i];
//Keep track of how many data points we've processed
_dataPointCount += dataPoints.Count;
//We don't want to pump data that we added just for currency conversions
if (config.IsInternalFeed)
{
continue;
}
//Create TradeBars Unified Data --> OR --> invoke generic data event. One loop.
foreach (var dataPoint in dataPoints)
{
//Update registered consolidators for this symbol index
try
{
for (var j = 0; j < config.Consolidators.Count; j++)
{
config.Consolidators[j].Update(dataPoint);
}
}
catch (Exception err)
{
algorithm.RunTimeError = err;
_algorithmState = AlgorithmStatus.RuntimeError;
Log.Error("AlgorithmManager.Run(): RuntimeError: Consolidators update: " + err.Message);
return;
}
switch (config.Type.Name)
{
case "TradeBar":
var bar = dataPoint as TradeBar;
try
{
if (bar != null)
{
if (backwardsCompatibilityMode)
{
if (!oldBars.ContainsKey(bar.Symbol))
{
oldBars.Add(bar.Symbol, bar);
}
}
else
{
if (!newBars.ContainsKey(bar.Symbol))
{
newBars.Add(bar.Symbol, bar);
}
}
}
}
catch (Exception err)
{
Log.Error(time.ToLongTimeString() + " >> " + bar.Time.ToLongTimeString() + " >> " + bar.Symbol + " >> " + bar.Value.ToString("C"));
Log.Error("AlgorithmManager.Run(): Failed to add TradeBar (" + bar.Symbol + ") Time: (" + time.ToLongTimeString() + ") Count:(" + newBars.Count + ") " + err.Message);
}
break;
case "Tick":
var tick = dataPoint as Tick;
if (tick != null)
{
if (backwardsCompatibilityMode)
{
if (!oldTicks.ContainsKey(tick.Symbol))
{
oldTicks.Add(tick.Symbol, new List <Tick>());
}
oldTicks[tick.Symbol].Add(tick);
}
else
{
if (!newTicks.ContainsKey(tick.Symbol))
{
newTicks.Add(tick.Symbol, new List <Tick>());
}
newTicks[tick.Symbol].Add(tick);
}
}
break;
default:
//Send data into the generic algorithm event handlers
try
{
methodInvokers[config.Type](algorithm, dataPoint);
}
catch (Exception err)
{
algorithm.RunTimeError = err;
_algorithmState = AlgorithmStatus.RuntimeError;
Log.Debug("AlgorithmManager.Run(): RuntimeError: Custom Data: " + err.Message + " STACK >>> " + err.StackTrace);
return;
}
break;
}
}
}
//After we've fired all other events in this second, fire the pricing events:
if (backwardsCompatibilityMode)
{
//Log.Debug("AlgorithmManager.Run(): Invoking v1.0 Event Handlers...");
try
{
if (oldTradeBarsMethodInfo != null && oldBars.Count > 0)
{
methodInvokers[tradebarsType](algorithm, oldBars);
}
if (oldTicksMethodInfo != null && oldTicks.Count > 0)
{
methodInvokers[ticksType](algorithm, oldTicks);
}
}
catch (Exception err)
{
algorithm.RunTimeError = err;
_algorithmState = AlgorithmStatus.RuntimeError;
Log.Debug("AlgorithmManager.Run(): RuntimeError: Backwards Compatibility Mode: " + err.Message + " STACK >>> " + err.StackTrace);
return;
}
}
else
{
//Log.Debug("AlgorithmManager.Run(): Invoking v2.0 Event Handlers...");
try
{
if (newTradeBarsMethodInfo != null && newBars.Count > 0)
{
methodInvokers[tradebarsType](algorithm, newBars);
}
if (newTicksMethodInfo != null && newTicks.Count > 0)
{
methodInvokers[ticksType](algorithm, newTicks);
}
}
catch (Exception err)
{
algorithm.RunTimeError = err;
_algorithmState = AlgorithmStatus.RuntimeError;
Log.Debug("AlgorithmManager.Run(): RuntimeError: New Style Mode: " + err.Message + " STACK >>> " + err.StackTrace);
return;
}
}
//If its the historical/paper trading models, wait until market orders have been "filled"
// Manually trigger the event handler to prevent thread switch.
transactions.ProcessSynchronousEvents();
//Save the previous time for the sample calculations
_previousTime = time;
// Process any required events of the results handler such as sampling assets, equity, or stock prices.
results.ProcessSynchronousEvents();
} // End of Time Loop
} // End of ForEach DataStream
//Stream over:: Send the final packet and fire final events:
Log.Trace("AlgorithmManager.Run(): Firing On End Of Algorithm...");
try
{
algorithm.OnEndOfAlgorithm();
}
catch (Exception err)
{
_algorithmState = AlgorithmStatus.RuntimeError;
algorithm.RunTimeError = new Exception("Error running OnEndOfAlgorithm(): " + err.Message, err.InnerException);
Log.Debug("AlgorithmManager.OnEndOfAlgorithm(): " + err.Message + " STACK >>> " + err.StackTrace);
return;
}
// Process any required events of the results handler such as sampling assets, equity, or stock prices.
results.ProcessSynchronousEvents(forceProcess: true);
//Liquidate Holdings for Calculations:
if (_algorithmState == AlgorithmStatus.Liquidated || !Engine.LiveMode)
{
Log.Trace("AlgorithmManager.Run(): Liquidating algorithm holdings...");
algorithm.Liquidate();
results.LogMessage("Algorithm Liquidated");
results.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.Liquidated);
}
//Manually stopped the algorithm
if (_algorithmState == AlgorithmStatus.Stopped)
{
Log.Trace("AlgorithmManager.Run(): Stopping algorithm...");
results.LogMessage("Algorithm Stopped");
results.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.Stopped);
}
//Backtest deleted.
if (_algorithmState == AlgorithmStatus.Deleted)
{
Log.Trace("AlgorithmManager.Run(): Deleting algorithm...");
results.DebugMessage("Algorithm Id:(" + job.AlgorithmId + ") Deleted by request.");
results.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.Deleted);
}
//Algorithm finished, send regardless of commands:
results.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.Completed);
//Take final samples:
results.SampleRange(algorithm.GetChartUpdates());
results.SampleEquity(_frontier, Math.Round(algorithm.Portfolio.TotalPortfolioValue, 4));
results.SamplePerformance(_frontier, Math.Round((algorithm.Portfolio.TotalPortfolioValue - startingPortfolioValue) * 100 / startingPortfolioValue, 10));
} // End of Run();
/// <summary>
/// Primary entry point to setup a new algorithm
/// </summary>
/// <param name="algorithm">Algorithm instance</param>
/// <param name="brokerage">New brokerage output instance</param>
/// <param name="job">Algorithm job task</param>
/// <param name="resultHandler">The configured result handler</param>
/// <param name="transactionHandler">The configurated transaction handler</param>
/// <param name="realTimeHandler">The configured real time handler</param>
/// <returns>True on successfully setting up the algorithm state, or false on error.</returns>
public bool Setup(IAlgorithm algorithm, IBrokerage brokerage, AlgorithmNodePacket job, IResultHandler resultHandler, ITransactionHandler transactionHandler, IRealTimeHandler realTimeHandler)
{
// verify we were given the correct job packet type
var liveJob = job as LiveNodePacket;
if (liveJob == null)
{
AddInitializationError("BrokerageSetupHandler requires a LiveNodePacket");
return(false);
}
algorithm.Name = liveJob.GetAlgorithmName();
// verify the brokerage was specified
if (string.IsNullOrWhiteSpace(liveJob.Brokerage))
{
AddInitializationError("A brokerage must be specified");
return(false);
}
// attach to the message event to relay brokerage specific initialization messages
EventHandler <BrokerageMessageEvent> brokerageOnMessage = (sender, args) =>
{
if (args.Type == BrokerageMessageType.Error)
{
AddInitializationError($"Brokerage Error Code: {args.Code} - {args.Message}");
}
};
try
{
Log.Trace("BrokerageSetupHandler.Setup(): Initializing algorithm...");
resultHandler.SendStatusUpdate(AlgorithmStatus.Initializing, "Initializing algorithm...");
//Execute the initialize code:
var controls = job.Controls;
var isolator = new Isolator();
var initializeComplete = isolator.ExecuteWithTimeLimit(TimeSpan.FromSeconds(300), () =>
{
try
{
//Set the default brokerage model before initialize
algorithm.SetBrokerageModel(_factory.BrokerageModel);
//Margin calls are disabled by default in live mode
algorithm.Portfolio.MarginCallModel = MarginCallModel.Null;
//Set our parameters
algorithm.SetParameters(job.Parameters);
algorithm.SetAvailableDataTypes(GetConfiguredDataFeeds());
//Algorithm is live, not backtesting:
algorithm.SetLiveMode(true);
//Initialize the algorithm's starting date
algorithm.SetDateTime(DateTime.UtcNow);
//Set the source impl for the event scheduling
algorithm.Schedule.SetEventSchedule(realTimeHandler);
// set the option chain provider
algorithm.SetOptionChainProvider(new CachingOptionChainProvider(new LiveOptionChainProvider()));
// set the future chain provider
algorithm.SetFutureChainProvider(new CachingFutureChainProvider(new LiveFutureChainProvider()));
// If we're going to receive market data from IB,
// set the default subscription limit to 100,
// algorithms can override this setting in the Initialize method
if (brokerage is InteractiveBrokersBrokerage &&
liveJob.DataQueueHandler.EndsWith("InteractiveBrokersBrokerage"))
{
algorithm.Settings.DataSubscriptionLimit = 100;
}
//Initialise the algorithm, get the required data:
algorithm.Initialize();
//Finalize Initialization
algorithm.PostInitialize();
if (liveJob.Brokerage != "PaperBrokerage")
{
//Zero the CashBook - we'll populate directly from brokerage
foreach (var kvp in algorithm.Portfolio.CashBook)
{
kvp.Value.SetAmount(0);
}
}
}
catch (Exception err)
{
AddInitializationError(err.ToString(), err);
}
}, controls.RamAllocation);
if (!initializeComplete)
{
AddInitializationError("Initialization timed out.");
return(false);
}
// let the world know what we're doing since logging in can take a minute
resultHandler.SendStatusUpdate(AlgorithmStatus.LoggingIn, "Logging into brokerage...");
brokerage.Message += brokerageOnMessage;
Log.Trace("BrokerageSetupHandler.Setup(): Connecting to brokerage...");
try
{
// this can fail for various reasons, such as already being logged in somewhere else
brokerage.Connect();
}
catch (Exception err)
{
Log.Error(err);
AddInitializationError(
$"Error connecting to brokerage: {err.Message}. " +
"This may be caused by incorrect login credentials or an unsupported account type.", err);
return(false);
}
if (!brokerage.IsConnected)
{
// if we're reporting that we're not connected, bail
AddInitializationError("Unable to connect to brokerage.");
return(false);
}
Log.Trace("BrokerageSetupHandler.Setup(): Fetching cash balance from brokerage...");
try
{
// set the algorithm's cash balance for each currency
var cashBalance = brokerage.GetCashBalance();
foreach (var cash in cashBalance)
{
Log.Trace("BrokerageSetupHandler.Setup(): Setting " + cash.Symbol + " cash to " + cash.Amount);
algorithm.Portfolio.SetCash(cash.Symbol, cash.Amount, cash.ConversionRate);
}
}
catch (Exception err)
{
Log.Error(err);
AddInitializationError("Error getting cash balance from brokerage: " + err.Message, err);
return(false);
}
var supportedSecurityTypes = new HashSet <SecurityType>
{
SecurityType.Equity, SecurityType.Forex, SecurityType.Cfd, SecurityType.Option, SecurityType.Future, SecurityType.Crypto
};
var minResolution = new Lazy <Resolution>(() => algorithm.Securities.Select(x => x.Value.Resolution).DefaultIfEmpty(Resolution.Second).Min());
Log.Trace("BrokerageSetupHandler.Setup(): Fetching open orders from brokerage...");
try
{
GetOpenOrders(algorithm, resultHandler, transactionHandler, brokerage, supportedSecurityTypes, minResolution.Value);
}
catch (Exception err)
{
Log.Error(err);
AddInitializationError("Error getting open orders from brokerage: " + err.Message, err);
return(false);
}
Log.Trace("BrokerageSetupHandler.Setup(): Fetching holdings from brokerage...");
try
{
// populate the algorithm with the account's current holdings
var holdings = brokerage.GetAccountHoldings();
foreach (var holding in holdings)
{
Log.Trace("BrokerageSetupHandler.Setup(): Has existing holding: " + holding);
// verify existing holding security type
if (!supportedSecurityTypes.Contains(holding.Type))
{
Log.Error("BrokerageSetupHandler.Setup(): Unsupported security type: " + holding.Type + "-" + holding.Symbol.Value);
AddInitializationError("Found unsupported security type in existing brokerage holdings: " + holding.Type + ". " +
"QuantConnect currently supports the following security types: " + string.Join(",", supportedSecurityTypes));
// keep aggregating these errors
continue;
}
AddUnrequestedSecurity(algorithm, holding.Symbol, minResolution.Value);
algorithm.Portfolio[holding.Symbol].SetHoldings(holding.AveragePrice, holding.Quantity);
algorithm.Securities[holding.Symbol].SetMarketPrice(new TradeBar
{
Time = DateTime.Now,
Open = holding.MarketPrice,
High = holding.MarketPrice,
Low = holding.MarketPrice,
Close = holding.MarketPrice,
Volume = 0,
Symbol = holding.Symbol,
DataType = MarketDataType.TradeBar
});
}
}
catch (Exception err)
{
Log.Error(err);
AddInitializationError("Error getting account holdings from brokerage: " + err.Message, err);
return(false);
}
//Set the starting portfolio value for the strategy to calculate performance:
StartingPortfolioValue = algorithm.Portfolio.TotalPortfolioValue;
StartingDate = DateTime.Now;
}
catch (Exception err)
{
AddInitializationError(err.ToString(), err);
}
finally
{
if (brokerage != null)
{
brokerage.Message -= brokerageOnMessage;
}
}
return(Errors.Count == 0);
}
/// <summary>
/// Primary entry point to setup a new algorithm
/// </summary>
/// <param name="algorithm">Algorithm instance</param>
/// <param name="brokerage">New brokerage output instance</param>
/// <param name="job">Algorithm job task</param>
/// <param name="resultHandler">The configured result handler</param>
/// <param name="transactionHandler">The configurated transaction handler</param>
/// <param name="realTimeHandler">The configured real time handler</param>
/// <returns>True on successfully setting up the algorithm state, or false on error.</returns>
public bool Setup(IAlgorithm algorithm, IBrokerage brokerage, AlgorithmNodePacket job, IResultHandler resultHandler, ITransactionHandler transactionHandler, IRealTimeHandler realTimeHandler)
{
_algorithm = algorithm;
// verify we were given the correct job packet type
var liveJob = job as LiveNodePacket;
if (liveJob == null)
{
AddInitializationError("BrokerageSetupHandler requires a LiveNodePacket");
return(false);
}
// verify the brokerage was specified
if (string.IsNullOrWhiteSpace(liveJob.Brokerage))
{
AddInitializationError("A brokerage must be specified");
return(false);
}
// attach to the message event to relay brokerage specific initialization messages
EventHandler <BrokerageMessageEvent> brokerageOnMessage = (sender, args) =>
{
if (args.Type == BrokerageMessageType.Error)
{
AddInitializationError(string.Format("Brokerage Error Code: {0} - {1}", args.Code, args.Message));
}
};
try
{
Log.Trace("BrokerageSetupHandler.Setup(): Initializing algorithm...");
resultHandler.SendStatusUpdate(AlgorithmStatus.Initializing, "Initializing algorithm...");
//Execute the initialize code:
var controls = job.Controls;
var isolator = new Isolator();
var initializeComplete = isolator.ExecuteWithTimeLimit(TimeSpan.FromSeconds(300), () =>
{
try
{
//Set the default brokerage model before initialize
algorithm.SetBrokerageModel(_factory.BrokerageModel);
//Set our parameters
algorithm.SetParameters(job.Parameters);
//Algorithm is live, not backtesting:
algorithm.SetLiveMode(true);
//Initialize the algorithm's starting date
algorithm.SetDateTime(DateTime.UtcNow);
//Set the source impl for the event scheduling
algorithm.Schedule.SetEventSchedule(realTimeHandler);
//Initialise the algorithm, get the required data:
algorithm.Initialize();
if (liveJob.Brokerage != "PaperBrokerage")
{
//Zero the CashBook - we'll populate directly from brokerage
foreach (var kvp in algorithm.Portfolio.CashBook)
{
kvp.Value.SetAmount(0);
}
}
}
catch (Exception err)
{
AddInitializationError(err.Message);
}
});
if (!initializeComplete)
{
AddInitializationError("Initialization timed out.");
return(false);
}
// let the world know what we're doing since logging in can take a minute
resultHandler.SendStatusUpdate(AlgorithmStatus.LoggingIn, "Logging into brokerage...");
brokerage.Message += brokerageOnMessage;
algorithm.Transactions.SetOrderProcessor(transactionHandler);
algorithm.PostInitialize();
Log.Trace("BrokerageSetupHandler.Setup(): Connecting to brokerage...");
try
{
// this can fail for various reasons, such as already being logged in somewhere else
brokerage.Connect();
}
catch (Exception err)
{
Log.Error(err);
AddInitializationError(string.Format("Error connecting to brokerage: {0}. " +
"This may be caused by incorrect login credentials or an unsupported account type.", err.Message));
return(false);
}
if (!brokerage.IsConnected)
{
// if we're reporting that we're not connected, bail
AddInitializationError("Unable to connect to brokerage.");
return(false);
}
Log.Trace("BrokerageSetupHandler.Setup(): Fetching cash balance from brokerage...");
try
{
// set the algorithm's cash balance for each currency
var cashBalance = brokerage.GetCashBalance();
foreach (var cash in cashBalance)
{
Log.Trace("BrokerageSetupHandler.Setup(): Setting " + cash.Symbol + " cash to " + cash.Amount);
algorithm.Portfolio.SetCash(cash.Symbol, cash.Amount, cash.ConversionRate);
}
}
catch (Exception err)
{
Log.Error(err);
AddInitializationError("Error getting cash balance from brokerage: " + err.Message);
return(false);
}
Log.Trace("BrokerageSetupHandler.Setup(): Fetching open orders from brokerage...");
try
{
// populate the algorithm with the account's outstanding orders
var openOrders = brokerage.GetOpenOrders();
foreach (var order in openOrders)
{
// be sure to assign order IDs such that we increment from the SecurityTransactionManager to avoid ID collisions
Log.Trace("BrokerageSetupHandler.Setup(): Has open order: " + order.Symbol.ToString() + " - " + order.Quantity);
order.Id = algorithm.Transactions.GetIncrementOrderId();
transactionHandler.Orders.AddOrUpdate(order.Id, order, (i, o) => order);
}
}
catch (Exception err)
{
Log.Error(err);
AddInitializationError("Error getting open orders from brokerage: " + err.Message);
return(false);
}
Log.Trace("BrokerageSetupHandler.Setup(): Fetching holdings from brokerage...");
try
{
// populate the algorithm with the account's current holdings
var holdings = brokerage.GetAccountHoldings();
var supportedSecurityTypes = new HashSet <SecurityType> {
SecurityType.Equity, SecurityType.Forex, SecurityType.Cfd
};
var minResolution = new Lazy <Resolution>(() => algorithm.Securities.Select(x => x.Value.Resolution).DefaultIfEmpty(Resolution.Second).Min());
foreach (var holding in holdings)
{
Log.Trace("BrokerageSetupHandler.Setup(): Has existing holding: " + holding);
// verify existing holding security type
if (!supportedSecurityTypes.Contains(holding.Type))
{
Log.Error("BrokerageSetupHandler.Setup(): Unsupported security type: " + holding.Type + "-" + holding.Symbol.Value);
AddInitializationError("Found unsupported security type in existing brokerage holdings: " + holding.Type + ". " +
"VigiothCapital.QuantTrader currently supports the following security types: " + string.Join(",", supportedSecurityTypes));
// keep aggregating these errors
continue;
}
if (!algorithm.Portfolio.ContainsKey(holding.Symbol))
{
Log.Trace("BrokerageSetupHandler.Setup(): Adding unrequested security: " + holding.Symbol.ToString());
// for items not directly requested set leverage to 1 and at the min resolution
algorithm.AddSecurity(holding.Type, holding.Symbol.Value, minResolution.Value, null, true, 1.0m, false);
}
algorithm.Portfolio[holding.Symbol].SetHoldings(holding.AveragePrice, (int)holding.Quantity);
algorithm.Securities[holding.Symbol].SetMarketPrice(new TradeBar
{
Time = DateTime.Now,
Open = holding.MarketPrice,
High = holding.MarketPrice,
Low = holding.MarketPrice,
Close = holding.MarketPrice,
Volume = 0,
Symbol = holding.Symbol,
DataType = MarketDataType.TradeBar
});
}
}
catch (Exception err)
{
Log.Error(err);
AddInitializationError("Error getting account holdings from brokerage: " + err.Message);
return(false);
}
//Set the starting portfolio value for the strategy to calculate performance:
StartingPortfolioValue = algorithm.Portfolio.TotalPortfolioValue;
StartingDate = DateTime.Now;
}
catch (Exception err)
{
AddInitializationError(err.Message);
}
finally
{
if (brokerage != null)
{
brokerage.Message -= brokerageOnMessage;
}
}
return(Errors.Count == 0);
}
/********************************************************
* CLASS METHODS
*********************************************************/
/// <summary>
/// Launch the algorithm manager to run this strategy
/// </summary>
/// <param name="job">Algorithm job</param>
/// <param name="algorithm">Algorithm instance</param>
/// <param name="feed">Datafeed object</param>
/// <param name="transactions">Transaction manager object</param>
/// <param name="results">Result handler object</param>
/// <param name="setup">Setup handler object</param>
/// <param name="realtime">Realtime processing object</param>
/// <remarks>Modify with caution</remarks>
public static void Run(AlgorithmNodePacket job, IAlgorithm algorithm, IDataFeed feed, ITransactionHandler transactions, IResultHandler results, ISetupHandler setup, IRealTimeHandler realtime)
{
//Initialize:
var backwardsCompatibilityMode = false;
var tradebarsType = typeof (TradeBars);
var ticksType = typeof(Ticks);
var startingPerformance = setup.StartingCapital;
var backtestMode = (job.Type == PacketType.BacktestNode);
var methodInvokers = new Dictionary<Type, MethodInvoker>();
//Initialize Properties:
_frontier = setup.StartingDate;
_runtimeError = null;
_algorithmId = job.AlgorithmId;
_algorithmState = AlgorithmStatus.Running;
_previousTime = setup.StartingDate.Date;
//Create the method accessors to push generic types into algorithm: Find all OnData events:
//Algorithm 1.0 Data Accessors.
//If the users defined these methods, add them in manually. This allows keeping backwards compatibility to algorithm 1.0.
var oldTradeBarsMethodInfo = (algorithm.GetType()).GetMethod("OnTradeBar", new[] { typeof(Dictionary<string, TradeBar>) });
var oldTicksMethodInfo = (algorithm.GetType()).GetMethod("OnTick", new[] { typeof(Dictionary<string, List<Tick>>) });
//Algorithm 2.0 Data Generics Accessors.
//New hidden access to tradebars with custom type.
var newTradeBarsMethodInfo = (algorithm.GetType()).GetMethod("OnData", new[] { tradebarsType });
var newTicksMethodInfo = (algorithm.GetType()).GetMethod("OnData", new[] { ticksType });
if (newTradeBarsMethodInfo == null && newTicksMethodInfo == null)
{
backwardsCompatibilityMode = true;
if (oldTradeBarsMethodInfo != null) methodInvokers.Add(tradebarsType, oldTradeBarsMethodInfo.DelegateForCallMethod());
if (oldTradeBarsMethodInfo != null) methodInvokers.Add(ticksType, oldTicksMethodInfo.DelegateForCallMethod());
}
else
{
backwardsCompatibilityMode = false;
if (newTradeBarsMethodInfo != null) methodInvokers.Add(tradebarsType, newTradeBarsMethodInfo.DelegateForCallMethod());
if (newTicksMethodInfo != null) methodInvokers.Add(ticksType, newTicksMethodInfo.DelegateForCallMethod());
}
//Go through the subscription types and create invokers to trigger the event handlers for each custom type:
foreach (var config in feed.Subscriptions)
{
//If type is a tradebar, combine tradebars and ticks into unified array:
if (config.Type.Name != "TradeBar" && config.Type.Name != "Tick")
{
//Get the matching method for this event handler - e.g. public void OnData(Quandl data) { .. }
var genericMethod = (algorithm.GetType()).GetMethod("OnData", new[] { config.Type });
//Is we already have this Type-handler then don't add it to invokers again.
if (methodInvokers.ContainsKey(config.Type)) continue;
//If we couldnt find the event handler, let the user know we can't fire that event.
if (genericMethod == null)
{
_runtimeError = new Exception("Data event handler not found, please create a function matching this template: public void OnData(" + config.Type.Name + " data) { }");
_algorithmState = AlgorithmStatus.RuntimeError;
return;
}
methodInvokers.Add(config.Type, genericMethod.DelegateForCallMethod());
}
}
//Loop over the queues: get a data collection, then pass them all into relevent methods in the algorithm.
Log.Debug("AlgorithmManager.Run(): Algorithm initialized, launching time loop.");
foreach (var newData in DataStream.GetData(feed, setup.StartingDate))
{
//Check this backtest is still running:
if (_algorithmState != AlgorithmStatus.Running) break;
//Go over each time stamp we've collected, pass it into the algorithm in order:
foreach (var time in newData.Keys)
{
//Set the time frontier:
_frontier = time;
//Execute with TimeLimit Monitor:
if (Isolator.IsCancellationRequested) return;
//Refresh the realtime event monitor:
realtime.SetTime(time);
//Fire EOD if the time packet we just processed is greater
if (backtestMode && _previousTime.Date != time.Date)
{
//Sample the portfolio value over time for chart.
results.SampleEquity(_previousTime, Math.Round(algorithm.Portfolio.TotalPortfolioValue, 4));
if (startingPerformance == 0)
{
results.SamplePerformance(_previousTime.Date, 0);
}
else
{
results.SamplePerformance(_previousTime.Date, Math.Round((algorithm.Portfolio.TotalPortfolioValue - startingPerformance) * 100 / startingPerformance, 10));
}
startingPerformance = algorithm.Portfolio.TotalPortfolioValue;
}
//Check if the user's signalled Quit: loop over data until day changes.
if (algorithm.GetQuit())
{
_algorithmState = AlgorithmStatus.Quit;
break;
}
//Pass in the new time first:
algorithm.SetDateTime(time);
//Trigger the data events: Invoke the types we have data for:
var oldBars = new Dictionary<string, TradeBar>();
var oldTicks = new Dictionary<string, List<Tick>>();
var newBars = new TradeBars(time);
var newTicks = new Ticks(time);
//Invoke all non-tradebars, non-ticks methods:
// --> i == Subscription Configuration Index, so we don't need to compare types.
foreach (var i in newData[time].Keys)
{
//Data point and config of this point:
var dataPoints = newData[time][i];
var config = feed.Subscriptions[i];
//Create TradeBars Unified Data --> OR --> invoke generic data event. One loop.
foreach (var dataPoint in dataPoints)
{
//Update the securities properties: first before calling user code to avoid issues with data
algorithm.Securities.Update(time, dataPoint);
//Update registered consolidators for this symbol index
for (var j = 0; j < config.Consolidators.Count; j++)
{
config.Consolidators[j].Update(dataPoint);
}
switch (config.Type.Name)
{
case "TradeBar":
var bar = dataPoint as TradeBar;
try
{
if (bar != null)
{
if (backwardsCompatibilityMode)
{
if (!oldBars.ContainsKey(bar.Symbol)) oldBars.Add(bar.Symbol, bar);
}
else
{
if (!newBars.ContainsKey(bar.Symbol)) newBars.Add(bar.Symbol, bar);
}
}
}
catch (Exception err)
{
Log.Error(time.ToLongTimeString() + " >> " + bar.Time.ToLongTimeString() + " >> " + bar.Symbol + " >> " + bar.Value.ToString("C"));
Log.Error("AlgorithmManager.Run(): Failed to add TradeBar (" + bar.Symbol + ") Time: (" + time.ToLongTimeString() + ") Count:(" + newBars.Count + ") " + err.Message);
}
break;
case "Tick":
var tick = dataPoint as Tick;
if (tick != null)
{
if (backwardsCompatibilityMode) {
if (!oldTicks.ContainsKey(tick.Symbol)) { oldTicks.Add(tick.Symbol, new List<Tick>()); }
oldTicks[tick.Symbol].Add(tick);
}
else
{
if (!newTicks.ContainsKey(tick.Symbol)) { newTicks.Add(tick.Symbol, new List<Tick>()); }
newTicks[tick.Symbol].Add(tick);
}
}
break;
default:
//Send data into the generic algorithm event handlers
try
{
methodInvokers[config.Type](algorithm, dataPoint);
}
catch (Exception err)
{
_runtimeError = err;
_algorithmState = AlgorithmStatus.RuntimeError;
Log.Debug("AlgorithmManager.Run(): RuntimeError: Custom Data: " + err.Message + " STACK >>> " + err.StackTrace);
return;
}
break;
}
}
}
//After we've fired all other events in this second, fire the pricing events:
if (backwardsCompatibilityMode)
{
//Log.Debug("AlgorithmManager.Run(): Invoking v1.0 Event Handlers...");
try
{
if (oldTradeBarsMethodInfo != null && oldBars.Count > 0) methodInvokers[tradebarsType](algorithm, oldBars);
if (oldTicksMethodInfo != null && oldTicks.Count > 0) methodInvokers[ticksType](algorithm, oldTicks);
}
catch (Exception err)
{
_runtimeError = err;
_algorithmState = AlgorithmStatus.RuntimeError;
Log.Debug("AlgorithmManager.Run(): RuntimeError: Backwards Compatibility Mode: " + err.Message + " STACK >>> " + err.StackTrace);
return;
}
}
else
{
//Log.Debug("AlgorithmManager.Run(): Invoking v2.0 Event Handlers...");
try
{
if (newTradeBarsMethodInfo != null && newBars.Count > 0) methodInvokers[tradebarsType](algorithm, newBars);
if (newTicksMethodInfo != null && newTicks.Count > 0) methodInvokers[ticksType](algorithm, newTicks);
}
catch (Exception err)
{
_runtimeError = err;
_algorithmState = AlgorithmStatus.RuntimeError;
Log.Debug("AlgorithmManager.Run(): RuntimeError: New Style Mode: " + err.Message + " STACK >>> " + err.StackTrace);
return;
}
}
//If its the historical/paper trading models, wait until market orders have been "filled"
// Manually trigger the event handler to prevent thread switch.
transactions.ProcessSynchronousEvents();
//Save the previous time for the sample calculations
_previousTime = time;
} // End of Time Loop
// Process any required events of the results handler such as sampling assets, equity, or stock prices.
results.ProcessSynchronousEvents();
} // End of ForEach DataStream
//Stream over:: Send the final packet and fire final events:
Log.Trace("AlgorithmManager.Run(): Firing On End Of Algorithm...");
try
{
algorithm.OnEndOfAlgorithm();
}
catch (Exception err)
{
_algorithmState = AlgorithmStatus.RuntimeError;
_runtimeError = new Exception("Error running OnEndOfAlgorithm(): " + err.Message, err.InnerException);
Log.Debug("AlgorithmManager.OnEndOfAlgorithm(): " + err.Message + " STACK >>> " + err.StackTrace);
return;
}
// Process any required events of the results handler such as sampling assets, equity, or stock prices.
results.ProcessSynchronousEvents(forceProcess: true);
//Liquidate Holdings for Calculations:
if (_algorithmState == AlgorithmStatus.Liquidated || !Engine.LiveMode)
{
Log.Trace("AlgorithmManager.Run(): Liquidating algorithm holdings...");
algorithm.Liquidate();
results.LogMessage("Algorithm Liquidated");
results.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.Liquidated);
}
//Manually stopped the algorithm
if (_algorithmState == AlgorithmStatus.Stopped)
{
Log.Trace("AlgorithmManager.Run(): Stopping algorithm...");
results.LogMessage("Algorithm Stopped");
results.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.Stopped);
}
//Backtest deleted.
if (_algorithmState == AlgorithmStatus.Deleted)
{
Log.Trace("AlgorithmManager.Run(): Deleting algorithm...");
results.DebugMessage("Algorithm Id:(" + job.AlgorithmId + ") Deleted by request.");
results.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.Deleted);
}
//Algorithm finished, send regardless of commands:
results.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.Completed);
//Take final samples:
results.SampleRange(algorithm.GetChartUpdates());
results.SampleEquity(_frontier, Math.Round(algorithm.Portfolio.TotalPortfolioValue, 4));
results.SamplePerformance(_frontier, Math.Round((algorithm.Portfolio.TotalPortfolioValue - startingPerformance) * 100 / startingPerformance, 10));
}
/********************************************************
* CLASS METHODS
*********************************************************/
/// <summary>
/// Launch the algorithm manager to run this strategy
/// </summary>
/// <param name="job">Algorithm job</param>
/// <param name="algorithm">Algorithm instance</param>
/// <param name="feed">Datafeed object</param>
/// <param name="transactions">Transaction manager object</param>
/// <param name="results">Result handler object</param>
/// <param name="setup">Setup handler object</param>
/// <param name="realtime">Realtime processing object</param>
/// <remarks>Modify with caution</remarks>
public static void Run(AlgorithmNodePacket job, IAlgorithm algorithm, IDataFeed feed, ITransactionHandler transactions, IResultHandler results, ISetupHandler setup, IRealTimeHandler realtime)
{
//Initialize:
var backwardsCompatibilityMode = false;
var tradebarsType = typeof(TradeBars);
var ticksType = typeof(Ticks);
var startingPerformance = setup.StartingCapital;
var backtestMode = (job.Type == PacketType.BacktestNode);
var methodInvokers = new Dictionary <Type, MethodInvoker>();
//Initialize Properties:
_nextSample = new DateTime();
_frontier = setup.StartingDate;
_runtimeError = null;
_algorithmId = job.AlgorithmId;
_algorithmState = AlgorithmStatus.Running;
_previousTime = setup.StartingDate.Date;
//Create the method accessors to push generic types into algorithm: Find all OnData events:
//Algorithm 1.0 Data Accessors.
//If the users defined these methods, add them in manually. This allows keeping backwards compatibility to algorithm 1.0.
var oldTradeBarsMethodInfo = (algorithm.GetType()).GetMethod("OnTradeBar", new[] { typeof(Dictionary <string, TradeBar>) });
var oldTicksMethodInfo = (algorithm.GetType()).GetMethod("OnTick", new[] { typeof(Dictionary <string, List <Tick> >) });
//Algorithm 2.0 Data Generics Accessors.
//New hidden access to tradebars with custom type.
var newTradeBarsMethodInfo = (algorithm.GetType()).GetMethod("OnData", new[] { tradebarsType });
var newTicksMethodInfo = (algorithm.GetType()).GetMethod("OnData", new[] { ticksType });
if (newTradeBarsMethodInfo == null && newTicksMethodInfo == null)
{
backwardsCompatibilityMode = true;
if (oldTradeBarsMethodInfo != null)
{
methodInvokers.Add(tradebarsType, oldTradeBarsMethodInfo.DelegateForCallMethod());
}
if (oldTradeBarsMethodInfo != null)
{
methodInvokers.Add(ticksType, oldTicksMethodInfo.DelegateForCallMethod());
}
}
else
{
backwardsCompatibilityMode = false;
if (newTradeBarsMethodInfo != null)
{
methodInvokers.Add(tradebarsType, newTradeBarsMethodInfo.DelegateForCallMethod());
}
if (newTicksMethodInfo != null)
{
methodInvokers.Add(ticksType, newTicksMethodInfo.DelegateForCallMethod());
}
}
//Go through the subscription types and create invokers to trigger the event handlers for each custom type:
foreach (var config in feed.Subscriptions)
{
//If type is a tradebar, combine tradebars and ticks into unified array:
if (config.Type.Name != "TradeBar" && config.Type.Name != "Tick")
{
//Get the matching method for this event handler - e.g. public void OnData(Quandl data) { .. }
var genericMethod = (algorithm.GetType()).GetMethod("OnData", new[] { config.Type });
//Is we already have this Type-handler then don't add it to invokers again.
if (methodInvokers.ContainsKey(config.Type))
{
continue;
}
//If we couldnt find the event handler, let the user know we can't fire that event.
if (genericMethod == null)
{
_runtimeError = new Exception("Data event handler not found, please create a function matching this template: public void OnData(" + config.Type.Name + " data) { }");
return;
}
methodInvokers.Add(config.Type, genericMethod.DelegateForCallMethod());
}
}
//Loop over the queues: get a data collection, then pass them all into relevent methods in the algorithm.
Log.Debug("AlgorithmManager.Run(): Algorithm initialized, launching time loop.");
foreach (var newData in DataStream.GetData(feed, setup.StartingDate))
{
//Check this backtest is still running:
if (_algorithmState != AlgorithmStatus.Running)
{
break;
}
//Go over each time stamp we've collected, pass it into the algorithm in order:
foreach (var time in newData.Keys)
{
//Set the time frontier:
_frontier = time;
//Execute with TimeLimit Monitor:
if (Isolator.IsCancellationRequested)
{
return;
}
//Refresh the realtime event monitor:
realtime.SetTime(time);
//Fire EOD if the time packet we just processed is greater
if (backtestMode && _previousTime.Date != time.Date)
{
//Sample the portfolio value over time for chart.
results.SampleEquity(_previousTime, Math.Round(algorithm.Portfolio.TotalPortfolioValue, 4));
if (startingPerformance == 0)
{
results.SamplePerformance(_previousTime.Date, 0);
}
else
{
results.SamplePerformance(_previousTime.Date, Math.Round((algorithm.Portfolio.TotalPortfolioValue - startingPerformance) * 100 / startingPerformance, 10));
}
startingPerformance = algorithm.Portfolio.TotalPortfolioValue;
//Log.Trace("AlgorithmManager.Run(): Date Changed: " + time.ToShortDateString());
if (time.Date.DayOfYear % 10 == 0 && OS.TotalPhysicalMemoryUsed > 500)
{
results.DebugMessage("Algorithm Notice: Memory " + OS.TotalPhysicalMemoryUsed + "Mb Used of Maxiumum " + Engine.MaximumRamAllocation + "Mb. Try not to store data in your algorithm.");
}
}
//Check if the user's signalled Quit: loop over data until day changes.
if (algorithm.GetQuit())
{
_algorithmState = AlgorithmStatus.Quit;
break;
}
//Pass in the new time first:
algorithm.SetDateTime(time);
//Trigger the data events: Invoke the types we have data for:
var oldBars = new Dictionary <string, TradeBar>();
var oldTicks = new Dictionary <string, List <Tick> >();
var newBars = new TradeBars(time);
var newTicks = new Ticks(time);
//Invoke all non-tradebars, non-ticks methods:
// --> i == Subscription Configuration Index, so we don't need to compare types.
foreach (var i in newData[time].Keys)
{
//Data point and config of this point:
var dataPoints = newData[time][i];
var config = feed.Subscriptions[i];
//Create TradeBars Unified Data --> OR --> invoke generic data event. One loop.
foreach (var dataPoint in dataPoints)
{
//Update the securities properties: first before calling user code to avoid issues with data
algorithm.Securities.Update(time, dataPoint);
//Update registered consolidators for this symbol index
for (var j = 0; j < config.Consolidators.Count; j++)
{
config.Consolidators[j].Update(dataPoint);
}
switch (config.Type.Name)
{
case "TradeBar":
var bar = dataPoint as TradeBar;
try
{
if (bar != null)
{
if (backwardsCompatibilityMode)
{
if (!oldBars.ContainsKey(bar.Symbol))
{
oldBars.Add(bar.Symbol, bar);
}
}
else
{
if (!newBars.ContainsKey(bar.Symbol))
{
newBars.Add(bar.Symbol, bar);
}
}
}
}
catch (Exception err)
{
Log.Error(time.ToLongTimeString() + " >> " + bar.Time.ToLongTimeString() + " >> " + bar.Symbol + " >> " + bar.Value.ToString("C"));
Log.Error("AlgorithmManager.Run(): Failed to add TradeBar (" + bar.Symbol + ") Time: (" + time.ToLongTimeString() + ") Count:(" + newBars.Count + ") " + err.Message);
}
break;
case "Tick":
var tick = dataPoint as Tick;
if (tick != null)
{
if (backwardsCompatibilityMode)
{
if (!oldTicks.ContainsKey(tick.Symbol))
{
oldTicks.Add(tick.Symbol, new List <Tick>());
}
oldTicks[tick.Symbol].Add(tick);
}
else
{
if (!newTicks.ContainsKey(tick.Symbol))
{
newTicks.Add(tick.Symbol, new List <Tick>());
}
newTicks[tick.Symbol].Add(tick);
}
}
break;
default:
//Send data into the generic algorithm event handlers
try
{
methodInvokers[config.Type](algorithm, dataPoint);
}
catch (Exception err)
{
_runtimeError = err;
Log.Error("AlgorithmManager.Run(): RuntimeError: Custom Data: " + err.Message + " STACK >>> " + err.StackTrace);
return;
}
break;
}
}
}
//After we've fired all other events in this second, fire the pricing events:
if (backwardsCompatibilityMode)
{
//Log.Debug("AlgorithmManager.Run(): Invoking v1.0 Event Handlers...");
try
{
if (oldTradeBarsMethodInfo != null && oldBars.Count > 0)
{
methodInvokers[tradebarsType](algorithm, oldBars);
}
if (oldTicksMethodInfo != null && oldTicks.Count > 0)
{
methodInvokers[ticksType](algorithm, oldTicks);
}
}
catch (Exception err)
{
_runtimeError = err;
Log.Error("AlgorithmManager.Run(): RuntimeError: Backwards Compatibility Mode: " + err.Message + " STACK >>> " + err.StackTrace);
return;
}
}
else
{
//Log.Debug("AlgorithmManager.Run(): Invoking v2.0 Event Handlers...");
try
{
if (newTradeBarsMethodInfo != null && newBars.Count > 0)
{
methodInvokers[tradebarsType](algorithm, newBars);
}
if (newTicksMethodInfo != null && newTicks.Count > 0)
{
methodInvokers[ticksType](algorithm, newTicks);
}
}
catch (Exception err)
{
_runtimeError = err;
Log.Error("AlgorithmManager.Run(): RuntimeError: New Style Mode: " + err.Message + " STACK >>> " + err.StackTrace);
return;
}
}
//If its the historical/paper trading models, wait until its flagged as "ready"
if (job.TransactionEndpoint == TransactionHandlerEndpoint.Backtesting || job.TransactionEndpoint == TransactionHandlerEndpoint.PaperTrading)
{
while (!transactions.Ready)
{
Thread.Yield();
}
}
if (time > _nextSample)
{
//Set next sample time: 4000 samples per backtest
_nextSample = time.Add(results.ResamplePeriod);
//Sample the portfolio value over time for chart.
results.SampleEquity(time, Math.Round(algorithm.Portfolio.TotalPortfolioValue, 4));
//Also add the user samples / plots to the result handler tracking:
results.SampleRange(algorithm.GetChartUpdates());
//Sample the asset pricing:
foreach (var security in algorithm.Securities.Values)
{
results.SampleAssetPrices(security.Symbol, time, security.Price);
}
}
ProcessMessages(results, algorithm);
//Save the previous time for the sample calculations
_previousTime = time;
} // End of Time Loop
} // End of ForEach DataStream
//Stream over:: Send the final packet and fire final events:
Log.Trace("AlgorithmManager.Run(): Firing On End Of Algorithm...");
try
{
algorithm.OnEndOfAlgorithm();
}
catch (Exception err)
{
_runtimeError = new Exception("Error running OnEndOfAlgorithm(): " + err.Message, err.InnerException);
return;
}
//Process the final messages from the algorithm
ProcessMessages(results, algorithm);
//Liquidate Holdings for Calculations:
if (_algorithmState == AlgorithmStatus.Liquidated || Engine.IsLocal || job.TransactionEndpoint == TransactionHandlerEndpoint.Backtesting)
{
Log.Trace("AlgorithmManager.Run(): Liquidating algorithm holdings...");
algorithm.Liquidate();
results.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.Liquidated);
}
//Manually stopped the algorithm
if (_algorithmState == AlgorithmStatus.Stopped)
{
Log.Trace("AlgorithmManager.Run(): Stopping algorithm...");
results.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.Stopped);
}
//Backtest deleted.
if (_algorithmState == AlgorithmStatus.Deleted)
{
Log.Trace("AlgorithmManager.Run(): Deleting algorithm...");
results.DebugMessage("Algorithm Id:(" + job.AlgorithmId + ") Deleted by request.");
results.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.Deleted);
}
//Algorithm finished, send regardless of commands:
results.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.Completed);
//Take final samples:
results.SampleRange(algorithm.GetChartUpdates());
results.SampleEquity(_frontier, Math.Round(algorithm.Portfolio.TotalPortfolioValue, 4));
results.SamplePerformance(_frontier, Math.Round((algorithm.Portfolio.TotalPortfolioValue - startingPerformance) * 100 / startingPerformance, 10));
} // End of Run();
/// <summary>
/// Setup the algorithm cash, dates and data subscriptions as desired.
/// </summary>
/// <param name="algorithm">Algorithm instance</param>
/// <param name="brokerage">Brokerage instance</param>
/// <param name="baseJob">Algorithm job</param>
/// <param name="resultHandler">The configured result handler</param>
/// <param name="transactionHandler">The configurated transaction handler</param>
/// <param name="realTimeHandler">The configured real time handler</param>
/// <returns>Boolean true on successfully initializing the algorithm</returns>
public bool Setup(IAlgorithm algorithm, IBrokerage brokerage, AlgorithmNodePacket baseJob, IResultHandler resultHandler, ITransactionHandler transactionHandler, IRealTimeHandler realTimeHandler)
{
var job = baseJob as BacktestNodePacket;
if (job == null)
{
throw new ArgumentException("Expected BacktestNodePacket but received " + baseJob.GetType().Name);
}
Log.Trace(string.Format("BacktestingSetupHandler.Setup(): Setting up job: Plan: {0}, UID: {1}, PID: {2}, Version: {3}, Source: {4}", job.UserPlan, job.UserId, job.ProjectId, job.Version, job.RequestSource));
if (algorithm == null)
{
Errors.Add("Could not create instance of algorithm");
return false;
}
//Make sure the algorithm start date ok.
if (job.PeriodStart == default(DateTime))
{
Errors.Add("Algorithm start date was never set");
return false;
}
var controls = job.Controls;
var isolator = new Isolator();
var initializeComplete = isolator.ExecuteWithTimeLimit(TimeSpan.FromMinutes(5), () =>
{
try
{
//Set our parameters
algorithm.SetParameters(job.Parameters);
//Algorithm is backtesting, not live:
algorithm.SetLiveMode(false);
//Set the algorithm time before we even initialize:
algorithm.SetDateTime(job.PeriodStart.ConvertToUtc(algorithm.TimeZone));
//Set the source impl for the event scheduling
algorithm.Schedule.SetEventSchedule(realTimeHandler);
//Initialise the algorithm, get the required data:
algorithm.Initialize();
}
catch (Exception err)
{
Errors.Add("Failed to initialize algorithm: Initialize(): " + err.Message);
}
});
//Before continuing, detect if this is ready:
if (!initializeComplete) return false;
algorithm.Transactions.SetOrderProcessor(transactionHandler);
algorithm.PostInitialize();
//Calculate the max runtime for the strategy
_maxRuntime = GetMaximumRuntime(job.PeriodStart, job.PeriodFinish, algorithm.SubscriptionManager.Count);
//Get starting capital:
_startingCaptial = algorithm.Portfolio.Cash;
//Max Orders: 10k per backtest:
if (job.UserPlan == UserPlan.Free)
{
_maxOrders = 10000;
}
else
{
_maxOrders = int.MaxValue;
_maxRuntime += _maxRuntime;
}
//Set back to the algorithm,
algorithm.SetMaximumOrders(_maxOrders);
//Starting date of the algorithm:
_startingDate = job.PeriodStart;
//Put into log for debugging:
Log.Trace("SetUp Backtesting: User: "******" ProjectId: " + job.ProjectId + " AlgoId: " + job.AlgorithmId);
Log.Trace("Dates: Start: " + job.PeriodStart.ToShortDateString() + " End: " + job.PeriodFinish.ToShortDateString() + " Cash: " + _startingCaptial.ToString("C"));
if (Errors.Count > 0)
{
initializeComplete = false;
}
return initializeComplete;
}
/// <summary>
/// Setup the algorithm cash, dates and portfolio as desired.
/// </summary>
/// <param name="algorithm">Existing algorithm instance</param>
/// <param name="brokerage">New brokerage instance</param>
/// <param name="baseJob">Backtesting job</param>
/// <param name="resultHandler">The configured result handler</param>
/// <param name="transactionHandler">The configuration transaction handler</param>
/// <param name="realTimeHandler">The configured real time handler</param>
/// <returns>Boolean true on successfully setting up the console.</returns>
public bool Setup(IAlgorithm algorithm, IBrokerage brokerage, AlgorithmNodePacket baseJob, IResultHandler resultHandler, ITransactionHandler transactionHandler, IRealTimeHandler realTimeHandler)
{
var initializeComplete = false;
try
{
//Set common variables for console programs:
if (baseJob.Type == PacketType.BacktestNode)
{
var backtestJob = baseJob as BacktestNodePacket;
algorithm.SetMaximumOrders(int.MaxValue);
// set our parameters
algorithm.SetParameters(baseJob.Parameters);
algorithm.SetLiveMode(false);
algorithm.SetAvailableDataTypes(GetConfiguredDataFeeds());
//Set the source impl for the event scheduling
algorithm.Schedule.SetEventSchedule(realTimeHandler);
//Setup Base Algorithm:
algorithm.Initialize();
//Set the time frontier of the algorithm
algorithm.SetDateTime(algorithm.StartDate.ConvertToUtc(algorithm.TimeZone));
//Construct the backtest job packet:
backtestJob.PeriodStart = algorithm.StartDate;
backtestJob.PeriodFinish = algorithm.EndDate;
backtestJob.BacktestId = algorithm.GetType().Name;
backtestJob.Type = PacketType.BacktestNode;
backtestJob.UserId = baseJob.UserId;
backtestJob.Channel = baseJob.Channel;
//Backtest Specific Parameters:
StartingDate = backtestJob.PeriodStart;
StartingPortfolioValue = algorithm.Portfolio.Cash;
}
else
{
throw new Exception("The ConsoleSetupHandler is for backtests only. Use the BrokerageSetupHandler.");
}
}
catch (Exception err)
{
Log.Error(err);
Errors.Add("Failed to initialize algorithm: Initialize(): " + err);
}
if (Errors.Count == 0)
{
initializeComplete = true;
}
algorithm.Transactions.SetOrderProcessor(transactionHandler);
algorithm.PostInitialize();
return(initializeComplete);
}
