/******************************************************** * CLASS METHODS *********************************************************/ /// <summary> /// Primary Analysis Thread: /// </summary> public static void Main(string[] args) { //Initialize: AlgorithmNodePacket job = null; var timer = Stopwatch.StartNew(); var algorithm = default(IAlgorithm); //Name thread for the profiler: Thread.CurrentThread.Name = "Algorithm Analysis Thread"; Log.Trace("Engine.Main(): Started " + DateTime.Now.ToShortTimeString()); Log.Trace("Engine.Main(): Memory " + OS.ApplicationMemoryUsed + "Mb-App " + +OS.TotalPhysicalMemoryUsed + "Mb-Used " + OS.TotalPhysicalMemory + "Mb-Total"); //Import external libraries specific to physical server location (cloud/local) var catalog = new AggregateCatalog(); catalog.Catalogs.Add(new DirectoryCatalog(@"../../Extensions")); var container = new CompositionContainer(catalog); try { // grab the right export based on configuration Notify = container.GetExportedValueByTypeName <IMessagingHandler>(Config.Get("messaging-handler")); Tasks = container.GetExportedValueByTypeName <ITaskHandler>(Config.Get("task-handler")); Controls = container.GetExportedValueByTypeName <IControls>(Config.Get("controls-handler")); } catch (CompositionException compositionException) { Log.Error("Engine.Main(): Failed to load library: " + compositionException); } //Setup packeting, queue and controls system: These don't do much locally. Controls.Initialize(); Notify.Initialize(); Tasks.Initialize(_liveMode); //Start monitoring the backtest active status: var statusPingThread = new Thread(StateCheck.Ping.Run); statusPingThread.Start(); do { try { //Clean up cache directories: CleanUpDirectories(); //Reset algo manager internal variables preparing for a new algorithm. AlgorithmManager.ResetManager(); //Reset thread holders. var initializeComplete = false; Thread threadFeed = null; Thread threadTransactions = null; Thread threadResults = null; Thread threadRealTime = null; //-> Pull job from QuantConnect job queue, or, pull local build: var algorithmPath = ""; job = Tasks.NextTask(out algorithmPath); // Blocking. //-> Initialize messaging system Notify.SetChannel(job.Channel); //-> Reset the backtest stopwatch; we're now running the algorithm. timer.Restart(); //-> Create SetupHandler to configure internal algorithm state: SetupHandler = GetSetupHandler(job.SetupEndpoint); //-> Set the result handler type for this algorithm job, and launch the associated result thread. ResultHandler = GetResultHandler(job); threadResults = new Thread(ResultHandler.Run, 0) { Name = "Result Thread" }; threadResults.Start(); try { // Save algorithm to cache, load algorithm instance: algorithm = SetupHandler.CreateAlgorithmInstance(algorithmPath); //Initialize the internal state of algorithm and job: executes the algorithm.Initialize() method. initializeComplete = SetupHandler.Setup(algorithm, out _brokerage, job); //If there are any reasons it failed, pass these back to the IDE. if (!initializeComplete || algorithm.ErrorMessages.Count > 0 || SetupHandler.Errors.Count > 0) { initializeComplete = false; //Get all the error messages: internal in algorithm and external in setup handler. var errorMessage = String.Join(",", algorithm.ErrorMessages); errorMessage += String.Join(",", SetupHandler.Errors); throw new Exception(errorMessage); } } catch (Exception err) { ResultHandler.RuntimeError("Algorithm.Initialize() Error: " + err.Message, err.StackTrace); } //-> Using the job + initialization: load the designated handlers: if (initializeComplete) { //Set algorithm as locked; set it to live mode if we're trading live, and set it to locked for no further updates. algorithm.SetAlgorithmId(job.AlgorithmId); algorithm.SetLiveMode(LiveMode); algorithm.SetLocked(); //Load the associated handlers for data, transaction and realtime events: ResultHandler.SetAlgorithm(algorithm); DataFeed = GetDataFeedHandler(algorithm, _brokerage, job); TransactionHandler = GetTransactionHandler(algorithm, _brokerage, ResultHandler, job); RealTimeHandler = GetRealTimeHandler(algorithm, _brokerage, DataFeed, ResultHandler, job); //Set the error handlers for the brokerage asynchronous errors. SetupHandler.SetupErrorHandler(ResultHandler, _brokerage); //Send status to user the algorithm is now executing. ResultHandler.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.Running); //Launch the data, transaction and realtime handlers into dedicated threads threadFeed = new Thread(DataFeed.Run, 0) { Name = "DataFeed Thread" }; threadTransactions = new Thread(TransactionHandler.Run, 0) { Name = "Transaction Thread" }; threadRealTime = new Thread(RealTimeHandler.Run, 0) { Name = "RealTime Thread" }; //Launch the data feed, result sending, and transaction models/handlers in separate threads. threadFeed.Start(); // Data feed pushing data packets into thread bridge; threadTransactions.Start(); // Transaction modeller scanning new order requests threadRealTime.Start(); // RealTime scan time for time based events: // Result manager scanning message queue: (started earlier) try { // Execute the Algorithm Code: var complete = Isolator.ExecuteWithTimeLimit(SetupHandler.MaximumRuntime, () => { try { //Run Algorithm Job: // -> Using this Data Feed, // -> Send Orders to this TransactionHandler, // -> Send Results to ResultHandler. AlgorithmManager.Run(job, algorithm, DataFeed, TransactionHandler, ResultHandler, SetupHandler, RealTimeHandler); } catch (Exception err) { //Debugging at this level is difficult, stack trace needed. Log.Error("Engine.Run(): Error in Algo Manager: " + err.Message + " ST >> " + err.StackTrace); } Log.Trace("Engine.Run(): Exiting Algorithm Manager"); }, MaximumRamAllocation); if (!complete) { Log.Error("Engine.Main(): Failed to complete in time: " + SetupHandler.MaximumRuntime.ToString("F")); throw new Exception("Failed to complete algorithm within " + SetupHandler.MaximumRuntime.ToString("F") + " seconds. Please make it run faster."); } // Algorithm runtime error: if (AlgorithmManager.RunTimeError != null) { throw AlgorithmManager.RunTimeError; } } catch (Exception err) { //Log: Log.Error("Engine.Run(): Breaking out of parent try-catch: " + err.Message + " " + err.StackTrace); //Error running the user algorithm: purge datafeed. if (DataFeed != null) { DataFeed.Exit(); } //Send the error message: if (ResultHandler != null) { ResultHandler.RuntimeError("Runtime Error: " + err.Message, err.StackTrace); } } //Send result data back: this entire code block could be rewritten. // todo: - Split up statistics class, its enormous. // todo: - Make a dedicated Statistics.Benchmark class. // todo: - Elegently manage failure scenarios where no equity present. // todo: - Move all creation and transmission of statistics out of primary engine loop. // todo: - Statistics.Generate(algorithm, resulthandler, transactionhandler); try { var charts = new Dictionary <string, Chart>(ResultHandler.Charts); var orders = new Dictionary <int, Order>(TransactionHandler.Orders); var holdings = new Dictionary <string, Holding>(); var statistics = new Dictionary <string, string>(); var banner = new Dictionary <string, string>(); try { //Generates error when things don't exist (no charting logged, runtime errors in main algo execution) var equity = charts["Strategy Equity"].Series["Equity"].Values; var performance = charts["Daily Performance"].Series["Performance"].Values; var profitLoss = new SortedDictionary <DateTime, decimal>(algorithm.Transactions.TransactionRecord); statistics = Statistics.Statistics.Generate(equity, profitLoss, performance, SetupHandler.StartingCapital, 252); } catch (Exception err) { Log.Error("Algorithm.Node.Engine(): Error generating result packet: " + err.Message); } //Diagnostics Completed: ResultHandler.DebugMessage("Algorithm Id:(" + job.AlgorithmId + ") completed analysis in " + timer.Elapsed.TotalSeconds.ToString("F2") + " seconds"); //Send the result packet: ResultHandler.SendFinalResult(job, orders, algorithm.Transactions.TransactionRecord, holdings, statistics, banner); } catch (Exception err) { Log.Error("Engine.Main(): Error sending analysis result: " + err.Message + " ST >> " + err.StackTrace); } //Before we return, send terminate commands to close up the threads timer.Stop(); //Algorithm finished running. TransactionHandler.Exit(); DataFeed.Exit(); RealTimeHandler.Exit(); AlgorithmManager.ResetManager(); } //Close result handler: ResultHandler.Exit(); //Wait for the threads to complete: Log.Trace("Engine.Main(): Waiting for threads to deactivate..."); var ts = Stopwatch.StartNew(); while ((ResultHandler.IsActive || (TransactionHandler != null && TransactionHandler.IsActive) || (DataFeed != null && DataFeed.IsActive)) && ts.ElapsedMilliseconds < 60 * 1000) { Thread.Sleep(100); DataFeed.Exit(); Log.Trace("WAITING >> Result: " + ResultHandler.IsActive + " Transaction: " + TransactionHandler.IsActive + " DataFeed: " + DataFeed.IsActive + " RealTime: " + RealTimeHandler.IsActive); } Log.Trace("Engine.Main(): Closing Threads..."); if (threadFeed != null && threadFeed.IsAlive) { threadFeed.Abort(); } if (threadTransactions != null && threadTransactions.IsAlive) { threadTransactions.Abort(); } if (threadResults != null && threadResults.IsAlive) { threadResults.Abort(); } Log.Trace("Engine.Main(): Analysis Completed and Results Posted."); } catch (Exception err) { Log.Error("Engine.Main(): Error running algorithm: " + err.Message + " >> " + err.StackTrace); } finally { //Delete the message from the queue before another worker picks it up: Tasks.AcknowledgeTask(job); Log.Trace("Engine.Main(): Packet removed from queue: " + job.AlgorithmId); GC.Collect(); } //If we're running locally will execute just once. } while (!IsLocal); // Send the exit signal and then kill the thread StateCheck.Ping.Exit(); // Make the console window pause so we can read log output before exiting and killing the application completely System.Console.ReadKey(); //Finally if ping thread still not complete, kill. if (statusPingThread != null && statusPingThread.IsAlive) { statusPingThread.Abort(); } }