public static void RunLocalBacktest(string algorithm, Dictionary <string, string> expectedStatistics, Language language) { var statistics = new Dictionary <string, string>(); Composer.Instance.Reset(); try { // set the configuration up Config.Set("algorithm-type-name", algorithm); Config.Set("live-mode", "false"); Config.Set("environment", ""); Config.Set("messaging-handler", "QuantConnect.Messaging.Messaging"); Config.Set("job-queue-handler", "QuantConnect.Queues.JobQueue"); Config.Set("api-handler", "QuantConnect.Api.Api"); Config.Set("result-handler", "QuantConnect.Lean.Engine.Results.BacktestingResultHandler"); Config.Set("algorithm-language", language.ToString()); Config.Set("algorithm-location", "QuantConnect.Algorithm." + language + ".dll"); var debugEnabled = Log.DebuggingEnabled; var logHandlers = new ILogHandler[] { new ConsoleLogHandler(), new FileLogHandler("regression.log", false) }; using (Log.LogHandler = new CompositeLogHandler(logHandlers)) using (var algorithmHandlers = LeanEngineAlgorithmHandlers.FromConfiguration(Composer.Instance)) using (var systemHandlers = LeanEngineSystemHandlers.FromConfiguration(Composer.Instance)) { Log.DebuggingEnabled = true; Log.LogHandler.Trace(""); Log.LogHandler.Trace("{0}: Running " + algorithm + "...", DateTime.UtcNow); Log.LogHandler.Trace(""); // run the algorithm in its own thread var engine = new Lean.Engine.Engine(systemHandlers, algorithmHandlers, false); Task.Factory.StartNew(() => { string algorithmPath; var job = systemHandlers.JobQueue.NextJob(out algorithmPath); var algorithmManager = new AlgorithmManager(false); engine.Run(job, algorithmManager, algorithmPath); }).Wait(); var backtestingResultHandler = (BacktestingResultHandler)algorithmHandlers.Results; statistics = backtestingResultHandler.FinalStatistics; Log.DebuggingEnabled = debugEnabled; } } catch (Exception ex) { Log.LogHandler.Error("{0} {1}", ex.Message, ex.StackTrace); } foreach (var stat in expectedStatistics) { Assert.AreEqual(true, statistics.ContainsKey(stat.Key), "Missing key: " + stat.Key); Assert.AreEqual(stat.Value, statistics[stat.Key], "Failed on " + stat.Key); } }
public static void RunLocalBacktest(string algorithm, Dictionary<string, string> expectedStatistics, Language language) { var statistics = new Dictionary<string, string>(); Composer.Instance.Reset(); try { // set the configuration up Config.Set("algorithm-type-name", algorithm); Config.Set("live-mode", "false"); Config.Set("environment", ""); Config.Set("messaging-handler", "QuantConnect.Messaging.Messaging"); Config.Set("job-queue-handler", "QuantConnect.Queues.JobQueue"); Config.Set("api-handler", "QuantConnect.Api.Api"); Config.Set("result-handler", "QuantConnect.Lean.Engine.Results.BacktestingResultHandler"); Config.Set("algorithm-language", language.ToString()); Config.Set("algorithm-location", "QuantConnect.Algorithm." + language + ".dll"); var debugEnabled = Log.DebuggingEnabled; var logHandlers = new ILogHandler[] {new ConsoleLogHandler(), new FileLogHandler("regression.log", false)}; using (Log.LogHandler = new CompositeLogHandler(logHandlers)) using (var algorithmHandlers = LeanEngineAlgorithmHandlers.FromConfiguration(Composer.Instance)) using (var systemHandlers = LeanEngineSystemHandlers.FromConfiguration(Composer.Instance)) { Log.DebuggingEnabled = true; Log.LogHandler.Trace(""); Log.LogHandler.Trace("{0}: Running " + algorithm + "...", DateTime.UtcNow); Log.LogHandler.Trace(""); // run the algorithm in its own thread var engine = new Lean.Engine.Engine(systemHandlers, algorithmHandlers, false); Task.Factory.StartNew(() => { string algorithmPath; var job = systemHandlers.JobQueue.NextJob(out algorithmPath); engine.Run(job, algorithmPath); }).Wait(); var backtestingResultHandler = (BacktestingResultHandler)algorithmHandlers.Results; statistics = backtestingResultHandler.FinalStatistics; Log.DebuggingEnabled = debugEnabled; } } catch (Exception ex) { Log.LogHandler.Error("{0} {1}", ex.Message, ex.StackTrace); } foreach (var stat in expectedStatistics) { Assert.AreEqual(true, statistics.ContainsKey(stat.Key), "Missing key: " + stat.Key); Assert.AreEqual(stat.Value, statistics[stat.Key], "Failed on " + stat.Key); } }
public static void RunLocalBacktest(string algorithm, Dictionary <string, string> expectedStatistics) { var statistics = new Dictionary <string, string>(); Composer.Instance.Reset(); try { using (Log.LogHandler = new CompositeLogHandler(new ILogHandler[] { new ConsoleLogHandler(), new FileLogHandler("regression.log") })) using (var algorithmHandlers = LeanEngineAlgorithmHandlers.FromConfiguration(Composer.Instance)) using (var systemHandlers = LeanEngineSystemHandlers.FromConfiguration(Composer.Instance)) { Console.WriteLine("Running " + algorithm + "..."); // set the configuration up Config.Set("algorithm-type-name", algorithm); Config.Set("live-mode", "false"); Config.Set("environment", ""); Config.Set("messaging-handler", "QuantConnect.Messaging.Messaging"); Config.Set("job-queue-handler", "QuantConnect.Queues.JobQueue"); Config.Set("api-handler", "QuantConnect.Api.Api"); // run the algorithm in its own thread var engine = new Lean.Engine.Engine(systemHandlers, algorithmHandlers, false); Task.Factory.StartNew(() => { string algorithmPath; var job = systemHandlers.JobQueue.NextJob(out algorithmPath); engine.Run(job, algorithmPath); }).Wait(); var consoleResultHandler = (ConsoleResultHandler)algorithmHandlers.Results; statistics = consoleResultHandler.FinalStatistics; } } catch (Exception ex) { Log.LogHandler.Error("{0} {1}", ex.Message, ex.StackTrace); } foreach (var stat in expectedStatistics) { Assert.AreEqual(true, statistics.ContainsKey(stat.Key), "Missing key: " + stat.Key); Assert.AreEqual(stat.Value, statistics[stat.Key], "Failed on " + stat.Key); } }
public static void RunLocalBacktest(string algorithm, Dictionary<string, string> expectedStatistics) { Log.LogHandler = new CompositeLogHandler(new ILogHandler[] { new ConsoleLogHandler(), new FileLogHandler("regression.log") }); Console.WriteLine("Running " + algorithm + "..."); // set the configuration up Config.Set("algorithm-type-name", algorithm); Config.Set("live-mode", "false"); Config.Set("environment", ""); Config.Set("messaging-handler", "QuantConnect.Messaging.Messaging"); Config.Set("job-queue-handler", "QuantConnect.Queues.JobQueue"); Config.Set("api-handler", "QuantConnect.Api.Api"); // run the algorithm in its own thread var systemHandlers = LeanEngineSystemHandlers.FromConfiguration(Composer.Instance); var algorithmHandlers = LeanEngineAlgorithmHandlers.FromConfiguration(Composer.Instance); var engine = new Lean.Engine.Engine(systemHandlers, algorithmHandlers, false); Task.Factory.StartNew(() => { string algorithmPath; var job = systemHandlers.JobQueue.NextJob(out algorithmPath); engine.Run(job, algorithmPath); systemHandlers.JobQueue.AcknowledgeJob(job); }).Wait(); var consoleResultHandler = (ConsoleResultHandler)algorithmHandlers.Results; var statistics = consoleResultHandler.FinalStatistics; foreach (var stat in expectedStatistics) { Assert.AreEqual(stat.Value, statistics[stat.Key], "Failed on " + stat.Key); } systemHandlers.Dispose(); algorithmHandlers.Dispose(); }
public static void RunLocalBacktest(string algorithm, Dictionary <string, string> expectedStatistics, AlphaRuntimeStatistics expectedAlphaStatistics, Language language) { var statistics = new Dictionary <string, string>(); var alphaStatistics = new AlphaRuntimeStatistics(); Composer.Instance.Reset(); var ordersLogFile = string.Empty; var logFile = $"./regression/{algorithm}.{language.ToLower()}.log"; Directory.CreateDirectory(Path.GetDirectoryName(logFile)); File.Delete(logFile); try { // set the configuration up Config.Set("algorithm-type-name", algorithm); Config.Set("live-mode", "false"); Config.Set("environment", ""); Config.Set("messaging-handler", "QuantConnect.Messaging.Messaging"); Config.Set("job-queue-handler", "QuantConnect.Queues.JobQueue"); Config.Set("setup-handler", "RegressionSetupHandlerWrapper"); Config.Set("history-provider", "RegressionHistoryProviderWrapper"); Config.Set("api-handler", "QuantConnect.Api.Api"); Config.Set("result-handler", "QuantConnect.Lean.Engine.Results.RegressionResultHandler"); Config.Set("algorithm-language", language.ToString()); Config.Set("algorithm-location", language == Language.Python ? "../../../Algorithm.Python/" + algorithm + ".py" : "QuantConnect.Algorithm." + language + ".dll"); var debugEnabled = Log.DebuggingEnabled; var logHandlers = new ILogHandler[] { new ConsoleLogHandler(), new FileLogHandler(logFile, false) }; using (Log.LogHandler = new CompositeLogHandler(logHandlers)) using (var algorithmHandlers = LeanEngineAlgorithmHandlers.FromConfiguration(Composer.Instance)) using (var systemHandlers = LeanEngineSystemHandlers.FromConfiguration(Composer.Instance)) { Log.DebuggingEnabled = true; Log.LogHandler.Trace(""); Log.LogHandler.Trace("{0}: Running " + algorithm + "...", DateTime.UtcNow); Log.LogHandler.Trace(""); // run the algorithm in its own thread var engine = new Lean.Engine.Engine(systemHandlers, algorithmHandlers, false); Task.Factory.StartNew(() => { string algorithmPath; var job = systemHandlers.JobQueue.NextJob(out algorithmPath); var algorithmManager = new AlgorithmManager(false); engine.Run(job, algorithmManager, algorithmPath); ordersLogFile = ((RegressionResultHandler)algorithmHandlers.Results).OrdersLogFilePath; }).Wait(); var backtestingResultHandler = (BacktestingResultHandler)algorithmHandlers.Results; statistics = backtestingResultHandler.FinalStatistics; var defaultAlphaHandler = (DefaultAlphaHandler)algorithmHandlers.Alphas; alphaStatistics = defaultAlphaHandler.RuntimeStatistics; Log.DebuggingEnabled = debugEnabled; } } catch (Exception ex) { Log.LogHandler.Error("{0} {1}", ex.Message, ex.StackTrace); } foreach (var stat in expectedStatistics) { Assert.AreEqual(true, statistics.ContainsKey(stat.Key), "Missing key: " + stat.Key); Assert.AreEqual(stat.Value, statistics[stat.Key], "Failed on " + stat.Key); } if (expectedAlphaStatistics != null) { AssertAlphaStatistics(expectedAlphaStatistics, alphaStatistics, s => s.MeanPopulationScore.Direction); AssertAlphaStatistics(expectedAlphaStatistics, alphaStatistics, s => s.MeanPopulationScore.Magnitude); AssertAlphaStatistics(expectedAlphaStatistics, alphaStatistics, s => s.RollingAveragedPopulationScore.Direction); AssertAlphaStatistics(expectedAlphaStatistics, alphaStatistics, s => s.RollingAveragedPopulationScore.Magnitude); AssertAlphaStatistics(expectedAlphaStatistics, alphaStatistics, s => s.LongShortRatio); AssertAlphaStatistics(expectedAlphaStatistics, alphaStatistics, s => s.TotalInsightsClosed); AssertAlphaStatistics(expectedAlphaStatistics, alphaStatistics, s => s.TotalInsightsGenerated); AssertAlphaStatistics(expectedAlphaStatistics, alphaStatistics, s => s.TotalAccumulatedEstimatedAlphaValue); AssertAlphaStatistics(expectedAlphaStatistics, alphaStatistics, s => s.TotalInsightsAnalysisCompleted); } // we successfully passed the regression test, copy the log file so we don't have to continually // re-run master in order to compare against a passing run var passedFile = logFile.Replace("./regression/", "./passed/"); Directory.CreateDirectory(Path.GetDirectoryName(passedFile)); File.Delete(passedFile); File.Copy(logFile, passedFile); var passedOrderLogFile = ordersLogFile.Replace("./regression/", "./passed/"); Directory.CreateDirectory(Path.GetDirectoryName(passedFile)); File.Delete(passedOrderLogFile); if (File.Exists(ordersLogFile)) { File.Copy(ordersLogFile, passedOrderLogFile); } }
public static AlgorithmRunnerResults RunLocalBacktest( string algorithm, Dictionary <string, string> expectedStatistics, AlphaRuntimeStatistics expectedAlphaStatistics, Language language, AlgorithmStatus expectedFinalStatus, DateTime?startDate = null, DateTime?endDate = null, string setupHandler = "RegressionSetupHandlerWrapper", decimal?initialCash = null) { AlgorithmManager algorithmManager = null; var statistics = new Dictionary <string, string>(); var alphaStatistics = new AlphaRuntimeStatistics(new TestAccountCurrencyProvider()); BacktestingResultHandler results = null; Composer.Instance.Reset(); SymbolCache.Clear(); var ordersLogFile = string.Empty; var logFile = $"./regression/{algorithm}.{language.ToLower()}.log"; Directory.CreateDirectory(Path.GetDirectoryName(logFile)); File.Delete(logFile); try { // set the configuration up Config.Set("algorithm-type-name", algorithm); Config.Set("live-mode", "false"); Config.Set("environment", ""); Config.Set("messaging-handler", "QuantConnect.Messaging.Messaging"); Config.Set("job-queue-handler", "QuantConnect.Queues.JobQueue"); Config.Set("setup-handler", setupHandler); Config.Set("history-provider", "RegressionHistoryProviderWrapper"); Config.Set("api-handler", "QuantConnect.Api.Api"); Config.Set("result-handler", "QuantConnect.Lean.Engine.Results.RegressionResultHandler"); Config.Set("algorithm-language", language.ToString()); Config.Set("algorithm-location", language == Language.Python ? "../../../Algorithm.Python/" + algorithm + ".py" : "QuantConnect.Algorithm." + language + ".dll"); // Store initial log variables var initialLogHandler = Log.LogHandler; var initialDebugEnabled = Log.DebuggingEnabled; // Log handlers specific to this test function var newLogHandlers = new ILogHandler[] { new ConsoleErrorLogHandler(), new FileLogHandler(logFile, false) }; using (Log.LogHandler = new CompositeLogHandler(newLogHandlers)) using (var algorithmHandlers = LeanEngineAlgorithmHandlers.FromConfiguration(Composer.Instance)) using (var systemHandlers = LeanEngineSystemHandlers.FromConfiguration(Composer.Instance)) using (var workerThread = new TestWorkerThread()) { Log.DebuggingEnabled = true; Log.Trace(""); Log.Trace("{0}: Running " + algorithm + "...", DateTime.UtcNow); Log.Trace(""); // run the algorithm in its own thread var engine = new Lean.Engine.Engine(systemHandlers, algorithmHandlers, false); Task.Factory.StartNew(() => { try { string algorithmPath; var job = (BacktestNodePacket)systemHandlers.JobQueue.NextJob(out algorithmPath); job.BacktestId = algorithm; job.PeriodStart = startDate; job.PeriodFinish = endDate; if (initialCash.HasValue) { job.CashAmount = new CashAmount(initialCash.Value, Currencies.USD); } algorithmManager = new AlgorithmManager(false, job); systemHandlers.LeanManager.Initialize(systemHandlers, algorithmHandlers, job, algorithmManager); engine.Run(job, algorithmManager, algorithmPath, workerThread); ordersLogFile = ((RegressionResultHandler)algorithmHandlers.Results).LogFilePath; } catch (Exception e) { Log.Trace($"Error in AlgorithmRunner task: {e}"); } }).Wait(); var backtestingResultHandler = (BacktestingResultHandler)algorithmHandlers.Results; results = backtestingResultHandler; statistics = backtestingResultHandler.FinalStatistics; var defaultAlphaHandler = (DefaultAlphaHandler)algorithmHandlers.Alphas; alphaStatistics = defaultAlphaHandler.RuntimeStatistics; } // Reset settings to initial values Log.LogHandler = initialLogHandler; Log.DebuggingEnabled = initialDebugEnabled; } catch (Exception ex) { if (expectedFinalStatus != AlgorithmStatus.RuntimeError) { Log.Error("{0} {1}", ex.Message, ex.StackTrace); } } if (algorithmManager?.State != expectedFinalStatus) { Assert.Fail($"Algorithm state should be {expectedFinalStatus} and is: {algorithmManager?.State}"); } foreach (var stat in expectedStatistics) { Assert.AreEqual(true, statistics.ContainsKey(stat.Key), "Missing key: " + stat.Key); Assert.AreEqual(stat.Value, statistics[stat.Key], "Failed on " + stat.Key); } if (expectedAlphaStatistics != null) { AssertAlphaStatistics(expectedAlphaStatistics, alphaStatistics, s => s.MeanPopulationScore.Direction); AssertAlphaStatistics(expectedAlphaStatistics, alphaStatistics, s => s.MeanPopulationScore.Magnitude); AssertAlphaStatistics(expectedAlphaStatistics, alphaStatistics, s => s.RollingAveragedPopulationScore.Direction); AssertAlphaStatistics(expectedAlphaStatistics, alphaStatistics, s => s.RollingAveragedPopulationScore.Magnitude); AssertAlphaStatistics(expectedAlphaStatistics, alphaStatistics, s => s.LongShortRatio); AssertAlphaStatistics(expectedAlphaStatistics, alphaStatistics, s => s.TotalInsightsClosed); AssertAlphaStatistics(expectedAlphaStatistics, alphaStatistics, s => s.TotalInsightsGenerated); AssertAlphaStatistics(expectedAlphaStatistics, alphaStatistics, s => s.TotalAccumulatedEstimatedAlphaValue); AssertAlphaStatistics(expectedAlphaStatistics, alphaStatistics, s => s.TotalInsightsAnalysisCompleted); } // we successfully passed the regression test, copy the log file so we don't have to continually // re-run master in order to compare against a passing run var passedFile = logFile.Replace("./regression/", "./passed/"); Directory.CreateDirectory(Path.GetDirectoryName(passedFile)); File.Delete(passedFile); File.Copy(logFile, passedFile); var passedOrderLogFile = ordersLogFile.Replace("./regression/", "./passed/"); Directory.CreateDirectory(Path.GetDirectoryName(passedFile)); File.Delete(passedOrderLogFile); if (File.Exists(ordersLogFile)) { File.Copy(ordersLogFile, passedOrderLogFile); } return(new AlgorithmRunnerResults(algorithm, language, algorithmManager, results)); }