public static void RunLocalBacktest(string algorithm, Dictionary <string, string> expectedStatistics, Language language)
{
var statistics = new Dictionary <string, string>();
Composer.Instance.Reset();
try
{
// set the configuration up
Config.Set("algorithm-type-name", algorithm);
Config.Set("live-mode", "false");
Config.Set("environment", "");
Config.Set("messaging-handler", "QuantConnect.Messaging.Messaging");
Config.Set("job-queue-handler", "QuantConnect.Queues.JobQueue");
Config.Set("api-handler", "QuantConnect.Api.Api");
Config.Set("result-handler", "QuantConnect.Lean.Engine.Results.BacktestingResultHandler");
Config.Set("algorithm-language", language.ToString());
Config.Set("algorithm-location", "QuantConnect.Algorithm." + language + ".dll");
var debugEnabled = Log.DebuggingEnabled;
var logHandlers = new ILogHandler[] { new ConsoleLogHandler(), new FileLogHandler("regression.log", false) };
using (Log.LogHandler = new CompositeLogHandler(logHandlers))
using (var algorithmHandlers = LeanEngineAlgorithmHandlers.FromConfiguration(Composer.Instance))
using (var systemHandlers = LeanEngineSystemHandlers.FromConfiguration(Composer.Instance))
{
Log.DebuggingEnabled = true;
Log.LogHandler.Trace("");
Log.LogHandler.Trace("{0}: Running " + algorithm + "...", DateTime.UtcNow);
Log.LogHandler.Trace("");
// run the algorithm in its own thread
var engine = new Lean.Engine.Engine(systemHandlers, algorithmHandlers, false);
Task.Factory.StartNew(() =>
{
string algorithmPath;
var job = systemHandlers.JobQueue.NextJob(out algorithmPath);
var algorithmManager = new AlgorithmManager(false);
engine.Run(job, algorithmManager, algorithmPath);
}).Wait();
var backtestingResultHandler = (BacktestingResultHandler)algorithmHandlers.Results;
statistics = backtestingResultHandler.FinalStatistics;
Log.DebuggingEnabled = debugEnabled;
}
}
catch (Exception ex)
{
Log.LogHandler.Error("{0} {1}", ex.Message, ex.StackTrace);
}
foreach (var stat in expectedStatistics)
{
Assert.AreEqual(true, statistics.ContainsKey(stat.Key), "Missing key: " + stat.Key);
Assert.AreEqual(stat.Value, statistics[stat.Key], "Failed on " + stat.Key);
}
}
public static void RunLocalBacktest(string algorithm, Dictionary<string, string> expectedStatistics, Language language)
{
var statistics = new Dictionary<string, string>();
Composer.Instance.Reset();
try
{
// set the configuration up
Config.Set("algorithm-type-name", algorithm);
Config.Set("live-mode", "false");
Config.Set("environment", "");
Config.Set("messaging-handler", "QuantConnect.Messaging.Messaging");
Config.Set("job-queue-handler", "QuantConnect.Queues.JobQueue");
Config.Set("api-handler", "QuantConnect.Api.Api");
Config.Set("result-handler", "QuantConnect.Lean.Engine.Results.BacktestingResultHandler");
Config.Set("algorithm-language", language.ToString());
Config.Set("algorithm-location", "QuantConnect.Algorithm." + language + ".dll");
var debugEnabled = Log.DebuggingEnabled;
var logHandlers = new ILogHandler[] {new ConsoleLogHandler(), new FileLogHandler("regression.log", false)};
using (Log.LogHandler = new CompositeLogHandler(logHandlers))
using (var algorithmHandlers = LeanEngineAlgorithmHandlers.FromConfiguration(Composer.Instance))
using (var systemHandlers = LeanEngineSystemHandlers.FromConfiguration(Composer.Instance))
{
Log.DebuggingEnabled = true;
Log.LogHandler.Trace("");
Log.LogHandler.Trace("{0}: Running " + algorithm + "...", DateTime.UtcNow);
Log.LogHandler.Trace("");
// run the algorithm in its own thread
var engine = new Lean.Engine.Engine(systemHandlers, algorithmHandlers, false);
Task.Factory.StartNew(() =>
{
string algorithmPath;
var job = systemHandlers.JobQueue.NextJob(out algorithmPath);
engine.Run(job, algorithmPath);
}).Wait();
var backtestingResultHandler = (BacktestingResultHandler)algorithmHandlers.Results;
statistics = backtestingResultHandler.FinalStatistics;
Log.DebuggingEnabled = debugEnabled;
}
}
catch (Exception ex)
{
Log.LogHandler.Error("{0} {1}", ex.Message, ex.StackTrace);
}
foreach (var stat in expectedStatistics)
{
Assert.AreEqual(true, statistics.ContainsKey(stat.Key), "Missing key: " + stat.Key);
Assert.AreEqual(stat.Value, statistics[stat.Key], "Failed on " + stat.Key);
}
}
public static void RunLocalBacktest(string algorithm, Dictionary <string, string> expectedStatistics)
{
var statistics = new Dictionary <string, string>();
Composer.Instance.Reset();
try
{
using (Log.LogHandler = new CompositeLogHandler(new ILogHandler[]
{
new ConsoleLogHandler(),
new FileLogHandler("regression.log")
}))
using (var algorithmHandlers = LeanEngineAlgorithmHandlers.FromConfiguration(Composer.Instance))
using (var systemHandlers = LeanEngineSystemHandlers.FromConfiguration(Composer.Instance))
{
Console.WriteLine("Running " + algorithm + "...");
// set the configuration up
Config.Set("algorithm-type-name", algorithm);
Config.Set("live-mode", "false");
Config.Set("environment", "");
Config.Set("messaging-handler", "QuantConnect.Messaging.Messaging");
Config.Set("job-queue-handler", "QuantConnect.Queues.JobQueue");
Config.Set("api-handler", "QuantConnect.Api.Api");
// run the algorithm in its own thread
var engine = new Lean.Engine.Engine(systemHandlers, algorithmHandlers, false);
Task.Factory.StartNew(() =>
{
string algorithmPath;
var job = systemHandlers.JobQueue.NextJob(out algorithmPath);
engine.Run(job, algorithmPath);
}).Wait();
var consoleResultHandler = (ConsoleResultHandler)algorithmHandlers.Results;
statistics = consoleResultHandler.FinalStatistics;
}
}
catch (Exception ex)
{
Log.LogHandler.Error("{0} {1}", ex.Message, ex.StackTrace);
}
foreach (var stat in expectedStatistics)
{
Assert.AreEqual(true, statistics.ContainsKey(stat.Key), "Missing key: " + stat.Key);
Assert.AreEqual(stat.Value, statistics[stat.Key], "Failed on " + stat.Key);
}
}
public static void RunLocalBacktest(string algorithm, Dictionary<string, string> expectedStatistics)
{
Log.LogHandler = new CompositeLogHandler(new ILogHandler[]
{
new ConsoleLogHandler(),
new FileLogHandler("regression.log")
});
Console.WriteLine("Running " + algorithm + "...");
// set the configuration up
Config.Set("algorithm-type-name", algorithm);
Config.Set("live-mode", "false");
Config.Set("environment", "");
Config.Set("messaging-handler", "QuantConnect.Messaging.Messaging");
Config.Set("job-queue-handler", "QuantConnect.Queues.JobQueue");
Config.Set("api-handler", "QuantConnect.Api.Api");
// run the algorithm in its own thread
var systemHandlers = LeanEngineSystemHandlers.FromConfiguration(Composer.Instance);
var algorithmHandlers = LeanEngineAlgorithmHandlers.FromConfiguration(Composer.Instance);
var engine = new Lean.Engine.Engine(systemHandlers, algorithmHandlers, false);
Task.Factory.StartNew(() =>
{
string algorithmPath;
var job = systemHandlers.JobQueue.NextJob(out algorithmPath);
engine.Run(job, algorithmPath);
systemHandlers.JobQueue.AcknowledgeJob(job);
}).Wait();
var consoleResultHandler = (ConsoleResultHandler)algorithmHandlers.Results;
var statistics = consoleResultHandler.FinalStatistics;
foreach (var stat in expectedStatistics)
{
Assert.AreEqual(stat.Value, statistics[stat.Key], "Failed on " + stat.Key);
}
systemHandlers.Dispose();
algorithmHandlers.Dispose();
}
public static void RunLocalBacktest(string algorithm, Dictionary <string, string> expectedStatistics, AlphaRuntimeStatistics expectedAlphaStatistics, Language language)
{
var statistics = new Dictionary <string, string>();
var alphaStatistics = new AlphaRuntimeStatistics();
Composer.Instance.Reset();
var ordersLogFile = string.Empty;
var logFile = $"./regression/{algorithm}.{language.ToLower()}.log";
Directory.CreateDirectory(Path.GetDirectoryName(logFile));
File.Delete(logFile);
try
{
// set the configuration up
Config.Set("algorithm-type-name", algorithm);
Config.Set("live-mode", "false");
Config.Set("environment", "");
Config.Set("messaging-handler", "QuantConnect.Messaging.Messaging");
Config.Set("job-queue-handler", "QuantConnect.Queues.JobQueue");
Config.Set("setup-handler", "RegressionSetupHandlerWrapper");
Config.Set("history-provider", "RegressionHistoryProviderWrapper");
Config.Set("api-handler", "QuantConnect.Api.Api");
Config.Set("result-handler", "QuantConnect.Lean.Engine.Results.RegressionResultHandler");
Config.Set("algorithm-language", language.ToString());
Config.Set("algorithm-location",
language == Language.Python
? "../../../Algorithm.Python/" + algorithm + ".py"
: "QuantConnect.Algorithm." + language + ".dll");
var debugEnabled = Log.DebuggingEnabled;
var logHandlers = new ILogHandler[] { new ConsoleLogHandler(), new FileLogHandler(logFile, false) };
using (Log.LogHandler = new CompositeLogHandler(logHandlers))
using (var algorithmHandlers = LeanEngineAlgorithmHandlers.FromConfiguration(Composer.Instance))
using (var systemHandlers = LeanEngineSystemHandlers.FromConfiguration(Composer.Instance))
{
Log.DebuggingEnabled = true;
Log.LogHandler.Trace("");
Log.LogHandler.Trace("{0}: Running " + algorithm + "...", DateTime.UtcNow);
Log.LogHandler.Trace("");
// run the algorithm in its own thread
var engine = new Lean.Engine.Engine(systemHandlers, algorithmHandlers, false);
Task.Factory.StartNew(() =>
{
string algorithmPath;
var job = systemHandlers.JobQueue.NextJob(out algorithmPath);
var algorithmManager = new AlgorithmManager(false);
engine.Run(job, algorithmManager, algorithmPath);
ordersLogFile = ((RegressionResultHandler)algorithmHandlers.Results).OrdersLogFilePath;
}).Wait();
var backtestingResultHandler = (BacktestingResultHandler)algorithmHandlers.Results;
statistics = backtestingResultHandler.FinalStatistics;
var defaultAlphaHandler = (DefaultAlphaHandler)algorithmHandlers.Alphas;
alphaStatistics = defaultAlphaHandler.RuntimeStatistics;
Log.DebuggingEnabled = debugEnabled;
}
}
catch (Exception ex)
{
Log.LogHandler.Error("{0} {1}", ex.Message, ex.StackTrace);
}
foreach (var stat in expectedStatistics)
{
Assert.AreEqual(true, statistics.ContainsKey(stat.Key), "Missing key: " + stat.Key);
Assert.AreEqual(stat.Value, statistics[stat.Key], "Failed on " + stat.Key);
}
if (expectedAlphaStatistics != null)
{
AssertAlphaStatistics(expectedAlphaStatistics, alphaStatistics, s => s.MeanPopulationScore.Direction);
AssertAlphaStatistics(expectedAlphaStatistics, alphaStatistics, s => s.MeanPopulationScore.Magnitude);
AssertAlphaStatistics(expectedAlphaStatistics, alphaStatistics, s => s.RollingAveragedPopulationScore.Direction);
AssertAlphaStatistics(expectedAlphaStatistics, alphaStatistics, s => s.RollingAveragedPopulationScore.Magnitude);
AssertAlphaStatistics(expectedAlphaStatistics, alphaStatistics, s => s.LongShortRatio);
AssertAlphaStatistics(expectedAlphaStatistics, alphaStatistics, s => s.TotalInsightsClosed);
AssertAlphaStatistics(expectedAlphaStatistics, alphaStatistics, s => s.TotalInsightsGenerated);
AssertAlphaStatistics(expectedAlphaStatistics, alphaStatistics, s => s.TotalAccumulatedEstimatedAlphaValue);
AssertAlphaStatistics(expectedAlphaStatistics, alphaStatistics, s => s.TotalInsightsAnalysisCompleted);
}
// we successfully passed the regression test, copy the log file so we don't have to continually
// re-run master in order to compare against a passing run
var passedFile = logFile.Replace("./regression/", "./passed/");
Directory.CreateDirectory(Path.GetDirectoryName(passedFile));
File.Delete(passedFile);
File.Copy(logFile, passedFile);
var passedOrderLogFile = ordersLogFile.Replace("./regression/", "./passed/");
Directory.CreateDirectory(Path.GetDirectoryName(passedFile));
File.Delete(passedOrderLogFile);
if (File.Exists(ordersLogFile))
{
File.Copy(ordersLogFile, passedOrderLogFile);
}
}
public static AlgorithmRunnerResults RunLocalBacktest(
string algorithm,
Dictionary <string, string> expectedStatistics,
AlphaRuntimeStatistics expectedAlphaStatistics,
Language language,
AlgorithmStatus expectedFinalStatus,
DateTime?startDate = null,
DateTime?endDate = null,
string setupHandler = "RegressionSetupHandlerWrapper",
decimal?initialCash = null)
{
AlgorithmManager algorithmManager = null;
var statistics = new Dictionary <string, string>();
var alphaStatistics = new AlphaRuntimeStatistics(new TestAccountCurrencyProvider());
BacktestingResultHandler results = null;
Composer.Instance.Reset();
SymbolCache.Clear();
var ordersLogFile = string.Empty;
var logFile = $"./regression/{algorithm}.{language.ToLower()}.log";
Directory.CreateDirectory(Path.GetDirectoryName(logFile));
File.Delete(logFile);
try
{
// set the configuration up
Config.Set("algorithm-type-name", algorithm);
Config.Set("live-mode", "false");
Config.Set("environment", "");
Config.Set("messaging-handler", "QuantConnect.Messaging.Messaging");
Config.Set("job-queue-handler", "QuantConnect.Queues.JobQueue");
Config.Set("setup-handler", setupHandler);
Config.Set("history-provider", "RegressionHistoryProviderWrapper");
Config.Set("api-handler", "QuantConnect.Api.Api");
Config.Set("result-handler", "QuantConnect.Lean.Engine.Results.RegressionResultHandler");
Config.Set("algorithm-language", language.ToString());
Config.Set("algorithm-location",
language == Language.Python
? "../../../Algorithm.Python/" + algorithm + ".py"
: "QuantConnect.Algorithm." + language + ".dll");
// Store initial log variables
var initialLogHandler = Log.LogHandler;
var initialDebugEnabled = Log.DebuggingEnabled;
// Log handlers specific to this test function
var newLogHandlers = new ILogHandler[] { new ConsoleErrorLogHandler(), new FileLogHandler(logFile, false) };
using (Log.LogHandler = new CompositeLogHandler(newLogHandlers))
using (var algorithmHandlers = LeanEngineAlgorithmHandlers.FromConfiguration(Composer.Instance))
using (var systemHandlers = LeanEngineSystemHandlers.FromConfiguration(Composer.Instance))
using (var workerThread = new TestWorkerThread())
{
Log.DebuggingEnabled = true;
Log.Trace("");
Log.Trace("{0}: Running " + algorithm + "...", DateTime.UtcNow);
Log.Trace("");
// run the algorithm in its own thread
var engine = new Lean.Engine.Engine(systemHandlers, algorithmHandlers, false);
Task.Factory.StartNew(() =>
{
try
{
string algorithmPath;
var job = (BacktestNodePacket)systemHandlers.JobQueue.NextJob(out algorithmPath);
job.BacktestId = algorithm;
job.PeriodStart = startDate;
job.PeriodFinish = endDate;
if (initialCash.HasValue)
{
job.CashAmount = new CashAmount(initialCash.Value, Currencies.USD);
}
algorithmManager = new AlgorithmManager(false, job);
systemHandlers.LeanManager.Initialize(systemHandlers, algorithmHandlers, job, algorithmManager);
engine.Run(job, algorithmManager, algorithmPath, workerThread);
ordersLogFile = ((RegressionResultHandler)algorithmHandlers.Results).LogFilePath;
}
catch (Exception e)
{
Log.Trace($"Error in AlgorithmRunner task: {e}");
}
}).Wait();
var backtestingResultHandler = (BacktestingResultHandler)algorithmHandlers.Results;
results = backtestingResultHandler;
statistics = backtestingResultHandler.FinalStatistics;
var defaultAlphaHandler = (DefaultAlphaHandler)algorithmHandlers.Alphas;
alphaStatistics = defaultAlphaHandler.RuntimeStatistics;
}
// Reset settings to initial values
Log.LogHandler = initialLogHandler;
Log.DebuggingEnabled = initialDebugEnabled;
}
catch (Exception ex)
{
if (expectedFinalStatus != AlgorithmStatus.RuntimeError)
{
Log.Error("{0} {1}", ex.Message, ex.StackTrace);
}
}
if (algorithmManager?.State != expectedFinalStatus)
{
Assert.Fail($"Algorithm state should be {expectedFinalStatus} and is: {algorithmManager?.State}");
}
foreach (var stat in expectedStatistics)
{
Assert.AreEqual(true, statistics.ContainsKey(stat.Key), "Missing key: " + stat.Key);
Assert.AreEqual(stat.Value, statistics[stat.Key], "Failed on " + stat.Key);
}
if (expectedAlphaStatistics != null)
{
AssertAlphaStatistics(expectedAlphaStatistics, alphaStatistics, s => s.MeanPopulationScore.Direction);
AssertAlphaStatistics(expectedAlphaStatistics, alphaStatistics, s => s.MeanPopulationScore.Magnitude);
AssertAlphaStatistics(expectedAlphaStatistics, alphaStatistics, s => s.RollingAveragedPopulationScore.Direction);
AssertAlphaStatistics(expectedAlphaStatistics, alphaStatistics, s => s.RollingAveragedPopulationScore.Magnitude);
AssertAlphaStatistics(expectedAlphaStatistics, alphaStatistics, s => s.LongShortRatio);
AssertAlphaStatistics(expectedAlphaStatistics, alphaStatistics, s => s.TotalInsightsClosed);
AssertAlphaStatistics(expectedAlphaStatistics, alphaStatistics, s => s.TotalInsightsGenerated);
AssertAlphaStatistics(expectedAlphaStatistics, alphaStatistics, s => s.TotalAccumulatedEstimatedAlphaValue);
AssertAlphaStatistics(expectedAlphaStatistics, alphaStatistics, s => s.TotalInsightsAnalysisCompleted);
}
// we successfully passed the regression test, copy the log file so we don't have to continually
// re-run master in order to compare against a passing run
var passedFile = logFile.Replace("./regression/", "./passed/");
Directory.CreateDirectory(Path.GetDirectoryName(passedFile));
File.Delete(passedFile);
File.Copy(logFile, passedFile);
var passedOrderLogFile = ordersLogFile.Replace("./regression/", "./passed/");
Directory.CreateDirectory(Path.GetDirectoryName(passedFile));
File.Delete(passedOrderLogFile);
if (File.Exists(ordersLogFile))
{
File.Copy(ordersLogFile, passedOrderLogFile);
}
return(new AlgorithmRunnerResults(algorithm, language, algorithmManager, results));
}
