/// <summary>
/// <see cref = "QuantBook" /> constructor.
/// Provides access to data for quantitative analysis
/// </summary>
public QuantBook() : base()
{
try
{
using (Py.GIL())
{
_pandas = Py.Import("pandas");
}
// By default, set start date to end data which is yesterday
SetStartDate(EndDate);
// Sets PandasConverter
SetPandasConverter();
// Initialize History Provider
var composer = new Composer();
var algorithmHandlers = LeanEngineAlgorithmHandlers.FromConfiguration(composer);
var systemHandlers = LeanEngineSystemHandlers.FromConfiguration(composer);
systemHandlers.LeanManager.Initialize(systemHandlers,
algorithmHandlers,
new BacktestNodePacket(),
new AlgorithmManager(false));
systemHandlers.LeanManager.SetAlgorithm(this);
_dataCacheProvider = new ZipDataCacheProvider(algorithmHandlers.DataProvider);
var symbolPropertiesDataBase = SymbolPropertiesDatabase.FromDataFolder();
var securityService = new SecurityService(Portfolio.CashBook, MarketHoursDatabase, symbolPropertiesDataBase, this);
Securities.SetSecurityService(securityService);
SubscriptionManager.SetDataManager(
new DataManager(new NullDataFeed(),
new UniverseSelection(this, securityService),
this,
TimeKeeper,
MarketHoursDatabase,
false));
var mapFileProvider = algorithmHandlers.MapFileProvider;
HistoryProvider = composer.GetExportedValueByTypeName <IHistoryProvider>(Config.Get("history-provider", "SubscriptionDataReaderHistoryProvider"));
HistoryProvider.Initialize(
new HistoryProviderInitializeParameters(
null,
null,
algorithmHandlers.DataProvider,
_dataCacheProvider,
mapFileProvider,
algorithmHandlers.FactorFileProvider,
null
)
);
SetOptionChainProvider(new CachingOptionChainProvider(new BacktestingOptionChainProvider()));
SetFutureChainProvider(new CachingFutureChainProvider(new BacktestingFutureChainProvider()));
}
catch (Exception exception)
{
throw new Exception("QuantBook.Main(): " + exception);
}
}
public static void RunLocalBacktest(string algorithm, Dictionary <string, string> expectedStatistics, Language language)
{
var statistics = new Dictionary <string, string>();
Composer.Instance.Reset();
try
{
// set the configuration up
Config.Set("algorithm-type-name", algorithm);
Config.Set("live-mode", "false");
Config.Set("environment", "");
Config.Set("messaging-handler", "QuantConnect.Messaging.Messaging");
Config.Set("job-queue-handler", "QuantConnect.Queues.JobQueue");
Config.Set("api-handler", "QuantConnect.Api.Api");
Config.Set("result-handler", "QuantConnect.Lean.Engine.Results.BacktestingResultHandler");
Config.Set("algorithm-language", language.ToString());
Config.Set("algorithm-location", "QuantConnect.Algorithm." + language + ".dll");
var debugEnabled = Log.DebuggingEnabled;
var logHandlers = new ILogHandler[] { new ConsoleLogHandler(), new FileLogHandler("regression.log", false) };
using (Log.LogHandler = new CompositeLogHandler(logHandlers))
using (var algorithmHandlers = LeanEngineAlgorithmHandlers.FromConfiguration(Composer.Instance))
using (var systemHandlers = LeanEngineSystemHandlers.FromConfiguration(Composer.Instance))
{
Log.DebuggingEnabled = true;
Log.LogHandler.Trace("");
Log.LogHandler.Trace("{0}: Running " + algorithm + "...", DateTime.UtcNow);
Log.LogHandler.Trace("");
// run the algorithm in its own thread
var engine = new Lean.Engine.Engine(systemHandlers, algorithmHandlers, false);
Task.Factory.StartNew(() =>
{
string algorithmPath;
var job = systemHandlers.JobQueue.NextJob(out algorithmPath);
var algorithmManager = new AlgorithmManager(false);
engine.Run(job, algorithmManager, algorithmPath);
}).Wait();
var backtestingResultHandler = (BacktestingResultHandler)algorithmHandlers.Results;
statistics = backtestingResultHandler.FinalStatistics;
Log.DebuggingEnabled = debugEnabled;
}
}
catch (Exception ex)
{
Log.LogHandler.Error("{0} {1}", ex.Message, ex.StackTrace);
}
foreach (var stat in expectedStatistics)
{
Assert.AreEqual(true, statistics.ContainsKey(stat.Key), "Missing key: " + stat.Key);
Assert.AreEqual(stat.Value, statistics[stat.Key], "Failed on " + stat.Key);
}
}
/// <summary>
/// Launches a Lean Engine using a parameter
/// </summary>
/// <param name="val">The paramater to use when launching lean. </param>
private void LaunchLean(string val)
{
Config.Set("environment", "backtesting");
string algorithm = val;
// Set the algorithm in Config. Here is where you can customize Config settings
Config.Set("algorithm-type-name", algorithm);
_jobQueue = new JobQueue();
_notify = new Messaging();
_api = new Api();
/************ Comment one of the two following lines to select which ResultHandler to use ***********/
_resultshandler = new OptimizationResultHandler();
//_resultshandler = new ConsoleResultHandler();
_dataFeed = new FileSystemDataFeed();
_setup = new ConsoleSetupHandler();
_realTime = new BacktestingRealTimeHandler();
_historyProvider = new SubscriptionDataReaderHistoryProvider();
_transactions = new BacktestingTransactionHandler();
// Set the Log.LogHandler to only write to the log.txt file.
// This setting avoids writing Log messages to the console.
Log.LogHandler = (ILogHandler) new FileLogHandler();
Log.DebuggingEnabled = false; // Set this property to true for lots of messages
Log.DebuggingLevel = 1; // A reminder that the default level for Log.Debug message is 1
var systemHandlers = new LeanEngineSystemHandlers(_jobQueue, _api, _notify);
systemHandlers.Initialize();
var algorithmHandlers = new LeanEngineAlgorithmHandlers(_resultshandler, _setup, _dataFeed, _transactions, _realTime, _historyProvider);
string algorithmPath;
AlgorithmNodePacket job = systemHandlers.JobQueue.NextJob(out algorithmPath);
try
{
var _engine = new Engine(systemHandlers, algorithmHandlers, Config.GetBool("live-mode"));
_engine.Run(job, algorithmPath);
}
finally
{
/* The JobQueue.AcknowledgeJob only asks for any key to close the window.
* We do not want that behavior, so we comment out this line so that multiple Leans will run
*
* The alternative is to comment out Console.Read(); the line in JobQueue class.
*/
//systemHandlers.JobQueue.AcknowledgeJob(job);
Log.Trace("Engine.Main(): Packet removed from queue: " + job.AlgorithmId);
// clean up resources
systemHandlers.Dispose();
algorithmHandlers.Dispose();
Log.LogHandler.Dispose();
}
}
private void LaunchLean()
{
Config.Set("environment", "backtesting");
if (!string.IsNullOrEmpty(_config.AlgorithmTypeName))
{
Config.Set("algorithm-type-name", _config.AlgorithmTypeName);
}
if (!string.IsNullOrEmpty(_config.AlgorithmLocation))
{
Config.Set("algorithm-location", Path.GetFileName(_config.AlgorithmLocation));
}
if (!string.IsNullOrEmpty(_config.DataFolder))
{
Config.Set("data-folder", _config.DataFolder);
}
var systemHandlers = LeanEngineSystemHandlers.FromConfiguration(Composer.Instance);
systemHandlers.Initialize();
Log.LogHandler = Composer.Instance.GetExportedValueByTypeName <ILogHandler>(Config.Get("log-handler", "CompositeLogHandler"));
//Log.DebuggingEnabled = false;
//Log.DebuggingLevel = 1;
LeanEngineAlgorithmHandlers leanEngineAlgorithmHandlers;
try
{
leanEngineAlgorithmHandlers = LeanEngineAlgorithmHandlers.FromConfiguration(Composer.Instance);
_resultsHandler = (BacktestingResultHandler)leanEngineAlgorithmHandlers.Results;
}
catch (CompositionException compositionException)
{
Log.Error("Engine.Main(): Failed to load library: " + compositionException);
throw;
}
string algorithmPath;
AlgorithmNodePacket job = systemHandlers.JobQueue.NextJob(out algorithmPath);
try
{
var _engine = new Engine(systemHandlers, leanEngineAlgorithmHandlers, Config.GetBool("live-mode"));
_engine.Run(job, algorithmPath);
}
finally
{
Log.Trace("Engine.Main(): Packet removed from queue: " + job.AlgorithmId);
// clean up resources
systemHandlers.Dispose();
leanEngineAlgorithmHandlers.Dispose();
Log.LogHandler.Dispose();
}
}
/// <summary>
/// Empty implementation of the ILeanManager interface
/// </summary>
/// <param name="systemHandlers">Exposes lean engine system handlers running LEAN</param>
/// <param name="algorithmHandlers">Exposes the lean algorithm handlers running lean</param>
/// <param name="job">The job packet representing either a live or backtest Lean instance</param>
/// <param name="algorithmManager">The Algorithm manager</param>
public void Initialize(LeanEngineSystemHandlers systemHandlers, LeanEngineAlgorithmHandlers algorithmHandlers, AlgorithmNodePacket job, AlgorithmManager algorithmManager)
{
// NOP
var server = new RESTServer();
//var server = new RESTServer("*");
Console.WriteLine(String.Format("Core Router REST Server Listening On {0}:{1} @ {2}", server.Host, server.Port, server.BaseUrl));
server.Start();
Console.WriteLine("RaysLeanManager Initialized!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!");
}
private void LaunchLean()
{
Config.Set("environment", "backtesting");
string algorithm = "EMATest";
Config.Set("algorithm-type-name", algorithm);
_jobQueue = new JobQueue();
_notify = new Messaging();
_api = new Api();
_resultshandler = new DesktopResultHandler();
_dataFeed = new FileSystemDataFeed();
_setup = new ConsoleSetupHandler();
_realTime = new BacktestingRealTimeHandler();
_historyProvider = new SubscriptionDataReaderHistoryProvider();
_transactions = new BacktestingTransactionHandler();
var systemHandlers = new LeanEngineSystemHandlers(_jobQueue, _api, _notify);
systemHandlers.Initialize();
// var algorithmHandlers = new LeanEngineAlgorithmHandlers (_resultshandler, _setup, _dataFeed, _transactions, _realTime, _historyProvider);
Log.LogHandler = Composer.Instance.GetExportedValueByTypeName <ILogHandler>(Config.Get("log-handler", "CompositeLogHandler"));
LeanEngineAlgorithmHandlers leanEngineAlgorithmHandlers;
try
{
leanEngineAlgorithmHandlers = LeanEngineAlgorithmHandlers.FromConfiguration(Composer.Instance);
_resultshandler = leanEngineAlgorithmHandlers.Results;
}
catch (CompositionException compositionException)
{
Log.Error("Engine.Main(): Failed to load library: " + compositionException);
throw;
}
string algorithmPath;
AlgorithmNodePacket job = systemHandlers.JobQueue.NextJob(out algorithmPath);
try
{
var _engine = new Engine(systemHandlers, leanEngineAlgorithmHandlers, Config.GetBool("live-mode"));
_engine.Run(job, algorithmPath);
}
finally
{
//Delete the message from the job queue:
//systemHandlers.JobQueue.AcknowledgeJob(job);
Log.Trace("Engine.Main(): Packet removed from queue: " + job.AlgorithmId);
// clean up resources
systemHandlers.Dispose();
leanEngineAlgorithmHandlers.Dispose();
Log.LogHandler.Dispose();
}
}
private static void LaunchLean()
{
/* Config.Set("environment", "backtesting");
* string algorithm = (string)AppDomain.CurrentDomain.GetData("AlgorithmTypeName");
* string path = (string)AppDomain.CurrentDomain.GetData("AlgorithmLocation");
*
* Config.Set("algorithm-type-name", algorithm);
* if (!string.IsNullOrEmpty(path))
* {
* Config.Set("algorithm-location", path);
* }
*/
// Config.Set("result-handler", "BacktestingResultHandler");
Config.Set("messaging-handler", "QuantConnect.Lean.TestProject.Messaging");
var systemHandlers = LeanEngineSystemHandlers.FromConfiguration(Composer.Instance);
systemHandlers.Initialize();
Log.LogHandler = Composer.Instance.GetExportedValueByTypeName <ILogHandler>(Config.Get("log-handler", "CompositeLogHandler"));
//Log.DebuggingEnabled = false;
//Log.DebuggingLevel = 1;
LeanEngineAlgorithmHandlers leanEngineAlgorithmHandlers;
try
{
leanEngineAlgorithmHandlers = LeanEngineAlgorithmHandlers.FromConfiguration(Composer.Instance);
_resultsHandler = (BacktestingResultHandler)leanEngineAlgorithmHandlers.Results;
}
catch (CompositionException compositionException)
{
Log.Error("Engine.Main(): Failed to load library: " + compositionException);
throw;
}
string algorithmPath;
AlgorithmNodePacket job = systemHandlers.JobQueue.NextJob(out algorithmPath);
try
{
var _engine = new QuantConnect.Lean.Engine.Engine(systemHandlers, leanEngineAlgorithmHandlers, Config.GetBool("live-mode"));
_engine.Run(job, algorithmPath);
}
finally
{
Log.Trace("Engine.Main(): Packet removed from queue: " + job.AlgorithmId);
// clean up resources
systemHandlers.Dispose();
leanEngineAlgorithmHandlers.Dispose();
Log.LogHandler.Dispose();
}
}
private void LaunchLean()
{
ConfigMerger.Merge(_config, _id, this.GetType());
var systemHandlers = new LeanEngineSystemHandlers(
new JobQueue(),
new EmptyApiHandler(),
new Messaging(),
new LocalLeanManager());
systemHandlers.Initialize();
//separate log uniquely named
var logFileName = "log" + DateTime.Now.ToString("yyyyMMddssfffffff") + "_" + _id + ".txt";
using (Log.LogHandler = new FileLogHandler(logFileName, true))
{
var map = new LocalDiskMapFileProvider();
var leanEngineAlgorithmHandlers = new LeanEngineAlgorithmHandlers(
new OptimizerResultHandler(),
new ConsoleSetupHandler(),
new FileSystemDataFeed(),
new BacktestingTransactionHandler(),
new BacktestingRealTimeHandler(),
map,
new LocalDiskFactorFileProvider(map),
new DefaultDataProvider(),
new OptimizerAlphaHandler(),
new EmptyObjectStore(),
new DataPermissionManager());
_resultsHandler = (OptimizerResultHandler)leanEngineAlgorithmHandlers.Results;
var job = (BacktestNodePacket)systemHandlers.JobQueue.NextJob(out var algorithmPath);
//todo: pass period through job
//job.PeriodStart = _config.StartDate;
//job.PeriodFinish = _config.EndDate;
try
{
var algorithmManager = new AlgorithmManager(false);
systemHandlers.LeanManager.Initialize(systemHandlers, leanEngineAlgorithmHandlers, job, algorithmManager);
var engine = new Engine(systemHandlers, leanEngineAlgorithmHandlers, false);
engine.Run(job, algorithmManager, algorithmPath, WorkerThread.Instance);
}
finally
{
// clean up resources
systemHandlers.Dispose();
leanEngineAlgorithmHandlers.Dispose();
}
}
}
public Engine CreateEngine()
{
//Launch the Lean Engine in another thread: this will run the algorithm specified above.
// TODO > This should only be launched when clicking a backtest/trade live button provided in the UX.
Composer v = Composer.Instance;
//Log.LogHandler = new CompositeLogHandler(new ILogHandler[]
// {
// new ConsoleLogHandler(),
// new FileLogHandler("regression.log")
// });
var systemHandlers = LeanEngineSystemHandlers.FromConfiguration(Composer.Instance);
_algorithmHandlers = LeanEngineAlgorithmHandlers.FromConfiguration(Composer.Instance);
var engine = new Engine(systemHandlers, _algorithmHandlers, Config.GetBool("live-mode"));
Log("Engine.Main(): Memory " + OS.ApplicationMemoryUsed + "Mb-App " + +OS.TotalPhysicalMemoryUsed + "Mb-Used " + OS.TotalPhysicalMemory + "Mb-Total");
// log the job endpoints
Log("JOB HANDLERS: ");
Log(" DataFeed: " + _algorithmHandlers.DataFeed.GetType().FullName);
Log(" Setup: " + _algorithmHandlers.Setup.GetType().FullName);
Log(" RealTime: " + _algorithmHandlers.RealTime.GetType().FullName);
Log(" Results: " + _algorithmHandlers.Results.GetType().FullName);
Log(" Transactions: " + _algorithmHandlers.Transactions.GetType().FullName);
Log(" History: " + _algorithmHandlers.HistoryProvider.GetType().FullName);
Log(" Commands: " + _algorithmHandlers.CommandQueue.GetType().FullName);
Log(" FactorFileProvider: " + _algorithmHandlers.FactorFileProvider.GetType().FullName);
Log(" HistoryProvider: " + _algorithmHandlers.HistoryProvider.GetType().FullName);
Log(" MapFileProvider: " + _algorithmHandlers.MapFileProvider.GetType().FullName);
Log(" Transactions: " + _algorithmHandlers.Transactions.GetType().FullName);
_leanEngineThread = new Thread(() =>
{
string algorithmPath;
job = systemHandlers.JobQueue.NextJob(out algorithmPath);
if (job is LiveNodePacket)
{
Log(" Brokerage: " + ((LiveNodePacket)job).Brokerage);
}
engine.Run(job, algorithmPath);
systemHandlers.JobQueue.AcknowledgeJob(job);
});
_leanEngineThread.Start();
return(engine);
}
public void Run(Dictionary <string, string> parameters)
{
const string mode = "DEBUG";
Console.OutputEncoding = System.Text.Encoding.UTF8;
var liveMode = Config.GetBool("live-mode");
Log.DebuggingEnabled = Config.GetBool("debug-mode");
Log.FilePath = Path.Combine(Config.Get("results-destination-folder"), "log.txt");
Log.LogHandler = Composer.Instance.GetExportedValueByTypeName <ILogHandler>(Config.Get("log-handler", "CompositeLogHandler"));
Log.Trace($"Engine.Main(): LEAN ALGORITHMIC TRADING ENGINE v{Globals.Version} Mode: {mode} ({(Environment.Is64BitProcess ? "64" : "32")}bit)");
Log.Trace($"Engine.Main(): Started {DateTime.Now.ToShortTimeString()}");
var systemHandler = LeanEngineSystemHandlers.FromConfiguration(Composer.Instance);
systemHandler.Initialize();
var job = systemHandler.JobQueue.NextJob(out var assemblyPath);
foreach (var kvp in parameters)
{
job.Parameters[kvp.Key] = kvp.Value;
}
var algorithmHandler = LeanEngineAlgorithmHandlers.FromConfiguration(Composer.Instance);
try
{
var algorithmManager = new AlgorithmManager(liveMode, job);
systemHandler.LeanManager.Initialize(systemHandler, algorithmHandler, job, algorithmManager);
var engine = new Engine(systemHandler, algorithmHandler, liveMode);
engine.Run(job, algorithmManager, assemblyPath, WorkerThread.Instance);
}
finally
{
Console.WriteLine("Engine.Main(): Analysis Complete. Press any key to continue.");
Console.ReadKey(true);
systemHandler.Dispose();
algorithmHandler.Dispose();
Log.LogHandler.Dispose();
Log.Trace("Program.Main(): Exiting Lean...");
Thread.Sleep(2000);
Environment.Exit(0);
}
}
public static void RunLocalBacktest(string algorithm, Dictionary <string, string> expectedStatistics)
{
var statistics = new Dictionary <string, string>();
Composer.Instance.Reset();
try
{
using (Log.LogHandler = new CompositeLogHandler(new ILogHandler[]
{
new ConsoleLogHandler(),
new FileLogHandler("regression.log")
}))
using (var algorithmHandlers = LeanEngineAlgorithmHandlers.FromConfiguration(Composer.Instance))
using (var systemHandlers = LeanEngineSystemHandlers.FromConfiguration(Composer.Instance))
{
Console.WriteLine("Running " + algorithm + "...");
// set the configuration up
Config.Set("algorithm-type-name", algorithm);
Config.Set("live-mode", "false");
Config.Set("environment", "");
Config.Set("messaging-handler", "QuantConnect.Messaging.Messaging");
Config.Set("job-queue-handler", "QuantConnect.Queues.JobQueue");
Config.Set("api-handler", "QuantConnect.Api.Api");
// run the algorithm in its own thread
var engine = new Lean.Engine.Engine(systemHandlers, algorithmHandlers, false);
Task.Factory.StartNew(() =>
{
string algorithmPath;
var job = systemHandlers.JobQueue.NextJob(out algorithmPath);
engine.Run(job, algorithmPath);
}).Wait();
var consoleResultHandler = (ConsoleResultHandler)algorithmHandlers.Results;
statistics = consoleResultHandler.FinalStatistics;
}
}
catch (Exception ex)
{
Log.LogHandler.Error("{0} {1}", ex.Message, ex.StackTrace);
}
foreach (var stat in expectedStatistics)
{
Assert.AreEqual(true, statistics.ContainsKey(stat.Key), "Missing key: " + stat.Key);
Assert.AreEqual(stat.Value, statistics[stat.Key], "Failed on " + stat.Key);
}
}
public TrackModel Run(TrackModel model, AccountModel account, SettingModel settings, HostDomainLogger logger)
{
if (model == null)
{
throw new ArgumentNullException(nameof(model));
}
Log.LogHandler = logger;
if (!SetConfig(model, account, settings))
{
return(model);
}
// Register all data providers
ProviderFactory.RegisterProviders(settings);
Log.Trace("LeanLanucher: Memory " + OS.ApplicationMemoryUsed + "Mb-App " + OS.TotalPhysicalMemoryUsed + "Mb-Used");
try
{
var liveMode = Config.GetBool("live-mode");
using var algorithmHandlers = LeanEngineAlgorithmHandlers.FromConfiguration(Composer.Instance);
using var systemHandlers = LeanEngineSystemHandlers.FromConfiguration(Composer.Instance);
systemHandlers.Initialize();
var engine = new Engine(systemHandlers, algorithmHandlers, liveMode);
var algorithmManager = new AlgorithmManager(liveMode);
AlgorithmNodePacket job = systemHandlers.JobQueue.NextJob(out string assemblyPath);
job.UserPlan = UserPlan.Professional;
job.Language = model.AlgorithmLanguage;
if (job is BacktestNodePacket backtest)
{
backtest.PeriodStart = model.StartDate;
backtest.PeriodFinish = model.EndDate;
backtest.CashAmount = new CashAmount(model.InitialCapital, Currencies.USD);
}
systemHandlers.LeanManager.Initialize(systemHandlers, algorithmHandlers, job, algorithmManager);
engine.Run(job, algorithmManager, assemblyPath, WorkerThread.Instance);
BacktestResultHandler resultHandler = algorithmHandlers.Results as BacktestResultHandler;
model.Result = resultHandler?.JsonResult ?? string.Empty;
model.Logs = resultHandler?.Logs ?? string.Empty;
}
catch (Exception ex)
{
Log.Error("{0}: {1}", ex.GetType(), ex.Message);
}
NetMQConfig.Cleanup(false);
Log.LogHandler.Dispose();
return(model);
}
public TrackModel Run(TrackModel model, AccountModel account, SettingService settings, HostDomainLogger logger)
{
if (model == null)
{
throw new ArgumentNullException(nameof(model));
}
Log.LogHandler = logger;
if (!SetConfig(model, account, settings))
{
return(model);
}
// Register all data providers
ProviderFactory.RegisterProviders();
Log.Trace("LeanLanucher: Memory " + OS.ApplicationMemoryUsed + "Mb-App " + OS.TotalPhysicalMemoryUsed + "Mb-Used");
try
{
var liveMode = Config.GetBool("live-mode");
using (var algorithmHandlers = LeanEngineAlgorithmHandlers.FromConfiguration(Composer.Instance))
using (var systemHandlers = LeanEngineSystemHandlers.FromConfiguration(Composer.Instance))
{
string assemblyPath;
systemHandlers.Initialize();
var engine = new Engine(systemHandlers, algorithmHandlers, liveMode);
var algorithmManager = new AlgorithmManager(liveMode);
AlgorithmNodePacket job = systemHandlers.JobQueue.NextJob(out assemblyPath);
job.UserPlan = UserPlan.Professional;
systemHandlers.LeanManager.Initialize(systemHandlers, algorithmHandlers, job, algorithmManager);
engine.Run(job, algorithmManager, assemblyPath);
systemHandlers.JobQueue.AcknowledgeJob(job);
BacktestResultHandler resultHandler = algorithmHandlers.Results as BacktestResultHandler;
model.Result = resultHandler?.JsonResult ?? string.Empty;
model.Logs = resultHandler?.Logs ?? string.Empty;
}
}
catch (Exception ex)
{
Log.Error("{0}: {1}", ex.GetType(), ex.Message);
}
NetMQConfig.Cleanup(false);
Log.LogHandler.Dispose();
return(model);
}
/// <summary>
/// Launch the LEAN Engine in a separate thread.
/// </summary>
private static Engine LaunchLean()
{
//Launch the Lean Engine in another thread: this will run the algorithm specified above.
// TODO > This should only be launched when clicking a backtest/trade live button provided in the UX.
var systemHandlers = LeanEngineSystemHandlers.FromConfiguration(Composer.Instance);
var algorithmHandlers = LeanEngineAlgorithmHandlers.FromConfiguration(Composer.Instance);
var engine = new Engine(systemHandlers, algorithmHandlers, Config.GetBool("live-mode"));
_leanEngineThread = new Thread(() =>
{
string algorithmPath;
var job = systemHandlers.JobQueue.NextJob(out algorithmPath);
engine.Run(job, algorithmPath);
systemHandlers.JobQueue.AcknowledgeJob(job);
});
_leanEngineThread.Start();
return(engine);
}
public void RunJob()
{
try
{
EngineContext.SystemHandlers = LeanEngineSystemHandlers.FromConfiguration(Composer.Instance);
EngineContext.SystemHandlers.Initialize();
EngineContext.AlgorithmHandlers = LeanEngineAlgorithmHandlers.FromConfiguration(Composer.Instance);
_resultsHandler = (OptimizerResultHandler)EngineContext.AlgorithmHandlers.Results;
EngineContext.Engine = new Engine(EngineContext.SystemHandlers, EngineContext.AlgorithmHandlers, false);
}
catch (CompositionException compositionException)
{
LogProvider.ErrorLogger.Error("Engine.Main(): Failed to load library", compositionException);
throw;
}
string algorithmPath;
AlgorithmNodePacket job = EngineContext.SystemHandlers.JobQueue.NextJob(out algorithmPath);
EngineContext.Engine.Run(job, new AlgorithmManager(false), algorithmPath, WorkerThread.Instance);
}
/// <summary>
/// <see cref = "QuantBook" /> constructor.
/// Provides access to data for quantitative analysis
/// </summary>
public QuantBook() : base()
{
try
{
using (Py.GIL())
{
_pandas = Py.Import("pandas");
}
// By default, set start date to end data which is yesterday
SetStartDate(EndDate);
// Sets PandasConverter
SetPandasConverter();
// Initialize History Provider
var composer = new Composer();
var algorithmHandlers = LeanEngineAlgorithmHandlers.FromConfiguration(composer);
var systemHandlers = LeanEngineSystemHandlers.FromConfiguration(composer);
// init the API
systemHandlers.Initialize();
systemHandlers.LeanManager.Initialize(systemHandlers,
algorithmHandlers,
new BacktestNodePacket(),
new AlgorithmManager(false));
systemHandlers.LeanManager.SetAlgorithm(this);
algorithmHandlers.ObjectStore.Initialize("QuantBook",
Config.GetInt("job-user-id"),
Config.GetInt("project-id"),
Config.Get("api-access-token"),
new Controls
{
// if <= 0 we disable periodic persistence and make it synchronous
PersistenceIntervalSeconds = -1,
StorageLimitMB = Config.GetInt("storage-limit-mb", 5),
StorageFileCount = Config.GetInt("storage-file-count", 100),
StoragePermissions = (FileAccess)Config.GetInt("storage-permissions", (int)FileAccess.ReadWrite)
});
SetObjectStore(algorithmHandlers.ObjectStore);
_dataCacheProvider = new ZipDataCacheProvider(algorithmHandlers.DataProvider);
var symbolPropertiesDataBase = SymbolPropertiesDatabase.FromDataFolder();
var registeredTypes = new RegisteredSecurityDataTypesProvider();
var securityService = new SecurityService(Portfolio.CashBook,
MarketHoursDatabase,
symbolPropertiesDataBase,
this,
registeredTypes,
new SecurityCacheProvider(Portfolio));
Securities.SetSecurityService(securityService);
SubscriptionManager.SetDataManager(
new DataManager(new NullDataFeed(),
new UniverseSelection(this, securityService),
this,
TimeKeeper,
MarketHoursDatabase,
false,
registeredTypes));
var mapFileProvider = algorithmHandlers.MapFileProvider;
HistoryProvider = composer.GetExportedValueByTypeName <IHistoryProvider>(Config.Get("history-provider", "SubscriptionDataReaderHistoryProvider"));
HistoryProvider.Initialize(
new HistoryProviderInitializeParameters(
null,
null,
algorithmHandlers.DataProvider,
_dataCacheProvider,
mapFileProvider,
algorithmHandlers.FactorFileProvider,
null,
true
)
);
SetOptionChainProvider(new CachingOptionChainProvider(new BacktestingOptionChainProvider()));
SetFutureChainProvider(new CachingFutureChainProvider(new BacktestingFutureChainProvider()));
}
catch (Exception exception)
{
throw new Exception("QuantBook.Main(): " + exception);
}
}
public static void RunLocalBacktest(string algorithm, Dictionary <string, string> expectedStatistics, AlphaRuntimeStatistics expectedAlphaStatistics, Language language)
{
var statistics = new Dictionary <string, string>();
var alphaStatistics = new AlphaRuntimeStatistics();
Composer.Instance.Reset();
var ordersLogFile = string.Empty;
var logFile = $"./regression/{algorithm}.{language.ToLower()}.log";
Directory.CreateDirectory(Path.GetDirectoryName(logFile));
File.Delete(logFile);
try
{
// set the configuration up
Config.Set("algorithm-type-name", algorithm);
Config.Set("live-mode", "false");
Config.Set("environment", "");
Config.Set("messaging-handler", "QuantConnect.Messaging.Messaging");
Config.Set("job-queue-handler", "QuantConnect.Queues.JobQueue");
Config.Set("setup-handler", "RegressionSetupHandlerWrapper");
Config.Set("history-provider", "RegressionHistoryProviderWrapper");
Config.Set("api-handler", "QuantConnect.Api.Api");
Config.Set("result-handler", "QuantConnect.Lean.Engine.Results.RegressionResultHandler");
Config.Set("algorithm-language", language.ToString());
Config.Set("algorithm-location",
language == Language.Python
? "../../../Algorithm.Python/" + algorithm + ".py"
: "QuantConnect.Algorithm." + language + ".dll");
var debugEnabled = Log.DebuggingEnabled;
var logHandlers = new ILogHandler[] { new ConsoleLogHandler(), new FileLogHandler(logFile, false) };
using (Log.LogHandler = new CompositeLogHandler(logHandlers))
using (var algorithmHandlers = LeanEngineAlgorithmHandlers.FromConfiguration(Composer.Instance))
using (var systemHandlers = LeanEngineSystemHandlers.FromConfiguration(Composer.Instance))
{
Log.DebuggingEnabled = true;
Log.LogHandler.Trace("");
Log.LogHandler.Trace("{0}: Running " + algorithm + "...", DateTime.UtcNow);
Log.LogHandler.Trace("");
// run the algorithm in its own thread
var engine = new Lean.Engine.Engine(systemHandlers, algorithmHandlers, false);
Task.Factory.StartNew(() =>
{
string algorithmPath;
var job = systemHandlers.JobQueue.NextJob(out algorithmPath);
var algorithmManager = new AlgorithmManager(false);
engine.Run(job, algorithmManager, algorithmPath);
ordersLogFile = ((RegressionResultHandler)algorithmHandlers.Results).OrdersLogFilePath;
}).Wait();
var backtestingResultHandler = (BacktestingResultHandler)algorithmHandlers.Results;
statistics = backtestingResultHandler.FinalStatistics;
var defaultAlphaHandler = (DefaultAlphaHandler)algorithmHandlers.Alphas;
alphaStatistics = defaultAlphaHandler.RuntimeStatistics;
Log.DebuggingEnabled = debugEnabled;
}
}
catch (Exception ex)
{
Log.LogHandler.Error("{0} {1}", ex.Message, ex.StackTrace);
}
foreach (var stat in expectedStatistics)
{
Assert.AreEqual(true, statistics.ContainsKey(stat.Key), "Missing key: " + stat.Key);
Assert.AreEqual(stat.Value, statistics[stat.Key], "Failed on " + stat.Key);
}
if (expectedAlphaStatistics != null)
{
AssertAlphaStatistics(expectedAlphaStatistics, alphaStatistics, s => s.MeanPopulationScore.Direction);
AssertAlphaStatistics(expectedAlphaStatistics, alphaStatistics, s => s.MeanPopulationScore.Magnitude);
AssertAlphaStatistics(expectedAlphaStatistics, alphaStatistics, s => s.RollingAveragedPopulationScore.Direction);
AssertAlphaStatistics(expectedAlphaStatistics, alphaStatistics, s => s.RollingAveragedPopulationScore.Magnitude);
AssertAlphaStatistics(expectedAlphaStatistics, alphaStatistics, s => s.LongShortRatio);
AssertAlphaStatistics(expectedAlphaStatistics, alphaStatistics, s => s.TotalInsightsClosed);
AssertAlphaStatistics(expectedAlphaStatistics, alphaStatistics, s => s.TotalInsightsGenerated);
AssertAlphaStatistics(expectedAlphaStatistics, alphaStatistics, s => s.TotalAccumulatedEstimatedAlphaValue);
AssertAlphaStatistics(expectedAlphaStatistics, alphaStatistics, s => s.TotalInsightsAnalysisCompleted);
}
// we successfully passed the regression test, copy the log file so we don't have to continually
// re-run master in order to compare against a passing run
var passedFile = logFile.Replace("./regression/", "./passed/");
Directory.CreateDirectory(Path.GetDirectoryName(passedFile));
File.Delete(passedFile);
File.Copy(logFile, passedFile);
var passedOrderLogFile = ordersLogFile.Replace("./regression/", "./passed/");
Directory.CreateDirectory(Path.GetDirectoryName(passedFile));
File.Delete(passedOrderLogFile);
if (File.Exists(ordersLogFile))
{
File.Copy(ordersLogFile, passedOrderLogFile);
}
}
private void LaunchLean()
{
Config.Set("environment", "backtesting");
if (!string.IsNullOrEmpty(_config.AlgorithmTypeName))
{
Config.Set("algorithm-type-name", _config.AlgorithmTypeName);
}
if (!string.IsNullOrEmpty(_config.AlgorithmLocation))
{
Config.Set("algorithm-location", Path.GetFileName(_config.AlgorithmLocation));
}
if (!string.IsNullOrEmpty(_config.DataFolder))
{
Config.Set("data-folder", _config.DataFolder);
}
if (!string.IsNullOrEmpty(_config.TransactionLog))
{
var filename = _config.TransactionLog;
filename = Path.Combine(AppDomain.CurrentDomain.BaseDirectory,
Path.GetFileNameWithoutExtension(filename) + _id + Path.GetExtension(filename));
Config.Set("transaction-log", filename);
}
var systemHandlers = new LeanEngineSystemHandlers(
new JobQueue(),
new EmptyApiHandler(),
new Messaging(),
new LocalLeanManager());
systemHandlers.Initialize();
//separate log uniquely named
var logFileName = "log" + DateTime.Now.ToString("yyyyMMddssfffffff") + "_" + _id + ".txt";
using (Log.LogHandler = new FileLogHandler(logFileName, true))
{
//override config to use custom result handler
Config.Set("backtesting.result-handler", nameof(OptimizerResultHandler));
var map = new LocalDiskMapFileProvider();
var leanEngineAlgorithmHandlers = new LeanEngineAlgorithmHandlers(
new OptimizerResultHandler(),
new ConsoleSetupHandler(),
new FileSystemDataFeed(),
new BacktestingTransactionHandler(),
new BacktestingRealTimeHandler(),
map,
new LocalDiskFactorFileProvider(map),
new DefaultDataProvider(),
new OptimizerAlphaHandler(),
new EmptyObjectStore());
_resultsHandler = (OptimizerResultHandler)leanEngineAlgorithmHandlers.Results;
var job = (BacktestNodePacket)systemHandlers.JobQueue.NextJob(out var algorithmPath);
//todo: pass period through job
//job.PeriodStart = _config.StartDate;
//job.PeriodFinish = _config.EndDate;
try
{
var algorithmManager = new AlgorithmManager(false);
systemHandlers.LeanManager.Initialize(systemHandlers, leanEngineAlgorithmHandlers, job, algorithmManager);
var engine = new Engine(systemHandlers, leanEngineAlgorithmHandlers, false);
engine.Run(job, algorithmManager, algorithmPath, WorkerThread.Instance);
}
finally
{
// clean up resources
systemHandlers.Dispose();
leanEngineAlgorithmHandlers.Dispose();
}
}
}
public static AlgorithmRunnerResults RunLocalBacktest(
string algorithm,
Dictionary <string, string> expectedStatistics,
AlphaRuntimeStatistics expectedAlphaStatistics,
Language language,
AlgorithmStatus expectedFinalStatus,
DateTime?startDate = null,
DateTime?endDate = null,
string setupHandler = "RegressionSetupHandlerWrapper",
decimal?initialCash = null)
{
AlgorithmManager algorithmManager = null;
var statistics = new Dictionary <string, string>();
var alphaStatistics = new AlphaRuntimeStatistics(new TestAccountCurrencyProvider());
BacktestingResultHandler results = null;
Composer.Instance.Reset();
SymbolCache.Clear();
var ordersLogFile = string.Empty;
var logFile = $"./regression/{algorithm}.{language.ToLower()}.log";
Directory.CreateDirectory(Path.GetDirectoryName(logFile));
File.Delete(logFile);
try
{
// set the configuration up
Config.Set("algorithm-type-name", algorithm);
Config.Set("live-mode", "false");
Config.Set("environment", "");
Config.Set("messaging-handler", "QuantConnect.Messaging.Messaging");
Config.Set("job-queue-handler", "QuantConnect.Queues.JobQueue");
Config.Set("setup-handler", setupHandler);
Config.Set("history-provider", "RegressionHistoryProviderWrapper");
Config.Set("api-handler", "QuantConnect.Api.Api");
Config.Set("result-handler", "QuantConnect.Lean.Engine.Results.RegressionResultHandler");
Config.Set("algorithm-language", language.ToString());
Config.Set("algorithm-location",
language == Language.Python
? "../../../Algorithm.Python/" + algorithm + ".py"
: "QuantConnect.Algorithm." + language + ".dll");
// Store initial log variables
var initialLogHandler = Log.LogHandler;
var initialDebugEnabled = Log.DebuggingEnabled;
// Log handlers specific to this test function
var newLogHandlers = new ILogHandler[] { new ConsoleErrorLogHandler(), new FileLogHandler(logFile, false) };
using (Log.LogHandler = new CompositeLogHandler(newLogHandlers))
using (var algorithmHandlers = LeanEngineAlgorithmHandlers.FromConfiguration(Composer.Instance))
using (var systemHandlers = LeanEngineSystemHandlers.FromConfiguration(Composer.Instance))
using (var workerThread = new TestWorkerThread())
{
Log.DebuggingEnabled = true;
Log.Trace("");
Log.Trace("{0}: Running " + algorithm + "...", DateTime.UtcNow);
Log.Trace("");
// run the algorithm in its own thread
var engine = new Lean.Engine.Engine(systemHandlers, algorithmHandlers, false);
Task.Factory.StartNew(() =>
{
try
{
string algorithmPath;
var job = (BacktestNodePacket)systemHandlers.JobQueue.NextJob(out algorithmPath);
job.BacktestId = algorithm;
job.PeriodStart = startDate;
job.PeriodFinish = endDate;
if (initialCash.HasValue)
{
job.CashAmount = new CashAmount(initialCash.Value, Currencies.USD);
}
algorithmManager = new AlgorithmManager(false, job);
systemHandlers.LeanManager.Initialize(systemHandlers, algorithmHandlers, job, algorithmManager);
engine.Run(job, algorithmManager, algorithmPath, workerThread);
ordersLogFile = ((RegressionResultHandler)algorithmHandlers.Results).LogFilePath;
}
catch (Exception e)
{
Log.Trace($"Error in AlgorithmRunner task: {e}");
}
}).Wait();
var backtestingResultHandler = (BacktestingResultHandler)algorithmHandlers.Results;
results = backtestingResultHandler;
statistics = backtestingResultHandler.FinalStatistics;
var defaultAlphaHandler = (DefaultAlphaHandler)algorithmHandlers.Alphas;
alphaStatistics = defaultAlphaHandler.RuntimeStatistics;
}
// Reset settings to initial values
Log.LogHandler = initialLogHandler;
Log.DebuggingEnabled = initialDebugEnabled;
}
catch (Exception ex)
{
if (expectedFinalStatus != AlgorithmStatus.RuntimeError)
{
Log.Error("{0} {1}", ex.Message, ex.StackTrace);
}
}
if (algorithmManager?.State != expectedFinalStatus)
{
Assert.Fail($"Algorithm state should be {expectedFinalStatus} and is: {algorithmManager?.State}");
}
foreach (var stat in expectedStatistics)
{
Assert.AreEqual(true, statistics.ContainsKey(stat.Key), "Missing key: " + stat.Key);
Assert.AreEqual(stat.Value, statistics[stat.Key], "Failed on " + stat.Key);
}
if (expectedAlphaStatistics != null)
{
AssertAlphaStatistics(expectedAlphaStatistics, alphaStatistics, s => s.MeanPopulationScore.Direction);
AssertAlphaStatistics(expectedAlphaStatistics, alphaStatistics, s => s.MeanPopulationScore.Magnitude);
AssertAlphaStatistics(expectedAlphaStatistics, alphaStatistics, s => s.RollingAveragedPopulationScore.Direction);
AssertAlphaStatistics(expectedAlphaStatistics, alphaStatistics, s => s.RollingAveragedPopulationScore.Magnitude);
AssertAlphaStatistics(expectedAlphaStatistics, alphaStatistics, s => s.LongShortRatio);
AssertAlphaStatistics(expectedAlphaStatistics, alphaStatistics, s => s.TotalInsightsClosed);
AssertAlphaStatistics(expectedAlphaStatistics, alphaStatistics, s => s.TotalInsightsGenerated);
AssertAlphaStatistics(expectedAlphaStatistics, alphaStatistics, s => s.TotalAccumulatedEstimatedAlphaValue);
AssertAlphaStatistics(expectedAlphaStatistics, alphaStatistics, s => s.TotalInsightsAnalysisCompleted);
}
// we successfully passed the regression test, copy the log file so we don't have to continually
// re-run master in order to compare against a passing run
var passedFile = logFile.Replace("./regression/", "./passed/");
Directory.CreateDirectory(Path.GetDirectoryName(passedFile));
File.Delete(passedFile);
File.Copy(logFile, passedFile);
var passedOrderLogFile = ordersLogFile.Replace("./regression/", "./passed/");
Directory.CreateDirectory(Path.GetDirectoryName(passedFile));
File.Delete(passedOrderLogFile);
if (File.Exists(ordersLogFile))
{
File.Copy(ordersLogFile, passedOrderLogFile);
}
return(new AlgorithmRunnerResults(algorithm, language, algorithmManager, results));
}
/// <summary>
/// <see cref = "QuantBook" /> constructor.
/// Provides access to data for quantitative analysis
/// </summary>
public QuantBook() : base()
{
try
{
using (Py.GIL())
{
_pandas = Py.Import("pandas");
}
// Issue #4892 : Set start time relative to NY time
// when the data is available from the previous day
var newYorkTime = DateTime.UtcNow.ConvertFromUtc(TimeZones.NewYork);
var hourThreshold = Config.GetInt("qb-data-hour", 9);
// If it is after our hour threshold; then we can use today
if (newYorkTime.Hour >= hourThreshold)
{
SetStartDate(newYorkTime);
}
else
{
SetStartDate(newYorkTime - TimeSpan.FromDays(1));
}
// Sets PandasConverter
SetPandasConverter();
// Reset our composer; needed for re-creation of QuantBook
Composer.Instance.Reset();
var composer = Composer.Instance;
// Create our handlers with our composer instance
var algorithmHandlers = LeanEngineAlgorithmHandlers.FromConfiguration(composer);
var systemHandlers = LeanEngineSystemHandlers.FromConfiguration(composer);
// init the API
systemHandlers.Initialize();
systemHandlers.LeanManager.Initialize(systemHandlers,
algorithmHandlers,
new BacktestNodePacket(),
new AlgorithmManager(false));
systemHandlers.LeanManager.SetAlgorithm(this);
algorithmHandlers.ObjectStore.Initialize("QuantBook",
Config.GetInt("job-user-id"),
Config.GetInt("project-id"),
Config.Get("api-access-token"),
new Controls
{
// if <= 0 we disable periodic persistence and make it synchronous
PersistenceIntervalSeconds = -1,
StorageLimitMB = Config.GetInt("storage-limit-mb", 5),
StorageFileCount = Config.GetInt("storage-file-count", 100),
StoragePermissions = (FileAccess)Config.GetInt("storage-permissions", (int)FileAccess.ReadWrite)
});
SetObjectStore(algorithmHandlers.ObjectStore);
_dataCacheProvider = new ZipDataCacheProvider(algorithmHandlers.DataProvider);
_dataProvider = algorithmHandlers.DataProvider;
var symbolPropertiesDataBase = SymbolPropertiesDatabase.FromDataFolder();
var registeredTypes = new RegisteredSecurityDataTypesProvider();
var securityService = new SecurityService(Portfolio.CashBook,
MarketHoursDatabase,
symbolPropertiesDataBase,
this,
registeredTypes,
new SecurityCacheProvider(Portfolio));
Securities.SetSecurityService(securityService);
SubscriptionManager.SetDataManager(
new DataManager(new NullDataFeed(),
new UniverseSelection(this, securityService, algorithmHandlers.DataPermissionsManager, algorithmHandlers.DataProvider),
this,
TimeKeeper,
MarketHoursDatabase,
false,
registeredTypes,
algorithmHandlers.DataPermissionsManager));
var mapFileProvider = algorithmHandlers.MapFileProvider;
HistoryProvider = composer.GetExportedValueByTypeName <IHistoryProvider>(Config.Get("history-provider", "SubscriptionDataReaderHistoryProvider"));
HistoryProvider.Initialize(
new HistoryProviderInitializeParameters(
null,
null,
algorithmHandlers.DataProvider,
_dataCacheProvider,
mapFileProvider,
algorithmHandlers.FactorFileProvider,
null,
true,
algorithmHandlers.DataPermissionsManager
)
);
SetOptionChainProvider(new CachingOptionChainProvider(new BacktestingOptionChainProvider(_dataProvider)));
SetFutureChainProvider(new CachingFutureChainProvider(new BacktestingFutureChainProvider(_dataProvider)));
}
catch (Exception exception)
{
throw new Exception("QuantBook.Main(): " + exception);
}
}
static void Main(string[] args)
{
if (OS.IsWindows)
{
Console.OutputEncoding = System.Text.Encoding.UTF8;
}
// expect first argument to be config file name
if (args.Length > 0)
{
Config.MergeCommandLineArgumentsWithConfiguration(LeanArgumentParser.ParseArguments(args));
}
var liveMode = Config.GetBool("live-mode");
//Name thread for the profiler:
Thread.CurrentThread.Name = "Algorithm Analysis Thread";
Initializer.Start();
leanEngineSystemHandlers = Initializer.GetSystemHandlers();
//-> Pull job from QuantConnect job queue, or, pull local build:
string assemblyPath;
job = leanEngineSystemHandlers.JobQueue.NextJob(out assemblyPath);
leanEngineAlgorithmHandlers = Initializer.GetAlgorithmHandlers();
if (job == null)
{
const string jobNullMessage = "Engine.Main(): Sorry we could not process this algorithm request.";
Log.Error(jobNullMessage);
throw new ArgumentException(jobNullMessage);
}
// if the job version doesn't match this instance version then we can't process it
// we also don't want to reprocess redelivered jobs
if (job.Redelivered)
{
Log.Error("Engine.Run(): Job Version: " + job.Version + " Deployed Version: " + Globals.Version + " Redelivered: " + job.Redelivered);
//Tiny chance there was an uncontrolled collapse of a server, resulting in an old user task circulating.
//In this event kill the old algorithm and leave a message so the user can later review.
leanEngineSystemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, _collapseMessage);
leanEngineSystemHandlers.Notify.SetAuthentication(job);
leanEngineSystemHandlers.Notify.Send(new RuntimeErrorPacket(job.UserId, job.AlgorithmId, _collapseMessage));
leanEngineSystemHandlers.JobQueue.AcknowledgeJob(job);
Exit(1);
}
try
{
// Set our exit handler for the algorithm
Console.CancelKeyPress += new ConsoleCancelEventHandler(ExitKeyPress);
// Create the algorithm manager and start our engine
algorithmManager = new AlgorithmManager(liveMode, job);
leanEngineSystemHandlers.LeanManager.Initialize(leanEngineSystemHandlers, leanEngineAlgorithmHandlers, job, algorithmManager);
var engine = new Engine.Engine(leanEngineSystemHandlers, leanEngineAlgorithmHandlers, liveMode);
engine.Run(job, algorithmManager, assemblyPath, WorkerThread.Instance);
}
finally
{
var algorithmStatus = algorithmManager?.State ?? AlgorithmStatus.DeployError;
Exit(algorithmStatus != AlgorithmStatus.Completed ? 1 : 0);
}
}
static void Main(string[] args)
{
Log.LogHandler = Composer.Instance.GetExportedValueByTypeName <ILogHandler>(Config.Get("log-handler", "CompositeLogHandler"));
//Initialize:
string mode = "RELEASE";
var liveMode = Config.GetBool("live-mode");
Log.DebuggingEnabled = Config.GetBool("debug-mode");
#if DEBUG
mode = "DEBUG";
#endif
//Name thread for the profiler:
Thread.CurrentThread.Name = "Algorithm Analysis Thread";
Log.Trace("Engine.Main(): LEAN ALGORITHMIC TRADING ENGINE v" + Constants.Version + " Mode: " + mode);
Log.Trace("Engine.Main(): Started " + DateTime.Now.ToShortTimeString());
Log.Trace("Engine.Main(): Memory " + OS.ApplicationMemoryUsed + "Mb-App " + +OS.TotalPhysicalMemoryUsed + "Mb-Used " + OS.TotalPhysicalMemory + "Mb-Total");
//Import external libraries specific to physical server location (cloud/local)
LeanEngineSystemHandlers leanEngineSystemHandlers;
try
{
leanEngineSystemHandlers = LeanEngineSystemHandlers.FromConfiguration(Composer.Instance);
}
catch (CompositionException compositionException)
{
Log.Error("Engine.Main(): Failed to load library: " + compositionException);
throw;
}
//Setup packeting, queue and controls system: These don't do much locally.
leanEngineSystemHandlers.Initialize();
//-> Pull job from QuantConnect job queue, or, pull local build:
string assemblyPath;
var job = leanEngineSystemHandlers.JobQueue.NextJob(out assemblyPath);
if (job == null)
{
throw new Exception("Engine.Main(): Job was null.");
}
LeanEngineAlgorithmHandlers leanEngineAlgorithmHandlers;
try
{
leanEngineAlgorithmHandlers = LeanEngineAlgorithmHandlers.FromConfiguration(Composer.Instance);
}
catch (CompositionException compositionException)
{
Log.Error("Engine.Main(): Failed to load library: " + compositionException);
throw;
}
// log the job endpoints
Log.Trace("JOB HANDLERS: ");
Log.Trace(" DataFeed: " + leanEngineAlgorithmHandlers.DataFeed.GetType().FullName);
Log.Trace(" Setup: " + leanEngineAlgorithmHandlers.Setup.GetType().FullName);
Log.Trace(" RealTime: " + leanEngineAlgorithmHandlers.RealTime.GetType().FullName);
Log.Trace(" Results: " + leanEngineAlgorithmHandlers.Results.GetType().FullName);
Log.Trace(" Transactions: " + leanEngineAlgorithmHandlers.Transactions.GetType().FullName);
Log.Trace(" History: " + leanEngineAlgorithmHandlers.HistoryProvider.GetType().FullName);
Log.Trace(" Commands: " + leanEngineAlgorithmHandlers.CommandQueue.GetType().FullName);
if (job is LiveNodePacket)
{
Log.Trace(" Brokerage: " + ((LiveNodePacket)job).Brokerage);
}
// if the job version doesn't match this instance version then we can't process it
// we also don't want to reprocess redelivered jobs
if (job.Version != Constants.Version || job.Redelivered)
{
Log.Error("Engine.Run(): Job Version: " + job.Version + " Deployed Version: " + Constants.Version + " Redelivered: " + job.Redelivered);
//Tiny chance there was an uncontrolled collapse of a server, resulting in an old user task circulating.
//In this event kill the old algorithm and leave a message so the user can later review.
leanEngineSystemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, _collapseMessage);
leanEngineSystemHandlers.Notify.SetChannel(job.Channel);
leanEngineSystemHandlers.Notify.RuntimeError(job.AlgorithmId, _collapseMessage);
leanEngineSystemHandlers.JobQueue.AcknowledgeJob(job);
return;
}
try
{
var engine = new Engine.Engine(leanEngineSystemHandlers, leanEngineAlgorithmHandlers, liveMode);
engine.Run(job, assemblyPath);
}
finally
{
//Delete the message from the job queue:
leanEngineSystemHandlers.JobQueue.AcknowledgeJob(job);
Log.Trace("Engine.Main(): Packet removed from queue: " + job.AlgorithmId);
// clean up resources
leanEngineSystemHandlers.Dispose();
leanEngineAlgorithmHandlers.Dispose();
Log.LogHandler.Dispose();
}
}
/// <summary>
/// Method performs necessary initialization and starts and algorithm inside Lean Engine.
/// </summary>
public Dictionary <string, decimal> Run(Dictionary <string, string> alorithmInputs)
{
// Chromosome id must be there
var id = alorithmInputs["chromosome-id"];
// Set algorithm input variables
foreach (var pair in alorithmInputs.Where(i => i.Key != "chromosome-id"))
{
Config.Set(pair.Key, pair.Value);
}
// Lean general settings
Config.Set("environment", "backtesting");
Config.Set("algorithm-language", "CSharp");
Config.Set("result-handler", nameof(OptimizerResultHandler)); //override default result handler
// Create uniquely named log file for the backtest
var dirPath = Path.Combine(Directory.GetCurrentDirectory(), "OptimizationLogs");
var logFileName = "log" + "_" + id + ".txt";
var filePath = Path.Combine(dirPath, logFileName);
// Create directory if not exist
System.IO.Directory.CreateDirectory(dirPath);
Log.LogHandler = new FileLogHandler(filePath);
// LeanEngineSystemHandlers
LeanEngineSystemHandlers leanEngineSystemHandlers;
try
{
leanEngineSystemHandlers = LeanEngineSystemHandlers.FromConfiguration(Composer.Instance);
}
catch (CompositionException compositionException)
{
Log.Error("Engine.Main(): Failed to load library: " + compositionException);
throw;
}
// Setup packeting, queue and controls system: These don't do much locally.
leanEngineSystemHandlers.Initialize();
// Pull job from QuantConnect job queue, or, pull local build:
var job = leanEngineSystemHandlers.JobQueue.NextJob(out var assemblyPath);
if (job == null)
{
throw new Exception("Engine.Main(): Job was null.");
}
// LeanEngineSystemHandlers
LeanEngineAlgorithmHandlers leanEngineAlgorithmHandlers;
try
{
leanEngineAlgorithmHandlers = LeanEngineAlgorithmHandlers.FromConfiguration(Composer.Instance);
}
catch (CompositionException compositionException)
{
Log.Error("Engine.Main(): Failed to load library: " + compositionException);
throw;
}
// Engine
try
{
var algorithmManager = new AlgorithmManager(false, job);
leanEngineSystemHandlers.LeanManager.Initialize(leanEngineSystemHandlers, leanEngineAlgorithmHandlers, job, algorithmManager);
var engine = new Engine(leanEngineSystemHandlers, leanEngineAlgorithmHandlers, false);
// Run -->
engine.Run(job, algorithmManager, assemblyPath, WorkerThread.Instance);
}
finally
{
// do not Acknowledge Job, clean up resources
Log.Trace("Engine.Main(): Packet removed from queue: " + job.AlgorithmId);
leanEngineSystemHandlers.Dispose();
leanEngineAlgorithmHandlers.Dispose();
Log.LogHandler.Dispose();
}
// Results
_resultsHandler = (OptimizerResultHandler)leanEngineAlgorithmHandlers.Results;
return(_resultsHandler.FullResults);
}
private void LaunchLean()
{
Config.Set("environment", "backtesting");
if (!string.IsNullOrEmpty(_config.AlgorithmTypeName))
{
Config.Set("algorithm-type-name", _config.AlgorithmTypeName);
}
if (!string.IsNullOrEmpty(_config.AlgorithmLocation))
{
Config.Set("algorithm-location", Path.GetFileName(_config.AlgorithmLocation));
}
if (!string.IsNullOrEmpty(_config.DataFolder))
{
Config.Set("data-folder", _config.DataFolder);
}
if (!string.IsNullOrEmpty(_config.TransactionLog))
{
var filename = _config.TransactionLog;
filename = Path.Combine(AppDomain.CurrentDomain.BaseDirectory,
Path.GetFileNameWithoutExtension(filename) + _id + Path.GetExtension(filename));
Config.Set("transaction-log", filename);
}
//transaction-log
Config.Set("api-handler", nameof(EmptyApiHandler));
var systemHandlers = LeanEngineSystemHandlers.FromConfiguration(Composer.Instance);
systemHandlers.Initialize();
//separate log uniquely named
var logFileName = "log" + DateTime.Now.ToString("yyyyMMddssfffffff") + "_" + _id + ".txt";
using (Log.LogHandler = new FileLogHandler(logFileName, true))
{
LeanEngineAlgorithmHandlers leanEngineAlgorithmHandlers;
try
{
//override config to use custom result handler
Config.Set("backtesting.result-handler", nameof(OptimizerResultHandler));
leanEngineAlgorithmHandlers = LeanEngineAlgorithmHandlers.FromConfiguration(Composer.Instance);
_resultsHandler = (OptimizerResultHandler)leanEngineAlgorithmHandlers.Results;
}
catch (CompositionException compositionException)
{
Log.Error("Engine.Main(): Failed to load library: " + compositionException);
throw;
}
string algorithmPath;
AlgorithmNodePacket job = systemHandlers.JobQueue.NextJob(out algorithmPath);
try
{
var algorithmManager = new AlgorithmManager(false);
systemHandlers.LeanManager.Initialize(systemHandlers, leanEngineAlgorithmHandlers, job, algorithmManager);
var engine = new Engine(systemHandlers, leanEngineAlgorithmHandlers, false);
engine.Run(job, algorithmManager, algorithmPath);
}
finally
{
Log.Trace("Engine.Main(): Packet removed from queue: " + job.AlgorithmId);
// clean up resources
systemHandlers.Dispose();
leanEngineAlgorithmHandlers.Dispose();
}
}
}
static void Main(string[] args)
{
var mode = "RELEASE";
#if DEBUG
mode = "DEBUG";
#endif
var environment = Config.Get("environment");
var liveMode = Config.GetBool("live-mode");
Log.DebuggingEnabled = Config.GetBool("debug-mode");
Log.LogHandler = Composer.Instance.GetExportedValueByTypeName <ILogHandler>(Config.Get("log-handler", "CompositeLogHandler"));
Thread.CurrentThread.Name = "Algorithm Analysis Thread";
Log.Trace("Engine.Main(): LEAN ALGORITHMIC TRADING ENGINE v" + Globals.Version + " Mode: " + mode);
Log.Trace("Engine.Main(): Started " + DateTime.Now.ToShortTimeString());
Log.Trace("Engine.Main(): Memory " + OS.ApplicationMemoryUsed + "Mb-App " + +OS.TotalPhysicalMemoryUsed + "Mb-Used " + OS.TotalPhysicalMemory + "Mb-Total");
LeanEngineSystemHandlers leanEngineSystemHandlers;
try
{
leanEngineSystemHandlers = LeanEngineSystemHandlers.FromConfiguration(Composer.Instance);
}
catch (CompositionException compositionException)
{
Log.Error("Engine.Main(): Failed to load library: " + compositionException);
throw;
}
leanEngineSystemHandlers.Initialize();
string assemblyPath;
var job = leanEngineSystemHandlers.JobQueue.NextJob(out assemblyPath);
if (job == null)
{
throw new Exception("Engine.Main(): Job was null.");
}
LeanEngineAlgorithmHandlers leanEngineAlgorithmHandlers;
try
{
leanEngineAlgorithmHandlers = LeanEngineAlgorithmHandlers.FromConfiguration(Composer.Instance);
}
catch (CompositionException compositionException)
{
Log.Error("Engine.Main(): Failed to load library: " + compositionException);
throw;
}
if (environment == "backtesting-desktop")
{
Application.EnableVisualStyles();
var messagingHandler = leanEngineSystemHandlers.Notify;
var thread = new Thread(() => LaunchUX(messagingHandler, job));
thread.SetApartmentState(ApartmentState.STA);
thread.Start();
}
Log.Trace("JOB HANDLERS: ");
Log.Trace(" DataFeed: " + leanEngineAlgorithmHandlers.DataFeed.GetType().FullName);
Log.Trace(" Setup: " + leanEngineAlgorithmHandlers.Setup.GetType().FullName);
Log.Trace(" RealTime: " + leanEngineAlgorithmHandlers.RealTime.GetType().FullName);
Log.Trace(" Results: " + leanEngineAlgorithmHandlers.Results.GetType().FullName);
Log.Trace(" Transactions: " + leanEngineAlgorithmHandlers.Transactions.GetType().FullName);
Log.Trace(" History: " + leanEngineAlgorithmHandlers.HistoryProvider.GetType().FullName);
Log.Trace(" Commands: " + leanEngineAlgorithmHandlers.CommandQueue.GetType().FullName);
if (job is LiveNodePacket)
{
Log.Trace(" Brokerage: " + ((LiveNodePacket)job).Brokerage);
}
if (VersionHelper.IsNotEqualVersion(job.Version) || job.Redelivered)
{
Log.Error("Engine.Run(): Job Version: " + job.Version + " Deployed Version: " + Globals.Version + " Redelivered: " + job.Redelivered);
leanEngineSystemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, _collapseMessage);
leanEngineSystemHandlers.Notify.SetAuthentication(job);
leanEngineSystemHandlers.Notify.Send(new RuntimeErrorPacket(job.AlgorithmId, _collapseMessage));
leanEngineSystemHandlers.JobQueue.AcknowledgeJob(job);
return;
}
try
{
var engine = new Engine.Engine(leanEngineSystemHandlers, leanEngineAlgorithmHandlers, liveMode);
engine.Run(job, assemblyPath);
}
finally
{
leanEngineSystemHandlers.JobQueue.AcknowledgeJob(job);
Log.Trace("Engine.Main(): Packet removed from queue: " + job.AlgorithmId);
leanEngineSystemHandlers.Dispose();
leanEngineAlgorithmHandlers.Dispose();
Log.LogHandler.Dispose();
}
}
static void Main(string[] args)
{
//Initialize:
var mode = "RELEASE";
#if DEBUG
mode = "DEBUG";
#endif
if (OS.IsWindows)
{
Console.OutputEncoding = System.Text.Encoding.Unicode;
}
var environment = Config.Get("environment");
var liveMode = Config.GetBool("live-mode");
Log.DebuggingEnabled = Config.GetBool("debug-mode");
Log.LogHandler = Composer.Instance.GetExportedValueByTypeName <ILogHandler>(Config.Get("log-handler", "CompositeLogHandler"));
//Name thread for the profiler:
Thread.CurrentThread.Name = "Algorithm Analysis Thread";
Log.Trace("Engine.Main(): LEAN ALGORITHMIC TRADING ENGINE v" + Globals.Version + " Mode: " + mode + " (" + (Environment.Is64BitProcess ? "64" : "32") + "bit)");
Log.Trace("Engine.Main(): Started " + DateTime.Now.ToShortTimeString());
Log.Trace("Engine.Main(): Memory " + OS.ApplicationMemoryUsed + "Mb-App " + +OS.TotalPhysicalMemoryUsed + "Mb-Used " + OS.TotalPhysicalMemory + "Mb-Total");
//Import external libraries specific to physical server location (cloud/local)
LeanEngineSystemHandlers leanEngineSystemHandlers;
try
{
leanEngineSystemHandlers = LeanEngineSystemHandlers.FromConfiguration(Composer.Instance);
}
catch (CompositionException compositionException)
{
Log.Error("Engine.Main(): Failed to load library: " + compositionException);
throw;
}
//Setup packeting, queue and controls system: These don't do much locally.
leanEngineSystemHandlers.Initialize();
//-> Pull job from QuantConnect job queue, or, pull local build:
string assemblyPath;
var job = leanEngineSystemHandlers.JobQueue.NextJob(out assemblyPath);
if (job == null)
{
throw new Exception("Engine.Main(): Job was null.");
}
LeanEngineAlgorithmHandlers leanEngineAlgorithmHandlers;
try
{
leanEngineAlgorithmHandlers = LeanEngineAlgorithmHandlers.FromConfiguration(Composer.Instance);
}
catch (CompositionException compositionException)
{
Log.Error("Engine.Main(): Failed to load library: " + compositionException);
throw;
}
if (environment.EndsWith("-desktop"))
{
var info = new ProcessStartInfo
{
UseShellExecute = false,
FileName = Config.Get("desktop-exe"),
Arguments = Config.Get("desktop-http-port")
};
Process.Start(info);
}
// if the job version doesn't match this instance version then we can't process it
// we also don't want to reprocess redelivered jobs
if (VersionHelper.IsNotEqualVersion(job.Version) || job.Redelivered)
{
Log.Error("Engine.Run(): Job Version: " + job.Version + " Deployed Version: " + Globals.Version + " Redelivered: " + job.Redelivered);
//Tiny chance there was an uncontrolled collapse of a server, resulting in an old user task circulating.
//In this event kill the old algorithm and leave a message so the user can later review.
leanEngineSystemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, _collapseMessage);
leanEngineSystemHandlers.Notify.SetAuthentication(job);
leanEngineSystemHandlers.Notify.Send(new RuntimeErrorPacket(job.UserId, job.AlgorithmId, _collapseMessage));
leanEngineSystemHandlers.JobQueue.AcknowledgeJob(job);
return;
}
try
{
var engine = new Engine.Engine(leanEngineSystemHandlers, leanEngineAlgorithmHandlers, liveMode);
engine.Run(job, assemblyPath);
}
finally
{
//Delete the message from the job queue:
leanEngineSystemHandlers.JobQueue.AcknowledgeJob(job);
Log.Trace("Engine.Main(): Packet removed from queue: " + job.AlgorithmId);
// clean up resources
leanEngineSystemHandlers.Dispose();
leanEngineAlgorithmHandlers.Dispose();
Log.LogHandler.Dispose();
}
}
static void Main(string[] args)
{
//Initialize:
var mode = "RELEASE";
#if DEBUG
mode = "DEBUG";
#endif
if (OS.IsWindows)
{
Console.OutputEncoding = System.Text.Encoding.UTF8;
}
// expect first argument to be config file name
if (args.Length > 0)
{
Config.MergeCommandLineArgumentsWithConfiguration(LeanArgumentParser.ParseArguments(args));
}
var liveMode = Config.GetBool("live-mode");
Log.DebuggingEnabled = Config.GetBool("debug-mode");
Log.FilePath = Path.Combine(Config.Get("results-destination-folder"), "log.txt");
Log.LogHandler = Composer.Instance.GetExportedValueByTypeName <ILogHandler>(Config.Get("log-handler", "CompositeLogHandler"));
//Name thread for the profiler:
Thread.CurrentThread.Name = "Algorithm Analysis Thread";
Log.Trace($"Engine.Main(): LEAN ALGORITHMIC TRADING ENGINE v{Globals.Version} Mode: {mode} ({(Environment.Is64BitProcess ? "64" : "32")}bit) Host: {Environment.MachineName}");
Log.Trace("Engine.Main(): Started " + DateTime.Now.ToShortTimeString());
//Import external libraries specific to physical server location (cloud/local)
try
{
leanEngineSystemHandlers = LeanEngineSystemHandlers.FromConfiguration(Composer.Instance);
}
catch (CompositionException compositionException)
{
Log.Error("Engine.Main(): Failed to load library: " + compositionException);
throw;
}
//Setup packeting, queue and controls system: These don't do much locally.
leanEngineSystemHandlers.Initialize();
//-> Pull job from QuantConnect job queue, or, pull local build:
string assemblyPath;
job = leanEngineSystemHandlers.JobQueue.NextJob(out assemblyPath);
if (job == null)
{
const string jobNullMessage = "Engine.Main(): Sorry we could not process this algorithm request.";
Log.Error(jobNullMessage);
throw new ArgumentException(jobNullMessage);
}
try
{
leanEngineAlgorithmHandlers = LeanEngineAlgorithmHandlers.FromConfiguration(Composer.Instance);
}
catch (CompositionException compositionException)
{
Log.Error("Engine.Main(): Failed to load library: " + compositionException);
throw;
}
// if the job version doesn't match this instance version then we can't process it
// we also don't want to reprocess redelivered jobs
if (job.Redelivered)
{
Log.Error("Engine.Run(): Job Version: " + job.Version + " Deployed Version: " + Globals.Version + " Redelivered: " + job.Redelivered);
//Tiny chance there was an uncontrolled collapse of a server, resulting in an old user task circulating.
//In this event kill the old algorithm and leave a message so the user can later review.
leanEngineSystemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, _collapseMessage);
leanEngineSystemHandlers.Notify.SetAuthentication(job);
leanEngineSystemHandlers.Notify.Send(new RuntimeErrorPacket(job.UserId, job.AlgorithmId, _collapseMessage));
leanEngineSystemHandlers.JobQueue.AcknowledgeJob(job);
Exit(1);
}
try
{
// Set our exit handler for the algorithm
Console.CancelKeyPress += new ConsoleCancelEventHandler(ExitKeyPress);
// Create the algorithm manager and start our engine
algorithmManager = new AlgorithmManager(liveMode, job);
leanEngineSystemHandlers.LeanManager.Initialize(leanEngineSystemHandlers, leanEngineAlgorithmHandlers, job, algorithmManager);
var engine = new Engine.Engine(leanEngineSystemHandlers, leanEngineAlgorithmHandlers, liveMode);
engine.Run(job, algorithmManager, assemblyPath, WorkerThread.Instance);
}
finally
{
var algorithmStatus = algorithmManager?.State ?? AlgorithmStatus.DeployError;
Exit(algorithmStatus != AlgorithmStatus.Completed ? 1 : 0);
}
}
private void LaunchLean()
{
Config.Set("environment", "backtesting");
if (!string.IsNullOrEmpty(_config.AlgorithmTypeName))
{
Config.Set("algorithm-type-name", _config.AlgorithmTypeName);
}
if (!string.IsNullOrEmpty(_config.AlgorithmLocation))
{
Config.Set("algorithm-location", Path.GetFileName(_config.AlgorithmLocation));
}
if (!string.IsNullOrEmpty(_config.DataFolder))
{
Config.Set("data-folder", _config.DataFolder);
}
var systemHandlers = LeanEngineSystemHandlers.FromConfiguration(Composer.Instance);
systemHandlers.Initialize();
//separate log now uniquely named
var logFileName = "log" + DateTime.Now.ToString("yyyyMMddssfffffff");
if (File.Exists(logFileName + ".txt"))
{
logFileName += "_" + Guid.NewGuid().ToString();
}
logFileName += ".txt";
var logHandlers = new ILogHandler[] { new FileLogHandler(logFileName, true) };
using (Log.LogHandler = new CompositeLogHandler(logHandlers))
{
LeanEngineAlgorithmHandlers leanEngineAlgorithmHandlers;
try
{
//override config to use custom result handler
Config.Set("backtesting.result-handler", nameof(OptimizerResultHandler));
leanEngineAlgorithmHandlers = LeanEngineAlgorithmHandlers.FromConfiguration(Composer.Instance);
_resultsHandler = (OptimizerResultHandler)leanEngineAlgorithmHandlers.Results;
}
catch (CompositionException compositionException)
{
Log.Error("Engine.Main(): Failed to load library: " + compositionException);
throw;
}
string algorithmPath;
AlgorithmNodePacket job = systemHandlers.JobQueue.NextJob(out algorithmPath);
try
{
var _engine = new Engine(systemHandlers, leanEngineAlgorithmHandlers, Config.GetBool("live-mode"));
_engine.Run(job, algorithmPath);
}
finally
{
Log.Trace("Engine.Main(): Packet removed from queue: " + job.AlgorithmId);
// clean up resources
systemHandlers.Dispose();
leanEngineAlgorithmHandlers.Dispose();
}
}
}
/// <summary>
/// Empty implementation of the ILeanManager interface
/// </summary>
/// <param name="systemHandlers">Exposes lean engine system handlers running LEAN</param>
/// <param name="algorithmHandlers">Exposes the lean algorithm handlers running lean</param>
/// <param name="job">The job packet representing either a live or backtest Lean instance</param>
/// <param name="algorithmManager">The Algorithm manager</param>
public void Initialize(LeanEngineSystemHandlers systemHandlers, LeanEngineAlgorithmHandlers algorithmHandlers, AlgorithmNodePacket job, AlgorithmManager algorithmManager)
{
// NOP
}
private void LaunchLean(string id)
{
ConfigMerger.Merge(_config, id);
Config.Set("api-handler", nameof(EmptyApiHandler));
//todo: instance logging
//var logFileName = "log" + DateTime.Now.ToString("yyyyMMddssfffffff") + "_" + id + ".txt";
Log.LogHandler = LogSingleton.Instance;
var jobQueue = new JobQueue();
var manager = new LocalLeanManager();
var systemHandlers = new LeanEngineSystemHandlers(
jobQueue,
new EmptyApiHandler(),
new QuantConnect.Messaging.Messaging(),
manager);
systemHandlers.Initialize();
var map = new LocalDiskMapFileProvider();
var results = new OptimizerResultHandler();
var transactions = new BacktestingTransactionHandler();
var dataFeed = new FileSystemDataFeed();
var realTime = new BacktestingRealTimeHandler();
var data = new DefaultDataProvider();
var leanEngineAlgorithmHandlers = new LeanEngineAlgorithmHandlers(
results,
new ConsoleSetupHandler(),
dataFeed,
transactions,
realTime,
map,
new LocalDiskFactorFileProvider(map),
data,
new OptimizerAlphaHandler(),
new EmptyObjectStore());
_resultsHandler = (OptimizerResultHandler)leanEngineAlgorithmHandlers.Results;
var job = (BacktestNodePacket)systemHandlers.JobQueue.NextJob(out var algorithmPath);
//mark job with id. Is set on algorithm in OptimizerAlphaHandler
job.BacktestId = id;
//todo: pass period through job
//job.PeriodStart = _config.StartDate;
//job.PeriodFinish = _config.EndDate;
Engine engine;
AlgorithmManager algorithmManager;
try
{
algorithmManager = new AlgorithmManager(false);
systemHandlers.LeanManager.Initialize(systemHandlers, leanEngineAlgorithmHandlers, job, algorithmManager);
engine = new Engine(systemHandlers, leanEngineAlgorithmHandlers, false);
using (var workerThread = new MultipleWorkerThread())
{
engine.Run(job, algorithmManager, algorithmPath, workerThread);
}
}
finally
{
// clean up resources
Composer.Instance.Reset();
results.Charts.Clear();
results.Messages.Clear();
if (results.Algorithm != null)
{
results.Algorithm.Transactions.TransactionRecord.Clear();
results.Algorithm.SubscriptionManager.Subscriptions.SelectMany(s => s.Consolidators)?.ToList().ForEach(f =>
{
results.Algorithm.SubscriptionManager.RemoveConsolidator(f.WorkingData?.Symbol, f);
UnregisterAllEvents(f);
});
if (results.Algorithm is QCAlgorithm)
{
((QCAlgorithm)results.Algorithm).SubscriptionManager.Subscriptions.ToList().Clear();
}
if (_config.AlgorithmLanguage != "Python")
{
results.Algorithm.HistoryProvider = null;
}
var closedTrades = (List <Trade>) typeof(TradeBuilder).GetField("_closedTrades", BindingFlags.Instance | BindingFlags.NonPublic).GetValue(results.Algorithm.TradeBuilder);
closedTrades.Clear();
results.Algorithm = null;
}
transactions.Orders.Clear();
transactions.OrderTickets.Clear();
manager.Dispose();
systemHandlers.Dispose();
leanEngineAlgorithmHandlers.Dispose();
results = null;
dataFeed = null;
transactions = null;
realTime = null;
data = null;
map = null;
systemHandlers = null;
leanEngineAlgorithmHandlers = null;
algorithmManager = null;
engine = null;
job = null;
jobQueue = null;
manager = null;
}
}
