/// <summary>
/// <see cref = "QuantBook" /> constructor.
/// Provides access to data for quantitative analysis
/// </summary>
public QuantBook() : base()
{
try
{
using (Py.GIL())
{
_pandas = Py.Import("pandas");
}
_converter = new PandasConverter(_pandas);
// Create new instance of QCAlgorithm we are going to wrap
_algorithm = new QCAlgorithm();
// By default, set start date to end data which is yesterday
SetStartDate(_algorithm.EndDate);
// Initialize History Provider
var composer = new Composer();
var algorithmHandlers = LeanEngineAlgorithmHandlers.FromConfiguration(composer);
_dataCacheProvider = new ZipDataCacheProvider(algorithmHandlers.DataProvider);
var mapFileProvider = algorithmHandlers.MapFileProvider;
_algorithm.HistoryProvider = composer.GetExportedValueByTypeName <IHistoryProvider>(Config.Get("history-provider", "SubscriptionDataReaderHistoryProvider"));
_algorithm.HistoryProvider.Initialize(null, algorithmHandlers.DataProvider, _dataCacheProvider, mapFileProvider, algorithmHandlers.FactorFileProvider, null);
}
catch (Exception exception)
{
throw new Exception("QuantBook.Main(): " + exception);
}
}
/// <summary>
/// <see cref = "QuantBook" /> constructor.
/// Provides access to data for quantitative analysis
/// </summary>
public QuantBook() : base()
{
try
{
using (Py.GIL())
{
_pandas = Py.Import("pandas");
}
// By default, set start date to end data which is yesterday
SetStartDate(EndDate);
// Sets PandasConverter
SetPandasConverter();
// Initialize History Provider
var composer = new Composer();
var algorithmHandlers = LeanEngineAlgorithmHandlers.FromConfiguration(composer);
var systemHandlers = LeanEngineSystemHandlers.FromConfiguration(composer);
systemHandlers.LeanManager.Initialize(systemHandlers,
algorithmHandlers,
new BacktestNodePacket(),
new AlgorithmManager(false));
systemHandlers.LeanManager.SetAlgorithm(this);
_dataCacheProvider = new ZipDataCacheProvider(algorithmHandlers.DataProvider);
var symbolPropertiesDataBase = SymbolPropertiesDatabase.FromDataFolder();
var securityService = new SecurityService(Portfolio.CashBook, MarketHoursDatabase, symbolPropertiesDataBase, this);
Securities.SetSecurityService(securityService);
SubscriptionManager.SetDataManager(
new DataManager(new NullDataFeed(),
new UniverseSelection(this, securityService),
this,
TimeKeeper,
MarketHoursDatabase,
false));
var mapFileProvider = algorithmHandlers.MapFileProvider;
HistoryProvider = composer.GetExportedValueByTypeName <IHistoryProvider>(Config.Get("history-provider", "SubscriptionDataReaderHistoryProvider"));
HistoryProvider.Initialize(
new HistoryProviderInitializeParameters(
null,
null,
algorithmHandlers.DataProvider,
_dataCacheProvider,
mapFileProvider,
algorithmHandlers.FactorFileProvider,
null
)
);
SetOptionChainProvider(new CachingOptionChainProvider(new BacktestingOptionChainProvider()));
SetFutureChainProvider(new CachingFutureChainProvider(new BacktestingFutureChainProvider()));
}
catch (Exception exception)
{
throw new Exception("QuantBook.Main(): " + exception);
}
}
/// <summary>
/// <see cref = "QuantBook" /> constructor.
/// Provides access to data for quantitative analysis
/// </summary>
public QuantBook() : base()
{
try
{
using (Py.GIL())
{
_pandas = Py.Import("pandas");
}
// By default, set start date to end data which is yesterday
SetStartDate(EndDate);
// Sets PandasConverter
SetPandasConverter();
// Initialize History Provider
var composer = new Composer();
var algorithmHandlers = LeanEngineAlgorithmHandlers.FromConfiguration(composer);
_dataCacheProvider = new ZipDataCacheProvider(algorithmHandlers.DataProvider);
var nullDataFeed = new NullDataFeed();
SubscriptionManager.SetDataManager(new DataManager(nullDataFeed, new UniverseSelection(nullDataFeed, this), Settings, TimeKeeper));
var mapFileProvider = algorithmHandlers.MapFileProvider;
HistoryProvider = composer.GetExportedValueByTypeName <IHistoryProvider>(Config.Get("history-provider", "SubscriptionDataReaderHistoryProvider"));
HistoryProvider.Initialize(null, algorithmHandlers.DataProvider, _dataCacheProvider, mapFileProvider, algorithmHandlers.FactorFileProvider, null);
SetOptionChainProvider(new CachingOptionChainProvider(new BacktestingOptionChainProvider()));
SetFutureChainProvider(new CachingFutureChainProvider(new BacktestingFutureChainProvider()));
}
catch (Exception exception)
{
throw new Exception("QuantBook.Main(): " + exception);
}
}
public static void RunLocalBacktest(string algorithm, Dictionary <string, string> expectedStatistics, Language language)
{
var statistics = new Dictionary <string, string>();
Composer.Instance.Reset();
try
{
// set the configuration up
Config.Set("algorithm-type-name", algorithm);
Config.Set("live-mode", "false");
Config.Set("environment", "");
Config.Set("messaging-handler", "QuantConnect.Messaging.Messaging");
Config.Set("job-queue-handler", "QuantConnect.Queues.JobQueue");
Config.Set("api-handler", "QuantConnect.Api.Api");
Config.Set("result-handler", "QuantConnect.Lean.Engine.Results.BacktestingResultHandler");
Config.Set("algorithm-language", language.ToString());
Config.Set("algorithm-location", "QuantConnect.Algorithm." + language + ".dll");
var debugEnabled = Log.DebuggingEnabled;
var logHandlers = new ILogHandler[] { new ConsoleLogHandler(), new FileLogHandler("regression.log", false) };
using (Log.LogHandler = new CompositeLogHandler(logHandlers))
using (var algorithmHandlers = LeanEngineAlgorithmHandlers.FromConfiguration(Composer.Instance))
using (var systemHandlers = LeanEngineSystemHandlers.FromConfiguration(Composer.Instance))
{
Log.DebuggingEnabled = true;
Log.LogHandler.Trace("");
Log.LogHandler.Trace("{0}: Running " + algorithm + "...", DateTime.UtcNow);
Log.LogHandler.Trace("");
// run the algorithm in its own thread
var engine = new Lean.Engine.Engine(systemHandlers, algorithmHandlers, false);
Task.Factory.StartNew(() =>
{
string algorithmPath;
var job = systemHandlers.JobQueue.NextJob(out algorithmPath);
var algorithmManager = new AlgorithmManager(false);
engine.Run(job, algorithmManager, algorithmPath);
}).Wait();
var backtestingResultHandler = (BacktestingResultHandler)algorithmHandlers.Results;
statistics = backtestingResultHandler.FinalStatistics;
Log.DebuggingEnabled = debugEnabled;
}
}
catch (Exception ex)
{
Log.LogHandler.Error("{0} {1}", ex.Message, ex.StackTrace);
}
foreach (var stat in expectedStatistics)
{
Assert.AreEqual(true, statistics.ContainsKey(stat.Key), "Missing key: " + stat.Key);
Assert.AreEqual(stat.Value, statistics[stat.Key], "Failed on " + stat.Key);
}
}
private void LaunchLean()
{
Config.Set("environment", "backtesting");
if (!string.IsNullOrEmpty(_config.AlgorithmTypeName))
{
Config.Set("algorithm-type-name", _config.AlgorithmTypeName);
}
if (!string.IsNullOrEmpty(_config.AlgorithmLocation))
{
Config.Set("algorithm-location", Path.GetFileName(_config.AlgorithmLocation));
}
if (!string.IsNullOrEmpty(_config.DataFolder))
{
Config.Set("data-folder", _config.DataFolder);
}
var systemHandlers = LeanEngineSystemHandlers.FromConfiguration(Composer.Instance);
systemHandlers.Initialize();
Log.LogHandler = Composer.Instance.GetExportedValueByTypeName <ILogHandler>(Config.Get("log-handler", "CompositeLogHandler"));
//Log.DebuggingEnabled = false;
//Log.DebuggingLevel = 1;
LeanEngineAlgorithmHandlers leanEngineAlgorithmHandlers;
try
{
leanEngineAlgorithmHandlers = LeanEngineAlgorithmHandlers.FromConfiguration(Composer.Instance);
_resultsHandler = (BacktestingResultHandler)leanEngineAlgorithmHandlers.Results;
}
catch (CompositionException compositionException)
{
Log.Error("Engine.Main(): Failed to load library: " + compositionException);
throw;
}
string algorithmPath;
AlgorithmNodePacket job = systemHandlers.JobQueue.NextJob(out algorithmPath);
try
{
var _engine = new Engine(systemHandlers, leanEngineAlgorithmHandlers, Config.GetBool("live-mode"));
_engine.Run(job, algorithmPath);
}
finally
{
Log.Trace("Engine.Main(): Packet removed from queue: " + job.AlgorithmId);
// clean up resources
systemHandlers.Dispose();
leanEngineAlgorithmHandlers.Dispose();
Log.LogHandler.Dispose();
}
}
private void LaunchLean()
{
Config.Set("environment", "backtesting");
string algorithm = "EMATest";
Config.Set("algorithm-type-name", algorithm);
_jobQueue = new JobQueue();
_notify = new Messaging();
_api = new Api();
_resultshandler = new DesktopResultHandler();
_dataFeed = new FileSystemDataFeed();
_setup = new ConsoleSetupHandler();
_realTime = new BacktestingRealTimeHandler();
_historyProvider = new SubscriptionDataReaderHistoryProvider();
_transactions = new BacktestingTransactionHandler();
var systemHandlers = new LeanEngineSystemHandlers(_jobQueue, _api, _notify);
systemHandlers.Initialize();
// var algorithmHandlers = new LeanEngineAlgorithmHandlers (_resultshandler, _setup, _dataFeed, _transactions, _realTime, _historyProvider);
Log.LogHandler = Composer.Instance.GetExportedValueByTypeName <ILogHandler>(Config.Get("log-handler", "CompositeLogHandler"));
LeanEngineAlgorithmHandlers leanEngineAlgorithmHandlers;
try
{
leanEngineAlgorithmHandlers = LeanEngineAlgorithmHandlers.FromConfiguration(Composer.Instance);
_resultshandler = leanEngineAlgorithmHandlers.Results;
}
catch (CompositionException compositionException)
{
Log.Error("Engine.Main(): Failed to load library: " + compositionException);
throw;
}
string algorithmPath;
AlgorithmNodePacket job = systemHandlers.JobQueue.NextJob(out algorithmPath);
try
{
var _engine = new Engine(systemHandlers, leanEngineAlgorithmHandlers, Config.GetBool("live-mode"));
_engine.Run(job, algorithmPath);
}
finally
{
//Delete the message from the job queue:
//systemHandlers.JobQueue.AcknowledgeJob(job);
Log.Trace("Engine.Main(): Packet removed from queue: " + job.AlgorithmId);
// clean up resources
systemHandlers.Dispose();
leanEngineAlgorithmHandlers.Dispose();
Log.LogHandler.Dispose();
}
}
private static void LaunchLean()
{
/* Config.Set("environment", "backtesting");
* string algorithm = (string)AppDomain.CurrentDomain.GetData("AlgorithmTypeName");
* string path = (string)AppDomain.CurrentDomain.GetData("AlgorithmLocation");
*
* Config.Set("algorithm-type-name", algorithm);
* if (!string.IsNullOrEmpty(path))
* {
* Config.Set("algorithm-location", path);
* }
*/
// Config.Set("result-handler", "BacktestingResultHandler");
Config.Set("messaging-handler", "QuantConnect.Lean.TestProject.Messaging");
var systemHandlers = LeanEngineSystemHandlers.FromConfiguration(Composer.Instance);
systemHandlers.Initialize();
Log.LogHandler = Composer.Instance.GetExportedValueByTypeName <ILogHandler>(Config.Get("log-handler", "CompositeLogHandler"));
//Log.DebuggingEnabled = false;
//Log.DebuggingLevel = 1;
LeanEngineAlgorithmHandlers leanEngineAlgorithmHandlers;
try
{
leanEngineAlgorithmHandlers = LeanEngineAlgorithmHandlers.FromConfiguration(Composer.Instance);
_resultsHandler = (BacktestingResultHandler)leanEngineAlgorithmHandlers.Results;
}
catch (CompositionException compositionException)
{
Log.Error("Engine.Main(): Failed to load library: " + compositionException);
throw;
}
string algorithmPath;
AlgorithmNodePacket job = systemHandlers.JobQueue.NextJob(out algorithmPath);
try
{
var _engine = new QuantConnect.Lean.Engine.Engine(systemHandlers, leanEngineAlgorithmHandlers, Config.GetBool("live-mode"));
_engine.Run(job, algorithmPath);
}
finally
{
Log.Trace("Engine.Main(): Packet removed from queue: " + job.AlgorithmId);
// clean up resources
systemHandlers.Dispose();
leanEngineAlgorithmHandlers.Dispose();
Log.LogHandler.Dispose();
}
}
public Engine CreateEngine()
{
//Launch the Lean Engine in another thread: this will run the algorithm specified above.
// TODO > This should only be launched when clicking a backtest/trade live button provided in the UX.
Composer v = Composer.Instance;
//Log.LogHandler = new CompositeLogHandler(new ILogHandler[]
// {
// new ConsoleLogHandler(),
// new FileLogHandler("regression.log")
// });
var systemHandlers = LeanEngineSystemHandlers.FromConfiguration(Composer.Instance);
_algorithmHandlers = LeanEngineAlgorithmHandlers.FromConfiguration(Composer.Instance);
var engine = new Engine(systemHandlers, _algorithmHandlers, Config.GetBool("live-mode"));
Log("Engine.Main(): Memory " + OS.ApplicationMemoryUsed + "Mb-App " + +OS.TotalPhysicalMemoryUsed + "Mb-Used " + OS.TotalPhysicalMemory + "Mb-Total");
// log the job endpoints
Log("JOB HANDLERS: ");
Log(" DataFeed: " + _algorithmHandlers.DataFeed.GetType().FullName);
Log(" Setup: " + _algorithmHandlers.Setup.GetType().FullName);
Log(" RealTime: " + _algorithmHandlers.RealTime.GetType().FullName);
Log(" Results: " + _algorithmHandlers.Results.GetType().FullName);
Log(" Transactions: " + _algorithmHandlers.Transactions.GetType().FullName);
Log(" History: " + _algorithmHandlers.HistoryProvider.GetType().FullName);
Log(" Commands: " + _algorithmHandlers.CommandQueue.GetType().FullName);
Log(" FactorFileProvider: " + _algorithmHandlers.FactorFileProvider.GetType().FullName);
Log(" HistoryProvider: " + _algorithmHandlers.HistoryProvider.GetType().FullName);
Log(" MapFileProvider: " + _algorithmHandlers.MapFileProvider.GetType().FullName);
Log(" Transactions: " + _algorithmHandlers.Transactions.GetType().FullName);
_leanEngineThread = new Thread(() =>
{
string algorithmPath;
job = systemHandlers.JobQueue.NextJob(out algorithmPath);
if (job is LiveNodePacket)
{
Log(" Brokerage: " + ((LiveNodePacket)job).Brokerage);
}
engine.Run(job, algorithmPath);
systemHandlers.JobQueue.AcknowledgeJob(job);
});
_leanEngineThread.Start();
return(engine);
}
public void Run(Dictionary <string, string> parameters)
{
const string mode = "DEBUG";
Console.OutputEncoding = System.Text.Encoding.UTF8;
var liveMode = Config.GetBool("live-mode");
Log.DebuggingEnabled = Config.GetBool("debug-mode");
Log.FilePath = Path.Combine(Config.Get("results-destination-folder"), "log.txt");
Log.LogHandler = Composer.Instance.GetExportedValueByTypeName <ILogHandler>(Config.Get("log-handler", "CompositeLogHandler"));
Log.Trace($"Engine.Main(): LEAN ALGORITHMIC TRADING ENGINE v{Globals.Version} Mode: {mode} ({(Environment.Is64BitProcess ? "64" : "32")}bit)");
Log.Trace($"Engine.Main(): Started {DateTime.Now.ToShortTimeString()}");
var systemHandler = LeanEngineSystemHandlers.FromConfiguration(Composer.Instance);
systemHandler.Initialize();
var job = systemHandler.JobQueue.NextJob(out var assemblyPath);
foreach (var kvp in parameters)
{
job.Parameters[kvp.Key] = kvp.Value;
}
var algorithmHandler = LeanEngineAlgorithmHandlers.FromConfiguration(Composer.Instance);
try
{
var algorithmManager = new AlgorithmManager(liveMode, job);
systemHandler.LeanManager.Initialize(systemHandler, algorithmHandler, job, algorithmManager);
var engine = new Engine(systemHandler, algorithmHandler, liveMode);
engine.Run(job, algorithmManager, assemblyPath, WorkerThread.Instance);
}
finally
{
Console.WriteLine("Engine.Main(): Analysis Complete. Press any key to continue.");
Console.ReadKey(true);
systemHandler.Dispose();
algorithmHandler.Dispose();
Log.LogHandler.Dispose();
Log.Trace("Program.Main(): Exiting Lean...");
Thread.Sleep(2000);
Environment.Exit(0);
}
}
public static void RunLocalBacktest(string algorithm, Dictionary <string, string> expectedStatistics)
{
var statistics = new Dictionary <string, string>();
Composer.Instance.Reset();
try
{
using (Log.LogHandler = new CompositeLogHandler(new ILogHandler[]
{
new ConsoleLogHandler(),
new FileLogHandler("regression.log")
}))
using (var algorithmHandlers = LeanEngineAlgorithmHandlers.FromConfiguration(Composer.Instance))
using (var systemHandlers = LeanEngineSystemHandlers.FromConfiguration(Composer.Instance))
{
Console.WriteLine("Running " + algorithm + "...");
// set the configuration up
Config.Set("algorithm-type-name", algorithm);
Config.Set("live-mode", "false");
Config.Set("environment", "");
Config.Set("messaging-handler", "QuantConnect.Messaging.Messaging");
Config.Set("job-queue-handler", "QuantConnect.Queues.JobQueue");
Config.Set("api-handler", "QuantConnect.Api.Api");
// run the algorithm in its own thread
var engine = new Lean.Engine.Engine(systemHandlers, algorithmHandlers, false);
Task.Factory.StartNew(() =>
{
string algorithmPath;
var job = systemHandlers.JobQueue.NextJob(out algorithmPath);
engine.Run(job, algorithmPath);
}).Wait();
var consoleResultHandler = (ConsoleResultHandler)algorithmHandlers.Results;
statistics = consoleResultHandler.FinalStatistics;
}
}
catch (Exception ex)
{
Log.LogHandler.Error("{0} {1}", ex.Message, ex.StackTrace);
}
foreach (var stat in expectedStatistics)
{
Assert.AreEqual(true, statistics.ContainsKey(stat.Key), "Missing key: " + stat.Key);
Assert.AreEqual(stat.Value, statistics[stat.Key], "Failed on " + stat.Key);
}
}
public TrackModel Run(TrackModel model, AccountModel account, SettingModel settings, HostDomainLogger logger)
{
if (model == null)
{
throw new ArgumentNullException(nameof(model));
}
Log.LogHandler = logger;
if (!SetConfig(model, account, settings))
{
return(model);
}
// Register all data providers
ProviderFactory.RegisterProviders(settings);
Log.Trace("LeanLanucher: Memory " + OS.ApplicationMemoryUsed + "Mb-App " + OS.TotalPhysicalMemoryUsed + "Mb-Used");
try
{
var liveMode = Config.GetBool("live-mode");
using var algorithmHandlers = LeanEngineAlgorithmHandlers.FromConfiguration(Composer.Instance);
using var systemHandlers = LeanEngineSystemHandlers.FromConfiguration(Composer.Instance);
systemHandlers.Initialize();
var engine = new Engine(systemHandlers, algorithmHandlers, liveMode);
var algorithmManager = new AlgorithmManager(liveMode);
AlgorithmNodePacket job = systemHandlers.JobQueue.NextJob(out string assemblyPath);
job.UserPlan = UserPlan.Professional;
job.Language = model.AlgorithmLanguage;
if (job is BacktestNodePacket backtest)
{
backtest.PeriodStart = model.StartDate;
backtest.PeriodFinish = model.EndDate;
backtest.CashAmount = new CashAmount(model.InitialCapital, Currencies.USD);
}
systemHandlers.LeanManager.Initialize(systemHandlers, algorithmHandlers, job, algorithmManager);
engine.Run(job, algorithmManager, assemblyPath, WorkerThread.Instance);
BacktestResultHandler resultHandler = algorithmHandlers.Results as BacktestResultHandler;
model.Result = resultHandler?.JsonResult ?? string.Empty;
model.Logs = resultHandler?.Logs ?? string.Empty;
}
catch (Exception ex)
{
Log.Error("{0}: {1}", ex.GetType(), ex.Message);
}
NetMQConfig.Cleanup(false);
Log.LogHandler.Dispose();
return(model);
}
public TrackModel Run(TrackModel model, AccountModel account, SettingService settings, HostDomainLogger logger)
{
if (model == null)
{
throw new ArgumentNullException(nameof(model));
}
Log.LogHandler = logger;
if (!SetConfig(model, account, settings))
{
return(model);
}
// Register all data providers
ProviderFactory.RegisterProviders();
Log.Trace("LeanLanucher: Memory " + OS.ApplicationMemoryUsed + "Mb-App " + OS.TotalPhysicalMemoryUsed + "Mb-Used");
try
{
var liveMode = Config.GetBool("live-mode");
using (var algorithmHandlers = LeanEngineAlgorithmHandlers.FromConfiguration(Composer.Instance))
using (var systemHandlers = LeanEngineSystemHandlers.FromConfiguration(Composer.Instance))
{
string assemblyPath;
systemHandlers.Initialize();
var engine = new Engine(systemHandlers, algorithmHandlers, liveMode);
var algorithmManager = new AlgorithmManager(liveMode);
AlgorithmNodePacket job = systemHandlers.JobQueue.NextJob(out assemblyPath);
job.UserPlan = UserPlan.Professional;
systemHandlers.LeanManager.Initialize(systemHandlers, algorithmHandlers, job, algorithmManager);
engine.Run(job, algorithmManager, assemblyPath);
systemHandlers.JobQueue.AcknowledgeJob(job);
BacktestResultHandler resultHandler = algorithmHandlers.Results as BacktestResultHandler;
model.Result = resultHandler?.JsonResult ?? string.Empty;
model.Logs = resultHandler?.Logs ?? string.Empty;
}
}
catch (Exception ex)
{
Log.Error("{0}: {1}", ex.GetType(), ex.Message);
}
NetMQConfig.Cleanup(false);
Log.LogHandler.Dispose();
return(model);
}
/// <summary>
/// Launch the LEAN Engine in a separate thread.
/// </summary>
private static Engine LaunchLean()
{
//Launch the Lean Engine in another thread: this will run the algorithm specified above.
// TODO > This should only be launched when clicking a backtest/trade live button provided in the UX.
var systemHandlers = LeanEngineSystemHandlers.FromConfiguration(Composer.Instance);
var algorithmHandlers = LeanEngineAlgorithmHandlers.FromConfiguration(Composer.Instance);
var engine = new Engine(systemHandlers, algorithmHandlers, Config.GetBool("live-mode"));
_leanEngineThread = new Thread(() =>
{
string algorithmPath;
var job = systemHandlers.JobQueue.NextJob(out algorithmPath);
engine.Run(job, algorithmPath);
systemHandlers.JobQueue.AcknowledgeJob(job);
});
_leanEngineThread.Start();
return(engine);
}
public void RunJob()
{
try
{
EngineContext.SystemHandlers = LeanEngineSystemHandlers.FromConfiguration(Composer.Instance);
EngineContext.SystemHandlers.Initialize();
EngineContext.AlgorithmHandlers = LeanEngineAlgorithmHandlers.FromConfiguration(Composer.Instance);
_resultsHandler = (OptimizerResultHandler)EngineContext.AlgorithmHandlers.Results;
EngineContext.Engine = new Engine(EngineContext.SystemHandlers, EngineContext.AlgorithmHandlers, false);
}
catch (CompositionException compositionException)
{
LogProvider.ErrorLogger.Error("Engine.Main(): Failed to load library", compositionException);
throw;
}
string algorithmPath;
AlgorithmNodePacket job = EngineContext.SystemHandlers.JobQueue.NextJob(out algorithmPath);
EngineContext.Engine.Run(job, new AlgorithmManager(false), algorithmPath, WorkerThread.Instance);
}
/// <summary>
/// <see cref = "QuantBook" /> constructor.
/// Provides access to data for quantitative analysis
/// </summary>
public QuantBook()
{
try
{
using (Py.GIL())
{
_pandas = Py.Import("pandas");
}
var composer = new Composer();
var algorithmHandlers = LeanEngineAlgorithmHandlers.FromConfiguration(composer);
_dataCacheProvider = new SingleEntryDataCacheProvider(algorithmHandlers.DataProvider);
var mapFileProvider = algorithmHandlers.MapFileProvider;
_historyProvider = composer.GetExportedValueByTypeName <IHistoryProvider>(Config.Get("history-provider", "SubscriptionDataReaderHistoryProvider"));
_historyProvider.Initialize(null, algorithmHandlers.DataProvider, _dataCacheProvider, mapFileProvider, algorithmHandlers.FactorFileProvider, null);
}
catch (Exception exception)
{
throw new Exception("QuantBook.Main(): " + exception);
}
}
/// <summary>
/// Method performs necessary initialization and starts and algorithm inside Lean Engine.
/// </summary>
public static Dictionary <string, decimal> Run(Dictionary <string, string> inputs)
{
// Set the algorithm input variables
foreach (var pair in inputs)
{
Config.Set(pair.Key, pair.Value);
}
// Common settings:
Config.Set("environment", "backtesting");
Config.Set("algorithm-language", "CSharp");
Config.Set("result-handler", nameof(OptimizerResultHandler)); //override default result handler
// Log file location
var logFileName = Config.Get("log-file");
Log.LogHandler = new FileLogHandler(logFileName);
// LeanEngineSystemHandlers
LeanEngineSystemHandlers leanEngineSystemHandlers;
try
{
leanEngineSystemHandlers = LeanEngineSystemHandlers.FromConfiguration(Composer.Instance);
}
catch (CompositionException compositionException)
{
Log.Error("Engine.Main(): Failed to load library: " + compositionException);
throw;
}
// Setup packeting, queue and controls system: These don't do much locally.
leanEngineSystemHandlers.Initialize();
// Pull job from QuantConnect job queue, or, pull local build:
var job = leanEngineSystemHandlers.JobQueue.NextJob(out var assemblyPath);
if (job == null)
{
throw new Exception("Engine.Main(): Job was null.");
}
// LeanEngineSystemHandlers
LeanEngineAlgorithmHandlers leanEngineAlgorithmHandlers;
try
{
leanEngineAlgorithmHandlers = LeanEngineAlgorithmHandlers.FromConfiguration(Composer.Instance);
}
catch (CompositionException compositionException)
{
Log.Error("Engine.Main(): Failed to load library: " + compositionException);
throw;
}
// Engine
try
{
var algorithmManager = new AlgorithmManager(false, job);
leanEngineSystemHandlers.LeanManager.Initialize(leanEngineSystemHandlers, leanEngineAlgorithmHandlers, job, algorithmManager);
var engine = new Engine(leanEngineSystemHandlers, leanEngineAlgorithmHandlers, false);
engine.Run(job, algorithmManager, assemblyPath, WorkerThread.Instance);
}
finally
{
// do not Acknowledge Job, clean up resources
Log.Trace("Engine.Main(): Packet removed from queue: " + job.AlgorithmId);
leanEngineSystemHandlers.Dispose();
leanEngineAlgorithmHandlers.Dispose();
Log.LogHandler.Dispose();
}
// Results
ResultHandler = (OptimizerResultHandler)leanEngineAlgorithmHandlers.Results;
return(ResultHandler.FullResults);
}
public static void RunLocalBacktest(string algorithm, Dictionary <string, string> expectedStatistics, AlphaRuntimeStatistics expectedAlphaStatistics, Language language)
{
var statistics = new Dictionary <string, string>();
var alphaStatistics = new AlphaRuntimeStatistics();
Composer.Instance.Reset();
var ordersLogFile = string.Empty;
var logFile = $"./regression/{algorithm}.{language.ToLower()}.log";
Directory.CreateDirectory(Path.GetDirectoryName(logFile));
File.Delete(logFile);
try
{
// set the configuration up
Config.Set("algorithm-type-name", algorithm);
Config.Set("live-mode", "false");
Config.Set("environment", "");
Config.Set("messaging-handler", "QuantConnect.Messaging.Messaging");
Config.Set("job-queue-handler", "QuantConnect.Queues.JobQueue");
Config.Set("setup-handler", "RegressionSetupHandlerWrapper");
Config.Set("history-provider", "RegressionHistoryProviderWrapper");
Config.Set("api-handler", "QuantConnect.Api.Api");
Config.Set("result-handler", "QuantConnect.Lean.Engine.Results.RegressionResultHandler");
Config.Set("algorithm-language", language.ToString());
Config.Set("algorithm-location",
language == Language.Python
? "../../../Algorithm.Python/" + algorithm + ".py"
: "QuantConnect.Algorithm." + language + ".dll");
var debugEnabled = Log.DebuggingEnabled;
var logHandlers = new ILogHandler[] { new ConsoleLogHandler(), new FileLogHandler(logFile, false) };
using (Log.LogHandler = new CompositeLogHandler(logHandlers))
using (var algorithmHandlers = LeanEngineAlgorithmHandlers.FromConfiguration(Composer.Instance))
using (var systemHandlers = LeanEngineSystemHandlers.FromConfiguration(Composer.Instance))
{
Log.DebuggingEnabled = true;
Log.LogHandler.Trace("");
Log.LogHandler.Trace("{0}: Running " + algorithm + "...", DateTime.UtcNow);
Log.LogHandler.Trace("");
// run the algorithm in its own thread
var engine = new Lean.Engine.Engine(systemHandlers, algorithmHandlers, false);
Task.Factory.StartNew(() =>
{
string algorithmPath;
var job = systemHandlers.JobQueue.NextJob(out algorithmPath);
var algorithmManager = new AlgorithmManager(false);
engine.Run(job, algorithmManager, algorithmPath);
ordersLogFile = ((RegressionResultHandler)algorithmHandlers.Results).OrdersLogFilePath;
}).Wait();
var backtestingResultHandler = (BacktestingResultHandler)algorithmHandlers.Results;
statistics = backtestingResultHandler.FinalStatistics;
var defaultAlphaHandler = (DefaultAlphaHandler)algorithmHandlers.Alphas;
alphaStatistics = defaultAlphaHandler.RuntimeStatistics;
Log.DebuggingEnabled = debugEnabled;
}
}
catch (Exception ex)
{
Log.LogHandler.Error("{0} {1}", ex.Message, ex.StackTrace);
}
foreach (var stat in expectedStatistics)
{
Assert.AreEqual(true, statistics.ContainsKey(stat.Key), "Missing key: " + stat.Key);
Assert.AreEqual(stat.Value, statistics[stat.Key], "Failed on " + stat.Key);
}
if (expectedAlphaStatistics != null)
{
AssertAlphaStatistics(expectedAlphaStatistics, alphaStatistics, s => s.MeanPopulationScore.Direction);
AssertAlphaStatistics(expectedAlphaStatistics, alphaStatistics, s => s.MeanPopulationScore.Magnitude);
AssertAlphaStatistics(expectedAlphaStatistics, alphaStatistics, s => s.RollingAveragedPopulationScore.Direction);
AssertAlphaStatistics(expectedAlphaStatistics, alphaStatistics, s => s.RollingAveragedPopulationScore.Magnitude);
AssertAlphaStatistics(expectedAlphaStatistics, alphaStatistics, s => s.LongShortRatio);
AssertAlphaStatistics(expectedAlphaStatistics, alphaStatistics, s => s.TotalInsightsClosed);
AssertAlphaStatistics(expectedAlphaStatistics, alphaStatistics, s => s.TotalInsightsGenerated);
AssertAlphaStatistics(expectedAlphaStatistics, alphaStatistics, s => s.TotalAccumulatedEstimatedAlphaValue);
AssertAlphaStatistics(expectedAlphaStatistics, alphaStatistics, s => s.TotalInsightsAnalysisCompleted);
}
// we successfully passed the regression test, copy the log file so we don't have to continually
// re-run master in order to compare against a passing run
var passedFile = logFile.Replace("./regression/", "./passed/");
Directory.CreateDirectory(Path.GetDirectoryName(passedFile));
File.Delete(passedFile);
File.Copy(logFile, passedFile);
var passedOrderLogFile = ordersLogFile.Replace("./regression/", "./passed/");
Directory.CreateDirectory(Path.GetDirectoryName(passedFile));
File.Delete(passedOrderLogFile);
if (File.Exists(ordersLogFile))
{
File.Copy(ordersLogFile, passedOrderLogFile);
}
}
/// <summary>
/// <see cref = "QuantBook" /> constructor.
/// Provides access to data for quantitative analysis
/// </summary>
public QuantBook() : base()
{
try
{
using (Py.GIL())
{
_pandas = Py.Import("pandas");
}
// Issue #4892 : Set start time relative to NY time
// when the data is available from the previous day
var newYorkTime = DateTime.UtcNow.ConvertFromUtc(TimeZones.NewYork);
var hourThreshold = Config.GetInt("qb-data-hour", 9);
// If it is after our hour threshold; then we can use today
if (newYorkTime.Hour >= hourThreshold)
{
SetStartDate(newYorkTime);
}
else
{
SetStartDate(newYorkTime - TimeSpan.FromDays(1));
}
// Sets PandasConverter
SetPandasConverter();
// Reset our composer; needed for re-creation of QuantBook
Composer.Instance.Reset();
var composer = Composer.Instance;
// Create our handlers with our composer instance
var algorithmHandlers = LeanEngineAlgorithmHandlers.FromConfiguration(composer);
var systemHandlers = LeanEngineSystemHandlers.FromConfiguration(composer);
// init the API
systemHandlers.Initialize();
systemHandlers.LeanManager.Initialize(systemHandlers,
algorithmHandlers,
new BacktestNodePacket(),
new AlgorithmManager(false));
systemHandlers.LeanManager.SetAlgorithm(this);
algorithmHandlers.ObjectStore.Initialize("QuantBook",
Config.GetInt("job-user-id"),
Config.GetInt("project-id"),
Config.Get("api-access-token"),
new Controls
{
// if <= 0 we disable periodic persistence and make it synchronous
PersistenceIntervalSeconds = -1,
StorageLimitMB = Config.GetInt("storage-limit-mb", 5),
StorageFileCount = Config.GetInt("storage-file-count", 100),
StoragePermissions = (FileAccess)Config.GetInt("storage-permissions", (int)FileAccess.ReadWrite)
});
SetObjectStore(algorithmHandlers.ObjectStore);
_dataCacheProvider = new ZipDataCacheProvider(algorithmHandlers.DataProvider);
_dataProvider = algorithmHandlers.DataProvider;
var symbolPropertiesDataBase = SymbolPropertiesDatabase.FromDataFolder();
var registeredTypes = new RegisteredSecurityDataTypesProvider();
var securityService = new SecurityService(Portfolio.CashBook,
MarketHoursDatabase,
symbolPropertiesDataBase,
this,
registeredTypes,
new SecurityCacheProvider(Portfolio));
Securities.SetSecurityService(securityService);
SubscriptionManager.SetDataManager(
new DataManager(new NullDataFeed(),
new UniverseSelection(this, securityService, algorithmHandlers.DataPermissionsManager, algorithmHandlers.DataProvider),
this,
TimeKeeper,
MarketHoursDatabase,
false,
registeredTypes,
algorithmHandlers.DataPermissionsManager));
var mapFileProvider = algorithmHandlers.MapFileProvider;
HistoryProvider = composer.GetExportedValueByTypeName <IHistoryProvider>(Config.Get("history-provider", "SubscriptionDataReaderHistoryProvider"));
HistoryProvider.Initialize(
new HistoryProviderInitializeParameters(
null,
null,
algorithmHandlers.DataProvider,
_dataCacheProvider,
mapFileProvider,
algorithmHandlers.FactorFileProvider,
null,
true,
algorithmHandlers.DataPermissionsManager
)
);
SetOptionChainProvider(new CachingOptionChainProvider(new BacktestingOptionChainProvider(_dataProvider)));
SetFutureChainProvider(new CachingFutureChainProvider(new BacktestingFutureChainProvider(_dataProvider)));
}
catch (Exception exception)
{
throw new Exception("QuantBook.Main(): " + exception);
}
}
public static AlgorithmRunnerResults RunLocalBacktest(
string algorithm,
Dictionary <string, string> expectedStatistics,
AlphaRuntimeStatistics expectedAlphaStatistics,
Language language,
AlgorithmStatus expectedFinalStatus,
DateTime?startDate = null,
DateTime?endDate = null,
string setupHandler = "RegressionSetupHandlerWrapper",
decimal?initialCash = null)
{
AlgorithmManager algorithmManager = null;
var statistics = new Dictionary <string, string>();
var alphaStatistics = new AlphaRuntimeStatistics(new TestAccountCurrencyProvider());
BacktestingResultHandler results = null;
Composer.Instance.Reset();
SymbolCache.Clear();
var ordersLogFile = string.Empty;
var logFile = $"./regression/{algorithm}.{language.ToLower()}.log";
Directory.CreateDirectory(Path.GetDirectoryName(logFile));
File.Delete(logFile);
try
{
// set the configuration up
Config.Set("algorithm-type-name", algorithm);
Config.Set("live-mode", "false");
Config.Set("environment", "");
Config.Set("messaging-handler", "QuantConnect.Messaging.Messaging");
Config.Set("job-queue-handler", "QuantConnect.Queues.JobQueue");
Config.Set("setup-handler", setupHandler);
Config.Set("history-provider", "RegressionHistoryProviderWrapper");
Config.Set("api-handler", "QuantConnect.Api.Api");
Config.Set("result-handler", "QuantConnect.Lean.Engine.Results.RegressionResultHandler");
Config.Set("algorithm-language", language.ToString());
Config.Set("algorithm-location",
language == Language.Python
? "../../../Algorithm.Python/" + algorithm + ".py"
: "QuantConnect.Algorithm." + language + ".dll");
// Store initial log variables
var initialLogHandler = Log.LogHandler;
var initialDebugEnabled = Log.DebuggingEnabled;
// Log handlers specific to this test function
var newLogHandlers = new ILogHandler[] { new ConsoleErrorLogHandler(), new FileLogHandler(logFile, false) };
using (Log.LogHandler = new CompositeLogHandler(newLogHandlers))
using (var algorithmHandlers = LeanEngineAlgorithmHandlers.FromConfiguration(Composer.Instance))
using (var systemHandlers = LeanEngineSystemHandlers.FromConfiguration(Composer.Instance))
using (var workerThread = new TestWorkerThread())
{
Log.DebuggingEnabled = true;
Log.Trace("");
Log.Trace("{0}: Running " + algorithm + "...", DateTime.UtcNow);
Log.Trace("");
// run the algorithm in its own thread
var engine = new Lean.Engine.Engine(systemHandlers, algorithmHandlers, false);
Task.Factory.StartNew(() =>
{
try
{
string algorithmPath;
var job = (BacktestNodePacket)systemHandlers.JobQueue.NextJob(out algorithmPath);
job.BacktestId = algorithm;
job.PeriodStart = startDate;
job.PeriodFinish = endDate;
if (initialCash.HasValue)
{
job.CashAmount = new CashAmount(initialCash.Value, Currencies.USD);
}
algorithmManager = new AlgorithmManager(false, job);
systemHandlers.LeanManager.Initialize(systemHandlers, algorithmHandlers, job, algorithmManager);
engine.Run(job, algorithmManager, algorithmPath, workerThread);
ordersLogFile = ((RegressionResultHandler)algorithmHandlers.Results).LogFilePath;
}
catch (Exception e)
{
Log.Trace($"Error in AlgorithmRunner task: {e}");
}
}).Wait();
var backtestingResultHandler = (BacktestingResultHandler)algorithmHandlers.Results;
results = backtestingResultHandler;
statistics = backtestingResultHandler.FinalStatistics;
var defaultAlphaHandler = (DefaultAlphaHandler)algorithmHandlers.Alphas;
alphaStatistics = defaultAlphaHandler.RuntimeStatistics;
}
// Reset settings to initial values
Log.LogHandler = initialLogHandler;
Log.DebuggingEnabled = initialDebugEnabled;
}
catch (Exception ex)
{
if (expectedFinalStatus != AlgorithmStatus.RuntimeError)
{
Log.Error("{0} {1}", ex.Message, ex.StackTrace);
}
}
if (algorithmManager?.State != expectedFinalStatus)
{
Assert.Fail($"Algorithm state should be {expectedFinalStatus} and is: {algorithmManager?.State}");
}
foreach (var stat in expectedStatistics)
{
Assert.AreEqual(true, statistics.ContainsKey(stat.Key), "Missing key: " + stat.Key);
Assert.AreEqual(stat.Value, statistics[stat.Key], "Failed on " + stat.Key);
}
if (expectedAlphaStatistics != null)
{
AssertAlphaStatistics(expectedAlphaStatistics, alphaStatistics, s => s.MeanPopulationScore.Direction);
AssertAlphaStatistics(expectedAlphaStatistics, alphaStatistics, s => s.MeanPopulationScore.Magnitude);
AssertAlphaStatistics(expectedAlphaStatistics, alphaStatistics, s => s.RollingAveragedPopulationScore.Direction);
AssertAlphaStatistics(expectedAlphaStatistics, alphaStatistics, s => s.RollingAveragedPopulationScore.Magnitude);
AssertAlphaStatistics(expectedAlphaStatistics, alphaStatistics, s => s.LongShortRatio);
AssertAlphaStatistics(expectedAlphaStatistics, alphaStatistics, s => s.TotalInsightsClosed);
AssertAlphaStatistics(expectedAlphaStatistics, alphaStatistics, s => s.TotalInsightsGenerated);
AssertAlphaStatistics(expectedAlphaStatistics, alphaStatistics, s => s.TotalAccumulatedEstimatedAlphaValue);
AssertAlphaStatistics(expectedAlphaStatistics, alphaStatistics, s => s.TotalInsightsAnalysisCompleted);
}
// we successfully passed the regression test, copy the log file so we don't have to continually
// re-run master in order to compare against a passing run
var passedFile = logFile.Replace("./regression/", "./passed/");
Directory.CreateDirectory(Path.GetDirectoryName(passedFile));
File.Delete(passedFile);
File.Copy(logFile, passedFile);
var passedOrderLogFile = ordersLogFile.Replace("./regression/", "./passed/");
Directory.CreateDirectory(Path.GetDirectoryName(passedFile));
File.Delete(passedOrderLogFile);
if (File.Exists(ordersLogFile))
{
File.Copy(ordersLogFile, passedOrderLogFile);
}
return(new AlgorithmRunnerResults(algorithm, language, algorithmManager, results));
}
public decimal Run(ConfigVars vars)
{
foreach (var kvp in vars.Vars)
{
Config.Set(kvp.Key, kvp.Value.ToString());
}
// settings
Config.Set("environment", "backtesting");
Config.Set("algorithm-type-name", "BitfinexSuperTrend");
Config.Set("algorithm-language", "CSharp");
Config.Set("algorithm-location", "Optimization.exe");
Config.Set("data-folder", "C:/Users/stranger/Google Drive/Data/");
// default value set in QuantConnect.Configuration.Config is invalid. specify explicitely.
Config.Set("job-queue-handler", "QuantConnect.Queues.JobQueue");
// log handler
Log.LogHandler = Composer.Instance.GetExportedValueByTypeName <ILogHandler>(Config.Get("log-handler", "CompositeLogHandler"));
// == LeanEngineSystemHandlers ==
LeanEngineSystemHandlers leanEngineSystemHandlers;
try
{
leanEngineSystemHandlers = LeanEngineSystemHandlers.FromConfiguration(Composer.Instance);
}
catch (CompositionException compositionException)
{
Log.Error("Engine.Main(): Failed to load library: " + compositionException);
throw;
}
// can this be omitted?
leanEngineSystemHandlers.Initialize();
string assemblyPath;
var job = leanEngineSystemHandlers.JobQueue.NextJob(out assemblyPath);
if (job == null)
{
throw new Exception("Engine.Main(): Job was null.");
}
// == LeanEngineAlgorithmHandlers ==
LeanEngineAlgorithmHandlers leanEngineAlgorithmHandlers;
try
{
leanEngineAlgorithmHandlers = LeanEngineAlgorithmHandlers.FromConfiguration(Composer.Instance);
}
catch (CompositionException compositionException)
{
Log.Error("Engine.Main(): Failed to load library: " + compositionException);
throw;
}
// == Engine ==
try
{
var liveMode = Config.GetBool("live-mode");
var algorithmManager = new AlgorithmManager(liveMode);
// can this be omitted?
leanEngineSystemHandlers.LeanManager.Initialize(leanEngineSystemHandlers, leanEngineAlgorithmHandlers, job, algorithmManager);
var engine = new Engine(leanEngineSystemHandlers, leanEngineAlgorithmHandlers, liveMode);
engine.Run(job, algorithmManager, assemblyPath);
}
finally
{
// no Acknowledge Job, clean up resources
Log.Trace("Engine.Main(): Packet removed from queue: " + job.AlgorithmId);
leanEngineSystemHandlers.Dispose();
leanEngineAlgorithmHandlers.Dispose();
Log.LogHandler.Dispose();
}
// obtain results
var sharpeRatio = 0.0m;
var resultshandler = leanEngineAlgorithmHandlers.Results as BacktestingResultHandler;
if (resultshandler != null)
{
var ratio = resultshandler.FinalStatistics["Sharpe Ratio"];
Decimal.TryParse(ratio, out sharpeRatio);
}
else
{
Log.Error("Unable to cast: BacktestingResultHandler");
}
return(sharpeRatio);
}
static void Main(string[] args)
{
Log.LogHandler = Composer.Instance.GetExportedValueByTypeName <ILogHandler>(Config.Get("log-handler", "CompositeLogHandler"));
//Initialize:
string mode = "RELEASE";
var liveMode = Config.GetBool("live-mode");
Log.DebuggingEnabled = Config.GetBool("debug-mode");
#if DEBUG
mode = "DEBUG";
#endif
//Name thread for the profiler:
Thread.CurrentThread.Name = "Algorithm Analysis Thread";
Log.Trace("Engine.Main(): LEAN ALGORITHMIC TRADING ENGINE v" + Constants.Version + " Mode: " + mode);
Log.Trace("Engine.Main(): Started " + DateTime.Now.ToShortTimeString());
Log.Trace("Engine.Main(): Memory " + OS.ApplicationMemoryUsed + "Mb-App " + +OS.TotalPhysicalMemoryUsed + "Mb-Used " + OS.TotalPhysicalMemory + "Mb-Total");
//Import external libraries specific to physical server location (cloud/local)
LeanEngineSystemHandlers leanEngineSystemHandlers;
try
{
leanEngineSystemHandlers = LeanEngineSystemHandlers.FromConfiguration(Composer.Instance);
}
catch (CompositionException compositionException)
{
Log.Error("Engine.Main(): Failed to load library: " + compositionException);
throw;
}
//Setup packeting, queue and controls system: These don't do much locally.
leanEngineSystemHandlers.Initialize();
//-> Pull job from QuantConnect job queue, or, pull local build:
string assemblyPath;
var job = leanEngineSystemHandlers.JobQueue.NextJob(out assemblyPath);
if (job == null)
{
throw new Exception("Engine.Main(): Job was null.");
}
LeanEngineAlgorithmHandlers leanEngineAlgorithmHandlers;
try
{
leanEngineAlgorithmHandlers = LeanEngineAlgorithmHandlers.FromConfiguration(Composer.Instance);
}
catch (CompositionException compositionException)
{
Log.Error("Engine.Main(): Failed to load library: " + compositionException);
throw;
}
// log the job endpoints
Log.Trace("JOB HANDLERS: ");
Log.Trace(" DataFeed: " + leanEngineAlgorithmHandlers.DataFeed.GetType().FullName);
Log.Trace(" Setup: " + leanEngineAlgorithmHandlers.Setup.GetType().FullName);
Log.Trace(" RealTime: " + leanEngineAlgorithmHandlers.RealTime.GetType().FullName);
Log.Trace(" Results: " + leanEngineAlgorithmHandlers.Results.GetType().FullName);
Log.Trace(" Transactions: " + leanEngineAlgorithmHandlers.Transactions.GetType().FullName);
Log.Trace(" History: " + leanEngineAlgorithmHandlers.HistoryProvider.GetType().FullName);
Log.Trace(" Commands: " + leanEngineAlgorithmHandlers.CommandQueue.GetType().FullName);
if (job is LiveNodePacket)
{
Log.Trace(" Brokerage: " + ((LiveNodePacket)job).Brokerage);
}
// if the job version doesn't match this instance version then we can't process it
// we also don't want to reprocess redelivered jobs
if (job.Version != Constants.Version || job.Redelivered)
{
Log.Error("Engine.Run(): Job Version: " + job.Version + " Deployed Version: " + Constants.Version + " Redelivered: " + job.Redelivered);
//Tiny chance there was an uncontrolled collapse of a server, resulting in an old user task circulating.
//In this event kill the old algorithm and leave a message so the user can later review.
leanEngineSystemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, _collapseMessage);
leanEngineSystemHandlers.Notify.SetChannel(job.Channel);
leanEngineSystemHandlers.Notify.RuntimeError(job.AlgorithmId, _collapseMessage);
leanEngineSystemHandlers.JobQueue.AcknowledgeJob(job);
return;
}
try
{
var engine = new Engine.Engine(leanEngineSystemHandlers, leanEngineAlgorithmHandlers, liveMode);
engine.Run(job, assemblyPath);
}
finally
{
//Delete the message from the job queue:
leanEngineSystemHandlers.JobQueue.AcknowledgeJob(job);
Log.Trace("Engine.Main(): Packet removed from queue: " + job.AlgorithmId);
// clean up resources
leanEngineSystemHandlers.Dispose();
leanEngineAlgorithmHandlers.Dispose();
Log.LogHandler.Dispose();
}
}
static void Main(string[] args)
{
var mode = "RELEASE";
#if DEBUG
mode = "DEBUG";
#endif
var environment = Config.Get("environment");
var liveMode = Config.GetBool("live-mode");
Log.DebuggingEnabled = Config.GetBool("debug-mode");
Log.LogHandler = Composer.Instance.GetExportedValueByTypeName <ILogHandler>(Config.Get("log-handler", "CompositeLogHandler"));
Thread.CurrentThread.Name = "Algorithm Analysis Thread";
Log.Trace("Engine.Main(): LEAN ALGORITHMIC TRADING ENGINE v" + Globals.Version + " Mode: " + mode);
Log.Trace("Engine.Main(): Started " + DateTime.Now.ToShortTimeString());
Log.Trace("Engine.Main(): Memory " + OS.ApplicationMemoryUsed + "Mb-App " + +OS.TotalPhysicalMemoryUsed + "Mb-Used " + OS.TotalPhysicalMemory + "Mb-Total");
LeanEngineSystemHandlers leanEngineSystemHandlers;
try
{
leanEngineSystemHandlers = LeanEngineSystemHandlers.FromConfiguration(Composer.Instance);
}
catch (CompositionException compositionException)
{
Log.Error("Engine.Main(): Failed to load library: " + compositionException);
throw;
}
leanEngineSystemHandlers.Initialize();
string assemblyPath;
var job = leanEngineSystemHandlers.JobQueue.NextJob(out assemblyPath);
if (job == null)
{
throw new Exception("Engine.Main(): Job was null.");
}
LeanEngineAlgorithmHandlers leanEngineAlgorithmHandlers;
try
{
leanEngineAlgorithmHandlers = LeanEngineAlgorithmHandlers.FromConfiguration(Composer.Instance);
}
catch (CompositionException compositionException)
{
Log.Error("Engine.Main(): Failed to load library: " + compositionException);
throw;
}
if (environment == "backtesting-desktop")
{
Application.EnableVisualStyles();
var messagingHandler = leanEngineSystemHandlers.Notify;
var thread = new Thread(() => LaunchUX(messagingHandler, job));
thread.SetApartmentState(ApartmentState.STA);
thread.Start();
}
Log.Trace("JOB HANDLERS: ");
Log.Trace(" DataFeed: " + leanEngineAlgorithmHandlers.DataFeed.GetType().FullName);
Log.Trace(" Setup: " + leanEngineAlgorithmHandlers.Setup.GetType().FullName);
Log.Trace(" RealTime: " + leanEngineAlgorithmHandlers.RealTime.GetType().FullName);
Log.Trace(" Results: " + leanEngineAlgorithmHandlers.Results.GetType().FullName);
Log.Trace(" Transactions: " + leanEngineAlgorithmHandlers.Transactions.GetType().FullName);
Log.Trace(" History: " + leanEngineAlgorithmHandlers.HistoryProvider.GetType().FullName);
Log.Trace(" Commands: " + leanEngineAlgorithmHandlers.CommandQueue.GetType().FullName);
if (job is LiveNodePacket)
{
Log.Trace(" Brokerage: " + ((LiveNodePacket)job).Brokerage);
}
if (VersionHelper.IsNotEqualVersion(job.Version) || job.Redelivered)
{
Log.Error("Engine.Run(): Job Version: " + job.Version + " Deployed Version: " + Globals.Version + " Redelivered: " + job.Redelivered);
leanEngineSystemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, _collapseMessage);
leanEngineSystemHandlers.Notify.SetAuthentication(job);
leanEngineSystemHandlers.Notify.Send(new RuntimeErrorPacket(job.AlgorithmId, _collapseMessage));
leanEngineSystemHandlers.JobQueue.AcknowledgeJob(job);
return;
}
try
{
var engine = new Engine.Engine(leanEngineSystemHandlers, leanEngineAlgorithmHandlers, liveMode);
engine.Run(job, assemblyPath);
}
finally
{
leanEngineSystemHandlers.JobQueue.AcknowledgeJob(job);
Log.Trace("Engine.Main(): Packet removed from queue: " + job.AlgorithmId);
leanEngineSystemHandlers.Dispose();
leanEngineAlgorithmHandlers.Dispose();
Log.LogHandler.Dispose();
}
}
private void LaunchLean()
{
Config.Set("environment", "backtesting");
if (!string.IsNullOrEmpty(_config.AlgorithmTypeName))
{
Config.Set("algorithm-type-name", _config.AlgorithmTypeName);
}
if (!string.IsNullOrEmpty(_config.AlgorithmLocation))
{
Config.Set("algorithm-location", Path.GetFileName(_config.AlgorithmLocation));
}
if (!string.IsNullOrEmpty(_config.DataFolder))
{
Config.Set("data-folder", _config.DataFolder);
}
if (!string.IsNullOrEmpty(_config.TransactionLog))
{
var filename = _config.TransactionLog;
filename = Path.Combine(AppDomain.CurrentDomain.BaseDirectory,
Path.GetFileNameWithoutExtension(filename) + _id + Path.GetExtension(filename));
Config.Set("transaction-log", filename);
}
//transaction-log
Config.Set("api-handler", nameof(EmptyApiHandler));
var systemHandlers = LeanEngineSystemHandlers.FromConfiguration(Composer.Instance);
systemHandlers.Initialize();
//separate log uniquely named
var logFileName = "log" + DateTime.Now.ToString("yyyyMMddssfffffff") + "_" + _id + ".txt";
using (Log.LogHandler = new FileLogHandler(logFileName, true))
{
LeanEngineAlgorithmHandlers leanEngineAlgorithmHandlers;
try
{
//override config to use custom result handler
Config.Set("backtesting.result-handler", nameof(OptimizerResultHandler));
leanEngineAlgorithmHandlers = LeanEngineAlgorithmHandlers.FromConfiguration(Composer.Instance);
_resultsHandler = (OptimizerResultHandler)leanEngineAlgorithmHandlers.Results;
}
catch (CompositionException compositionException)
{
Log.Error("Engine.Main(): Failed to load library: " + compositionException);
throw;
}
string algorithmPath;
AlgorithmNodePacket job = systemHandlers.JobQueue.NextJob(out algorithmPath);
try
{
var algorithmManager = new AlgorithmManager(false);
systemHandlers.LeanManager.Initialize(systemHandlers, leanEngineAlgorithmHandlers, job, algorithmManager);
var engine = new Engine(systemHandlers, leanEngineAlgorithmHandlers, false);
engine.Run(job, algorithmManager, algorithmPath);
}
finally
{
Log.Trace("Engine.Main(): Packet removed from queue: " + job.AlgorithmId);
// clean up resources
systemHandlers.Dispose();
leanEngineAlgorithmHandlers.Dispose();
}
}
}
static void Main(string[] args)
{
//Initialize:
var mode = "RELEASE";
#if DEBUG
mode = "DEBUG";
#endif
if (OS.IsWindows)
{
Console.OutputEncoding = System.Text.Encoding.Unicode;
}
var environment = Config.Get("environment");
var liveMode = Config.GetBool("live-mode");
Log.DebuggingEnabled = Config.GetBool("debug-mode");
Log.LogHandler = Composer.Instance.GetExportedValueByTypeName <ILogHandler>(Config.Get("log-handler", "CompositeLogHandler"));
//Name thread for the profiler:
Thread.CurrentThread.Name = "Algorithm Analysis Thread";
Log.Trace("Engine.Main(): LEAN ALGORITHMIC TRADING ENGINE v" + Globals.Version + " Mode: " + mode + " (" + (Environment.Is64BitProcess ? "64" : "32") + "bit)");
Log.Trace("Engine.Main(): Started " + DateTime.Now.ToShortTimeString());
Log.Trace("Engine.Main(): Memory " + OS.ApplicationMemoryUsed + "Mb-App " + +OS.TotalPhysicalMemoryUsed + "Mb-Used " + OS.TotalPhysicalMemory + "Mb-Total");
//Import external libraries specific to physical server location (cloud/local)
LeanEngineSystemHandlers leanEngineSystemHandlers;
try
{
leanEngineSystemHandlers = LeanEngineSystemHandlers.FromConfiguration(Composer.Instance);
}
catch (CompositionException compositionException)
{
Log.Error("Engine.Main(): Failed to load library: " + compositionException);
throw;
}
//Setup packeting, queue and controls system: These don't do much locally.
leanEngineSystemHandlers.Initialize();
//-> Pull job from QuantConnect job queue, or, pull local build:
string assemblyPath;
var job = leanEngineSystemHandlers.JobQueue.NextJob(out assemblyPath);
if (job == null)
{
throw new Exception("Engine.Main(): Job was null.");
}
LeanEngineAlgorithmHandlers leanEngineAlgorithmHandlers;
try
{
leanEngineAlgorithmHandlers = LeanEngineAlgorithmHandlers.FromConfiguration(Composer.Instance);
}
catch (CompositionException compositionException)
{
Log.Error("Engine.Main(): Failed to load library: " + compositionException);
throw;
}
if (environment.EndsWith("-desktop"))
{
var info = new ProcessStartInfo
{
UseShellExecute = false,
FileName = Config.Get("desktop-exe"),
Arguments = Config.Get("desktop-http-port")
};
Process.Start(info);
}
// if the job version doesn't match this instance version then we can't process it
// we also don't want to reprocess redelivered jobs
if (VersionHelper.IsNotEqualVersion(job.Version) || job.Redelivered)
{
Log.Error("Engine.Run(): Job Version: " + job.Version + " Deployed Version: " + Globals.Version + " Redelivered: " + job.Redelivered);
//Tiny chance there was an uncontrolled collapse of a server, resulting in an old user task circulating.
//In this event kill the old algorithm and leave a message so the user can later review.
leanEngineSystemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, _collapseMessage);
leanEngineSystemHandlers.Notify.SetAuthentication(job);
leanEngineSystemHandlers.Notify.Send(new RuntimeErrorPacket(job.UserId, job.AlgorithmId, _collapseMessage));
leanEngineSystemHandlers.JobQueue.AcknowledgeJob(job);
return;
}
try
{
var engine = new Engine.Engine(leanEngineSystemHandlers, leanEngineAlgorithmHandlers, liveMode);
engine.Run(job, assemblyPath);
}
finally
{
//Delete the message from the job queue:
leanEngineSystemHandlers.JobQueue.AcknowledgeJob(job);
Log.Trace("Engine.Main(): Packet removed from queue: " + job.AlgorithmId);
// clean up resources
leanEngineSystemHandlers.Dispose();
leanEngineAlgorithmHandlers.Dispose();
Log.LogHandler.Dispose();
}
}
static void Main(string[] args)
{
Console.WriteLine("THIS WORKS");
// Create new connection factory
var factory = new ConnectionFactory()
{
HostName = "localhost"
};
var connection = factory.CreateConnection();
var channel = connection.CreateModel();
//Initialize:
var mode = "RELEASE";
#if DEBUG
mode = "DEBUG";
#endif
if (OS.IsWindows)
{
Console.OutputEncoding = System.Text.Encoding.UTF8;
}
// expect first argument to be config file name
if (args.Length > 0)
{
Config.MergeCommandLineArgumentsWithConfiguration(LeanArgumentParser.ParseArguments(args));
}
var environment = Config.Get("environment");
var liveMode = Config.GetBool("live-mode");
Log.DebuggingEnabled = Config.GetBool("debug-mode");
Log.LogHandler = Composer.Instance.GetExportedValueByTypeName <ILogHandler>(Config.Get("log-handler", "CompositeLogHandler"));
//Name thread for the profiler:
Thread.CurrentThread.Name = "Algorithm Analysis Thread";
Log.Trace("Engine.Main(): LEAN ALGORITHMIC TRADING ENGINE v" + Globals.Version + " Mode: " + mode + " (" + (Environment.Is64BitProcess ? "64" : "32") + "bit)");
Log.Trace("Engine.Main(): Started " + DateTime.Now.ToShortTimeString());
//Import external libraries specific to physical server location (cloud/local)
LeanEngineSystemHandlers leanEngineSystemHandlers;
try
{
leanEngineSystemHandlers = LeanEngineSystemHandlers.FromConfiguration(Composer.Instance);
}
catch (CompositionException compositionException)
{
Log.Error("Engine.Main(): Failed to load library: " + compositionException);
throw;
}
//Setup packeting, queue and controls system: These don't do much locally.
leanEngineSystemHandlers.Initialize();
LeanEngineAlgorithmHandlers leanEngineAlgorithmHandlers;
try
{
leanEngineAlgorithmHandlers = LeanEngineAlgorithmHandlers.FromConfiguration(Composer.Instance);
}
catch (CompositionException compositionException)
{
Log.Error("Engine.Main(): Failed to load library: " + compositionException);
throw;
}
//-> Pull job from QuantConnect job queue, or, pull local build:
string assemblyPath;
var job = leanEngineSystemHandlers.JobQueue.NextJob(out assemblyPath);
if (job == null)
{
const string jobNullMessage = "Engine.Main(): Sorry we could not process this algorithm request.";
Log.Error(jobNullMessage);
throw new ArgumentException(jobNullMessage);
}
// if the job version doesn't match this instance version then we can't process it
// we also don't want to reprocess redelivered jobs
if (VersionHelper.IsNotEqualVersion(job.Version) || job.Redelivered)
{
Log.Error("Engine.Run(): Job Version: " + job.Version + " Deployed Version: " + Globals.Version + " Redelivered: " + job.Redelivered);
//Tiny chance there was an uncontrolled collapse of a server, resulting in an old user task circulating.
//In this event kill the old algorithm and leave a message so the user can later review.
leanEngineSystemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, _collapseMessage);
leanEngineSystemHandlers.Notify.SetAuthentication(job);
leanEngineSystemHandlers.Notify.Send(new RuntimeErrorPacket(job.UserId, job.AlgorithmId, _collapseMessage));
leanEngineSystemHandlers.JobQueue.AcknowledgeJob(job);
return;
}
if (liveMode)
{
try
{
WorkerThread curThread = new WorkerThread();
//WorkerThread curThread = WorkerThread.Instance;
var algorithmManager = new AlgorithmManager(liveMode, job);
leanEngineSystemHandlers.LeanManager.Initialize(leanEngineSystemHandlers, leanEngineAlgorithmHandlers, job, algorithmManager);
var engine = new Engine.Engine(leanEngineSystemHandlers, leanEngineAlgorithmHandlers, liveMode);
engine.Run(job, algorithmManager, assemblyPath, curThread);
int threadId = curThread._workerThread.GetHashCode();
//Console.WriteLine(curThread._workerThread.);
Log.Trace("\n\n\n\n");
}
finally
{
//Delete the message from the job queue:
leanEngineSystemHandlers.JobQueue.AcknowledgeJob(job);
Log.Trace("Engine.Main(): Packet removed from queue: " + job.AlgorithmId);
// clean up resources
leanEngineSystemHandlers.Dispose();
leanEngineAlgorithmHandlers.Dispose();
Log.LogHandler.Dispose();
//Log.Trace("Program.Main(): Exiting Lean...");
//Environment.Exit(0);
}
}
else
{
// Set up queue for RabbitMQ
channel.QueueDeclare(queue: "backtestTrigger",
durable: false,
exclusive: false,
autoDelete: false,
arguments: null);
// Create object for rabbitMQ consumer
var consumer = new EventingBasicConsumer(channel);
// Set up consumer message handler
consumer.Received += (model, ea) =>
{
var body = ea.Body;
var message = Encoding.UTF8.GetString(body);
JObject jsonmessage = JObject.Parse(message);
try
{
WorkerThread curThread = new WorkerThread();
//WorkerThread curThread = WorkerThread.Instance;
var algorithmManager = new AlgorithmManager(liveMode, job);
leanEngineSystemHandlers.LeanManager.Initialize(leanEngineSystemHandlers, leanEngineAlgorithmHandlers, job, algorithmManager);
var engine = new Engine.Engine(leanEngineSystemHandlers, leanEngineAlgorithmHandlers, liveMode);
engine.Run(job, algorithmManager, assemblyPath, curThread);
int threadId = curThread._workerThread.GetHashCode();
//Console.WriteLine(curThread._workerThread.);
Log.Trace("\n\n\n\n");
}
finally
{
//Delete the message from the job queue:
leanEngineSystemHandlers.JobQueue.AcknowledgeJob(job);
Log.Trace("Engine.Main(): Packet removed from queue: " + job.AlgorithmId);
// clean up resources
leanEngineSystemHandlers.Dispose();
leanEngineAlgorithmHandlers.Dispose();
Log.LogHandler.Dispose();
//Log.Trace("Program.Main(): Exiting Lean...");
//Environment.Exit(0);
}
//foreach (string element in jsonmessage["timeFrames"])
//{
// Log.Trace(element);
//}
};
channel.BasicConsume(queue: "backtestTrigger", autoAck: true, consumer: consumer);
}
}
/// <summary>
/// Runs an instance of a backtest.
/// </summary>
/// <param name="libraryPath"></param>
/// <param name="apiJobUserId"></param>
/// <param name="apiAccessToken"></param>
/// <param name="startDate"></param>
/// <param name="endDate"></param>
/// <param name="alphaModelName"></param>
/// <param name="symbol"></param>
/// <param name="minuteResolution"></param>
/// <param name="parametersSerialized"></param>
public static void Run(string libraryPath, string apiJobUserId, string apiAccessToken, DateTime startDate, DateTime endDate, string alphaModelName, string symbol, int minuteResolution, string parametersSerialized)
{
// Initiate a thread safe operation, as it seems we need to do all of the below in a thread safe manner
ThreadSafe.Execute("config", () =>
{
// Copy the config file thread safely
File.Copy(Path.Combine(libraryPath, "Launcher/config.json"), Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "config.json"), true);
Config.SetConfigurationFile(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "config.json"));
Config.Reset();
// Configure path to and name of algorithm
Config.Set("algorithm-type-name", "BasicTemplateFrameworkAlgorithm");
Config.Set("algorithm-location", "Algorithm.dll");
// Set some values local to this Launcher
Config.Set("environment", "backtesting");
Config.Set("data-folder", Path.Combine(libraryPath, "Data/"));
Config.Set("job-queue-handler", "LeanBatchLauncher.Launcher.Queue");
Config.Set("data-provider", "QuantConnect.Lean.Engine.DataFeeds.ApiDataProvider");
Config.Set("job-user-id", apiJobUserId);
Config.Set("api-access-token", apiAccessToken);
// Set start and end dates
Config.Set("LBL-start-date", startDate.ToString());
Config.Set("LBL-end-date", endDate.ToString());
// Parse alpha model and parameters
Config.Set("LBL-alpha-model-name", alphaModelName);
// Set symbol
Config.Set("LBL-symbol", symbol);
// Set minute resolution
Config.Set("LBL-minute-resolution", minuteResolution.ToString());
// Deserialize parameters
var parameters = JsonConvert.DeserializeObject <Dictionary <string, Parameter> >(parametersSerialized);
// Save parameters
foreach (KeyValuePair <string, Parameter> entry in parameters)
{
Config.Set(entry.Key, entry.Value.Current.ToString());
}
}
);
Log.DebuggingEnabled = false;
// We only need console output - no logging
using (Log.LogHandler = new ConsoleLogHandler()) {
Log.Trace("Engine.Main(): LEAN ALGORITHMIC TRADING ENGINE v" + Globals.Version + " Mode: *CUSTOM BATCH* (" + (Environment.Is64BitProcess ? "64" : "32") + "bit)");
Log.Trace("Engine.Main(): Started " + DateTime.Now.ToShortTimeString());
// Import external libraries specific to physical server location (cloud/local)
LeanEngineSystemHandlers leanEngineSystemHandlers;
try {
leanEngineSystemHandlers = LeanEngineSystemHandlers.FromConfiguration(Composer.Instance);
} catch (CompositionException compositionException) {
Log.Error("Engine.Main(): Failed to load library: " + compositionException);
throw;
}
// Setup packeting, queue and controls system: These don't do much locally.
leanEngineSystemHandlers.Initialize();
//-> Pull job from QuantConnect job queue, or, pull local build:
var job = leanEngineSystemHandlers.JobQueue.NextJob(out string assemblyPath);
if (job == null)
{
throw new Exception("Engine.Main(): Job was null.");
}
LeanEngineAlgorithmHandlers leanEngineAlgorithmHandlers;
try {
leanEngineAlgorithmHandlers = LeanEngineAlgorithmHandlers.FromConfiguration(Composer.Instance);
} catch (CompositionException compositionException) {
Log.Error("Engine.Main(): Failed to load library: " + compositionException);
throw;
}
try {
var algorithmManager = new AlgorithmManager(false);
leanEngineSystemHandlers.LeanManager.Initialize(leanEngineSystemHandlers, leanEngineAlgorithmHandlers, job, algorithmManager);
var engine = new Engine(leanEngineSystemHandlers, leanEngineAlgorithmHandlers, false);
engine.Run(job, algorithmManager, assemblyPath);
} finally {
// Delete the message from the job queue:
leanEngineSystemHandlers.JobQueue.AcknowledgeJob(job);
Log.Trace("Engine.Main(): Packet removed from queue: " + job.AlgorithmId);
// Clean up resources
leanEngineSystemHandlers.Dispose();
leanEngineAlgorithmHandlers.Dispose();
Log.LogHandler.Dispose();
Log.Trace("Program.Main(): Exiting Lean...");
}
}
}
/// <summary>
/// Method performs necessary initialization and starts and algorithm inside Lean Engine.
/// </summary>
public Dictionary <string, decimal> Run(Dictionary <string, string> alorithmInputs)
{
// Chromosome id must be there
var id = alorithmInputs["chromosome-id"];
// Set algorithm input variables
foreach (var pair in alorithmInputs.Where(i => i.Key != "chromosome-id"))
{
Config.Set(pair.Key, pair.Value);
}
// Lean general settings
Config.Set("environment", "backtesting");
Config.Set("algorithm-language", "CSharp");
Config.Set("result-handler", nameof(OptimizerResultHandler)); //override default result handler
// Create uniquely named log file for the backtest
var dirPath = Path.Combine(Directory.GetCurrentDirectory(), "OptimizationLogs");
var logFileName = "log" + "_" + id + ".txt";
var filePath = Path.Combine(dirPath, logFileName);
// Create directory if not exist
System.IO.Directory.CreateDirectory(dirPath);
Log.LogHandler = new FileLogHandler(filePath);
// LeanEngineSystemHandlers
LeanEngineSystemHandlers leanEngineSystemHandlers;
try
{
leanEngineSystemHandlers = LeanEngineSystemHandlers.FromConfiguration(Composer.Instance);
}
catch (CompositionException compositionException)
{
Log.Error("Engine.Main(): Failed to load library: " + compositionException);
throw;
}
// Setup packeting, queue and controls system: These don't do much locally.
leanEngineSystemHandlers.Initialize();
// Pull job from QuantConnect job queue, or, pull local build:
var job = leanEngineSystemHandlers.JobQueue.NextJob(out var assemblyPath);
if (job == null)
{
throw new Exception("Engine.Main(): Job was null.");
}
// LeanEngineSystemHandlers
LeanEngineAlgorithmHandlers leanEngineAlgorithmHandlers;
try
{
leanEngineAlgorithmHandlers = LeanEngineAlgorithmHandlers.FromConfiguration(Composer.Instance);
}
catch (CompositionException compositionException)
{
Log.Error("Engine.Main(): Failed to load library: " + compositionException);
throw;
}
// Engine
try
{
var algorithmManager = new AlgorithmManager(false, job);
leanEngineSystemHandlers.LeanManager.Initialize(leanEngineSystemHandlers, leanEngineAlgorithmHandlers, job, algorithmManager);
var engine = new Engine(leanEngineSystemHandlers, leanEngineAlgorithmHandlers, false);
// Run -->
engine.Run(job, algorithmManager, assemblyPath, WorkerThread.Instance);
}
finally
{
// do not Acknowledge Job, clean up resources
Log.Trace("Engine.Main(): Packet removed from queue: " + job.AlgorithmId);
leanEngineSystemHandlers.Dispose();
leanEngineAlgorithmHandlers.Dispose();
Log.LogHandler.Dispose();
}
// Results
_resultsHandler = (OptimizerResultHandler)leanEngineAlgorithmHandlers.Results;
return(_resultsHandler.FullResults);
}
/// <summary>
/// <see cref = "QuantBook" /> constructor.
/// Provides access to data for quantitative analysis
/// </summary>
public QuantBook() : base()
{
try
{
using (Py.GIL())
{
_pandas = Py.Import("pandas");
}
// By default, set start date to end data which is yesterday
SetStartDate(EndDate);
// Sets PandasConverter
SetPandasConverter();
// Initialize History Provider
var composer = new Composer();
var algorithmHandlers = LeanEngineAlgorithmHandlers.FromConfiguration(composer);
var systemHandlers = LeanEngineSystemHandlers.FromConfiguration(composer);
// init the API
systemHandlers.Initialize();
systemHandlers.LeanManager.Initialize(systemHandlers,
algorithmHandlers,
new BacktestNodePacket(),
new AlgorithmManager(false));
systemHandlers.LeanManager.SetAlgorithm(this);
algorithmHandlers.ObjectStore.Initialize("QuantBook",
Config.GetInt("job-user-id"),
Config.GetInt("project-id"),
Config.Get("api-access-token"),
new Controls
{
// if <= 0 we disable periodic persistence and make it synchronous
PersistenceIntervalSeconds = -1,
StorageLimitMB = Config.GetInt("storage-limit-mb", 5),
StorageFileCount = Config.GetInt("storage-file-count", 100),
StoragePermissions = (FileAccess)Config.GetInt("storage-permissions", (int)FileAccess.ReadWrite)
});
SetObjectStore(algorithmHandlers.ObjectStore);
_dataCacheProvider = new ZipDataCacheProvider(algorithmHandlers.DataProvider);
var symbolPropertiesDataBase = SymbolPropertiesDatabase.FromDataFolder();
var registeredTypes = new RegisteredSecurityDataTypesProvider();
var securityService = new SecurityService(Portfolio.CashBook,
MarketHoursDatabase,
symbolPropertiesDataBase,
this,
registeredTypes,
new SecurityCacheProvider(Portfolio));
Securities.SetSecurityService(securityService);
SubscriptionManager.SetDataManager(
new DataManager(new NullDataFeed(),
new UniverseSelection(this, securityService),
this,
TimeKeeper,
MarketHoursDatabase,
false,
registeredTypes));
var mapFileProvider = algorithmHandlers.MapFileProvider;
HistoryProvider = composer.GetExportedValueByTypeName <IHistoryProvider>(Config.Get("history-provider", "SubscriptionDataReaderHistoryProvider"));
HistoryProvider.Initialize(
new HistoryProviderInitializeParameters(
null,
null,
algorithmHandlers.DataProvider,
_dataCacheProvider,
mapFileProvider,
algorithmHandlers.FactorFileProvider,
null,
true
)
);
SetOptionChainProvider(new CachingOptionChainProvider(new BacktestingOptionChainProvider()));
SetFutureChainProvider(new CachingFutureChainProvider(new BacktestingFutureChainProvider()));
}
catch (Exception exception)
{
throw new Exception("QuantBook.Main(): " + exception);
}
}
static void Main(string[] args)
{
//Initialize:
var mode = "RELEASE";
#if DEBUG
mode = "DEBUG";
#endif
if (OS.IsWindows)
{
Console.OutputEncoding = System.Text.Encoding.UTF8;
}
// expect first argument to be config file name
if (args.Length > 0)
{
Config.MergeCommandLineArgumentsWithConfiguration(LeanArgumentParser.ParseArguments(args));
}
var liveMode = Config.GetBool("live-mode");
Log.DebuggingEnabled = Config.GetBool("debug-mode");
Log.FilePath = Path.Combine(Config.Get("results-destination-folder"), "log.txt");
Log.LogHandler = Composer.Instance.GetExportedValueByTypeName <ILogHandler>(Config.Get("log-handler", "CompositeLogHandler"));
//Name thread for the profiler:
Thread.CurrentThread.Name = "Algorithm Analysis Thread";
Log.Trace($"Engine.Main(): LEAN ALGORITHMIC TRADING ENGINE v{Globals.Version} Mode: {mode} ({(Environment.Is64BitProcess ? "64" : "32")}bit) Host: {Environment.MachineName}");
Log.Trace("Engine.Main(): Started " + DateTime.Now.ToShortTimeString());
//Import external libraries specific to physical server location (cloud/local)
try
{
leanEngineSystemHandlers = LeanEngineSystemHandlers.FromConfiguration(Composer.Instance);
}
catch (CompositionException compositionException)
{
Log.Error("Engine.Main(): Failed to load library: " + compositionException);
throw;
}
//Setup packeting, queue and controls system: These don't do much locally.
leanEngineSystemHandlers.Initialize();
//-> Pull job from QuantConnect job queue, or, pull local build:
string assemblyPath;
job = leanEngineSystemHandlers.JobQueue.NextJob(out assemblyPath);
if (job == null)
{
const string jobNullMessage = "Engine.Main(): Sorry we could not process this algorithm request.";
Log.Error(jobNullMessage);
throw new ArgumentException(jobNullMessage);
}
try
{
leanEngineAlgorithmHandlers = LeanEngineAlgorithmHandlers.FromConfiguration(Composer.Instance);
}
catch (CompositionException compositionException)
{
Log.Error("Engine.Main(): Failed to load library: " + compositionException);
throw;
}
// if the job version doesn't match this instance version then we can't process it
// we also don't want to reprocess redelivered jobs
if (job.Redelivered)
{
Log.Error("Engine.Run(): Job Version: " + job.Version + " Deployed Version: " + Globals.Version + " Redelivered: " + job.Redelivered);
//Tiny chance there was an uncontrolled collapse of a server, resulting in an old user task circulating.
//In this event kill the old algorithm and leave a message so the user can later review.
leanEngineSystemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, _collapseMessage);
leanEngineSystemHandlers.Notify.SetAuthentication(job);
leanEngineSystemHandlers.Notify.Send(new RuntimeErrorPacket(job.UserId, job.AlgorithmId, _collapseMessage));
leanEngineSystemHandlers.JobQueue.AcknowledgeJob(job);
Exit(1);
}
try
{
// Set our exit handler for the algorithm
Console.CancelKeyPress += new ConsoleCancelEventHandler(ExitKeyPress);
// Create the algorithm manager and start our engine
algorithmManager = new AlgorithmManager(liveMode, job);
leanEngineSystemHandlers.LeanManager.Initialize(leanEngineSystemHandlers, leanEngineAlgorithmHandlers, job, algorithmManager);
var engine = new Engine.Engine(leanEngineSystemHandlers, leanEngineAlgorithmHandlers, liveMode);
engine.Run(job, algorithmManager, assemblyPath, WorkerThread.Instance);
}
finally
{
var algorithmStatus = algorithmManager?.State ?? AlgorithmStatus.DeployError;
Exit(algorithmStatus != AlgorithmStatus.Completed ? 1 : 0);
}
}
private void LaunchLean()
{
Config.Set("environment", "backtesting");
if (!string.IsNullOrEmpty(_config.AlgorithmTypeName))
{
Config.Set("algorithm-type-name", _config.AlgorithmTypeName);
}
if (!string.IsNullOrEmpty(_config.AlgorithmLocation))
{
Config.Set("algorithm-location", Path.GetFileName(_config.AlgorithmLocation));
}
if (!string.IsNullOrEmpty(_config.DataFolder))
{
Config.Set("data-folder", _config.DataFolder);
}
var systemHandlers = LeanEngineSystemHandlers.FromConfiguration(Composer.Instance);
systemHandlers.Initialize();
//separate log now uniquely named
var logFileName = "log" + DateTime.Now.ToString("yyyyMMddssfffffff");
if (File.Exists(logFileName + ".txt"))
{
logFileName += "_" + Guid.NewGuid().ToString();
}
logFileName += ".txt";
var logHandlers = new ILogHandler[] { new FileLogHandler(logFileName, true) };
using (Log.LogHandler = new CompositeLogHandler(logHandlers))
{
LeanEngineAlgorithmHandlers leanEngineAlgorithmHandlers;
try
{
//override config to use custom result handler
Config.Set("backtesting.result-handler", nameof(OptimizerResultHandler));
leanEngineAlgorithmHandlers = LeanEngineAlgorithmHandlers.FromConfiguration(Composer.Instance);
_resultsHandler = (OptimizerResultHandler)leanEngineAlgorithmHandlers.Results;
}
catch (CompositionException compositionException)
{
Log.Error("Engine.Main(): Failed to load library: " + compositionException);
throw;
}
string algorithmPath;
AlgorithmNodePacket job = systemHandlers.JobQueue.NextJob(out algorithmPath);
try
{
var _engine = new Engine(systemHandlers, leanEngineAlgorithmHandlers, Config.GetBool("live-mode"));
_engine.Run(job, algorithmPath);
}
finally
{
Log.Trace("Engine.Main(): Packet removed from queue: " + job.AlgorithmId);
// clean up resources
systemHandlers.Dispose();
leanEngineAlgorithmHandlers.Dispose();
}
}
}