public override void RevertChanges() { Filter.RevertChanges(); Phaser.RevertChanges(); Flanger.RevertChanges(); Synth.RevertChanges(); LoFi.RevertChanges(); RingModulator.RevertChanges(); GuitarToBass.RevertChanges(); SlowGear.RevertChanges(); Transpose.RevertChanges(); PitchBend.RevertChanges(); Robot.RevertChanges(); VocalDist.RevertChanges(); Dynamics.RevertChanges(); Eq.RevertChanges(); Isolator.RevertChanges(); Octave.RevertChanges(); Pan.RevertChanges(); Tremolo.RevertChanges(); Slicer.RevertChanges(); Delay.RevertChanges(); PanningDelay.RevertChanges(); TapeEcho.RevertChanges(); GranularDelay.RevertChanges(); Roll.RevertChanges(); Chorus.RevertChanges(); Reverb.RevertChanges(); }
/// <summary> /// Initialize the debugger /// </summary> /// <param name="algorithmNodePacket">The algorithm node packet</param> /// <param name="workerThread">The worker thread instance to use</param> public static bool InitializeDebugging(AlgorithmNodePacket algorithmNodePacket, WorkerThread workerThread) { var isolator = new Isolator(); return(isolator.ExecuteWithTimeLimit(TimeSpan.FromMinutes(5), () => DebuggerHelper.Initialize(algorithmNodePacket.Language), algorithmNodePacket.RamAllocation, sleepIntervalMillis: 100, workerThread: workerThread)); }
public int SaveIsolator(Isolator isolator, string user) { if (isolator.IsolatorId > 0) { isolator.SetUpdateDetails(user); repository.SaveExisting(isolator); return(isolator.IsolatorId); } isolator.SetCreateDetails(user); return(repository.SaveNew(isolator).IsolatorId); }
private void UpdateDisplay() { Debug.Assert(!InvokeRequired); if (!_browserIsReadyToNavigate) { return; } if (_url != null) { _browser.Visible = true; Isolator.Navigate(_browser, _url); } }
private object MapIsolatorRepsonse(Isolator iso, DateTime requestDate) { var isoTotalAvailabilityMins = iso.StaffShiftAllocations.Where(a => a.AllocatedDate.Date.Equals(requestDate.Date) && !a.IsArchived).Sum(s => s.IsolatorShift.TotalShiftDurationInMins); var prepOrders = iso.PreperationOrders.Where(a => a.PreperationDateTime.Date.Equals(requestDate.Date) && !a.IsArchived).ToList(); var percent = isoTotalAvailabilityMins > 0 ? (prepOrders.Sum(p => p.IntegrationOrder.RequiredPreperationTimeInMins) * 100 / isoTotalAvailabilityMins) : 0; return(new { iso.IsolatorId, IsolatorName = iso.Abbriviation, SessionDose = iso.TotalNumberOfDosesPerSession, ScheduledOrders = prepOrders.Count(), ScheduledPercent = Math.Round(percent), IsolatorUnavailabe = isoTotalAvailabilityMins <= 0 }); }
public static GridBoxRow BindGridData(Isolator source) { var row = new GridBoxRow { IdentityValue = source.IsolatorId, BoxHeading = source.Abbriviation }; row.BoxBodyDictionary.Add("Total no. of doses per session", source.TotalNumberOfDosesPerSession.ToString()); row.AddActionIcon("fa fa-edit text-success", "Click to view/edit"); row.AddActionIcon("fa fa-trash text-danger", "Click to delete"); row.AddActionIcon("fa fa-calendar-times-o text-warning", "Click to manage isolator offline dates"); row.AddActionIcon("fa fa-list text-info", "Click to manage isolator start/stop procedures"); row.AddActionIcon("fa fa-users text-primary", "Click to manage staff allocation", "/isolator/isolatorstaffallocationindex?isoid=" + source.IsolatorId + "&nm=" + source.Abbriviation + ""); return(row); }
public void TimeOutWorkCorrectly(Language language, bool useWorker) { var worker = useWorker ? new TestWorkerThread() : null; using (worker) { var isolator = new Isolator(); bool result = false; try { if (language == Language.CSharp) { result = isolator.ExecuteWithTimeLimit( TimeSpan.FromMilliseconds(100), () => Thread.Sleep(10000), // 10s sleep 5000, workerThread: worker ); } else { result = isolator.ExecuteWithTimeLimit( TimeSpan.FromMilliseconds(100), () => { using (Py.GIL()) { // 10s sleep PythonEngine.RunSimpleString("import time; time.sleep(10)"); } }, 5000, workerThread: worker ); } Assert.Fail($"Was expecting {nameof(TimeoutException)}"); } catch (TimeoutException) { Assert.IsFalse(result); } } }
public void WorksCorrectlyUsingWorker() { using (var worker = new TestWorkerThread()) { var isolator = new Isolator(); var executed = false; var result = isolator.ExecuteWithTimeLimit( TimeSpan.FromMilliseconds(100), () => { executed = true; }, 5000, workerThread: worker ); Assert.IsTrue(result); Assert.IsTrue(executed); } }
/// <summary> /// Creates a new instance of the class in the library, safely. /// </summary> /// <param name="assemblyPath">Location of the DLL</param> /// <param name="algorithmInstance">Output algorithm instance</param> /// <param name="errorMessage">Output error message on failure</param> /// <returns>bool success</returns> public bool TryCreateAlgorithmInstanceWithIsolator(string assemblyPath, out IAlgorithm algorithmInstance, out string errorMessage) { IAlgorithm instance = null; string error = string.Empty; bool success = false; bool complete = Isolator.ExecuteWithTimeLimit(_loaderTimeLimit, () => { success = TryCreateAlgorithmInstance(assemblyPath, out instance, out error); }); algorithmInstance = instance; errorMessage = error; // if the isolator stopped us early add that to our error message if (!complete) { errorMessage = "Failed to create algorithm instance within 10 seconds. Try re-building algorithm. " + error; } return(complete && success && algorithmInstance != null); }
/// <summary> /// Creates a new instance of the class in the library, safely. /// </summary> /// <param name="assemblyPath">Location of the DLL</param> /// <param name="ramLimit">Limit of the RAM for this process</param> /// <param name="algorithmInstance">Output algorithm instance</param> /// <param name="errorMessage">Output error message on failure</param> /// <returns>bool success</returns> public bool TryCreateAlgorithmInstanceWithIsolator(string assemblyPath, int ramLimit, out IAlgorithm algorithmInstance, out string errorMessage) { IAlgorithm instance = null; var error = string.Empty; var success = false; var isolator = new Isolator(); var complete = isolator.ExecuteWithTimeLimit(_loaderTimeLimit, () => { success = TryCreateAlgorithmInstance(assemblyPath, out instance, out error); }, ramLimit, sleepIntervalMillis: 50, workerThread: _workerThread); algorithmInstance = instance; errorMessage = error; // if the isolator stopped us early add that to our error message if (!complete) { errorMessage = "Failed to create algorithm instance within 10 seconds. Try re-building algorithm. " + error; } return(complete && success && algorithmInstance != null); }
private async void StartFlowInternal() { Isolator isolator = new Isolator(); var remoteExcel = isolator.GetIsolatedInstance <IRemoteExeclController, RemoteExeclController>(); await Task.Run(() => { var remoteActual = remoteExcel.Item2; remoteActual.RunExcelInit(this, new DataUpdater()); }); isolator.UnloadIsolationContext(remoteExcel.Item1); // today's arrivals updateArrivalsGrid(); // yesterday's sails updateSailsGrid(); // destination port updateDestinationGrid(); }
public override void ApplyChanges <T>(T model) { if (model is SettingsMemoryFxModel memoryFx) { Filter.ApplyChanges(memoryFx); Phaser.ApplyChanges(memoryFx); Flanger.ApplyChanges(memoryFx); Synth.ApplyChanges(memoryFx); LoFi.ApplyChanges(memoryFx); RingModulator.ApplyChanges(memoryFx); GuitarToBass.ApplyChanges(memoryFx); SlowGear.ApplyChanges(memoryFx); Transpose.ApplyChanges(memoryFx); PitchBend.ApplyChanges(memoryFx); Robot.ApplyChanges(memoryFx); VocalDist.ApplyChanges(memoryFx); Dynamics.ApplyChanges(memoryFx); Eq.ApplyChanges(memoryFx); Isolator.ApplyChanges(memoryFx); Octave.ApplyChanges(memoryFx); Pan.ApplyChanges(memoryFx); Tremolo.ApplyChanges(memoryFx); Slicer.ApplyChanges(memoryFx); Delay.ApplyChanges(memoryFx); PanningDelay.ApplyChanges(memoryFx); TapeEcho.ApplyChanges(memoryFx); GranularDelay.ApplyChanges(memoryFx); Roll.ApplyChanges(memoryFx); Chorus.ApplyChanges(memoryFx); Reverb.ApplyChanges(memoryFx); } else { throw new ArgumentException("Model must be of type SettingsMemoryFxModel."); } }
/// <summary> /// Runs a single backtest/live job from the job queue /// </summary> /// <param name="job">The algorithm job to be processed</param> /// <param name="assemblyPath">The path to the algorithm's assembly</param> public void Run(AlgorithmNodePacket job, string assemblyPath) { var algorithm = default(IAlgorithm); var algorithmManager = new AlgorithmManager(_liveMode); //Start monitoring the backtest active status: var statusPing = new StateCheck.Ping(algorithmManager, _systemHandlers.Api, _algorithmHandlers.Results, _systemHandlers.Notify, job); var statusPingThread = new Thread(statusPing.Run); statusPingThread.Start(); try { //Reset thread holders. var initializeComplete = false; Thread threadFeed = null; Thread threadTransactions = null; Thread threadResults = null; Thread threadRealTime = null; //-> Initialize messaging system _systemHandlers.Notify.SetAuthentication(job); //-> Set the result handler type for this algorithm job, and launch the associated result thread. _algorithmHandlers.Results.Initialize(job, _systemHandlers.Notify, _systemHandlers.Api, _algorithmHandlers.DataFeed, _algorithmHandlers.Setup, _algorithmHandlers.Transactions); threadResults = new Thread(_algorithmHandlers.Results.Run, 0) {Name = "Result Thread"}; threadResults.Start(); IBrokerage brokerage = null; try { // Save algorithm to cache, load algorithm instance: algorithm = _algorithmHandlers.Setup.CreateAlgorithmInstance(assemblyPath, job.Language); // Initialize the brokerage brokerage = _algorithmHandlers.Setup.CreateBrokerage(job, algorithm); // Initialize the data feed before we initialize so he can intercept added securities/universes via events _algorithmHandlers.DataFeed.Initialize(algorithm, job, _algorithmHandlers.Results, _algorithmHandlers.MapFileProvider, _algorithmHandlers.FactorFileProvider); // initialize command queue system _algorithmHandlers.CommandQueue.Initialize(job, algorithm); // set the history provider before setting up the algorithm _algorithmHandlers.HistoryProvider.Initialize(job, _algorithmHandlers.MapFileProvider, _algorithmHandlers.FactorFileProvider, progress => { // send progress updates to the result handler only during initialization if (!algorithm.GetLocked() || algorithm.IsWarmingUp) { _algorithmHandlers.Results.SendStatusUpdate(AlgorithmStatus.History, string.Format("Processing history {0}%...", progress)); } }); algorithm.HistoryProvider = _algorithmHandlers.HistoryProvider; // initialize the default brokerage message handler algorithm.BrokerageMessageHandler = new DefaultBrokerageMessageHandler(algorithm, job, _algorithmHandlers.Results, _systemHandlers.Api); //Initialize the internal state of algorithm and job: executes the algorithm.Initialize() method. initializeComplete = _algorithmHandlers.Setup.Setup(algorithm, brokerage, job, _algorithmHandlers.Results, _algorithmHandlers.Transactions, _algorithmHandlers.RealTime); // set this again now that we've actually added securities _algorithmHandlers.Results.SetAlgorithm(algorithm); //If there are any reasons it failed, pass these back to the IDE. if (!initializeComplete || algorithm.ErrorMessages.Count > 0 || _algorithmHandlers.Setup.Errors.Count > 0) { initializeComplete = false; //Get all the error messages: internal in algorithm and external in setup handler. var errorMessage = String.Join(",", algorithm.ErrorMessages); errorMessage += String.Join(",", _algorithmHandlers.Setup.Errors); Log.Error("Engine.Run(): " + errorMessage); _algorithmHandlers.Results.RuntimeError(errorMessage); _systemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, errorMessage); } } catch (Exception err) { Log.Error(err); var runtimeMessage = "Algorithm.Initialize() Error: " + err.Message + " Stack Trace: " + err.StackTrace; _algorithmHandlers.Results.RuntimeError(runtimeMessage, err.StackTrace); _systemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, runtimeMessage); } //-> Using the job + initialization: load the designated handlers: if (initializeComplete) { //-> Reset the backtest stopwatch; we're now running the algorithm. var startTime = DateTime.Now; //Set algorithm as locked; set it to live mode if we're trading live, and set it to locked for no further updates. algorithm.SetAlgorithmId(job.AlgorithmId); algorithm.SetLocked(); //Load the associated handlers for transaction and realtime events: _algorithmHandlers.Transactions.Initialize(algorithm, brokerage, _algorithmHandlers.Results); _algorithmHandlers.RealTime.Setup(algorithm, job, _algorithmHandlers.Results, _systemHandlers.Api); // wire up the brokerage message handler brokerage.Message += (sender, message) => { algorithm.BrokerageMessageHandler.Handle(message); // fire brokerage message events algorithm.OnBrokerageMessage(message); switch (message.Type) { case BrokerageMessageType.Disconnect: algorithm.OnBrokerageDisconnect(); break; case BrokerageMessageType.Reconnect: algorithm.OnBrokerageReconnect(); break; } }; //Send status to user the algorithm is now executing. _algorithmHandlers.Results.SendStatusUpdate(AlgorithmStatus.Running); //Launch the data, transaction and realtime handlers into dedicated threads threadFeed = new Thread(_algorithmHandlers.DataFeed.Run) {Name = "DataFeed Thread"}; threadTransactions = new Thread(_algorithmHandlers.Transactions.Run) {Name = "Transaction Thread"}; threadRealTime = new Thread(_algorithmHandlers.RealTime.Run) {Name = "RealTime Thread"}; //Launch the data feed, result sending, and transaction models/handlers in separate threads. threadFeed.Start(); // Data feed pushing data packets into thread bridge; threadTransactions.Start(); // Transaction modeller scanning new order requests threadRealTime.Start(); // RealTime scan time for time based events: // Result manager scanning message queue: (started earlier) _algorithmHandlers.Results.DebugMessage(string.Format("Launching analysis for {0} with LEAN Engine v{1}", job.AlgorithmId, Globals.Version)); try { //Create a new engine isolator class var isolator = new Isolator(); // Execute the Algorithm Code: var complete = isolator.ExecuteWithTimeLimit(_algorithmHandlers.Setup.MaximumRuntime, algorithmManager.TimeLoopWithinLimits, () => { try { //Run Algorithm Job: // -> Using this Data Feed, // -> Send Orders to this TransactionHandler, // -> Send Results to ResultHandler. algorithmManager.Run(job, algorithm, _algorithmHandlers.DataFeed, _algorithmHandlers.Transactions, _algorithmHandlers.Results, _algorithmHandlers.RealTime, _algorithmHandlers.CommandQueue, isolator.CancellationToken); } catch (Exception err) { //Debugging at this level is difficult, stack trace needed. Log.Error(err); algorithm.RunTimeError = err; algorithmManager.SetStatus(AlgorithmStatus.RuntimeError); return; } Log.Trace("Engine.Run(): Exiting Algorithm Manager"); }, job.RamAllocation); if (!complete) { Log.Error("Engine.Main(): Failed to complete in time: " + _algorithmHandlers.Setup.MaximumRuntime.ToString("F")); throw new Exception("Failed to complete algorithm within " + _algorithmHandlers.Setup.MaximumRuntime.ToString("F") + " seconds. Please make it run faster."); } // Algorithm runtime error: if (algorithm.RunTimeError != null) { throw algorithm.RunTimeError; } } catch (Exception err) { //Error running the user algorithm: purge datafeed, send error messages, set algorithm status to failed. Log.Error(err, "Breaking out of parent try catch:"); if (_algorithmHandlers.DataFeed != null) _algorithmHandlers.DataFeed.Exit(); if (_algorithmHandlers.Results != null) { var message = "Runtime Error: " + err.Message; Log.Trace("Engine.Run(): Sending runtime error to user..."); _algorithmHandlers.Results.LogMessage(message); _algorithmHandlers.Results.RuntimeError(message, err.StackTrace); _systemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, message + " Stack Trace: " + err.StackTrace); } } try { var trades = algorithm.TradeBuilder.ClosedTrades; var charts = new Dictionary<string, Chart>(_algorithmHandlers.Results.Charts); var orders = new Dictionary<int, Order>(_algorithmHandlers.Transactions.Orders); var holdings = new Dictionary<string, Holding>(); var banner = new Dictionary<string, string>(); var statisticsResults = new StatisticsResults(); try { //Generates error when things don't exist (no charting logged, runtime errors in main algo execution) const string strategyEquityKey = "Strategy Equity"; const string equityKey = "Equity"; const string dailyPerformanceKey = "Daily Performance"; const string benchmarkKey = "Benchmark"; // make sure we've taken samples for these series before just blindly requesting them if (charts.ContainsKey(strategyEquityKey) && charts[strategyEquityKey].Series.ContainsKey(equityKey) && charts[strategyEquityKey].Series.ContainsKey(dailyPerformanceKey)) { var equity = charts[strategyEquityKey].Series[equityKey].Values; var performance = charts[strategyEquityKey].Series[dailyPerformanceKey].Values; var profitLoss = new SortedDictionary<DateTime, decimal>(algorithm.Transactions.TransactionRecord); var totalTransactions = algorithm.Transactions.GetOrders(x => x.Status.IsFill()).Count(); var benchmark = charts[benchmarkKey].Series[benchmarkKey].Values; statisticsResults = StatisticsBuilder.Generate(trades, profitLoss, equity, performance, benchmark, _algorithmHandlers.Setup.StartingPortfolioValue, algorithm.Portfolio.TotalFees, totalTransactions); //Some users have $0 in their brokerage account / starting cash of $0. Prevent divide by zero errors var netReturn = _algorithmHandlers.Setup.StartingPortfolioValue > 0 ? (algorithm.Portfolio.TotalPortfolioValue - _algorithmHandlers.Setup.StartingPortfolioValue) / _algorithmHandlers.Setup.StartingPortfolioValue : 0; //Add other fixed parameters. banner.Add("Unrealized:", "$" + algorithm.Portfolio.TotalUnrealizedProfit.ToString("N2")); banner.Add("Fees:", "-$" + algorithm.Portfolio.TotalFees.ToString("N2")); banner.Add("Net Profit:", "$" + algorithm.Portfolio.TotalProfit.ToString("N2")); banner.Add("Return:", netReturn.ToString("P")); banner.Add("Equity:", "$" + algorithm.Portfolio.TotalPortfolioValue.ToString("N2")); banner.Add("Holdings:", "$" + algorithm.Portfolio.TotalHoldingsValue.ToString("N2")); banner.Add("Volume:", "$" + algorithm.Portfolio.TotalSaleVolume.ToString("N2")); } } catch (Exception err) { Log.Error(err, "Error generating statistics packet"); } //Diagnostics Completed, Send Result Packet: var totalSeconds = (DateTime.Now - startTime).TotalSeconds; var dataPoints = algorithmManager.DataPoints + _algorithmHandlers.HistoryProvider.DataPointCount; _algorithmHandlers.Results.DebugMessage( string.Format("Algorithm Id:({0}) completed in {1} seconds at {2}k data points per second. Processing total of {3} data points.", job.AlgorithmId, totalSeconds.ToString("F2"), ((dataPoints/(double) 1000)/totalSeconds).ToString("F0"), dataPoints.ToString("N0"))); _algorithmHandlers.Results.SendFinalResult(job, orders, algorithm.Transactions.TransactionRecord, holdings, statisticsResults, banner); } catch (Exception err) { Log.Error(err, "Error sending analysis results"); } //Before we return, send terminate commands to close up the threads _algorithmHandlers.Transactions.Exit(); _algorithmHandlers.DataFeed.Exit(); _algorithmHandlers.RealTime.Exit(); } //Close result handler: _algorithmHandlers.Results.Exit(); statusPing.Exit(); //Wait for the threads to complete: var ts = Stopwatch.StartNew(); while ((_algorithmHandlers.Results.IsActive || (_algorithmHandlers.Transactions != null && _algorithmHandlers.Transactions.IsActive) || (_algorithmHandlers.DataFeed != null && _algorithmHandlers.DataFeed.IsActive) || (_algorithmHandlers.RealTime != null && _algorithmHandlers.RealTime.IsActive)) && ts.ElapsedMilliseconds < 30*1000) { Thread.Sleep(100); Log.Trace("Waiting for threads to exit..."); } //Terminate threads still in active state. if (threadFeed != null && threadFeed.IsAlive) threadFeed.Abort(); if (threadTransactions != null && threadTransactions.IsAlive) threadTransactions.Abort(); if (threadResults != null && threadResults.IsAlive) threadResults.Abort(); if (statusPingThread != null && statusPingThread.IsAlive) statusPingThread.Abort(); if (brokerage != null) { Log.Trace("Engine.Run(): Disconnecting from brokerage..."); brokerage.Disconnect(); } if (_algorithmHandlers.Setup != null) { Log.Trace("Engine.Run(): Disposing of setup handler..."); _algorithmHandlers.Setup.Dispose(); } Log.Trace("Engine.Main(): Analysis Completed and Results Posted."); } catch (Exception err) { Log.Error(err, "Error running algorithm"); } finally { //No matter what for live mode; make sure we've set algorithm status in the API for "not running" conditions: if (_liveMode && algorithmManager.State != AlgorithmStatus.Running && algorithmManager.State != AlgorithmStatus.RuntimeError) _systemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, algorithmManager.State); _algorithmHandlers.Results.Exit(); _algorithmHandlers.DataFeed.Exit(); _algorithmHandlers.Transactions.Exit(); _algorithmHandlers.RealTime.Exit(); } }
/// <summary> /// Runs a single backtest/live job from the job queue /// </summary> /// <param name="job">The algorithm job to be processed</param> /// <param name="manager">The algorithm manager instance</param> /// <param name="assemblyPath">The path to the algorithm's assembly</param> /// <param name="workerThread">The worker thread instance</param> public void Run(AlgorithmNodePacket job, AlgorithmManager manager, string assemblyPath, WorkerThread workerThread) { var marketHoursDatabaseTask = Task.Run(() => StaticInitializations()); var algorithm = default(IAlgorithm); var algorithmManager = manager; try { //Reset thread holders. var initializeComplete = false; //-> Initialize messaging system SystemHandlers.Notify.SetAuthentication(job); //-> Set the result handler type for this algorithm job, and launch the associated result thread. AlgorithmHandlers.Results.Initialize(job, SystemHandlers.Notify, SystemHandlers.Api, AlgorithmHandlers.Transactions); IBrokerage brokerage = null; DataManager dataManager = null; var synchronizer = _liveMode ? new LiveSynchronizer() : new Synchronizer(); try { // we get the mhdb before creating the algorithm instance, // since the algorithm constructor will use it var marketHoursDatabase = marketHoursDatabaseTask.Result; AlgorithmHandlers.Setup.WorkerThread = workerThread; // Save algorithm to cache, load algorithm instance: algorithm = AlgorithmHandlers.Setup.CreateAlgorithmInstance(job, assemblyPath); // Set algorithm in ILeanManager SystemHandlers.LeanManager.SetAlgorithm(algorithm); // initialize the alphas handler with the algorithm instance AlgorithmHandlers.Alphas.Initialize(job, algorithm, SystemHandlers.Notify, SystemHandlers.Api); // initialize the object store AlgorithmHandlers.ObjectStore.Initialize(algorithm.Name, job.UserId, job.ProjectId, job.UserToken, job.Controls); // notify the user of any errors w/ object store persistence AlgorithmHandlers.ObjectStore.ErrorRaised += (sender, args) => algorithm.Debug($"ObjectStore Persistence Error: {args.Error.Message}"); // Initialize the brokerage IBrokerageFactory factory; brokerage = AlgorithmHandlers.Setup.CreateBrokerage(job, algorithm, out factory); var symbolPropertiesDatabase = SymbolPropertiesDatabase.FromDataFolder(); var registeredTypesProvider = new RegisteredSecurityDataTypesProvider(); var securityService = new SecurityService(algorithm.Portfolio.CashBook, marketHoursDatabase, symbolPropertiesDatabase, algorithm, registeredTypesProvider, new SecurityCacheProvider(algorithm.Portfolio)); algorithm.Securities.SetSecurityService(securityService); dataManager = new DataManager(AlgorithmHandlers.DataFeed, new UniverseSelection( algorithm, securityService), algorithm, algorithm.TimeKeeper, marketHoursDatabase, _liveMode, registeredTypesProvider); AlgorithmHandlers.Results.SetDataManager(dataManager); algorithm.SubscriptionManager.SetDataManager(dataManager); synchronizer.Initialize(algorithm, dataManager); // Initialize the data feed before we initialize so he can intercept added securities/universes via events AlgorithmHandlers.DataFeed.Initialize( algorithm, job, AlgorithmHandlers.Results, AlgorithmHandlers.MapFileProvider, AlgorithmHandlers.FactorFileProvider, AlgorithmHandlers.DataProvider, dataManager, (IDataFeedTimeProvider)synchronizer); // set the order processor on the transaction manager (needs to be done before initializing BrokerageHistoryProvider) algorithm.Transactions.SetOrderProcessor(AlgorithmHandlers.Transactions); algorithm.SetOrderEventProvider(AlgorithmHandlers.Transactions); // set the history provider before setting up the algorithm var historyProvider = GetHistoryProvider(job.HistoryProvider); if (historyProvider is BrokerageHistoryProvider) { (historyProvider as BrokerageHistoryProvider).SetBrokerage(brokerage); } var historyDataCacheProvider = new ZipDataCacheProvider(AlgorithmHandlers.DataProvider, isDataEphemeral: _liveMode); historyProvider.Initialize( new HistoryProviderInitializeParameters( job, SystemHandlers.Api, AlgorithmHandlers.DataProvider, historyDataCacheProvider, AlgorithmHandlers.MapFileProvider, AlgorithmHandlers.FactorFileProvider, progress => { // send progress updates to the result handler only during initialization if (!algorithm.GetLocked() || algorithm.IsWarmingUp) { AlgorithmHandlers.Results.SendStatusUpdate(AlgorithmStatus.History, Invariant($"Processing history {progress}%...")); } }, // disable parallel history requests for live trading parallelHistoryRequestsEnabled: !_liveMode ) ); historyProvider.InvalidConfigurationDetected += (sender, args) => { AlgorithmHandlers.Results.ErrorMessage(args.Message); }; historyProvider.NumericalPrecisionLimited += (sender, args) => { AlgorithmHandlers.Results.DebugMessage(args.Message); }; historyProvider.DownloadFailed += (sender, args) => { AlgorithmHandlers.Results.ErrorMessage(args.Message, args.StackTrace); }; historyProvider.ReaderErrorDetected += (sender, args) => { AlgorithmHandlers.Results.RuntimeError(args.Message, args.StackTrace); }; algorithm.HistoryProvider = historyProvider; // initialize the default brokerage message handler algorithm.BrokerageMessageHandler = factory.CreateBrokerageMessageHandler(algorithm, job, SystemHandlers.Api); //Initialize the internal state of algorithm and job: executes the algorithm.Initialize() method. initializeComplete = AlgorithmHandlers.Setup.Setup(new SetupHandlerParameters(dataManager.UniverseSelection, algorithm, brokerage, job, AlgorithmHandlers.Results, AlgorithmHandlers.Transactions, AlgorithmHandlers.RealTime, AlgorithmHandlers.ObjectStore)); // set this again now that we've actually added securities AlgorithmHandlers.Results.SetAlgorithm(algorithm, AlgorithmHandlers.Setup.StartingPortfolioValue); // alpha handler needs start/end dates to determine sample step sizes AlgorithmHandlers.Alphas.OnAfterAlgorithmInitialized(algorithm); //If there are any reasons it failed, pass these back to the IDE. if (!initializeComplete || algorithm.ErrorMessages.Count > 0 || AlgorithmHandlers.Setup.Errors.Count > 0) { initializeComplete = false; //Get all the error messages: internal in algorithm and external in setup handler. var errorMessage = string.Join(",", algorithm.ErrorMessages); errorMessage += string.Join(",", AlgorithmHandlers.Setup.Errors.Select(e => { var message = e.Message; if (e.InnerException != null) { var err = _exceptionInterpreter.Value.Interpret(e.InnerException, _exceptionInterpreter.Value); message += _exceptionInterpreter.Value.GetExceptionMessageHeader(err); } return(message); })); Log.Error("Engine.Run(): " + errorMessage); AlgorithmHandlers.Results.RuntimeError(errorMessage); SystemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, errorMessage); } } catch (Exception err) { Log.Error(err); var runtimeMessage = "Algorithm.Initialize() Error: " + err.Message + " Stack Trace: " + err; AlgorithmHandlers.Results.RuntimeError(runtimeMessage, err.ToString()); SystemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, runtimeMessage); } // log the job endpoints Log.Trace("JOB HANDLERS: "); Log.Trace(" DataFeed: " + AlgorithmHandlers.DataFeed.GetType().FullName); Log.Trace(" Setup: " + AlgorithmHandlers.Setup.GetType().FullName); Log.Trace(" RealTime: " + AlgorithmHandlers.RealTime.GetType().FullName); Log.Trace(" Results: " + AlgorithmHandlers.Results.GetType().FullName); Log.Trace(" Transactions: " + AlgorithmHandlers.Transactions.GetType().FullName); Log.Trace(" Alpha: " + AlgorithmHandlers.Alphas.GetType().FullName); Log.Trace(" ObjectStore: " + AlgorithmHandlers.ObjectStore.GetType().FullName); if (algorithm?.HistoryProvider != null) { Log.Trace(" History Provider: " + algorithm.HistoryProvider.GetType().FullName); } if (job is LiveNodePacket) { Log.Trace(" Brokerage: " + brokerage?.GetType().FullName); } //-> Using the job + initialization: load the designated handlers: if (initializeComplete) { // notify the LEAN manager that the algorithm is initialized and starting SystemHandlers.LeanManager.OnAlgorithmStart(); //-> Reset the backtest stopwatch; we're now running the algorithm. var startTime = DateTime.Now; //Set algorithm as locked; set it to live mode if we're trading live, and set it to locked for no further updates. algorithm.SetAlgorithmId(job.AlgorithmId); algorithm.SetLocked(); //Load the associated handlers for transaction and realtime events: AlgorithmHandlers.Transactions.Initialize(algorithm, brokerage, AlgorithmHandlers.Results); AlgorithmHandlers.RealTime.Setup(algorithm, job, AlgorithmHandlers.Results, SystemHandlers.Api, algorithmManager.TimeLimit); // wire up the brokerage message handler brokerage.Message += (sender, message) => { algorithm.BrokerageMessageHandler.Handle(message); // fire brokerage message events algorithm.OnBrokerageMessage(message); switch (message.Type) { case BrokerageMessageType.Disconnect: algorithm.OnBrokerageDisconnect(); break; case BrokerageMessageType.Reconnect: algorithm.OnBrokerageReconnect(); break; } }; //Send status to user the algorithm is now executing. AlgorithmHandlers.Results.SendStatusUpdate(AlgorithmStatus.Running); // Result manager scanning message queue: (started earlier) AlgorithmHandlers.Results.DebugMessage( $"Launching analysis for {job.AlgorithmId} with LEAN Engine v{Globals.Version}"); try { //Create a new engine isolator class var isolator = new Isolator(); // Execute the Algorithm Code: var complete = isolator.ExecuteWithTimeLimit(AlgorithmHandlers.Setup.MaximumRuntime, algorithmManager.TimeLimit.IsWithinLimit, () => { try { //Run Algorithm Job: // -> Using this Data Feed, // -> Send Orders to this TransactionHandler, // -> Send Results to ResultHandler. algorithmManager.Run(job, algorithm, synchronizer, AlgorithmHandlers.Transactions, AlgorithmHandlers.Results, AlgorithmHandlers.RealTime, SystemHandlers.LeanManager, AlgorithmHandlers.Alphas, isolator.CancellationToken); } catch (Exception err) { //Debugging at this level is difficult, stack trace needed. Log.Error(err); algorithm.RunTimeError = err; algorithmManager.SetStatus(AlgorithmStatus.RuntimeError); return; } Log.Trace("Engine.Run(): Exiting Algorithm Manager"); }, job.Controls.RamAllocation, workerThread: workerThread); if (!complete) { Log.Error("Engine.Main(): Failed to complete in time: " + AlgorithmHandlers.Setup.MaximumRuntime.ToStringInvariant("F")); throw new Exception("Failed to complete algorithm within " + AlgorithmHandlers.Setup.MaximumRuntime.ToStringInvariant("F") + " seconds. Please make it run faster."); } // Algorithm runtime error: if (algorithm.RunTimeError != null) { HandleAlgorithmError(job, algorithm.RunTimeError); } } catch (Exception err) { //Error running the user algorithm: purge datafeed, send error messages, set algorithm status to failed. algorithm.RunTimeError = err; algorithm.SetStatus(AlgorithmStatus.RuntimeError); HandleAlgorithmError(job, err); } // notify the LEAN manager that the algorithm has finished SystemHandlers.LeanManager.OnAlgorithmEnd(); try { var csvTransactionsFileName = Config.Get("transaction-log"); if (!string.IsNullOrEmpty(csvTransactionsFileName)) { SaveListOfTrades(AlgorithmHandlers.Transactions, csvTransactionsFileName); } if (!_liveMode) { //Diagnostics Completed, Send Result Packet: var totalSeconds = (DateTime.Now - startTime).TotalSeconds; var dataPoints = algorithmManager.DataPoints + algorithm.HistoryProvider.DataPointCount; var kps = dataPoints / (double)1000 / totalSeconds; AlgorithmHandlers.Results.DebugMessage($"Algorithm Id:({job.AlgorithmId}) completed in {totalSeconds:F2} seconds at {kps:F0}k data points per second. Processing total of {dataPoints:N0} data points."); } } catch (Exception err) { Log.Error(err, "Error sending analysis results"); } //Before we return, send terminate commands to close up the threads AlgorithmHandlers.Transactions.Exit(); AlgorithmHandlers.RealTime.Exit(); dataManager?.RemoveAllSubscriptions(); workerThread?.Dispose(); } // Close data feed, alphas. Could be running even if algorithm initialization failed AlgorithmHandlers.DataFeed.Exit(); AlgorithmHandlers.Alphas.Exit(); //Close result handler: AlgorithmHandlers.Results.Exit(); //Wait for the threads to complete: var millisecondInterval = 10; var millisecondTotalWait = 0; while ((AlgorithmHandlers.Results.IsActive || (AlgorithmHandlers.Transactions != null && AlgorithmHandlers.Transactions.IsActive) || (AlgorithmHandlers.DataFeed != null && AlgorithmHandlers.DataFeed.IsActive) || (AlgorithmHandlers.RealTime != null && AlgorithmHandlers.RealTime.IsActive) || (AlgorithmHandlers.Alphas != null && AlgorithmHandlers.Alphas.IsActive)) && millisecondTotalWait < 30 * 1000) { Thread.Sleep(millisecondInterval); if (millisecondTotalWait % (millisecondInterval * 10) == 0) { Log.Trace("Waiting for threads to exit..."); } millisecondTotalWait += millisecondInterval; } if (brokerage != null) { Log.Trace("Engine.Run(): Disconnecting from brokerage..."); brokerage.Disconnect(); brokerage.Dispose(); } if (AlgorithmHandlers.Setup != null) { Log.Trace("Engine.Run(): Disposing of setup handler..."); AlgorithmHandlers.Setup.Dispose(); } Log.Trace("Engine.Main(): Analysis Completed and Results Posted."); } catch (Exception err) { Log.Error(err, "Error running algorithm"); } finally { //No matter what for live mode; make sure we've set algorithm status in the API for "not running" conditions: if (_liveMode && algorithmManager.State != AlgorithmStatus.Running && algorithmManager.State != AlgorithmStatus.RuntimeError) { SystemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, algorithmManager.State); } AlgorithmHandlers.Results.Exit(); AlgorithmHandlers.DataFeed.Exit(); AlgorithmHandlers.Transactions.Exit(); AlgorithmHandlers.RealTime.Exit(); } }
/// <summary> /// Primary entry point to setup a new algorithm /// </summary> /// <param name="algorithm">Algorithm instance</param> /// <param name="brokerage">New brokerage output instance</param> /// <param name="job">Algorithm job task</param> /// <param name="resultHandler">The configured result handler</param> /// <param name="transactionHandler">The configurated transaction handler</param> /// <param name="realTimeHandler">The configured real time handler</param> /// <returns>True on successfully setting up the algorithm state, or false on error.</returns> public bool Setup(IAlgorithm algorithm, out IBrokerage brokerage, AlgorithmNodePacket job, IResultHandler resultHandler, ITransactionHandler transactionHandler, IRealTimeHandler realTimeHandler) { _algorithm = algorithm; brokerage = default(IBrokerage); // verify we were given the correct job packet type var liveJob = job as LiveNodePacket; if (liveJob == null) { AddInitializationError("BrokerageSetupHandler requires a LiveNodePacket"); return false; } // verify the brokerage was specified if (string.IsNullOrWhiteSpace(liveJob.Brokerage)) { AddInitializationError("A brokerage must be specified"); return false; } // attach to the message event to relay brokerage specific initialization messages EventHandler<BrokerageMessageEvent> brokerageOnMessage = (sender, args) => { if (args.Type == BrokerageMessageType.Error) { AddInitializationError(string.Format("Brokerage Error Code: {0} - {1}", args.Code, args.Message)); } }; try { Log.Trace("BrokerageSetupHandler.Setup(): Initializing algorithm..."); //Execute the initialize code: var isolator = new Isolator(); var initializeComplete = isolator.ExecuteWithTimeLimit(TimeSpan.FromSeconds(10), () => { try { //Set the live trading level asset/ram allocation limits. //Protects algorithm from linux killing the job by excess memory: switch (job.ServerType) { case ServerType.Server1024: algorithm.SetAssetLimits(100, 20, 10); break; case ServerType.Server2048: algorithm.SetAssetLimits(400, 50, 30); break; default: //512 algorithm.SetAssetLimits(50, 25, 15); break; } //Algorithm is live, not backtesting: algorithm.SetLiveMode(true); //Initialize the algorithm's starting date algorithm.SetDateTime(DateTime.UtcNow); //Set the source impl for the event scheduling algorithm.Schedule.SetEventSchedule(realTimeHandler); //Initialise the algorithm, get the required data: algorithm.Initialize(); } catch (Exception err) { AddInitializationError(err.Message); } }); if (!initializeComplete) { AddInitializationError("Initialization timed out."); return false; } try { // find the correct brokerage factory based on the specified brokerage in the live job packet _factory = Composer.Instance.Single<IBrokerageFactory>(factory => factory.BrokerageType.MatchesTypeName(liveJob.Brokerage)); } catch (Exception err) { Log.Error("BrokerageSetupHandler.Setup(): Error resolving brokerage factory for " + liveJob.Brokerage + ". " + err.Message); AddInitializationError("Unable to locate factory for brokerage: " + liveJob.Brokerage); } // let the world know what we're doing since logging in can take a minute resultHandler.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.LoggingIn, "Logging into brokerage..."); // initialize the correct brokerage using the resolved factory brokerage = _factory.CreateBrokerage(liveJob, algorithm); if (brokerage == null) { AddInitializationError("Failed to create instance of brokerage: " + liveJob.Brokerage); return false; } brokerage.Message += brokerageOnMessage; // set the transaction models base on the brokerage properties SetupHandler.UpdateTransactionModels(algorithm, algorithm.BrokerageModel); algorithm.Transactions.SetOrderProcessor(transactionHandler); algorithm.PostInitialize(); try { // this can fail for various reasons, such as already being logged in somewhere else brokerage.Connect(); } catch (Exception err) { Log.Error(err); AddInitializationError("Error connecting to brokerage: " + err.Message); return false; } if (!brokerage.IsConnected) { // if we're reporting that we're not connected, bail AddInitializationError("Unable to connect to brokerage."); return false; } try { // set the algorithm's cash balance for each currency var cashBalance = brokerage.GetCashBalance(); foreach (var cash in cashBalance) { Log.Trace("BrokerageSetupHandler.Setup(): Setting " + cash.Symbol + " cash to " + cash.Quantity); algorithm.SetCash(cash.Symbol, cash.Quantity, cash.ConversionRate); } } catch (Exception err) { Log.Error(err); AddInitializationError("Error getting cash balance from brokerage: " + err.Message); return false; } try { // populate the algorithm with the account's outstanding orders var openOrders = brokerage.GetOpenOrders(); foreach (var order in openOrders) { // be sure to assign order IDs such that we increment from the SecurityTransactionManager to avoid ID collisions Log.Trace("BrokerageSetupHandler.Setup(): Has open order: " + order.Symbol + " - " + order.Quantity); order.Id = algorithm.Transactions.GetIncrementOrderId(); transactionHandler.Orders.AddOrUpdate(order.Id, order, (i, o) => order); } } catch (Exception err) { Log.Error(err); AddInitializationError("Error getting open orders from brokerage: " + err.Message); return false; } try { // populate the algorithm with the account's current holdings var holdings = brokerage.GetAccountHoldings(); var minResolution = new Lazy<Resolution>(() => algorithm.Securities.Min(x => x.Value.Resolution)); foreach (var holding in holdings) { Log.Trace("BrokerageSetupHandler.Setup(): Has existing holding: " + holding); if (!algorithm.Portfolio.ContainsKey(holding.Symbol)) { Log.Trace("BrokerageSetupHandler.Setup(): Adding unrequested security: " + holding.Symbol); // for items not directly requested set leverage to 1 and at the min resolution algorithm.AddSecurity(holding.Type, holding.Symbol, minResolution.Value, null, true, 1.0m, false); } algorithm.Portfolio[holding.Symbol].SetHoldings(holding.AveragePrice, (int) holding.Quantity); algorithm.Securities[holding.Symbol].SetMarketPrice(new TradeBar { Time = DateTime.Now, Open = holding.MarketPrice, High = holding.MarketPrice, Low = holding.MarketPrice, Close = holding.MarketPrice, Volume = 0, Symbol = holding.Symbol, DataType = MarketDataType.TradeBar }); } } catch (Exception err) { Log.Error(err); AddInitializationError("Error getting account holdings from brokerage: " + err.Message); return false; } // call this after we've initialized everything from the brokerage since we may have added some holdings/currencies algorithm.Portfolio.CashBook.EnsureCurrencyDataFeeds(algorithm.Securities, algorithm.SubscriptionManager, SecurityExchangeHoursProvider.FromDataFolder()); //Set the starting portfolio value for the strategy to calculate performance: StartingPortfolioValue = algorithm.Portfolio.TotalPortfolioValue; StartingDate = DateTime.Now; } catch (Exception err) { AddInitializationError(err.Message); } finally { if (brokerage != null) { brokerage.Message -= brokerageOnMessage; } } return Errors.Count == 0; }
/// <summary> /// Runs a single backtest/live job from the job queue /// </summary> /// <param name="job">The algorithm job to be processed</param> /// <param name="manager"></param> /// <param name="assemblyPath">The path to the algorithm's assembly</param> public void Run(AlgorithmNodePacket job, AlgorithmManager manager, string assemblyPath) { var marketHoursDatabaseTask = Task.Run(() => StaticInitializations()); var algorithm = default(IAlgorithm); var algorithmManager = manager; try { //Reset thread holders. var initializeComplete = false; Thread threadResults = null; Thread threadRealTime = null; Thread threadAlphas = null; WorkerThread workerThread = null; //-> Initialize messaging system SystemHandlers.Notify.SetAuthentication(job); //-> Set the result handler type for this algorithm job, and launch the associated result thread. AlgorithmHandlers.Results.Initialize(job, SystemHandlers.Notify, SystemHandlers.Api, AlgorithmHandlers.Transactions); threadResults = new Thread(AlgorithmHandlers.Results.Run, 0) { IsBackground = true, Name = "Result Thread" }; threadResults.Start(); IBrokerage brokerage = null; DataManager dataManager = null; var synchronizer = _liveMode ? new LiveSynchronizer() : new Synchronizer(); try { // we get the mhdb before creating the algorithm instance, // since the algorithm constructor will use it var marketHoursDatabase = marketHoursDatabaseTask.Result; // start worker thread workerThread = new WorkerThread(); AlgorithmHandlers.Setup.WorkerThread = workerThread; // Save algorithm to cache, load algorithm instance: algorithm = AlgorithmHandlers.Setup.CreateAlgorithmInstance(job, assemblyPath); // Set algorithm in ILeanManager SystemHandlers.LeanManager.SetAlgorithm(algorithm); // initialize the alphas handler with the algorithm instance AlgorithmHandlers.Alphas.Initialize(job, algorithm, SystemHandlers.Notify, SystemHandlers.Api); // Initialize the brokerage IBrokerageFactory factory; brokerage = AlgorithmHandlers.Setup.CreateBrokerage(job, algorithm, out factory); var symbolPropertiesDatabase = SymbolPropertiesDatabase.FromDataFolder(); var securityService = new SecurityService(algorithm.Portfolio.CashBook, marketHoursDatabase, symbolPropertiesDatabase, (ISecurityInitializerProvider)algorithm); algorithm.Securities.SetSecurityService(securityService); dataManager = new DataManager(AlgorithmHandlers.DataFeed, new UniverseSelection( algorithm, securityService), algorithm, algorithm.TimeKeeper, marketHoursDatabase, _liveMode); AlgorithmHandlers.Results.SetDataManager(dataManager); algorithm.SubscriptionManager.SetDataManager(dataManager); synchronizer.Initialize(algorithm, dataManager); // Initialize the data feed before we initialize so he can intercept added securities/universes via events AlgorithmHandlers.DataFeed.Initialize( algorithm, job, AlgorithmHandlers.Results, AlgorithmHandlers.MapFileProvider, AlgorithmHandlers.FactorFileProvider, AlgorithmHandlers.DataProvider, dataManager, (IDataFeedTimeProvider)synchronizer); // set the order processor on the transaction manager (needs to be done before initializing BrokerageHistoryProvider) algorithm.Transactions.SetOrderProcessor(AlgorithmHandlers.Transactions); algorithm.SetOrderEventProvider(AlgorithmHandlers.Transactions); // set the history provider before setting up the algorithm var historyProvider = GetHistoryProvider(job.HistoryProvider); if (historyProvider is BrokerageHistoryProvider) { (historyProvider as BrokerageHistoryProvider).SetBrokerage(brokerage); } var historyDataCacheProvider = new ZipDataCacheProvider(AlgorithmHandlers.DataProvider, isDataEphemeral: _liveMode); historyProvider.Initialize( new HistoryProviderInitializeParameters( job, SystemHandlers.Api, AlgorithmHandlers.DataProvider, historyDataCacheProvider, AlgorithmHandlers.MapFileProvider, AlgorithmHandlers.FactorFileProvider, progress => { // send progress updates to the result handler only during initialization if (!algorithm.GetLocked() || algorithm.IsWarmingUp) { AlgorithmHandlers.Results.SendStatusUpdate(AlgorithmStatus.History, $"Processing history {progress}%..."); } } ) ); historyProvider.InvalidConfigurationDetected += (sender, args) => { AlgorithmHandlers.Results.ErrorMessage(args.Message); }; historyProvider.NumericalPrecisionLimited += (sender, args) => { AlgorithmHandlers.Results.DebugMessage(args.Message); }; historyProvider.DownloadFailed += (sender, args) => { AlgorithmHandlers.Results.ErrorMessage(args.Message, args.StackTrace); }; historyProvider.ReaderErrorDetected += (sender, args) => { AlgorithmHandlers.Results.RuntimeError(args.Message, args.StackTrace); }; algorithm.HistoryProvider = historyProvider; // initialize the default brokerage message handler algorithm.BrokerageMessageHandler = factory.CreateBrokerageMessageHandler(algorithm, job, SystemHandlers.Api); //Initialize the internal state of algorithm and job: executes the algorithm.Initialize() method. initializeComplete = AlgorithmHandlers.Setup.Setup(new SetupHandlerParameters(dataManager.UniverseSelection, algorithm, brokerage, job, AlgorithmHandlers.Results, AlgorithmHandlers.Transactions, AlgorithmHandlers.RealTime)); // set this again now that we've actually added securities AlgorithmHandlers.Results.SetAlgorithm(algorithm, AlgorithmHandlers.Setup.StartingPortfolioValue); // alpha handler needs start/end dates to determine sample step sizes AlgorithmHandlers.Alphas.OnAfterAlgorithmInitialized(algorithm); //If there are any reasons it failed, pass these back to the IDE. if (!initializeComplete || algorithm.ErrorMessages.Count > 0 || AlgorithmHandlers.Setup.Errors.Count > 0) { initializeComplete = false; //Get all the error messages: internal in algorithm and external in setup handler. var errorMessage = string.Join(",", algorithm.ErrorMessages); errorMessage += string.Join(",", AlgorithmHandlers.Setup.Errors.Select(e => { var message = e.Message; if (e.InnerException != null) { var err = _exceptionInterpreter.Value.Interpret(e.InnerException, _exceptionInterpreter.Value); message += _exceptionInterpreter.Value.GetExceptionMessageHeader(err); } return(message); })); Log.Error("Engine.Run(): " + errorMessage); AlgorithmHandlers.Results.RuntimeError(errorMessage); SystemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, errorMessage); } } catch (Exception err) { Log.Error(err); var runtimeMessage = "Algorithm.Initialize() Error: " + err.Message + " Stack Trace: " + err; AlgorithmHandlers.Results.RuntimeError(runtimeMessage, err.ToString()); SystemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, runtimeMessage); } // log the job endpoints Log.Trace("JOB HANDLERS: "); Log.Trace(" DataFeed: " + AlgorithmHandlers.DataFeed.GetType().FullName); Log.Trace(" Setup: " + AlgorithmHandlers.Setup.GetType().FullName); Log.Trace(" RealTime: " + AlgorithmHandlers.RealTime.GetType().FullName); Log.Trace(" Results: " + AlgorithmHandlers.Results.GetType().FullName); Log.Trace(" Transactions: " + AlgorithmHandlers.Transactions.GetType().FullName); Log.Trace(" Alpha: " + AlgorithmHandlers.Alphas.GetType().FullName); if (algorithm?.HistoryProvider != null) { Log.Trace(" History Provider: " + algorithm.HistoryProvider.GetType().FullName); } if (job is LiveNodePacket) { Log.Trace(" Brokerage: " + brokerage?.GetType().FullName); } //-> Using the job + initialization: load the designated handlers: if (initializeComplete) { // notify the LEAN manager that the algorithm is initialized and starting SystemHandlers.LeanManager.OnAlgorithmStart(); //-> Reset the backtest stopwatch; we're now running the algorithm. var startTime = DateTime.Now; //Set algorithm as locked; set it to live mode if we're trading live, and set it to locked for no further updates. algorithm.SetAlgorithmId(job.AlgorithmId); algorithm.SetLocked(); //Load the associated handlers for transaction and realtime events: AlgorithmHandlers.Transactions.Initialize(algorithm, brokerage, AlgorithmHandlers.Results); AlgorithmHandlers.RealTime.Setup(algorithm, job, AlgorithmHandlers.Results, SystemHandlers.Api); // wire up the brokerage message handler brokerage.Message += (sender, message) => { algorithm.BrokerageMessageHandler.Handle(message); // fire brokerage message events algorithm.OnBrokerageMessage(message); switch (message.Type) { case BrokerageMessageType.Disconnect: algorithm.OnBrokerageDisconnect(); break; case BrokerageMessageType.Reconnect: algorithm.OnBrokerageReconnect(); break; } }; //Send status to user the algorithm is now executing. AlgorithmHandlers.Results.SendStatusUpdate(AlgorithmStatus.Running); //Launch the data, transaction and realtime handlers into dedicated threads threadRealTime = new Thread(AlgorithmHandlers.RealTime.Run) { IsBackground = true, Name = "RealTime Thread" }; threadAlphas = new Thread(() => AlgorithmHandlers.Alphas.Run()) { IsBackground = true, Name = "Alpha Thread" }; //Launch the data feed, result sending, and transaction models/handlers in separate threads. threadRealTime.Start(); // RealTime scan time for time based events: threadAlphas.Start(); // Alpha thread for processing algorithm alpha insights // Result manager scanning message queue: (started earlier) AlgorithmHandlers.Results.DebugMessage( $"Launching analysis for {job.AlgorithmId} with LEAN Engine v{Globals.Version}"); try { //Create a new engine isolator class var isolator = new Isolator(); // Execute the Algorithm Code: var complete = isolator.ExecuteWithTimeLimit(AlgorithmHandlers.Setup.MaximumRuntime, algorithmManager.TimeLoopWithinLimits, () => { try { //Run Algorithm Job: // -> Using this Data Feed, // -> Send Orders to this TransactionHandler, // -> Send Results to ResultHandler. algorithmManager.Run(job, algorithm, synchronizer, AlgorithmHandlers.Transactions, AlgorithmHandlers.Results, AlgorithmHandlers.RealTime, SystemHandlers.LeanManager, AlgorithmHandlers.Alphas, isolator.CancellationToken); } catch (Exception err) { //Debugging at this level is difficult, stack trace needed. Log.Error(err); algorithm.RunTimeError = err; algorithmManager.SetStatus(AlgorithmStatus.RuntimeError); return; } Log.Trace("Engine.Run(): Exiting Algorithm Manager"); }, job.Controls.RamAllocation, workerThread: workerThread); if (!complete) { Log.Error("Engine.Main(): Failed to complete in time: " + AlgorithmHandlers.Setup.MaximumRuntime.ToString("F")); throw new Exception("Failed to complete algorithm within " + AlgorithmHandlers.Setup.MaximumRuntime.ToString("F") + " seconds. Please make it run faster."); } // Algorithm runtime error: if (algorithm.RunTimeError != null) { HandleAlgorithmError(job, algorithm.RunTimeError); } } catch (Exception err) { //Error running the user algorithm: purge datafeed, send error messages, set algorithm status to failed. HandleAlgorithmError(job, err); } // notify the LEAN manager that the algorithm has finished SystemHandlers.LeanManager.OnAlgorithmEnd(); try { var trades = algorithm.TradeBuilder.ClosedTrades; var charts = new Dictionary <string, Chart>(AlgorithmHandlers.Results.Charts); var orders = new Dictionary <int, Order>(AlgorithmHandlers.Transactions.Orders); var holdings = new Dictionary <string, Holding>(); var banner = new Dictionary <string, string>(); var statisticsResults = new StatisticsResults(); var csvTransactionsFileName = Config.Get("transaction-log"); if (!string.IsNullOrEmpty(csvTransactionsFileName)) { SaveListOfTrades(AlgorithmHandlers.Transactions, csvTransactionsFileName); } try { //Generates error when things don't exist (no charting logged, runtime errors in main algo execution) const string strategyEquityKey = "Strategy Equity"; const string equityKey = "Equity"; const string dailyPerformanceKey = "Daily Performance"; const string benchmarkKey = "Benchmark"; // make sure we've taken samples for these series before just blindly requesting them if (charts.ContainsKey(strategyEquityKey) && charts[strategyEquityKey].Series.ContainsKey(equityKey) && charts[strategyEquityKey].Series.ContainsKey(dailyPerformanceKey) && charts.ContainsKey(benchmarkKey) && charts[benchmarkKey].Series.ContainsKey(benchmarkKey) ) { var equity = charts[strategyEquityKey].Series[equityKey].Values; var performance = charts[strategyEquityKey].Series[dailyPerformanceKey].Values; var profitLoss = new SortedDictionary <DateTime, decimal>(algorithm.Transactions.TransactionRecord); var totalTransactions = algorithm.Transactions.GetOrders(x => x.Status.IsFill()).Count(); var benchmark = charts[benchmarkKey].Series[benchmarkKey].Values; statisticsResults = StatisticsBuilder.Generate(trades, profitLoss, equity, performance, benchmark, AlgorithmHandlers.Setup.StartingPortfolioValue, algorithm.Portfolio.TotalFees, totalTransactions); //Some users have $0 in their brokerage account / starting cash of $0. Prevent divide by zero errors var netReturn = AlgorithmHandlers.Setup.StartingPortfolioValue > 0 ? (algorithm.Portfolio.TotalPortfolioValue - AlgorithmHandlers.Setup.StartingPortfolioValue) / AlgorithmHandlers.Setup.StartingPortfolioValue : 0; //Add other fixed parameters. banner.Add("Unrealized", "$" + algorithm.Portfolio.TotalUnrealizedProfit.ToString("N2")); banner.Add("Fees", "-$" + algorithm.Portfolio.TotalFees.ToString("N2")); banner.Add("Net Profit", "$" + algorithm.Portfolio.TotalProfit.ToString("N2")); banner.Add("Return", netReturn.ToString("P")); banner.Add("Equity", "$" + algorithm.Portfolio.TotalPortfolioValue.ToString("N2")); } } catch (Exception err) { Log.Error(err, "Error generating statistics packet"); } //Diagnostics Completed, Send Result Packet: var totalSeconds = (DateTime.Now - startTime).TotalSeconds; var dataPoints = algorithmManager.DataPoints + algorithm.HistoryProvider.DataPointCount; if (!_liveMode) { var kps = dataPoints / (double)1000 / totalSeconds; AlgorithmHandlers.Results.DebugMessage($"Algorithm Id:({job.AlgorithmId}) completed in {totalSeconds:F2} seconds at {kps:F0}k data points per second. Processing total of {dataPoints:N0} data points."); } AlgorithmHandlers.Results.SendFinalResult(job, orders, algorithm.Transactions.TransactionRecord, holdings, algorithm.Portfolio.CashBook, statisticsResults, banner); } catch (Exception err) { Log.Error(err, "Error sending analysis results"); } //Before we return, send terminate commands to close up the threads AlgorithmHandlers.Transactions.Exit(); AlgorithmHandlers.DataFeed.Exit(); AlgorithmHandlers.RealTime.Exit(); AlgorithmHandlers.Alphas.Exit(); dataManager?.RemoveAllSubscriptions(); workerThread?.Dispose(); } //Close result handler: AlgorithmHandlers.Results.Exit(); //Wait for the threads to complete: var millisecondInterval = 10; var millisecondTotalWait = 0; while ((AlgorithmHandlers.Results.IsActive || (AlgorithmHandlers.Transactions != null && AlgorithmHandlers.Transactions.IsActive) || (AlgorithmHandlers.DataFeed != null && AlgorithmHandlers.DataFeed.IsActive) || (AlgorithmHandlers.RealTime != null && AlgorithmHandlers.RealTime.IsActive) || (AlgorithmHandlers.Alphas != null && AlgorithmHandlers.Alphas.IsActive)) && millisecondTotalWait < 30 * 1000) { Thread.Sleep(millisecondInterval); if (millisecondTotalWait % (millisecondInterval * 10) == 0) { Log.Trace("Waiting for threads to exit..."); } millisecondTotalWait += millisecondInterval; } //Terminate threads still in active state. if (threadResults != null && threadResults.IsAlive) { threadResults.Abort(); } if (threadAlphas != null && threadAlphas.IsAlive) { threadAlphas.Abort(); } if (brokerage != null) { Log.Trace("Engine.Run(): Disconnecting from brokerage..."); brokerage.Disconnect(); brokerage.Dispose(); } if (AlgorithmHandlers.Setup != null) { Log.Trace("Engine.Run(): Disposing of setup handler..."); AlgorithmHandlers.Setup.Dispose(); } Log.Trace("Engine.Main(): Analysis Completed and Results Posted."); } catch (Exception err) { Log.Error(err, "Error running algorithm"); } finally { //No matter what for live mode; make sure we've set algorithm status in the API for "not running" conditions: if (_liveMode && algorithmManager.State != AlgorithmStatus.Running && algorithmManager.State != AlgorithmStatus.RuntimeError) { SystemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, algorithmManager.State); } AlgorithmHandlers.Results.Exit(); AlgorithmHandlers.DataFeed.Exit(); AlgorithmHandlers.Transactions.Exit(); AlgorithmHandlers.RealTime.Exit(); } }
/// <summary> /// Primary entry point to setup a new algorithm /// </summary> /// <param name="algorithm">Algorithm instance</param> /// <param name="brokerage">New brokerage output instance</param> /// <param name="job">Algorithm job task</param> /// <returns>True on successfully setting up the algorithm state, or false on error.</returns> public bool Setup(IAlgorithm algorithm, out IBrokerage brokerage, AlgorithmNodePacket job) { _algorithm = algorithm; brokerage = default(IBrokerage); // verify we were given the correct job packet type var liveJob = job as LiveNodePacket; if (liveJob == null) { AddInitializationError("BrokerageSetupHandler requires a LiveNodePacket"); return(false); } // verify the brokerage was specified if (string.IsNullOrWhiteSpace(liveJob.Brokerage)) { AddInitializationError("A brokerage must be specified"); return(false); } try { Log.Trace("BrokerageSetupHandler.Setup(): Initializing algorithm..."); //Execute the initialize code: var initializeComplete = Isolator.ExecuteWithTimeLimit(TimeSpan.FromSeconds(10), () => { try { //Set the live trading level asset/ram allocation limits. //Protects algorithm from linux killing the job by excess memory: switch (job.ServerType) { case ServerType.Server1024: algorithm.SetAssetLimits(100, 20, 10); break; case ServerType.Server2048: algorithm.SetAssetLimits(400, 50, 30); break; default: //512 algorithm.SetAssetLimits(50, 25, 15); break; } //Algorithm is live, not backtesting: algorithm.SetLiveMode(true); //Initialise the algorithm, get the required data: algorithm.Initialize(); } catch (Exception err) { AddInitializationError(err.Message); } }); if (!initializeComplete) { AddInitializationError("Initialization timed out."); return(false); } try { // find the correct brokerage factory based on the specified brokerage in the live job packet _factory = Composer.Instance.Single <IBrokerageFactory>(factory => factory.BrokerageType.MatchesTypeName(liveJob.Brokerage)); } catch (Exception err) { Log.Error("BrokerageSetupHandler.Setup(): Error resolving brokerage factory for " + liveJob.Brokerage + ". " + err.Message); AddInitializationError("Unable to locate factory for brokerage: " + liveJob.Brokerage); } // let the world know what we're doing since logging in can take a minute Engine.ResultHandler.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.LoggingIn, "Logging into brokerage..."); // initialize the correct brokerage using the resolved factory brokerage = _factory.CreateBrokerage(liveJob, algorithm); // set the transaction models base on the brokerage properties SetupHandler.UpdateTransactionModels(algorithm, algorithm.BrokerageModel); try { // this can fail for various reasons, such as already being logged in somewhere else brokerage.Connect(); } catch (Exception err) { Log.Error(err); AddInitializationError("Error connecting to brokerage."); return(false); } try { // set the algorithm's cash balance for each currency var cashBalance = brokerage.GetCashBalance(); foreach (var cash in cashBalance) { Log.Trace("BrokerageSetupHandler.Setup(): Setting " + cash.Symbol + " cash to " + cash.Quantity); algorithm.SetCash(cash.Symbol, cash.Quantity, cash.ConversionRate); } } catch (Exception err) { Log.Error(err); AddInitializationError("Error getting cash balance from brokerage."); return(false); } try { // populate the algorithm with the account's outstanding orders var openOrders = brokerage.GetOpenOrders(); foreach (var order in openOrders) { // be sure to assign order IDs such that we increment from the SecurityTransactionManager to avoid ID collisions Log.Trace("BrokerageSetupHandler.Setup(): Has open order: " + order.Symbol + " - " + order.Quantity); order.Id = algorithm.Transactions.GetIncrementOrderId(); algorithm.Orders.AddOrUpdate(order.Id, order, (i, o) => order); } } catch (Exception err) { Log.Error(err); AddInitializationError("Error getting open orders from brokerage."); return(false); } try { // populate the algorithm with the account's current holdings var holdings = brokerage.GetAccountHoldings(); var minResolution = new Lazy <Resolution>(() => algorithm.Securities.Min(x => x.Value.Resolution)); foreach (var holding in holdings) { Log.Trace("BrokerageSetupHandler.Setup(): Has existing holding: " + holding); if (!algorithm.Portfolio.ContainsKey(holding.Symbol)) { Log.Trace("BrokerageSetupHandler.Setup(): Adding unrequested security: " + holding.Symbol); // for items not directly requested set leverage to 1 and at the min resolution algorithm.AddSecurity(holding.Type, holding.Symbol, minResolution.Value, true, 1.0m, false); } algorithm.Portfolio[holding.Symbol].SetHoldings(holding.AveragePrice, (int)holding.Quantity); algorithm.Securities[holding.Symbol].SetMarketPrice(DateTime.Now, new TradeBar { Time = DateTime.Now, Open = holding.MarketPrice, High = holding.MarketPrice, Low = holding.MarketPrice, Close = holding.MarketPrice, Volume = 0, Symbol = holding.Symbol, DataType = MarketDataType.TradeBar }); } } catch (Exception err) { Log.Error(err); AddInitializationError("Error getting account holdings from brokerage."); return(false); } // call this after we've initialized everything from the brokerage since we may have added some holdings/currencies algorithm.Portfolio.CashBook.EnsureCurrencyDataFeeds(algorithm.Securities, algorithm.SubscriptionManager); //Set the starting portfolio value for the strategy to calculate performance: StartingPortfolioValue = algorithm.Portfolio.TotalPortfolioValue; StartingDate = DateTime.Now; } catch (Exception err) { AddInitializationError(err.Message); } return(Errors.Count == 0); }
/// <summary> /// Setup the algorithm cash, dates and portfolio as desired. /// </summary> /// <param name="parameters">The parameters object to use</param> /// <returns>Boolean true on successfully setting up the console.</returns> public bool Setup(SetupHandlerParameters parameters) { var algorithm = parameters.Algorithm; var baseJob = parameters.AlgorithmNodePacket; var initializeComplete = false; try { //Set common variables for console programs: if (baseJob.Type == PacketType.BacktestNode) { var backtestJob = baseJob as BacktestNodePacket; if (backtestJob == null) { throw new ArgumentException("Expected BacktestNodePacket but received " + baseJob.GetType().Name); } algorithm.SetMaximumOrders(int.MaxValue); // set our parameters algorithm.SetParameters(baseJob.Parameters); algorithm.SetLiveMode(false); algorithm.SetAvailableDataTypes(GetConfiguredDataFeeds()); //Set the source impl for the event scheduling algorithm.Schedule.SetEventSchedule(parameters.RealTimeHandler); // set the option chain provider algorithm.SetOptionChainProvider(new CachingOptionChainProvider(new BacktestingOptionChainProvider())); // set the future chain provider algorithm.SetFutureChainProvider(new CachingFutureChainProvider(new BacktestingFutureChainProvider())); var isolator = new Isolator(); isolator.ExecuteWithTimeLimit(TimeSpan.FromMinutes(5), () => { //Setup Base Algorithm: algorithm.Initialize(); }, baseJob.Controls.RamAllocation, sleepIntervalMillis: 50, workerThread: WorkerThread); //Finalize Initialization algorithm.PostInitialize(); //Set the time frontier of the algorithm algorithm.SetDateTime(algorithm.StartDate.ConvertToUtc(algorithm.TimeZone)); //Construct the backtest job packet: backtestJob.PeriodStart = algorithm.StartDate; backtestJob.PeriodFinish = algorithm.EndDate; //Backtest Specific Parameters: StartingDate = backtestJob.PeriodStart; BaseSetupHandler.SetupCurrencyConversions(algorithm, parameters.UniverseSelection); StartingPortfolioValue = algorithm.Portfolio.Cash; } else { throw new Exception("The ConsoleSetupHandler is for backtests only. Use the BrokerageSetupHandler."); } } catch (Exception err) { Log.Error(err); Errors.Add(new AlgorithmSetupException("During the algorithm initialization, the following exception has occurred: ", err)); } if (Errors.Count == 0) { initializeComplete = true; } return(initializeComplete); }
/// <summary> /// Runs a single backtest/live job from the job queue /// </summary> /// <param name="job">The algorithm job to be processed</param> /// <param name="assemblyPath">The path to the algorithm's assembly</param> public void Run(AlgorithmNodePacket job, string assemblyPath) { var algorithm = default(IAlgorithm); var algorithmManager = new AlgorithmManager(_liveMode); //Start monitoring the backtest active status: var statusPing = new StateCheck.Ping(algorithmManager, _systemHandlers.Api, _algorithmHandlers.Results, _systemHandlers.Notify, job); var statusPingThread = new Thread(statusPing.Run); statusPingThread.Start(); try { //Reset thread holders. var initializeComplete = false; Thread threadFeed = null; Thread threadTransactions = null; Thread threadResults = null; Thread threadRealTime = null; //-> Initialize messaging system _systemHandlers.Notify.SetChannel(job.Channel); //-> Set the result handler type for this algorithm job, and launch the associated result thread. _algorithmHandlers.Results.Initialize(job, _systemHandlers.Notify, _systemHandlers.Api, _algorithmHandlers.DataFeed, _algorithmHandlers.Setup, _algorithmHandlers.Transactions); threadResults = new Thread(_algorithmHandlers.Results.Run, 0) { Name = "Result Thread" }; threadResults.Start(); IBrokerage brokerage = null; try { // Save algorithm to cache, load algorithm instance: algorithm = _algorithmHandlers.Setup.CreateAlgorithmInstance(assemblyPath, job.Language); // set the history provider before setting up the algorithm _algorithmHandlers.HistoryProvider.Initialize(job, progress => { // send progress updates to the result handler only during initialization if (!algorithm.GetLocked() || algorithm.IsWarmingUp) { _algorithmHandlers.Results.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.History, string.Format("Processing history {0}%...", progress)); } }); algorithm.HistoryProvider = _algorithmHandlers.HistoryProvider; // initialize the default brokerage message handler algorithm.BrokerageMessageHandler = new DefaultBrokerageMessageHandler(algorithm, job, _algorithmHandlers.Results, _systemHandlers.Api); //Initialize the internal state of algorithm and job: executes the algorithm.Initialize() method. initializeComplete = _algorithmHandlers.Setup.Setup(algorithm, out brokerage, job, _algorithmHandlers.Results, _algorithmHandlers.Transactions, _algorithmHandlers.RealTime); // set this again now that we've actually added securities _algorithmHandlers.Results.SetAlgorithm(algorithm); //If there are any reasons it failed, pass these back to the IDE. if (!initializeComplete || algorithm.ErrorMessages.Count > 0 || _algorithmHandlers.Setup.Errors.Count > 0) { initializeComplete = false; //Get all the error messages: internal in algorithm and external in setup handler. var errorMessage = String.Join(",", algorithm.ErrorMessages); errorMessage += String.Join(",", _algorithmHandlers.Setup.Errors); _algorithmHandlers.Results.RuntimeError(errorMessage); _systemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, errorMessage); } } catch (Exception err) { var runtimeMessage = "Algorithm.Initialize() Error: " + err.Message + " Stack Trace: " + err.StackTrace; _algorithmHandlers.Results.RuntimeError(runtimeMessage, err.StackTrace); _systemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, runtimeMessage); } //-> Using the job + initialization: load the designated handlers: if (initializeComplete) { //-> Reset the backtest stopwatch; we're now running the algorithm. var startTime = DateTime.Now; //Set algorithm as locked; set it to live mode if we're trading live, and set it to locked for no further updates. algorithm.SetAlgorithmId(job.AlgorithmId); algorithm.SetLocked(); //Wire up the universe selection event handler before kicking off the data feed var universeSelection = new UniverseSelection(_algorithmHandlers.DataFeed, algorithm, _liveMode); _algorithmHandlers.DataFeed.UniverseSelection += (sender, args) => universeSelection.ApplyUniverseSelection(args); //Load the associated handlers for data, transaction and realtime events: _algorithmHandlers.DataFeed.Initialize(algorithm, job, _algorithmHandlers.Results); _algorithmHandlers.Transactions.Initialize(algorithm, brokerage, _algorithmHandlers.Results); _algorithmHandlers.RealTime.Setup(algorithm, job, _algorithmHandlers.Results, _systemHandlers.Api); // wire up the brokerage message handler brokerage.Message += (sender, message) => algorithm.BrokerageMessageHandler.Handle(message); //Send status to user the algorithm is now executing. _algorithmHandlers.Results.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.Running); //Launch the data, transaction and realtime handlers into dedicated threads threadFeed = new Thread(_algorithmHandlers.DataFeed.Run) { Name = "DataFeed Thread" }; threadTransactions = new Thread(_algorithmHandlers.Transactions.Run) { Name = "Transaction Thread" }; threadRealTime = new Thread(_algorithmHandlers.RealTime.Run) { Name = "RealTime Thread" }; //Launch the data feed, result sending, and transaction models/handlers in separate threads. threadFeed.Start(); // Data feed pushing data packets into thread bridge; threadTransactions.Start(); // Transaction modeller scanning new order requests threadRealTime.Start(); // RealTime scan time for time based events: // Result manager scanning message queue: (started earlier) _algorithmHandlers.Results.DebugMessage(string.Format("Launching analysis for {0} with LEAN Engine v{1}", job.AlgorithmId, Constants.Version)); try { //Create a new engine isolator class var isolator = new Isolator(); // Execute the Algorithm Code: var complete = isolator.ExecuteWithTimeLimit(_algorithmHandlers.Setup.MaximumRuntime, algorithmManager.TimeLoopWithinLimits, () => { try { //Run Algorithm Job: // -> Using this Data Feed, // -> Send Orders to this TransactionHandler, // -> Send Results to ResultHandler. algorithmManager.Run(job, algorithm, _algorithmHandlers.DataFeed, _algorithmHandlers.Transactions, _algorithmHandlers.Results, _algorithmHandlers.RealTime, isolator.CancellationToken); } catch (Exception err) { //Debugging at this level is difficult, stack trace needed. Log.Error(err); algorithm.RunTimeError = err; algorithmManager.SetStatus(AlgorithmStatus.RuntimeError); return; } Log.Trace("Engine.Run(): Exiting Algorithm Manager"); }, job.RamAllocation); if (!complete) { Log.Error("Engine.Main(): Failed to complete in time: " + _algorithmHandlers.Setup.MaximumRuntime.ToString("F")); throw new Exception("Failed to complete algorithm within " + _algorithmHandlers.Setup.MaximumRuntime.ToString("F") + " seconds. Please make it run faster."); } // Algorithm runtime error: if (algorithm.RunTimeError != null) { throw algorithm.RunTimeError; } } catch (Exception err) { //Error running the user algorithm: purge datafeed, send error messages, set algorithm status to failed. Log.Error("Engine.Run(): Breaking out of parent try-catch: " + err.Message + " " + err.StackTrace); if (_algorithmHandlers.DataFeed != null) { _algorithmHandlers.DataFeed.Exit(); } if (_algorithmHandlers.Results != null) { var message = "Runtime Error: " + err.Message; Log.Trace("Engine.Run(): Sending runtime error to user..."); _algorithmHandlers.Results.LogMessage(message); _algorithmHandlers.Results.RuntimeError(message, err.StackTrace); _systemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, message + " Stack Trace: " + err.StackTrace); } } //Send result data back: this entire code block could be rewritten. // todo: - Split up statistics class, its enormous. // todo: - Make a dedicated Statistics.Benchmark class. // todo: - Move all creation and transmission of statistics out of primary engine loop. // todo: - Statistics.Generate(algorithm, resulthandler, transactionhandler); try { var trades = algorithm.TradeBuilder.ClosedTrades; var charts = new Dictionary <string, Chart>(_algorithmHandlers.Results.Charts); var orders = new Dictionary <int, Order>(_algorithmHandlers.Transactions.Orders); var holdings = new Dictionary <string, Holding>(); var banner = new Dictionary <string, string>(); var statisticsResults = new StatisticsResults(); try { //Generates error when things don't exist (no charting logged, runtime errors in main algo execution) const string strategyEquityKey = "Strategy Equity"; const string equityKey = "Equity"; const string dailyPerformanceKey = "Daily Performance"; const string benchmarkKey = "Benchmark"; // make sure we've taken samples for these series before just blindly requesting them if (charts.ContainsKey(strategyEquityKey) && charts[strategyEquityKey].Series.ContainsKey(equityKey) && charts[strategyEquityKey].Series.ContainsKey(dailyPerformanceKey)) { var equity = charts[strategyEquityKey].Series[equityKey].Values; var performance = charts[strategyEquityKey].Series[dailyPerformanceKey].Values; var profitLoss = new SortedDictionary <DateTime, decimal>(algorithm.Transactions.TransactionRecord); var totalTransactions = algorithm.Transactions.GetOrders(x => x.Status.IsFill()).Count(); var benchmark = charts[benchmarkKey].Series[benchmarkKey].Values; statisticsResults = StatisticsBuilder.Generate(trades, profitLoss, equity, performance, benchmark, _algorithmHandlers.Setup.StartingPortfolioValue, algorithm.Portfolio.TotalFees, totalTransactions); } } catch (Exception err) { Log.Error("Algorithm.Node.Engine(): Error generating statistics packet: " + err.Message); } //Diagnostics Completed, Send Result Packet: var totalSeconds = (DateTime.Now - startTime).TotalSeconds; var dataPoints = algorithmManager.DataPoints + _algorithmHandlers.HistoryProvider.DataPointCount; _algorithmHandlers.Results.DebugMessage( string.Format("Algorithm Id:({0}) completed in {1} seconds at {2}k data points per second. Processing total of {3} data points.", job.AlgorithmId, totalSeconds.ToString("F2"), ((dataPoints / (double)1000) / totalSeconds).ToString("F0"), dataPoints.ToString("N0"))); _algorithmHandlers.Results.SendFinalResult(job, orders, algorithm.Transactions.TransactionRecord, holdings, statisticsResults, banner); } catch (Exception err) { Log.Error("Engine.Main(): Error sending analysis result: " + err.Message + " ST >> " + err.StackTrace); } //Before we return, send terminate commands to close up the threads _algorithmHandlers.Transactions.Exit(); _algorithmHandlers.DataFeed.Exit(); _algorithmHandlers.RealTime.Exit(); } //Close result handler: _algorithmHandlers.Results.Exit(); statusPing.Exit(); //Wait for the threads to complete: var ts = Stopwatch.StartNew(); while ((_algorithmHandlers.Results.IsActive || (_algorithmHandlers.Transactions != null && _algorithmHandlers.Transactions.IsActive) || (_algorithmHandlers.DataFeed != null && _algorithmHandlers.DataFeed.IsActive) || (_algorithmHandlers.RealTime != null && _algorithmHandlers.RealTime.IsActive)) && ts.ElapsedMilliseconds < 30 * 1000) { Thread.Sleep(100); Log.Trace("Waiting for threads to exit..."); } //Terminate threads still in active state. if (threadFeed != null && threadFeed.IsAlive) { threadFeed.Abort(); } if (threadTransactions != null && threadTransactions.IsAlive) { threadTransactions.Abort(); } if (threadResults != null && threadResults.IsAlive) { threadResults.Abort(); } if (statusPingThread != null && statusPingThread.IsAlive) { statusPingThread.Abort(); } if (brokerage != null) { brokerage.Disconnect(); } if (_algorithmHandlers.Setup != null) { _algorithmHandlers.Setup.Dispose(); } Log.Trace("Engine.Main(): Analysis Completed and Results Posted."); } catch (Exception err) { Log.Error("Engine.Main(): Error running algorithm: " + err.Message + " >> " + err.StackTrace); } finally { //No matter what for live mode; make sure we've set algorithm status in the API for "not running" conditions: if (_liveMode && algorithmManager.State != AlgorithmStatus.Running && algorithmManager.State != AlgorithmStatus.RuntimeError) { _systemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, algorithmManager.State); } _algorithmHandlers.Results.Exit(); _algorithmHandlers.DataFeed.Exit(); _algorithmHandlers.Transactions.Exit(); _algorithmHandlers.RealTime.Exit(); } }
/// <summary> /// Primary entry point to setup a new algorithm /// </summary> /// <param name="algorithm">Algorithm instance</param> /// <param name="brokerage">New brokerage output instance</param> /// <param name="job">Algorithm job task</param> /// <returns>True on successfully setting up the algorithm state, or false on error.</returns> public bool Setup(IAlgorithm algorithm, out IBrokerage brokerage, AlgorithmNodePacket job) { _algorithm = algorithm; brokerage = default(IBrokerage); // verify we were given the correct job packet type var liveJob = job as LiveNodePacket; if (liveJob == null) { AddInitializationError("BrokerageSetupHandler requires a LiveNodePacket"); return(false); } // verify the brokerage was specified if (string.IsNullOrWhiteSpace(liveJob.Brokerage)) { AddInitializationError("A brokerage must be specified"); return(false); } try { //Execute the initialize code: var initializeComplete = Isolator.ExecuteWithTimeLimit(TimeSpan.FromSeconds(10), () => { try { //Set the live trading level asset/ram allocation limits. //Protects algorithm from linux killing the job by excess memory: switch (job.ServerType) { case ServerType.Server1024: algorithm.SetAssetLimits(100, 20, 10); break; case ServerType.Server2048: algorithm.SetAssetLimits(400, 50, 30); break; default: //512 algorithm.SetAssetLimits(50, 10, 5); break; } //Algorithm is live, not backtesting: algorithm.SetLiveMode(true); //Initialise the algorithm, get the required data: algorithm.Initialize(); } catch (Exception err) { AddInitializationError(err.Message); } }); if (!initializeComplete) { AddInitializationError("Failed to initialize algorithm."); return(false); } // find the correct brokerage factory based on the specified brokerage in the live job packet var brokerageFactory = Composer.Instance.Single <IBrokerageFactory>(factory => factory.BrokerageType.MatchesTypeName(liveJob.Brokerage)); // initialize the correct brokerage using the resolved factory brokerage = brokerageFactory.CreateBrokerage(liveJob, algorithm); brokerage.Connect(); // set the algorithm's cash balance for each currency var cashBalance = brokerage.GetCashBalance(); foreach (var item in cashBalance) { algorithm.SetCash(item.Key, item.Value, 0); } // populate the algorithm with the account's outstanding orders var openOrders = brokerage.GetOpenOrders(); foreach (var order in openOrders) { // be sure to assign order IDs such that we increment from the SecurityTransactionManager to avoid ID collisions order.Id = algorithm.Transactions.GetIncrementOrderId(); algorithm.Orders.AddOrUpdate(order.Id, order, (i, o) => order); } // populate the algorithm with the account's current holdings var holdings = brokerage.GetAccountHoldings(); var minResolution = new Lazy <Resolution>(() => algorithm.Securities.Min(x => x.Value.Resolution)); foreach (var holding in holdings) { if (!algorithm.Portfolio.ContainsKey(holding.Symbol)) { // for items not directly requested set leverage to 1 and at the min resolution algorithm.AddSecurity(holding.Type, holding.Symbol, minResolution.Value, true, 1.0m, false); } algorithm.Portfolio[holding.Symbol].SetHoldings(holding.AveragePrice, (int)holding.Quantity); algorithm.Securities[holding.Symbol].Update(DateTime.Now, new TradeBar { Time = DateTime.Now, Open = holding.AveragePrice, High = holding.AveragePrice, Low = holding.AveragePrice, Close = holding.AveragePrice, Volume = 0, Symbol = holding.Symbol, DataType = MarketDataType.TradeBar }); } // call this after we've initialized everything from the brokerage since we may have added some holdings/currencies algorithm.Portfolio.CashBook.EnsureCurrencyDataFeeds(algorithm.Securities, algorithm.SubscriptionManager); //Set the starting capital for the strategy to calculate performance: StartingCapital = algorithm.Portfolio.Cash; StartingDate = DateTime.Now; } catch (Exception err) { AddInitializationError(err.Message); } return(Errors.Count == 0); }
/// <summary> /// Primary entry point to setup a new algorithm /// </summary> /// <param name="parameters">The parameters object to use</param> /// <returns>True on successfully setting up the algorithm state, or false on error.</returns> public bool Setup(SetupHandlerParameters parameters) { var algorithm = parameters.Algorithm; var brokerage = parameters.Brokerage; // verify we were given the correct job packet type var liveJob = parameters.AlgorithmNodePacket as LiveNodePacket; if (liveJob == null) { AddInitializationError("BrokerageSetupHandler requires a LiveNodePacket"); return(false); } algorithm.Name = liveJob.GetAlgorithmName(); // verify the brokerage was specified if (string.IsNullOrWhiteSpace(liveJob.Brokerage)) { AddInitializationError("A brokerage must be specified"); return(false); } // attach to the message event to relay brokerage specific initialization messages EventHandler <BrokerageMessageEvent> brokerageOnMessage = (sender, args) => { if (args.Type == BrokerageMessageType.Error) { AddInitializationError($"Brokerage Error Code: {args.Code} - {args.Message}"); } }; try { // let the world know what we're doing since logging in can take a minute parameters.ResultHandler.SendStatusUpdate(AlgorithmStatus.LoggingIn, "Logging into brokerage..."); brokerage.Message += brokerageOnMessage; Log.Trace("BrokerageSetupHandler.Setup(): Connecting to brokerage..."); try { // this can fail for various reasons, such as already being logged in somewhere else brokerage.Connect(); } catch (Exception err) { Log.Error(err); AddInitializationError( $"Error connecting to brokerage: {err.Message}. " + "This may be caused by incorrect login credentials or an unsupported account type.", err); return(false); } if (!brokerage.IsConnected) { // if we're reporting that we're not connected, bail AddInitializationError("Unable to connect to brokerage."); return(false); } var message = $"{brokerage.Name} account base currency: {brokerage.AccountBaseCurrency ?? algorithm.AccountCurrency}"; var accountCurrency = brokerage.AccountBaseCurrency; if (liveJob.BrokerageData.ContainsKey(MaxAllocationLimitConfig)) { accountCurrency = Currencies.USD; message += ". Allocation limited, will use 'USD' account currency"; } Log.Trace($"BrokerageSetupHandler.Setup(): {message}"); algorithm.Debug(message); if (accountCurrency != null && accountCurrency != algorithm.AccountCurrency) { algorithm.SetAccountCurrency(accountCurrency); } Log.Trace("BrokerageSetupHandler.Setup(): Initializing algorithm..."); parameters.ResultHandler.SendStatusUpdate(AlgorithmStatus.Initializing, "Initializing algorithm..."); //Execute the initialize code: var controls = liveJob.Controls; var isolator = new Isolator(); var initializeComplete = isolator.ExecuteWithTimeLimit(TimeSpan.FromSeconds(300), () => { try { //Set the default brokerage model before initialize algorithm.SetBrokerageModel(_factory.GetBrokerageModel(algorithm.Transactions)); //Margin calls are disabled by default in live mode algorithm.Portfolio.MarginCallModel = MarginCallModel.Null; //Set our parameters algorithm.SetParameters(liveJob.Parameters); algorithm.SetAvailableDataTypes(BaseSetupHandler.GetConfiguredDataFeeds()); //Algorithm is live, not backtesting: algorithm.SetLiveMode(true); //Initialize the algorithm's starting date algorithm.SetDateTime(DateTime.UtcNow); //Set the source impl for the event scheduling algorithm.Schedule.SetEventSchedule(parameters.RealTimeHandler); var optionChainProvider = Composer.Instance.GetPart <IOptionChainProvider>(); if (optionChainProvider == null) { optionChainProvider = new CachingOptionChainProvider(new LiveOptionChainProvider()); } // set the option chain provider algorithm.SetOptionChainProvider(optionChainProvider); var futureChainProvider = Composer.Instance.GetPart <IFutureChainProvider>(); if (futureChainProvider == null) { futureChainProvider = new CachingFutureChainProvider(new LiveFutureChainProvider()); } // set the future chain provider algorithm.SetFutureChainProvider(futureChainProvider); // set the object store algorithm.SetObjectStore(parameters.ObjectStore); // If we're going to receive market data from IB, // set the default subscription limit to 100, // algorithms can override this setting in the Initialize method if (brokerage is InteractiveBrokersBrokerage && liveJob.DataQueueHandler.EndsWith("InteractiveBrokersBrokerage")) { algorithm.Settings.DataSubscriptionLimit = 100; } //Initialise the algorithm, get the required data: algorithm.Initialize(); if (liveJob.Brokerage != "PaperBrokerage") { //Zero the CashBook - we'll populate directly from brokerage foreach (var kvp in algorithm.Portfolio.CashBook) { kvp.Value.SetAmount(0); } } } catch (Exception err) { AddInitializationError(err.ToString(), err); } }, controls.RamAllocation, sleepIntervalMillis: 100); // entire system is waiting on this, so be as fast as possible if (Errors.Count != 0) { // if we already got an error just exit right away return(false); } if (!initializeComplete) { AddInitializationError("Initialization timed out."); return(false); } if (!LoadCashBalance(brokerage, algorithm)) { return(false); } if (!LoadExistingHoldingsAndOrders(brokerage, algorithm, parameters)) { return(false); } //Finalize Initialization algorithm.PostInitialize(); BaseSetupHandler.SetupCurrencyConversions(algorithm, parameters.UniverseSelection); if (algorithm.Portfolio.TotalPortfolioValue == 0) { algorithm.Debug("Warning: No cash balances or holdings were found in the brokerage account."); } string maxCashLimitStr; if (liveJob.BrokerageData.TryGetValue(MaxAllocationLimitConfig, out maxCashLimitStr)) { var maxCashLimit = decimal.Parse(maxCashLimitStr, NumberStyles.Any, CultureInfo.InvariantCulture); // If allocation exceeded by more than $10,000; block deployment if (algorithm.Portfolio.TotalPortfolioValue > (maxCashLimit + 10000m)) { var exceptionMessage = $"TotalPortfolioValue '{algorithm.Portfolio.TotalPortfolioValue}' exceeds allocation limit '{maxCashLimit}'"; algorithm.Debug(exceptionMessage); throw new ArgumentException(exceptionMessage); } } //Set the starting portfolio value for the strategy to calculate performance: StartingPortfolioValue = algorithm.Portfolio.TotalPortfolioValue; StartingDate = DateTime.Now; // we set the free portfolio value based on the initial total value and the free percentage value algorithm.Settings.FreePortfolioValue = algorithm.Portfolio.TotalPortfolioValue * algorithm.Settings.FreePortfolioValuePercentage; } catch (Exception err) { AddInitializationError(err.ToString(), err); } finally { if (brokerage != null) { brokerage.Message -= brokerageOnMessage; } } return(Errors.Count == 0); }
/// <summary> /// Setup the algorithm cash, dates and data subscriptions as desired. /// </summary> /// <param name="algorithm">Algorithm instance</param> /// <param name="brokerage">Brokerage instance</param> /// <param name="baseJob">Algorithm job</param> /// <param name="resultHandler">The configured result handler</param> /// <param name="transactionHandler">The configurated transaction handler</param> /// <returns>Boolean true on successfully initializing the algorithm</returns> public bool Setup(IAlgorithm algorithm, out IBrokerage brokerage, AlgorithmNodePacket baseJob, IResultHandler resultHandler, ITransactionHandler transactionHandler) { var job = baseJob as BacktestNodePacket; if (job == null) { throw new ArgumentException("Expected BacktestNodePacket but received " + baseJob.GetType().Name); } Log.Trace(string.Format("BacktestingSetupHandler.Setup(): Setting up job: Plan: {0}, UID: {1}, PID: {2}, Version: {3}, Source: {4}", job.UserPlan, job.UserId, job.ProjectId, job.Version, job.RequestSource)); brokerage = null; if (algorithm == null) { Errors.Add("Could not create instance of algorithm"); return false; } //Make sure the algorithm start date ok. if (job.PeriodStart == default(DateTime)) { Errors.Add("Algorithm start date was never set"); return false; } //Execute the initialize code: var isolator = new Isolator(); var initializeComplete = isolator.ExecuteWithTimeLimit(TimeSpan.FromSeconds(10), () => { try { //Algorithm is backtesting, not live: algorithm.SetLiveMode(false); //Set the backtest level asset ram allocation limits algorithm.SetAssetLimits(500, 100, 30); //Set the algorithm time before we even initialize: algorithm.SetDateTime(job.PeriodStart); //Initialise the algorithm, get the required data: algorithm.Initialize(); //Add currency data feeds that weren't explicity added in Initialize algorithm.Portfolio.CashBook.EnsureCurrencyDataFeeds(algorithm.Securities, algorithm.SubscriptionManager); } catch (Exception err) { Errors.Add("Failed to initialize algorithm: Initialize(): " + err.Message); } }); //Before continuing, detect if this is ready: if (!initializeComplete) return false; // this needs to be done after algorithm initialization brokerage = new BacktestingBrokerage(algorithm); SetupHandler.UpdateTransactionModels(algorithm, algorithm.BrokerageModel); algorithm.Transactions.SetOrderProcessor(transactionHandler); //Calculate the max runtime for the strategy _maxRuntime = GetMaximumRuntime(job.PeriodStart, job.PeriodFinish, algorithm.SubscriptionManager.Count); //Get starting capital: _startingCaptial = algorithm.Portfolio.Cash; //Max Orders: 10k per backtest: if (job.UserPlan == UserPlan.Free) { _maxOrders = 10000; } else { _maxOrders = int.MaxValue; _maxRuntime += _maxRuntime; } //Set back to the algorithm, algorithm.SetMaximumOrders(_maxOrders); //Starting date of the algorithm: _startingDate = job.PeriodStart; //Put into log for debugging: Log.Trace("SetUp Backtesting: User: "******" ProjectId: " + job.ProjectId + " AlgoId: " + job.AlgorithmId); Log.Trace("Dates: Start: " + job.PeriodStart.ToShortDateString() + " End: " + job.PeriodFinish.ToShortDateString() + " Cash: " + _startingCaptial.ToString("C")); if (Errors.Count > 0) { initializeComplete = false; } return initializeComplete; }
/// <summary> /// Primary entry point to setup a new algorithm /// </summary> /// <param name="algorithm">Algorithm instance</param> /// <param name="brokerage">New brokerage output instance</param> /// <param name="job">Algorithm job task</param> /// <param name="resultHandler">The configured result handler</param> /// <param name="transactionHandler">The configurated transaction handler</param> /// <param name="realTimeHandler">The configured real time handler</param> /// <returns>True on successfully setting up the algorithm state, or false on error.</returns> public bool Setup(IAlgorithm algorithm, out IBrokerage brokerage, AlgorithmNodePacket job, IResultHandler resultHandler, ITransactionHandler transactionHandler, IRealTimeHandler realTimeHandler) { _algorithm = algorithm; brokerage = default(IBrokerage); // verify we were given the correct job packet type var liveJob = job as LiveNodePacket; if (liveJob == null) { AddInitializationError("BrokerageSetupHandler requires a LiveNodePacket"); return(false); } // verify the brokerage was specified if (string.IsNullOrWhiteSpace(liveJob.Brokerage)) { AddInitializationError("A brokerage must be specified"); return(false); } // attach to the message event to relay brokerage specific initialization messages EventHandler <BrokerageMessageEvent> brokerageOnMessage = (sender, args) => { if (args.Type == BrokerageMessageType.Error) { AddInitializationError(string.Format("Brokerage Error Code: {0} - {1}", args.Code, args.Message)); } }; try { Log.Trace("BrokerageSetupHandler.Setup(): Initializing algorithm..."); resultHandler.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.Initializing, "Initializing algorithm..."); //Execute the initialize code: var isolator = new Isolator(); var initializeComplete = isolator.ExecuteWithTimeLimit(TimeSpan.FromSeconds(300), () => { try { //Set the live trading level asset/ram allocation limits. //Protects algorithm from linux killing the job by excess memory: switch (job.ServerType) { case ServerType.Server1024: algorithm.SetAssetLimits(100, 20, 10); break; case ServerType.Server2048: algorithm.SetAssetLimits(400, 50, 30); break; default: //512 algorithm.SetAssetLimits(50, 25, 15); break; } //Algorithm is live, not backtesting: algorithm.SetLiveMode(true); //Initialize the algorithm's starting date algorithm.SetDateTime(DateTime.UtcNow); //Set the source impl for the event scheduling algorithm.Schedule.SetEventSchedule(realTimeHandler); //Initialise the algorithm, get the required data: algorithm.Initialize(); } catch (Exception err) { AddInitializationError(err.Message); } }); if (!initializeComplete) { AddInitializationError("Initialization timed out."); return(false); } try { // find the correct brokerage factory based on the specified brokerage in the live job packet _factory = Composer.Instance.Single <IBrokerageFactory>(factory => factory.BrokerageType.MatchesTypeName(liveJob.Brokerage)); } catch (Exception err) { Log.Error("BrokerageSetupHandler.Setup(): Error resolving brokerage factory for " + liveJob.Brokerage + ". " + err.Message); AddInitializationError("Unable to locate factory for brokerage: " + liveJob.Brokerage); } // let the world know what we're doing since logging in can take a minute resultHandler.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.LoggingIn, "Logging into brokerage..."); // initialize the correct brokerage using the resolved factory brokerage = _factory.CreateBrokerage(liveJob, algorithm); if (brokerage == null) { AddInitializationError("Failed to create instance of brokerage: " + liveJob.Brokerage); return(false); } brokerage.Message += brokerageOnMessage; // set the transaction and settlement models based on the brokerage properties SetupHandler.UpdateModels(algorithm, algorithm.BrokerageModel); algorithm.Transactions.SetOrderProcessor(transactionHandler); algorithm.PostInitialize(); Log.Trace("BrokerageSetupHandler.Setup(): Connecting to brokerage..."); try { // this can fail for various reasons, such as already being logged in somewhere else brokerage.Connect(); } catch (Exception err) { Log.Error(err); AddInitializationError(string.Format("Error connecting to brokerage: {0}. " + "This may be caused by incorrect login credentials or an unsupported account type.", err.Message)); return(false); } if (!brokerage.IsConnected) { // if we're reporting that we're not connected, bail AddInitializationError("Unable to connect to brokerage."); return(false); } Log.Trace("BrokerageSetupHandler.Setup(): Fetching cash balance from brokerage..."); try { // set the algorithm's cash balance for each currency var cashBalance = brokerage.GetCashBalance(); foreach (var cash in cashBalance) { Log.Trace("BrokerageSetupHandler.Setup(): Setting " + cash.Symbol + " cash to " + cash.Quantity); algorithm.SetCash(cash.Symbol, cash.Quantity, cash.ConversionRate); } } catch (Exception err) { Log.Error(err); AddInitializationError("Error getting cash balance from brokerage: " + err.Message); return(false); } Log.Trace("BrokerageSetupHandler.Setup(): Fetching open orders from brokerage..."); try { // populate the algorithm with the account's outstanding orders var openOrders = brokerage.GetOpenOrders(); foreach (var order in openOrders) { // be sure to assign order IDs such that we increment from the SecurityTransactionManager to avoid ID collisions Log.Trace("BrokerageSetupHandler.Setup(): Has open order: " + order.Symbol + " - " + order.Quantity); order.Id = algorithm.Transactions.GetIncrementOrderId(); transactionHandler.Orders.AddOrUpdate(order.Id, order, (i, o) => order); } } catch (Exception err) { Log.Error(err); AddInitializationError("Error getting open orders from brokerage: " + err.Message); return(false); } Log.Trace("BrokerageSetupHandler.Setup(): Fetching holdings from brokerage..."); try { // populate the algorithm with the account's current holdings var holdings = brokerage.GetAccountHoldings(); var supportedSecurityTypes = new HashSet <SecurityType> { SecurityType.Equity, SecurityType.Forex }; var minResolution = new Lazy <Resolution>(() => algorithm.Securities.Min(x => x.Value.Resolution)); foreach (var holding in holdings) { var symbol = new Symbol(holding.Symbol); Log.Trace("BrokerageSetupHandler.Setup(): Has existing holding: " + holding); // verify existing holding security type if (!supportedSecurityTypes.Contains(holding.Type)) { Log.Error("BrokerageSetupHandler.Setup(): Unsupported security type: " + holding.Type + "-" + holding.Symbol.ToUpper()); AddInitializationError("Found unsupported security type in existing brokerage holdings: " + holding.Type + ". " + "QuantConnect currently supports the following security types: " + string.Join(",", supportedSecurityTypes)); // keep aggregating these errors continue; } if (!algorithm.Portfolio.ContainsKey(symbol)) { Log.Trace("BrokerageSetupHandler.Setup(): Adding unrequested security: " + holding.Symbol); // for items not directly requested set leverage to 1 and at the min resolution algorithm.AddSecurity(holding.Type, symbol, minResolution.Value, null, true, 1.0m, false); } algorithm.Portfolio[symbol].SetHoldings(holding.AveragePrice, (int)holding.Quantity); algorithm.Securities[symbol].SetMarketPrice(new TradeBar { Time = DateTime.Now, Open = holding.MarketPrice, High = holding.MarketPrice, Low = holding.MarketPrice, Close = holding.MarketPrice, Volume = 0, Symbol = symbol, DataType = MarketDataType.TradeBar }); } } catch (Exception err) { Log.Error(err); AddInitializationError("Error getting account holdings from brokerage: " + err.Message); return(false); } Log.Trace("BrokerageSetupHandler.Setup(): Ensuring currency data feeds present..."); // call this after we've initialized everything from the brokerage since we may have added some holdings/currencies algorithm.Portfolio.CashBook.EnsureCurrencyDataFeeds(algorithm.Securities, algorithm.SubscriptionManager, SecurityExchangeHoursProvider.FromDataFolder()); //Set the starting portfolio value for the strategy to calculate performance: StartingPortfolioValue = algorithm.Portfolio.TotalPortfolioValue; StartingDate = DateTime.Now; } catch (Exception err) { AddInitializationError(err.Message); } finally { if (brokerage != null) { brokerage.Message -= brokerageOnMessage; } } return(Errors.Count == 0); }
/// <summary> /// Setup the algorithm cash, dates and data subscriptions as desired. /// </summary> /// <param name="algorithm">Algorithm instance</param> /// <param name="brokerage">Brokerage instance</param> /// <param name="baseJob">Algorithm job</param> /// <param name="resultHandler">The configured result handler</param> /// <param name="transactionHandler">The configurated transaction handler</param> /// <param name="realTimeHandler">The configured real time handler</param> /// <returns>Boolean true on successfully initializing the algorithm</returns> public bool Setup(IAlgorithm algorithm, IBrokerage brokerage, AlgorithmNodePacket baseJob, IResultHandler resultHandler, ITransactionHandler transactionHandler, IRealTimeHandler realTimeHandler) { var job = baseJob as BacktestNodePacket; if (job == null) { throw new ArgumentException("Expected BacktestNodePacket but received " + baseJob.GetType().Name); } Log.Trace(string.Format("BacktestingSetupHandler.Setup(): Setting up job: Plan: {0}, UID: {1}, PID: {2}, Version: {3}, Source: {4}", job.UserPlan, job.UserId, job.ProjectId, job.Version, job.RequestSource)); if (algorithm == null) { Errors.Add("Could not create instance of algorithm"); return false; } //Make sure the algorithm start date ok. if (job.PeriodStart == default(DateTime)) { Errors.Add("Algorithm start date was never set"); return false; } var controls = job.Controls; var isolator = new Isolator(); var initializeComplete = isolator.ExecuteWithTimeLimit(TimeSpan.FromMinutes(5), () => { try { //Set our parameters algorithm.SetParameters(job.Parameters); //Algorithm is backtesting, not live: algorithm.SetLiveMode(false); //Set the algorithm time before we even initialize: algorithm.SetDateTime(job.PeriodStart.ConvertToUtc(algorithm.TimeZone)); //Set the source impl for the event scheduling algorithm.Schedule.SetEventSchedule(realTimeHandler); //Initialise the algorithm, get the required data: algorithm.Initialize(); } catch (Exception err) { Errors.Add("Failed to initialize algorithm: Initialize(): " + err.Message); } }); //Before continuing, detect if this is ready: if (!initializeComplete) return false; algorithm.Transactions.SetOrderProcessor(transactionHandler); algorithm.PostInitialize(); //Calculate the max runtime for the strategy _maxRuntime = GetMaximumRuntime(job.PeriodStart, job.PeriodFinish, algorithm.SubscriptionManager.Count); //Get starting capital: _startingCaptial = algorithm.Portfolio.Cash; //Max Orders: 10k per backtest: if (job.UserPlan == UserPlan.Free) { _maxOrders = 10000; } else { _maxOrders = int.MaxValue; _maxRuntime += _maxRuntime; } //Set back to the algorithm, algorithm.SetMaximumOrders(_maxOrders); //Starting date of the algorithm: _startingDate = job.PeriodStart; //Put into log for debugging: Log.Trace("SetUp Backtesting: User: "******" ProjectId: " + job.ProjectId + " AlgoId: " + job.AlgorithmId); Log.Trace("Dates: Start: " + job.PeriodStart.ToShortDateString() + " End: " + job.PeriodFinish.ToShortDateString() + " Cash: " + _startingCaptial.ToString("C")); if (Errors.Count > 0) { initializeComplete = false; } return initializeComplete; }
/// <summary> /// Creates a new instance of the class in the library, safely. /// </summary> /// <param name="assemblyPath">Location of the DLL</param> /// <param name="algorithmInstance">Output algorithm instance</param> /// <param name="errorMessage">Output error message on failure</param> /// <returns>bool success</returns> public bool TryCreateAlgorithmInstanceWithIsolator(string assemblyPath, out IAlgorithm algorithmInstance, out string errorMessage) { IAlgorithm instance = null; var error = string.Empty; var success = false; var isolator = new Isolator(); var complete = isolator.ExecuteWithTimeLimit(_loaderTimeLimit, () => { success = TryCreateAlgorithmInstance(assemblyPath, out instance, out error); }); algorithmInstance = instance; errorMessage = error; // if the isolator stopped us early add that to our error message if (!complete) { errorMessage = "Failed to create algorithm instance within 10 seconds. Try re-building algorithm. " + error; } return complete && success && algorithmInstance != null; }
/******************************************************** * CLASS METHODS *********************************************************/ /// <summary> /// Primary Analysis Thread: /// </summary> public static void Main(string[] args) { //Initialize: AlgorithmNodePacket job = null; var timer = Stopwatch.StartNew(); var algorithm = default(IAlgorithm); _version = DateTime.ParseExact(Config.Get("version", DateTime.Now.ToString(DateFormat.UI)), DateFormat.UI, CultureInfo.InvariantCulture); //Name thread for the profiler: Thread.CurrentThread.Name = "Algorithm Analysis Thread"; Log.Trace("Engine.Main(): LEAN ALGORITHMIC TRADING ENGINE v" + _version); Log.Trace("Engine.Main(): Started " + DateTime.Now.ToShortTimeString()); Log.Trace("Engine.Main(): Memory " + OS.ApplicationMemoryUsed + "Mb-App " + +OS.TotalPhysicalMemoryUsed + "Mb-Used " + OS.TotalPhysicalMemory + "Mb-Total"); //Import external libraries specific to physical server location (cloud/local) var catalog = new AggregateCatalog(); catalog.Catalogs.Add(new DirectoryCatalog(@"../../Extensions")); var container = new CompositionContainer(catalog); try { // grab the right export based on configuration Notify = container.GetExportedValueByTypeName <IMessagingHandler>(Config.Get("messaging-handler")); Queue = container.GetExportedValueByTypeName <IQueueHandler>(Config.Get("queue-handler")); Api = container.GetExportedValueByTypeName <IApi>(Config.Get("api-handler")); } catch (CompositionException compositionException) { Log.Error("Engine.Main(): Failed to load library: " + compositionException); } //Setup packeting, queue and controls system: These don't do much locally. Api.Initialize(); Notify.Initialize(); Queue.Initialize(_liveMode); //Start monitoring the backtest active status: var statusPingThread = new Thread(StateCheck.Ping.Run); statusPingThread.Start(); do { try { //Reset algo manager internal variables preparing for a new algorithm. AlgorithmManager.ResetManager(); //Reset thread holders. var initializeComplete = false; Thread threadFeed = null; Thread threadTransactions = null; Thread threadResults = null; Thread threadRealTime = null; //-> Pull job from QuantConnect job queue, or, pull local build: var algorithmPath = ""; job = Queue.NextJob(out algorithmPath); // Blocking. //-> Initialize messaging system Notify.SetChannel(job.Channel); //-> Reset the backtest stopwatch; we're now running the algorithm. timer.Restart(); //-> Create SetupHandler to configure internal algorithm state: SetupHandler = GetSetupHandler(job.SetupEndpoint); //-> Set the result handler type for this algorithm job, and launch the associated result thread. ResultHandler = GetResultHandler(job); threadResults = new Thread(ResultHandler.Run, 0) { Name = "Result Thread" }; threadResults.Start(); try { // Save algorithm to cache, load algorithm instance: algorithm = SetupHandler.CreateAlgorithmInstance(algorithmPath); //Initialize the internal state of algorithm and job: executes the algorithm.Initialize() method. initializeComplete = SetupHandler.Setup(algorithm, out _brokerage, job); //If there are any reasons it failed, pass these back to the IDE. if (!initializeComplete || algorithm.ErrorMessages.Count > 0 || SetupHandler.Errors.Count > 0) { initializeComplete = false; //Get all the error messages: internal in algorithm and external in setup handler. var errorMessage = String.Join(",", algorithm.ErrorMessages); errorMessage += String.Join(",", SetupHandler.Errors); throw new Exception(errorMessage); } } catch (Exception err) { var runtimeMessage = "Algorithm.Initialize() Error: " + err.Message + " Stack Trace: " + err.StackTrace; ResultHandler.RuntimeError(runtimeMessage, err.StackTrace); Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, runtimeMessage); } //-> Using the job + initialization: load the designated handlers: if (initializeComplete) { //Set algorithm as locked; set it to live mode if we're trading live, and set it to locked for no further updates. algorithm.SetAlgorithmId(job.AlgorithmId); algorithm.SetLiveMode(LiveMode); algorithm.SetLocked(); //Load the associated handlers for data, transaction and realtime events: ResultHandler.SetAlgorithm(algorithm); DataFeed = GetDataFeedHandler(algorithm, job); TransactionHandler = GetTransactionHandler(algorithm, _brokerage, ResultHandler, job); RealTimeHandler = GetRealTimeHandler(algorithm, _brokerage, DataFeed, ResultHandler, job); //Set the error handlers for the brokerage asynchronous errors. SetupHandler.SetupErrorHandler(ResultHandler, _brokerage); //Send status to user the algorithm is now executing. ResultHandler.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.Running); //Launch the data, transaction and realtime handlers into dedicated threads threadFeed = new Thread(DataFeed.Run, 0) { Name = "DataFeed Thread" }; threadTransactions = new Thread(TransactionHandler.Run, 0) { Name = "Transaction Thread" }; threadRealTime = new Thread(RealTimeHandler.Run, 0) { Name = "RealTime Thread" }; //Launch the data feed, result sending, and transaction models/handlers in separate threads. threadFeed.Start(); // Data feed pushing data packets into thread bridge; threadTransactions.Start(); // Transaction modeller scanning new order requests threadRealTime.Start(); // RealTime scan time for time based events: // Result manager scanning message queue: (started earlier) try { // Execute the Algorithm Code: var complete = Isolator.ExecuteWithTimeLimit(SetupHandler.MaximumRuntime, () => { try { //Run Algorithm Job: // -> Using this Data Feed, // -> Send Orders to this TransactionHandler, // -> Send Results to ResultHandler. AlgorithmManager.Run(job, algorithm, DataFeed, TransactionHandler, ResultHandler, SetupHandler, RealTimeHandler); } catch (Exception err) { //Debugging at this level is difficult, stack trace needed. Log.Error("Engine.Run(): Error in Algo Manager: " + err.Message + " ST >> " + err.StackTrace); } Log.Trace("Engine.Run(): Exiting Algorithm Manager"); }, MaximumRamAllocation); if (!complete) { Log.Error("Engine.Main(): Failed to complete in time: " + SetupHandler.MaximumRuntime.ToString("F")); throw new Exception("Failed to complete algorithm within " + SetupHandler.MaximumRuntime.ToString("F") + " seconds. Please make it run faster."); } // Algorithm runtime error: if (AlgorithmManager.RunTimeError != null) { throw AlgorithmManager.RunTimeError; } } catch (Exception err) { //Error running the user algorithm: purge datafeed, send error messages, set algorithm status to failed. Log.Error("Engine.Run(): Breaking out of parent try-catch: " + err.Message + " " + err.StackTrace); if (DataFeed != null) { DataFeed.Exit(); } if (ResultHandler != null) { var message = "Runtime Error: " + err.Message; Log.Trace("Engine.Run(): Sending runtime error to user..."); ResultHandler.LogMessage(message); ResultHandler.RuntimeError(message, err.StackTrace); Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, message + " Stack Trace: " + err.StackTrace); } } //Send result data back: this entire code block could be rewritten. // todo: - Split up statistics class, its enormous. // todo: - Make a dedicated Statistics.Benchmark class. // todo: - Move all creation and transmission of statistics out of primary engine loop. // todo: - Statistics.Generate(algorithm, resulthandler, transactionhandler); try { var charts = new Dictionary <string, Chart>(ResultHandler.Charts); var orders = new Dictionary <int, Order>(algorithm.Transactions.Orders); var holdings = new Dictionary <string, Holding>(); var statistics = new Dictionary <string, string>(); var banner = new Dictionary <string, string>(); try { //Generates error when things don't exist (no charting logged, runtime errors in main algo execution) const string strategyEquityKey = "Strategy Equity"; const string equityKey = "Equity"; const string dailyPerformanceKey = "Daily Performance"; // make sure we've taken samples for these series before just blindly requesting them if (charts.ContainsKey(strategyEquityKey) && charts[strategyEquityKey].Series.ContainsKey(equityKey) && charts[strategyEquityKey].Series.ContainsKey(dailyPerformanceKey)) { var equity = charts[strategyEquityKey].Series[equityKey].Values; var performance = charts[strategyEquityKey].Series[dailyPerformanceKey].Values; var profitLoss = new SortedDictionary <DateTime, decimal>(algorithm.Transactions.TransactionRecord); statistics = Statistics.Statistics.Generate(equity, profitLoss, performance, SetupHandler.StartingCapital, 252); } } catch (Exception err) { Log.Error("Algorithm.Node.Engine(): Error generating statistics packet: " + err.Message); } //Diagnostics Completed, Send Result Packet: ResultHandler.DebugMessage("Algorithm Id:(" + job.AlgorithmId + ") completed analysis in " + timer.Elapsed.TotalSeconds.ToString("F2") + " seconds"); ResultHandler.SendFinalResult(job, orders, algorithm.Transactions.TransactionRecord, holdings, statistics, banner); } catch (Exception err) { Log.Error("Engine.Main(): Error sending analysis result: " + err.Message + " ST >> " + err.StackTrace); } //Before we return, send terminate commands to close up the threads timer.Stop(); //Algorithm finished running. TransactionHandler.Exit(); DataFeed.Exit(); RealTimeHandler.Exit(); } //Close result handler: ResultHandler.Exit(); //Wait for the threads to complete: var ts = Stopwatch.StartNew(); while ((ResultHandler.IsActive || (TransactionHandler != null && TransactionHandler.IsActive) || (DataFeed != null && DataFeed.IsActive)) && ts.ElapsedMilliseconds < 30 * 1000) { Thread.Sleep(100); Log.Trace("Waiting for threads to exit..."); } if (threadFeed != null && threadFeed.IsAlive) { threadFeed.Abort(); } if (threadTransactions != null && threadTransactions.IsAlive) { threadTransactions.Abort(); } if (threadResults != null && threadResults.IsAlive) { threadResults.Abort(); } Log.Trace("Engine.Main(): Analysis Completed and Results Posted."); } catch (Exception err) { Log.Error("Engine.Main(): Error running algorithm: " + err.Message + " >> " + err.StackTrace); } finally { //Delete the message from the job queue: Queue.AcknowledgeJob(job); Log.Trace("Engine.Main(): Packet removed from queue: " + job.AlgorithmId); //No matter what for live mode; make sure we've set algorithm status in the API for "not running" conditions: if (LiveMode && AlgorithmManager.State != AlgorithmStatus.Running && AlgorithmManager.State != AlgorithmStatus.RuntimeError) { Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmManager.State); } //Attempt to clean up ram usage: GC.Collect(); } //If we're running locally will execute just once. } while (!IsLocal); // Send the exit signal and then kill the thread StateCheck.Ping.Exit(); // Make the console window pause so we can read log output before exiting and killing the application completely Console.ReadKey(); //Finally if ping thread still not complete, kill. if (statusPingThread != null && statusPingThread.IsAlive) { statusPingThread.Abort(); } }
/// <summary> /// Primary entry point to setup a new algorithm /// </summary> /// <param name="algorithm">Algorithm instance</param> /// <param name="brokerage">New brokerage output instance</param> /// <param name="job">Algorithm job task</param> /// <param name="resultHandler">The configured result handler</param> /// <param name="transactionHandler">The configurated transaction handler</param> /// <param name="realTimeHandler">The configured real time handler</param> /// <returns>True on successfully setting up the algorithm state, or false on error.</returns> public bool Setup(IAlgorithm algorithm, IBrokerage brokerage, AlgorithmNodePacket job, IResultHandler resultHandler, ITransactionHandler transactionHandler, IRealTimeHandler realTimeHandler) { _algorithm = algorithm; // verify we were given the correct job packet type var liveJob = job as LiveNodePacket; if (liveJob == null) { AddInitializationError("BrokerageSetupHandler requires a LiveNodePacket"); return(false); } // verify the brokerage was specified if (string.IsNullOrWhiteSpace(liveJob.Brokerage)) { AddInitializationError("A brokerage must be specified"); return(false); } // attach to the message event to relay brokerage specific initialization messages EventHandler <BrokerageMessageEvent> brokerageOnMessage = (sender, args) => { if (args.Type == BrokerageMessageType.Error) { AddInitializationError(string.Format("Brokerage Error Code: {0} - {1}", args.Code, args.Message)); } }; try { Log.Trace("BrokerageSetupHandler.Setup(): Initializing algorithm..."); resultHandler.SendStatusUpdate(AlgorithmStatus.Initializing, "Initializing algorithm..."); //Execute the initialize code: var controls = job.Controls; var isolator = new Isolator(); var initializeComplete = isolator.ExecuteWithTimeLimit(TimeSpan.FromSeconds(300), () => { try { //Set the default brokerage model before initialize algorithm.SetBrokerageModel(_factory.BrokerageModel); //Set our default markets algorithm.SetDefaultMarkets(_factory.DefaultMarkets.ToDictionary()); //Set our parameters algorithm.SetParameters(job.Parameters); //Algorithm is live, not backtesting: algorithm.SetLiveMode(true); //Initialize the algorithm's starting date algorithm.SetDateTime(DateTime.UtcNow); //Set the source impl for the event scheduling algorithm.Schedule.SetEventSchedule(realTimeHandler); //Initialise the algorithm, get the required data: algorithm.Initialize(); if (liveJob.Brokerage != "PaperBrokerage") { //Zero the CashBook - we'll populate directly from brokerage foreach (var kvp in algorithm.Portfolio.CashBook) { kvp.Value.SetAmount(0); } } } catch (Exception err) { AddInitializationError(err.Message); } }); if (!initializeComplete) { AddInitializationError("Initialization timed out."); return(false); } // let the world know what we're doing since logging in can take a minute resultHandler.SendStatusUpdate(AlgorithmStatus.LoggingIn, "Logging into brokerage..."); brokerage.Message += brokerageOnMessage; algorithm.Transactions.SetOrderProcessor(transactionHandler); algorithm.PostInitialize(); Log.Trace("BrokerageSetupHandler.Setup(): Connecting to brokerage..."); try { // this can fail for various reasons, such as already being logged in somewhere else brokerage.Connect(); } catch (Exception err) { Log.Error(err); AddInitializationError(string.Format("Error connecting to brokerage: {0}. " + "This may be caused by incorrect login credentials or an unsupported account type.", err.Message)); return(false); } if (!brokerage.IsConnected) { // if we're reporting that we're not connected, bail AddInitializationError("Unable to connect to brokerage."); return(false); } Log.Trace("BrokerageSetupHandler.Setup(): Fetching cash balance from brokerage..."); try { // set the algorithm's cash balance for each currency var cashBalance = brokerage.GetCashBalance(); foreach (var cash in cashBalance) { Log.Trace("BrokerageSetupHandler.Setup(): Setting " + cash.Symbol + " cash to " + cash.Amount); algorithm.Portfolio.SetCash(cash.Symbol, cash.Amount, cash.ConversionRate); } } catch (Exception err) { Log.Error(err); AddInitializationError("Error getting cash balance from brokerage: " + err.Message); return(false); } Log.Trace("BrokerageSetupHandler.Setup(): Fetching open orders from brokerage..."); try { // populate the algorithm with the account's outstanding orders var openOrders = brokerage.GetOpenOrders(); foreach (var order in openOrders) { // be sure to assign order IDs such that we increment from the SecurityTransactionManager to avoid ID collisions Log.Trace("BrokerageSetupHandler.Setup(): Has open order: " + order.Symbol.ToString() + " - " + order.Quantity); order.Id = algorithm.Transactions.GetIncrementOrderId(); transactionHandler.Orders.AddOrUpdate(order.Id, order, (i, o) => order); } } catch (Exception err) { Log.Error(err); AddInitializationError("Error getting open orders from brokerage: " + err.Message); return(false); } Log.Trace("BrokerageSetupHandler.Setup(): Fetching holdings from brokerage..."); try { // populate the algorithm with the account's current holdings var holdings = brokerage.GetAccountHoldings(); var supportedSecurityTypes = new HashSet <SecurityType> { SecurityType.Equity, SecurityType.Forex }; var minResolution = new Lazy <Resolution>(() => algorithm.Securities.Select(x => x.Value.Resolution).DefaultIfEmpty(Resolution.Second).Min()); foreach (var holding in holdings) { Log.Trace("BrokerageSetupHandler.Setup(): Has existing holding: " + holding); // verify existing holding security type if (!supportedSecurityTypes.Contains(holding.Type)) { Log.Error("BrokerageSetupHandler.Setup(): Unsupported security type: " + holding.Type + "-" + holding.Symbol.Value); AddInitializationError("Found unsupported security type in existing brokerage holdings: " + holding.Type + ". " + "QuantConnect currently supports the following security types: " + string.Join(",", supportedSecurityTypes)); // keep aggregating these errors continue; } if (!algorithm.Portfolio.ContainsKey(holding.Symbol)) { Log.Trace("BrokerageSetupHandler.Setup(): Adding unrequested security: " + holding.Symbol.ToString()); // for items not directly requested set leverage to 1 and at the min resolution algorithm.AddSecurity(holding.Type, holding.Symbol.Value, minResolution.Value, null, true, 1.0m, false); } algorithm.Portfolio[holding.Symbol].SetHoldings(holding.AveragePrice, (int)holding.Quantity); algorithm.Securities[holding.Symbol].SetMarketPrice(new TradeBar { Time = DateTime.Now, Open = holding.MarketPrice, High = holding.MarketPrice, Low = holding.MarketPrice, Close = holding.MarketPrice, Volume = 0, Symbol = holding.Symbol, DataType = MarketDataType.TradeBar }); } } catch (Exception err) { Log.Error(err); AddInitializationError("Error getting account holdings from brokerage: " + err.Message); return(false); } //Set the starting portfolio value for the strategy to calculate performance: StartingPortfolioValue = algorithm.Portfolio.TotalPortfolioValue; StartingDate = DateTime.Now; } catch (Exception err) { AddInitializationError(err.Message); } finally { if (brokerage != null) { brokerage.Message -= brokerageOnMessage; } } return(Errors.Count == 0); }
/// <summary> /// Setup the algorithm cash, dates and data subscriptions as desired. /// </summary> /// <param name="parameters">The parameters object to use</param> /// <returns>Boolean true on successfully initializing the algorithm</returns> public bool Setup(SetupHandlerParameters parameters) { var algorithm = parameters.Algorithm; var job = parameters.AlgorithmNodePacket as BacktestNodePacket; if (job == null) { throw new ArgumentException("Expected BacktestNodePacket but received " + parameters.AlgorithmNodePacket.GetType().Name); } Log.Trace(string.Format("BacktestingSetupHandler.Setup(): Setting up job: Plan: {0}, UID: {1}, PID: {2}, Version: {3}, Source: {4}", job.UserPlan, job.UserId, job.ProjectId, job.Version, job.RequestSource)); if (algorithm == null) { Errors.Add(new AlgorithmSetupException("Could not create instance of algorithm")); return(false); } algorithm.Name = job.GetAlgorithmName(); //Make sure the algorithm start date ok. if (job.PeriodStart == default(DateTime)) { Errors.Add(new AlgorithmSetupException("Algorithm start date was never set")); return(false); } var controls = job.Controls; var isolator = new Isolator(); var initializeComplete = isolator.ExecuteWithTimeLimit(TimeSpan.FromMinutes(5), () => { try { parameters.ResultHandler.SendStatusUpdate(AlgorithmStatus.Initializing, "Initializing algorithm..."); //Set our parameters algorithm.SetParameters(job.Parameters); //Algorithm is backtesting, not live: algorithm.SetLiveMode(false); //Set the source impl for the event scheduling algorithm.Schedule.SetEventSchedule(parameters.RealTimeHandler); // set the option chain provider algorithm.SetOptionChainProvider(new CachingOptionChainProvider(new BacktestingOptionChainProvider())); // set the future chain provider algorithm.SetFutureChainProvider(new CachingFutureChainProvider(new BacktestingFutureChainProvider())); //Initialise the algorithm, get the required data: algorithm.Initialize(); // finalize initialization algorithm.PostInitialize(); } catch (Exception err) { Log.Error(err); Errors.Add(new AlgorithmSetupException("During the algorithm initialization, the following exception has occurred: ", err)); } }, controls.RamAllocation, sleepIntervalMillis: 50); // entire system is waiting on this, so be as fast as possible //Before continuing, detect if this is ready: if (!initializeComplete) { return(false); } // TODO: Refactor the BacktestResultHandler to use algorithm not job to set times job.PeriodStart = algorithm.StartDate; job.PeriodFinish = algorithm.EndDate; //Calculate the max runtime for the strategy _maxRuntime = GetMaximumRuntime(job.PeriodStart, job.PeriodFinish, algorithm.SubscriptionManager, algorithm.UniverseManager, parameters.AlgorithmNodePacket.Controls); // Python takes forever; lets give it 10x longer to finish. if (job.Language == Language.Python) { _maxRuntime = _maxRuntime.Add(TimeSpan.FromSeconds(_maxRuntime.TotalSeconds * 9)); } BaseSetupHandler.SetupCurrencyConversions(algorithm, parameters.UniverseSelection); StartingPortfolioValue = algorithm.Portfolio.Cash; //Max Orders: 10k per backtest: if (job.UserPlan == UserPlan.Free) { _maxOrders = 10000; } else { _maxOrders = int.MaxValue; _maxRuntime += _maxRuntime; } //Set back to the algorithm, algorithm.SetMaximumOrders(_maxOrders); //Starting date of the algorithm: _startingDate = job.PeriodStart; //Put into log for debugging: Log.Trace("SetUp Backtesting: User: "******" ProjectId: " + job.ProjectId + " AlgoId: " + job.AlgorithmId); Log.Trace("Dates: Start: " + job.PeriodStart.ToShortDateString() + " End: " + job.PeriodFinish.ToShortDateString() + " Cash: " + StartingPortfolioValue.ToString("C")); if (Errors.Count > 0) { initializeComplete = false; } return(initializeComplete); }
/// <summary> /// Primary entry point to setup a new algorithm /// </summary> /// <param name="parameters">The parameters object to use</param> /// <returns>True on successfully setting up the algorithm state, or false on error.</returns> public bool Setup(SetupHandlerParameters parameters) { var algorithm = parameters.Algorithm; var brokerage = parameters.Brokerage; // verify we were given the correct job packet type var liveJob = parameters.AlgorithmNodePacket as LiveNodePacket; if (liveJob == null) { AddInitializationError("BrokerageSetupHandler requires a LiveNodePacket"); return(false); } algorithm.Name = liveJob.GetAlgorithmName(); // verify the brokerage was specified if (string.IsNullOrWhiteSpace(liveJob.Brokerage)) { AddInitializationError("A brokerage must be specified"); return(false); } // attach to the message event to relay brokerage specific initialization messages EventHandler <BrokerageMessageEvent> brokerageOnMessage = (sender, args) => { if (args.Type == BrokerageMessageType.Error) { AddInitializationError($"Brokerage Error Code: {args.Code} - {args.Message}"); } }; try { Log.Trace("BrokerageSetupHandler.Setup(): Initializing algorithm..."); parameters.ResultHandler.SendStatusUpdate(AlgorithmStatus.Initializing, "Initializing algorithm..."); //Execute the initialize code: var controls = liveJob.Controls; var isolator = new Isolator(); var initializeComplete = isolator.ExecuteWithTimeLimit(TimeSpan.FromSeconds(300), () => { try { //Set the default brokerage model before initialize algorithm.SetBrokerageModel(_factory.BrokerageModel); //Margin calls are disabled by default in live mode algorithm.Portfolio.MarginCallModel = MarginCallModel.Null; //Set our parameters algorithm.SetParameters(liveJob.Parameters); algorithm.SetAvailableDataTypes(GetConfiguredDataFeeds()); //Algorithm is live, not backtesting: algorithm.SetLiveMode(true); //Initialize the algorithm's starting date algorithm.SetDateTime(DateTime.UtcNow); //Set the source impl for the event scheduling algorithm.Schedule.SetEventSchedule(parameters.RealTimeHandler); // set the option chain provider algorithm.SetOptionChainProvider(new CachingOptionChainProvider(new LiveOptionChainProvider())); // set the future chain provider algorithm.SetFutureChainProvider(new CachingFutureChainProvider(new LiveFutureChainProvider())); // If we're going to receive market data from IB, // set the default subscription limit to 100, // algorithms can override this setting in the Initialize method if (brokerage is InteractiveBrokersBrokerage && liveJob.DataQueueHandler.EndsWith("InteractiveBrokersBrokerage")) { algorithm.Settings.DataSubscriptionLimit = 100; } //Initialise the algorithm, get the required data: algorithm.Initialize(); if (algorithm.AccountCurrency != Currencies.USD) { throw new NotImplementedException( "BrokerageSetupHandler.Setup(): calling 'QCAlgorithm.SetAccountCurrency()' " + "is only supported in backtesting for now."); } if (liveJob.Brokerage != "PaperBrokerage") { //Zero the CashBook - we'll populate directly from brokerage foreach (var kvp in algorithm.Portfolio.CashBook) { kvp.Value.SetAmount(0); } } } catch (Exception err) { AddInitializationError(err.ToString(), err); } }, controls.RamAllocation, sleepIntervalMillis: 50); // entire system is waiting on this, so be as fast as possible if (!initializeComplete) { AddInitializationError("Initialization timed out."); return(false); } // let the world know what we're doing since logging in can take a minute parameters.ResultHandler.SendStatusUpdate(AlgorithmStatus.LoggingIn, "Logging into brokerage..."); brokerage.Message += brokerageOnMessage; Log.Trace("BrokerageSetupHandler.Setup(): Connecting to brokerage..."); try { // this can fail for various reasons, such as already being logged in somewhere else brokerage.Connect(); } catch (Exception err) { Log.Error(err); AddInitializationError( $"Error connecting to brokerage: {err.Message}. " + "This may be caused by incorrect login credentials or an unsupported account type.", err); return(false); } if (!brokerage.IsConnected) { // if we're reporting that we're not connected, bail AddInitializationError("Unable to connect to brokerage."); return(false); } Log.Trace("BrokerageSetupHandler.Setup(): Fetching cash balance from brokerage..."); try { // set the algorithm's cash balance for each currency var cashBalance = brokerage.GetCashBalance(); foreach (var cash in cashBalance) { Log.Trace("BrokerageSetupHandler.Setup(): Setting " + cash.Currency + " cash to " + cash.Amount); algorithm.Portfolio.SetCash(cash.Currency, cash.Amount, 0); } } catch (Exception err) { Log.Error(err); AddInitializationError("Error getting cash balance from brokerage: " + err.Message, err); return(false); } var supportedSecurityTypes = new HashSet <SecurityType> { SecurityType.Equity, SecurityType.Forex, SecurityType.Cfd, SecurityType.Option, SecurityType.Future, SecurityType.Crypto }; var minResolution = new Lazy <Resolution>(() => algorithm.Securities.Select(x => x.Value.Resolution).DefaultIfEmpty(Resolution.Second).Min()); Log.Trace("BrokerageSetupHandler.Setup(): Fetching open orders from brokerage..."); try { GetOpenOrders(algorithm, parameters.ResultHandler, parameters.TransactionHandler, brokerage, supportedSecurityTypes, minResolution.Value); } catch (Exception err) { Log.Error(err); AddInitializationError("Error getting open orders from brokerage: " + err.Message, err); return(false); } Log.Trace("BrokerageSetupHandler.Setup(): Fetching holdings from brokerage..."); try { // populate the algorithm with the account's current holdings var holdings = brokerage.GetAccountHoldings(); // add options first to ensure raw data normalization mode is set on the equity underlyings foreach (var holding in holdings.OrderByDescending(x => x.Type)) { Log.Trace("BrokerageSetupHandler.Setup(): Has existing holding: " + holding); // verify existing holding security type if (!supportedSecurityTypes.Contains(holding.Type)) { Log.Error("BrokerageSetupHandler.Setup(): Unsupported security type: " + holding.Type + "-" + holding.Symbol.Value); AddInitializationError("Found unsupported security type in existing brokerage holdings: " + holding.Type + ". " + "QuantConnect currently supports the following security types: " + string.Join(",", supportedSecurityTypes)); // keep aggregating these errors continue; } AddUnrequestedSecurity(algorithm, holding.Symbol, minResolution.Value); algorithm.Portfolio[holding.Symbol].SetHoldings(holding.AveragePrice, holding.Quantity); algorithm.Securities[holding.Symbol].SetMarketPrice(new TradeBar { Time = DateTime.Now, Open = holding.MarketPrice, High = holding.MarketPrice, Low = holding.MarketPrice, Close = holding.MarketPrice, Volume = 0, Symbol = holding.Symbol, DataType = MarketDataType.TradeBar }); } } catch (Exception err) { Log.Error(err); AddInitializationError("Error getting account holdings from brokerage: " + err.Message, err); return(false); } //Finalize Initialization algorithm.PostInitialize(); BaseSetupHandler.SetupCurrencyConversions(algorithm, parameters.UniverseSelection); //Set the starting portfolio value for the strategy to calculate performance: StartingPortfolioValue = algorithm.Portfolio.TotalPortfolioValue; StartingDate = DateTime.Now; } catch (Exception err) { AddInitializationError(err.ToString(), err); } finally { if (brokerage != null) { brokerage.Message -= brokerageOnMessage; } } return(Errors.Count == 0); }
/// <summary> /// Primary entry point to setup a new algorithm /// </summary> /// <param name="algorithm">Algorithm instance</param> /// <param name="brokerage">New brokerage output instance</param> /// <param name="job">Algorithm job task</param> /// <param name="resultHandler">The configured result handler</param> /// <param name="transactionHandler">The configurated transaction handler</param> /// <param name="realTimeHandler">The configured real time handler</param> /// <returns>True on successfully setting up the algorithm state, or false on error.</returns> public bool Setup(IAlgorithm algorithm, IBrokerage brokerage, AlgorithmNodePacket job, IResultHandler resultHandler, ITransactionHandler transactionHandler, IRealTimeHandler realTimeHandler) { _algorithm = algorithm; // verify we were given the correct job packet type var liveJob = job as LiveNodePacket; if (liveJob == null) { AddInitializationError("BrokerageSetupHandler requires a LiveNodePacket"); return false; } // verify the brokerage was specified if (string.IsNullOrWhiteSpace(liveJob.Brokerage)) { AddInitializationError("A brokerage must be specified"); return false; } // attach to the message event to relay brokerage specific initialization messages EventHandler<BrokerageMessageEvent> brokerageOnMessage = (sender, args) => { if (args.Type == BrokerageMessageType.Error) { AddInitializationError(string.Format("Brokerage Error Code: {0} - {1}", args.Code, args.Message)); } }; try { Log.Trace("BrokerageSetupHandler.Setup(): Initializing algorithm..."); resultHandler.SendStatusUpdate(AlgorithmStatus.Initializing, "Initializing algorithm..."); //Execute the initialize code: var controls = job.Controls; var isolator = new Isolator(); var initializeComplete = isolator.ExecuteWithTimeLimit(TimeSpan.FromSeconds(300), () => { try { //Set the default brokerage model before initialize algorithm.SetBrokerageModel(_factory.BrokerageModel); //Set our parameters algorithm.SetParameters(job.Parameters); //Algorithm is live, not backtesting: algorithm.SetLiveMode(true); //Initialize the algorithm's starting date algorithm.SetDateTime(DateTime.UtcNow); //Set the source impl for the event scheduling algorithm.Schedule.SetEventSchedule(realTimeHandler); //Initialise the algorithm, get the required data: algorithm.Initialize(); if (liveJob.Brokerage != "PaperBrokerage") { //Zero the CashBook - we'll populate directly from brokerage foreach (var kvp in algorithm.Portfolio.CashBook) { kvp.Value.SetAmount(0); } } } catch (Exception err) { AddInitializationError(err.Message); } }); if (!initializeComplete) { AddInitializationError("Initialization timed out."); return false; } // let the world know what we're doing since logging in can take a minute resultHandler.SendStatusUpdate(AlgorithmStatus.LoggingIn, "Logging into brokerage..."); brokerage.Message += brokerageOnMessage; algorithm.Transactions.SetOrderProcessor(transactionHandler); Log.Trace("BrokerageSetupHandler.Setup(): Connecting to brokerage..."); try { // this can fail for various reasons, such as already being logged in somewhere else brokerage.Connect(); } catch (Exception err) { Log.Error(err); AddInitializationError(string.Format("Error connecting to brokerage: {0}. " + "This may be caused by incorrect login credentials or an unsupported account type.", err.Message)); return false; } if (!brokerage.IsConnected) { // if we're reporting that we're not connected, bail AddInitializationError("Unable to connect to brokerage."); return false; } Log.Trace("BrokerageSetupHandler.Setup(): Fetching cash balance from brokerage..."); try { // set the algorithm's cash balance for each currency var cashBalance = brokerage.GetCashBalance(); foreach (var cash in cashBalance) { Log.Trace("BrokerageSetupHandler.Setup(): Setting " + cash.Symbol + " cash to " + cash.Amount); algorithm.Portfolio.SetCash(cash.Symbol, cash.Amount, cash.ConversionRate); } } catch (Exception err) { Log.Error(err); AddInitializationError("Error getting cash balance from brokerage: " + err.Message); return false; } Log.Trace("BrokerageSetupHandler.Setup(): Fetching open orders from brokerage..."); try { // populate the algorithm with the account's outstanding orders var openOrders = brokerage.GetOpenOrders(); foreach (var order in openOrders) { // be sure to assign order IDs such that we increment from the SecurityTransactionManager to avoid ID collisions Log.Trace("BrokerageSetupHandler.Setup(): Has open order: " + order.Symbol.ToString() + " - " + order.Quantity); order.Id = algorithm.Transactions.GetIncrementOrderId(); transactionHandler.Orders.AddOrUpdate(order.Id, order, (i, o) => order); } } catch (Exception err) { Log.Error(err); AddInitializationError("Error getting open orders from brokerage: " + err.Message); return false; } Log.Trace("BrokerageSetupHandler.Setup(): Fetching holdings from brokerage..."); try { // populate the algorithm with the account's current holdings var holdings = brokerage.GetAccountHoldings(); var supportedSecurityTypes = new HashSet<SecurityType> { SecurityType.Equity, SecurityType.Forex, SecurityType.Cfd }; var minResolution = new Lazy<Resolution>(() => algorithm.Securities.Select(x => x.Value.Resolution).DefaultIfEmpty(Resolution.Second).Min()); foreach (var holding in holdings) { Log.Trace("BrokerageSetupHandler.Setup(): Has existing holding: " + holding); // verify existing holding security type if (!supportedSecurityTypes.Contains(holding.Type)) { Log.Error("BrokerageSetupHandler.Setup(): Unsupported security type: " + holding.Type + "-" + holding.Symbol.Value); AddInitializationError("Found unsupported security type in existing brokerage holdings: " + holding.Type + ". " + "QuantConnect currently supports the following security types: " + string.Join(",", supportedSecurityTypes)); // keep aggregating these errors continue; } if (!algorithm.Portfolio.ContainsKey(holding.Symbol)) { Log.Trace("BrokerageSetupHandler.Setup(): Adding unrequested security: " + holding.Symbol.ToString()); // for items not directly requested set leverage to 1 and at the min resolution algorithm.AddSecurity(holding.Type, holding.Symbol.Value, minResolution.Value, null, true, 1.0m, false); } algorithm.Portfolio[holding.Symbol].SetHoldings(holding.AveragePrice, (int) holding.Quantity); algorithm.Securities[holding.Symbol].SetMarketPrice(new TradeBar { Time = DateTime.Now, Open = holding.MarketPrice, High = holding.MarketPrice, Low = holding.MarketPrice, Close = holding.MarketPrice, Volume = 0, Symbol = holding.Symbol, DataType = MarketDataType.TradeBar }); } } catch (Exception err) { Log.Error(err); AddInitializationError("Error getting account holdings from brokerage: " + err.Message); return false; } algorithm.PostInitialize(); //Set the starting portfolio value for the strategy to calculate performance: StartingPortfolioValue = algorithm.Portfolio.TotalPortfolioValue; StartingDate = DateTime.Now; } catch (Exception err) { AddInitializationError(err.Message); } finally { if (brokerage != null) { brokerage.Message -= brokerageOnMessage; } } return Errors.Count == 0; }
/// <summary> /// Setup the algorithm cash, dates and data subscriptions as desired. /// </summary> /// <param name="algorithm">Algorithm instance</param> /// <param name="brokerage">Brokerage instance</param> /// <param name="baseJob">Algorithm job</param> /// <param name="resultHandler">The configured result handler</param> /// <param name="transactionHandler">The configurated transaction handler</param> /// <returns>Boolean true on successfully initializing the algorithm</returns> public bool Setup(IAlgorithm algorithm, out IBrokerage brokerage, AlgorithmNodePacket baseJob, IResultHandler resultHandler, ITransactionHandler transactionHandler) { var job = baseJob as BacktestNodePacket; if (job == null) { throw new ArgumentException("Expected BacktestNodePacket but received " + baseJob.GetType().Name); } Log.Trace(string.Format("BacktestingSetupHandler.Setup(): Setting up job: Plan: {0}, UID: {1}, PID: {2}, Version: {3}, Source: {4}", job.UserPlan, job.UserId, job.ProjectId, job.Version, job.RequestSource)); brokerage = null; if (algorithm == null) { Errors.Add("Could not create instance of algorithm"); return(false); } //Make sure the algorithm start date ok. if (job.PeriodStart == default(DateTime)) { Errors.Add("Algorithm start date was never set"); return(false); } //Execute the initialize code: var isolator = new Isolator(); var initializeComplete = isolator.ExecuteWithTimeLimit(TimeSpan.FromSeconds(10), () => { try { //Algorithm is backtesting, not live: algorithm.SetLiveMode(false); //Set the backtest level asset ram allocation limits algorithm.SetAssetLimits(500, 100, 30); //Set the algorithm time before we even initialize: algorithm.SetDateTime(job.PeriodStart); //Initialise the algorithm, get the required data: algorithm.Initialize(); //Add currency data feeds that weren't explicity added in Initialize algorithm.Portfolio.CashBook.EnsureCurrencyDataFeeds(algorithm.Securities, algorithm.SubscriptionManager, SecurityExchangeHoursProvider.FromDataFolder()); } catch (Exception err) { Errors.Add("Failed to initialize algorithm: Initialize(): " + err.Message); } }); //Before continuing, detect if this is ready: if (!initializeComplete) { return(false); } // this needs to be done after algorithm initialization brokerage = new BacktestingBrokerage(algorithm); SetupHandler.UpdateTransactionModels(algorithm, algorithm.BrokerageModel); algorithm.Transactions.SetOrderProcessor(transactionHandler); //Calculate the max runtime for the strategy _maxRuntime = GetMaximumRuntime(job.PeriodStart, job.PeriodFinish, algorithm.SubscriptionManager.Count); //Get starting capital: _startingCaptial = algorithm.Portfolio.Cash; //Max Orders: 10k per backtest: if (job.UserPlan == UserPlan.Free) { _maxOrders = 10000; } else { _maxOrders = int.MaxValue; _maxRuntime += _maxRuntime; } //Set back to the algorithm, algorithm.SetMaximumOrders(_maxOrders); //Starting date of the algorithm: _startingDate = job.PeriodStart; //Put into log for debugging: Log.Trace("SetUp Backtesting: User: "******" ProjectId: " + job.ProjectId + " AlgoId: " + job.AlgorithmId); Log.Trace("Dates: Start: " + job.PeriodStart.ToShortDateString() + " End: " + job.PeriodFinish.ToShortDateString() + " Cash: " + _startingCaptial.ToString("C")); if (Errors.Count > 0) { initializeComplete = false; } return(initializeComplete); }
/// <summary> /// Setup the algorithm cash, dates and data subscriptions as desired. /// </summary> /// <param name="algorithm">Algorithm instance</param> /// <param name="brokerage">Brokerage instance</param> /// <param name="baseJob">Algorithm job</param> /// <returns>Boolean true on successfully initializing the algorithm</returns> public bool Setup(IAlgorithm algorithm, out IBrokerage brokerage, AlgorithmNodePacket baseJob) { var job = baseJob as BacktestNodePacket; if (job == null) { throw new ArgumentException("Expected BacktestNodePacket but received " + baseJob.GetType().Name); } // Must be set since its defined as an out parameters brokerage = new BacktestingBrokerage(algorithm); if (algorithm == null) { Errors.Add("Could not create instance of algorithm"); return(false); } //Make sure the algorithm start date ok. if (job.PeriodStart == default(DateTime)) { Errors.Add("Algorithm start date was never set"); return(false); } //Execute the initialize code: var initializeComplete = Isolator.ExecuteWithTimeLimit(TimeSpan.FromSeconds(10), () => { try { //0.0 Set the algorithm time before we even initialize: algorithm.SetDateTime(job.PeriodStart); //1.0 Initialise the algorithm, get the required data: algorithm.Initialize(); //1.2 Set the algorithm to locked to avoid messing with cash: algorithm.SetLocked(); } catch (Exception err) { Errors.Add("Failed to initialize algorithm: Initialize(): " + err.Message); } }); //Before continuing, detect if this is ready: if (!initializeComplete) { return(false); } //Calculate the max runtime for the strategy _maxRuntime = GetMaximumRuntime(job.PeriodStart, job.PeriodFinish, algorithm.SubscriptionManager.Count); //Get starting capital: _startingCaptial = algorithm.Portfolio.Cash; //Max Orders: 100 per day: _maxOrders = (int)(job.PeriodFinish - job.PeriodStart).TotalDays * 100; //Starting date of the algorithm: _startingDate = job.PeriodStart; //Put into log for debugging: Log.Trace("SetUp Backtesting: User: "******" ProjectId: " + job.ProjectId + " AlgoId: " + job.AlgorithmId); Log.Trace("Dates: Start: " + job.PeriodStart.ToShortDateString() + " End: " + job.PeriodFinish.ToShortDateString() + " Cash: " + _startingCaptial.ToString("C")); if (Errors.Count > 0) { initializeComplete = false; } return(initializeComplete); }
/// <summary> /// Runs a single backtest/live job from the job queue /// </summary> /// <param name="job">The algorithm job to be processed</param> /// <param name="assemblyPath">The path to the algorithm's assembly</param> public void Run(AlgorithmNodePacket job, string assemblyPath) { var algorithm = default(IAlgorithm); var algorithmManager = new AlgorithmManager(_liveMode); //Start monitoring the backtest active status: var statusPing = new StateCheck.Ping(algorithmManager, _systemHandlers.Api, _algorithmHandlers.Results); var statusPingThread = new Thread(statusPing.Run); statusPingThread.Start(); try { //Reset thread holders. var initializeComplete = false; Thread threadFeed = null; Thread threadTransactions = null; Thread threadResults = null; Thread threadRealTime = null; //-> Initialize messaging system _systemHandlers.Notify.SetChannel(job.Channel); //-> Set the result handler type for this algorithm job, and launch the associated result thread. _algorithmHandlers.Results.Initialize(job, _systemHandlers.Notify, _systemHandlers.Api, _algorithmHandlers.DataFeed, _algorithmHandlers.Setup, _algorithmHandlers.Transactions); threadResults = new Thread(_algorithmHandlers.Results.Run, 0) {Name = "Result Thread"}; threadResults.Start(); IBrokerage brokerage = null; try { // Save algorithm to cache, load algorithm instance: algorithm = _algorithmHandlers.Setup.CreateAlgorithmInstance(assemblyPath, job.Language); //Initialize the internal state of algorithm and job: executes the algorithm.Initialize() method. initializeComplete = _algorithmHandlers.Setup.Setup(algorithm, out brokerage, job, _algorithmHandlers.Results, _algorithmHandlers.Transactions); //If there are any reasons it failed, pass these back to the IDE. if (!initializeComplete || algorithm.ErrorMessages.Count > 0 || _algorithmHandlers.Setup.Errors.Count > 0) { initializeComplete = false; //Get all the error messages: internal in algorithm and external in setup handler. var errorMessage = String.Join(",", algorithm.ErrorMessages); errorMessage += String.Join(",", _algorithmHandlers.Setup.Errors); _algorithmHandlers.Results.RuntimeError(errorMessage); _systemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, errorMessage); } } catch (Exception err) { var runtimeMessage = "Algorithm.Initialize() Error: " + err.Message + " Stack Trace: " + err.StackTrace; _algorithmHandlers.Results.RuntimeError(runtimeMessage, err.StackTrace); _systemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, runtimeMessage); } //-> Using the job + initialization: load the designated handlers: if (initializeComplete) { //-> Reset the backtest stopwatch; we're now running the algorithm. var startTime = DateTime.Now; //Set algorithm as locked; set it to live mode if we're trading live, and set it to locked for no further updates. algorithm.SetAlgorithmId(job.AlgorithmId); algorithm.SetLiveMode(_liveMode); algorithm.SetLocked(); //Load the associated handlers for data, transaction and realtime events: _algorithmHandlers.Results.SetAlgorithm(algorithm); _algorithmHandlers.DataFeed.Initialize(algorithm, job, _algorithmHandlers.Results); _algorithmHandlers.Transactions.Initialize(algorithm, brokerage, _algorithmHandlers.Results); _algorithmHandlers.RealTime.Initialize(algorithm, job, _algorithmHandlers.Results, _systemHandlers.Api); //Set the error handlers for the brokerage asynchronous errors. _algorithmHandlers.Setup.SetupErrorHandler(_algorithmHandlers.Results, brokerage); //Send status to user the algorithm is now executing. _algorithmHandlers.Results.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.Running); //Launch the data, transaction and realtime handlers into dedicated threads threadFeed = new Thread(_algorithmHandlers.DataFeed.Run) {Name = "DataFeed Thread"}; threadTransactions = new Thread(_algorithmHandlers.Transactions.Run) {Name = "Transaction Thread"}; threadRealTime = new Thread(_algorithmHandlers.RealTime.Run) {Name = "RealTime Thread"}; //Launch the data feed, result sending, and transaction models/handlers in separate threads. threadFeed.Start(); // Data feed pushing data packets into thread bridge; threadTransactions.Start(); // Transaction modeller scanning new order requests threadRealTime.Start(); // RealTime scan time for time based events: // Result manager scanning message queue: (started earlier) _algorithmHandlers.Results.DebugMessage(string.Format("Launching analysis for {0} with LEAN Engine v{1}", job.AlgorithmId, Constants.Version)); try { //Create a new engine isolator class var isolator = new Isolator(); // Execute the Algorithm Code: var complete = isolator.ExecuteWithTimeLimit(_algorithmHandlers.Setup.MaximumRuntime, algorithmManager.TimeLoopWithinLimits, () => { try { //Run Algorithm Job: // -> Using this Data Feed, // -> Send Orders to this TransactionHandler, // -> Send Results to ResultHandler. algorithmManager.Run(job, algorithm, _algorithmHandlers.DataFeed, _algorithmHandlers.Transactions, _algorithmHandlers.Results, _algorithmHandlers.RealTime, isolator.CancellationToken); } catch (Exception err) { //Debugging at this level is difficult, stack trace needed. Log.Error("Engine.Run", err); algorithm.RunTimeError = err; algorithmManager.SetStatus(AlgorithmStatus.RuntimeError); return; } Log.Trace("Engine.Run(): Exiting Algorithm Manager"); }, job.RamAllocation); if (!complete) { Log.Error("Engine.Main(): Failed to complete in time: " + _algorithmHandlers.Setup.MaximumRuntime.ToString("F")); throw new Exception("Failed to complete algorithm within " + _algorithmHandlers.Setup.MaximumRuntime.ToString("F") + " seconds. Please make it run faster."); } // Algorithm runtime error: if (algorithm.RunTimeError != null) { throw algorithm.RunTimeError; } } catch (Exception err) { //Error running the user algorithm: purge datafeed, send error messages, set algorithm status to failed. Log.Error("Engine.Run(): Breaking out of parent try-catch: " + err.Message + " " + err.StackTrace); if (_algorithmHandlers.DataFeed != null) _algorithmHandlers.DataFeed.Exit(); if (_algorithmHandlers.Results != null) { var message = "Runtime Error: " + err.Message; Log.Trace("Engine.Run(): Sending runtime error to user..."); _algorithmHandlers.Results.LogMessage(message); _algorithmHandlers.Results.RuntimeError(message, err.StackTrace); _systemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, message + " Stack Trace: " + err.StackTrace); } } //Send result data back: this entire code block could be rewritten. // todo: - Split up statistics class, its enormous. // todo: - Make a dedicated Statistics.Benchmark class. // todo: - Move all creation and transmission of statistics out of primary engine loop. // todo: - Statistics.Generate(algorithm, resulthandler, transactionhandler); try { var charts = new Dictionary<string, Chart>(_algorithmHandlers.Results.Charts); var orders = new Dictionary<int, Order>(_algorithmHandlers.Transactions.Orders); var holdings = new Dictionary<string, Holding>(); var statistics = new Dictionary<string, string>(); var banner = new Dictionary<string, string>(); try { //Generates error when things don't exist (no charting logged, runtime errors in main algo execution) const string strategyEquityKey = "Strategy Equity"; const string equityKey = "Equity"; const string dailyPerformanceKey = "Daily Performance"; const string benchmarkKey = "Benchmark"; // make sure we've taken samples for these series before just blindly requesting them if (charts.ContainsKey(strategyEquityKey) && charts[strategyEquityKey].Series.ContainsKey(equityKey) && charts[strategyEquityKey].Series.ContainsKey(dailyPerformanceKey)) { var equity = charts[strategyEquityKey].Series[equityKey].Values; var performance = charts[strategyEquityKey].Series[dailyPerformanceKey].Values; var profitLoss = new SortedDictionary<DateTime, decimal>(algorithm.Transactions.TransactionRecord); var numberOfTrades = algorithm.Transactions.GetOrders(x => x.Status.IsFill()).Count(); var benchmark = charts[benchmarkKey].Series[benchmarkKey].Values.ToDictionary(chartPoint => Time.UnixTimeStampToDateTime(chartPoint.x), chartPoint => chartPoint.y); statistics = Statistics.Statistics.Generate(equity, profitLoss, performance, benchmark, _algorithmHandlers.Setup.StartingPortfolioValue, algorithm.Portfolio.TotalFees, numberOfTrades, 252); } } catch (Exception err) { Log.Error("Algorithm.Node.Engine(): Error generating statistics packet: " + err.Message); } //Diagnostics Completed, Send Result Packet: var totalSeconds = (DateTime.Now - startTime).TotalSeconds; _algorithmHandlers.Results.DebugMessage( string.Format("Algorithm Id:({0}) completed in {1} seconds at {2}k data points per second. Processing total of {3} data points.", job.AlgorithmId, totalSeconds.ToString("F2"), ((algorithmManager.DataPoints/(double) 1000)/totalSeconds).ToString("F0"), algorithmManager.DataPoints.ToString("N0"))); _algorithmHandlers.Results.SendFinalResult(job, orders, algorithm.Transactions.TransactionRecord, holdings, statistics, banner); } catch (Exception err) { Log.Error("Engine.Main(): Error sending analysis result: " + err.Message + " ST >> " + err.StackTrace); } //Before we return, send terminate commands to close up the threads _algorithmHandlers.Transactions.Exit(); _algorithmHandlers.DataFeed.Exit(); _algorithmHandlers.RealTime.Exit(); } //Close result handler: _algorithmHandlers.Results.Exit(); statusPing.Exit(); //Wait for the threads to complete: var ts = Stopwatch.StartNew(); while ((_algorithmHandlers.Results.IsActive || (_algorithmHandlers.Transactions != null && _algorithmHandlers.Transactions.IsActive) || (_algorithmHandlers.DataFeed != null && _algorithmHandlers.DataFeed.IsActive) || (_algorithmHandlers.RealTime != null && _algorithmHandlers.RealTime.IsActive)) && ts.ElapsedMilliseconds < 30*1000) { Thread.Sleep(100); Log.Trace("Waiting for threads to exit..."); } //Terminate threads still in active state. if (threadFeed != null && threadFeed.IsAlive) threadFeed.Abort(); if (threadTransactions != null && threadTransactions.IsAlive) threadTransactions.Abort(); if (threadResults != null && threadResults.IsAlive) threadResults.Abort(); if (statusPingThread != null && statusPingThread.IsAlive) statusPingThread.Abort(); if (brokerage != null) { brokerage.Disconnect(); } if (_algorithmHandlers.Setup != null) { _algorithmHandlers.Setup.Dispose(); } Log.Trace("Engine.Main(): Analysis Completed and Results Posted."); } catch (Exception err) { Log.Error("Engine.Main(): Error running algorithm: " + err.Message + " >> " + err.StackTrace); } finally { //No matter what for live mode; make sure we've set algorithm status in the API for "not running" conditions: if (_liveMode && algorithmManager.State != AlgorithmStatus.Running && algorithmManager.State != AlgorithmStatus.RuntimeError) _systemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, algorithmManager.State); _algorithmHandlers.Results.Exit(); _algorithmHandlers.DataFeed.Exit(); _algorithmHandlers.Transactions.Exit(); _algorithmHandlers.RealTime.Exit(); } }
/// <summary> /// Setup the algorithm cash, dates and data subscriptions as desired. /// </summary> /// <param name="parameters">The parameters object to use</param> /// <returns>Boolean true on successfully initializing the algorithm</returns> public bool Setup(SetupHandlerParameters parameters) { var algorithm = parameters.Algorithm; var job = parameters.AlgorithmNodePacket as BacktestNodePacket; if (job == null) { throw new ArgumentException("Expected BacktestNodePacket but received " + parameters.AlgorithmNodePacket.GetType().Name); } Log.Trace($"BacktestingSetupHandler.Setup(): Setting up job: Plan: {job.UserPlan}, UID: {job.UserId.ToStringInvariant()}, " + $"PID: {job.ProjectId.ToStringInvariant()}, Version: {job.Version}, Source: {job.RequestSource}" ); if (algorithm == null) { Errors.Add(new AlgorithmSetupException("Could not create instance of algorithm")); return(false); } algorithm.Name = job.GetAlgorithmName(); //Make sure the algorithm start date ok. if (job.PeriodStart == default(DateTime)) { Errors.Add(new AlgorithmSetupException("Algorithm start date was never set")); return(false); } var controls = job.Controls; var isolator = new Isolator(); var initializeComplete = isolator.ExecuteWithTimeLimit(TimeSpan.FromMinutes(5), () => { try { parameters.ResultHandler.SendStatusUpdate(AlgorithmStatus.Initializing, "Initializing algorithm..."); //Set our parameters algorithm.SetParameters(job.Parameters); //Algorithm is backtesting, not live: algorithm.SetLiveMode(false); //Set the source impl for the event scheduling algorithm.Schedule.SetEventSchedule(parameters.RealTimeHandler); // set the option chain provider algorithm.SetOptionChainProvider(new CachingOptionChainProvider(new BacktestingOptionChainProvider())); // set the future chain provider algorithm.SetFutureChainProvider(new CachingFutureChainProvider(new BacktestingFutureChainProvider())); // set the object store algorithm.SetObjectStore(parameters.ObjectStore); // before we call initialize BaseSetupHandler.LoadBacktestJobAccountCurrency(algorithm, job); //Initialise the algorithm, get the required data: algorithm.Initialize(); // set start and end date if present in the job if (job.PeriodStart.HasValue) { algorithm.SetStartDate(job.PeriodStart.Value); } if (job.PeriodFinish.HasValue) { algorithm.SetEndDate(job.PeriodFinish.Value); } // after we call initialize BaseSetupHandler.LoadBacktestJobCashAmount(algorithm, job); // finalize initialization algorithm.PostInitialize(); } catch (Exception err) { Log.Error(err); Errors.Add(new AlgorithmSetupException("During the algorithm initialization, the following exception has occurred: ", err)); } }, controls.RamAllocation, sleepIntervalMillis: 10, // entire system is waiting on this, so be as fast as possible workerThread: WorkerThread); //Before continuing, detect if this is ready: if (!initializeComplete) { return(false); } //Calculate the max runtime for the strategy MaximumRuntime = GetMaximumRuntime(algorithm.StartDate, algorithm.EndDate, algorithm.SubscriptionManager, algorithm.UniverseManager, parameters.AlgorithmNodePacket.Controls); // Python takes forever; lets give it 10x longer to finish. if (job.Language == Language.Python) { MaximumRuntime = MaximumRuntime.Add(TimeSpan.FromSeconds(MaximumRuntime.TotalSeconds * 9)); } BaseSetupHandler.SetupCurrencyConversions(algorithm, parameters.UniverseSelection); StartingPortfolioValue = algorithm.Portfolio.Cash; // we set the free portfolio value based on the initial total value and the free percentage value algorithm.Settings.FreePortfolioValue = algorithm.Portfolio.TotalPortfolioValue * algorithm.Settings.FreePortfolioValuePercentage; //Max Orders: 10k per backtest: if (job.UserPlan == UserPlan.Free) { MaxOrders = 10000; } else { MaxOrders = int.MaxValue; MaximumRuntime += MaximumRuntime; } MaxOrders = job.Controls.BacktestingMaxOrders; //Set back to the algorithm, algorithm.SetMaximumOrders(MaxOrders); //Starting date of the algorithm: StartingDate = algorithm.StartDate; //Put into log for debugging: Log.Trace("SetUp Backtesting: User: "******" ProjectId: " + job.ProjectId + " AlgoId: " + job.AlgorithmId); Log.Trace($"Dates: Start: {algorithm.StartDate.ToStringInvariant("d")} " + $"End: {algorithm.EndDate.ToStringInvariant("d")} " + $"Cash: {StartingPortfolioValue.ToStringInvariant("C")}"); if (Errors.Count > 0) { initializeComplete = false; } return(initializeComplete); }
/// <summary> /// Setup the algorithm cash, dates and data subscriptions as desired. /// </summary> /// <param name="algorithm">Algorithm instance</param> /// <param name="brokerage">Brokerage instance</param> /// <param name="baseJob">Algorithm job</param> /// <param name="resultHandler">The configured result handler</param> /// <param name="transactionHandler">The configurated transaction handler</param> /// <param name="realTimeHandler">The configured real time handler</param> /// <returns>Boolean true on successfully initializing the algorithm</returns> public bool Setup(IAlgorithm algorithm, IBrokerage brokerage, AlgorithmNodePacket baseJob, IResultHandler resultHandler, ITransactionHandler transactionHandler, IRealTimeHandler realTimeHandler) { var job = baseJob as BacktestNodePacket; if (job == null) { throw new ArgumentException("Expected BacktestNodePacket but received " + baseJob.GetType().Name); } Log.Trace(string.Format("BacktestingSetupHandler.Setup(): Setting up job: Plan: {0}, UID: {1}, PID: {2}, Version: {3}, Source: {4}", job.UserPlan, job.UserId, job.ProjectId, job.Version, job.RequestSource)); if (algorithm == null) { Errors.Add("Could not create instance of algorithm"); return(false); } //Make sure the algorithm start date ok. if (job.PeriodStart == default(DateTime)) { Errors.Add("Algorithm start date was never set"); return(false); } var controls = job.Controls; var isolator = new Isolator(); var initializeComplete = isolator.ExecuteWithTimeLimit(TimeSpan.FromMinutes(5), () => { try { //Set our parameters algorithm.SetParameters(job.Parameters); //Algorithm is backtesting, not live: algorithm.SetLiveMode(false); //Set the algorithm time before we even initialize: algorithm.SetDateTime(job.PeriodStart.ConvertToUtc(algorithm.TimeZone)); //Set the source impl for the event scheduling algorithm.Schedule.SetEventSchedule(realTimeHandler); //Initialise the algorithm, get the required data: algorithm.Initialize(); } catch (Exception err) { Errors.Add("Failed to initialize algorithm: Initialize(): " + err.Message); } }); //Before continuing, detect if this is ready: if (!initializeComplete) { return(false); } algorithm.Transactions.SetOrderProcessor(transactionHandler); algorithm.PostInitialize(); //Calculate the max runtime for the strategy _maxRuntime = GetMaximumRuntime(job.PeriodStart, job.PeriodFinish, algorithm.SubscriptionManager.Count); //Get starting capital: _startingCaptial = algorithm.Portfolio.Cash; //Max Orders: 10k per backtest: if (job.UserPlan == UserPlan.Free) { _maxOrders = 10000; } else { _maxOrders = int.MaxValue; _maxRuntime += _maxRuntime; } //Set back to the algorithm, algorithm.SetMaximumOrders(_maxOrders); //Starting date of the algorithm: _startingDate = job.PeriodStart; //Put into log for debugging: Log.Trace("SetUp Backtesting: User: "******" ProjectId: " + job.ProjectId + " AlgoId: " + job.AlgorithmId); Log.Trace("Dates: Start: " + job.PeriodStart.ToShortDateString() + " End: " + job.PeriodFinish.ToShortDateString() + " Cash: " + _startingCaptial.ToString("C")); if (Errors.Count > 0) { initializeComplete = false; } return(initializeComplete); }
/// <summary> /// Runs a single backtest/live job from the job queue /// </summary> /// <param name="job">The algorithm job to be processed</param> /// <param name="manager"></param> /// <param name="assemblyPath">The path to the algorithm's assembly</param> public void Run(AlgorithmNodePacket job, AlgorithmManager manager, string assemblyPath) { var algorithm = default(IAlgorithm); var algorithmManager = manager; try { //Reset thread holders. var initializeComplete = false; Thread threadFeed = null; Thread threadTransactions = null; Thread threadResults = null; Thread threadRealTime = null; //-> Initialize messaging system _systemHandlers.Notify.SetAuthentication(job); //-> Set the result handler type for this algorithm job, and launch the associated result thread. _algorithmHandlers.Results.Initialize(job, _systemHandlers.Notify, _systemHandlers.Api, _algorithmHandlers.DataFeed, _algorithmHandlers.Setup, _algorithmHandlers.Transactions); threadResults = new Thread(_algorithmHandlers.Results.Run, 0) { IsBackground = true, Name = "Result Thread" }; threadResults.Start(); IBrokerage brokerage = null; try { // Save algorithm to cache, load algorithm instance: algorithm = _algorithmHandlers.Setup.CreateAlgorithmInstance(job, assemblyPath); // Set algorithm in ILeanManager _systemHandlers.LeanManager.SetAlgorithm(algorithm); // Initialize the brokerage IBrokerageFactory factory; brokerage = _algorithmHandlers.Setup.CreateBrokerage(job, algorithm, out factory); // Initialize the data feed before we initialize so he can intercept added securities/universes via events _algorithmHandlers.DataFeed.Initialize(algorithm, job, _algorithmHandlers.Results, _algorithmHandlers.MapFileProvider, _algorithmHandlers.FactorFileProvider, _algorithmHandlers.DataProvider); // set the order processor on the transaction manager (needs to be done before initializing BrokerageHistoryProvider) algorithm.Transactions.SetOrderProcessor(_algorithmHandlers.Transactions); // set the history provider before setting up the algorithm var historyProvider = GetHistoryProvider(job.HistoryProvider); if (historyProvider is BrokerageHistoryProvider) { (historyProvider as BrokerageHistoryProvider).SetBrokerage(brokerage); } var historyDataCacheProvider = new ZipDataCacheProvider(_algorithmHandlers.DataProvider); historyProvider.Initialize(job, _algorithmHandlers.DataProvider, historyDataCacheProvider, _algorithmHandlers.MapFileProvider, _algorithmHandlers.FactorFileProvider, progress => { // send progress updates to the result handler only during initialization if (!algorithm.GetLocked() || algorithm.IsWarmingUp) { _algorithmHandlers.Results.SendStatusUpdate(AlgorithmStatus.History, string.Format("Processing history {0}%...", progress)); } }); algorithm.HistoryProvider = historyProvider; // initialize the default brokerage message handler algorithm.BrokerageMessageHandler = factory.CreateBrokerageMessageHandler(algorithm, job, _systemHandlers.Api); //Initialize the internal state of algorithm and job: executes the algorithm.Initialize() method. initializeComplete = _algorithmHandlers.Setup.Setup(algorithm, brokerage, job, _algorithmHandlers.Results, _algorithmHandlers.Transactions, _algorithmHandlers.RealTime); // set this again now that we've actually added securities _algorithmHandlers.Results.SetAlgorithm(algorithm); //If there are any reasons it failed, pass these back to the IDE. if (!initializeComplete || algorithm.ErrorMessages.Count > 0 || _algorithmHandlers.Setup.Errors.Count > 0) { initializeComplete = false; //Get all the error messages: internal in algorithm and external in setup handler. var errorMessage = String.Join(",", algorithm.ErrorMessages); errorMessage += String.Join(",", _algorithmHandlers.Setup.Errors); Log.Error("Engine.Run(): " + errorMessage); _algorithmHandlers.Results.RuntimeError(errorMessage); _systemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, errorMessage); } } catch (Exception err) { Log.Error(err); var runtimeMessage = "Algorithm.Initialize() Error: " + err.Message + " Stack Trace: " + err.StackTrace; _algorithmHandlers.Results.RuntimeError(runtimeMessage, err.StackTrace); _systemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, runtimeMessage); } // log the job endpoints Log.Trace("JOB HANDLERS: "); Log.Trace(" DataFeed: " + _algorithmHandlers.DataFeed.GetType().FullName); Log.Trace(" Setup: " + _algorithmHandlers.Setup.GetType().FullName); Log.Trace(" RealTime: " + _algorithmHandlers.RealTime.GetType().FullName); Log.Trace(" Results: " + _algorithmHandlers.Results.GetType().FullName); Log.Trace(" Transactions: " + _algorithmHandlers.Transactions.GetType().FullName); if (algorithm != null && algorithm.HistoryProvider != null) { Log.Trace(" History Provider: " + algorithm.HistoryProvider.GetType().FullName); } if (job is LiveNodePacket) { Log.Trace(" Brokerage: " + brokerage.GetType().FullName); } //-> Using the job + initialization: load the designated handlers: if (initializeComplete) { //-> Reset the backtest stopwatch; we're now running the algorithm. var startTime = DateTime.Now; //Set algorithm as locked; set it to live mode if we're trading live, and set it to locked for no further updates. algorithm.SetAlgorithmId(job.AlgorithmId); algorithm.SetLocked(); //Load the associated handlers for transaction and realtime events: _algorithmHandlers.Transactions.Initialize(algorithm, brokerage, _algorithmHandlers.Results); _algorithmHandlers.RealTime.Setup(algorithm, job, _algorithmHandlers.Results, _systemHandlers.Api); // wire up the brokerage message handler brokerage.Message += (sender, message) => { algorithm.BrokerageMessageHandler.Handle(message); // fire brokerage message events algorithm.OnBrokerageMessage(message); switch (message.Type) { case BrokerageMessageType.Disconnect: algorithm.OnBrokerageDisconnect(); break; case BrokerageMessageType.Reconnect: algorithm.OnBrokerageReconnect(); break; } }; //Send status to user the algorithm is now executing. _algorithmHandlers.Results.SendStatusUpdate(AlgorithmStatus.Running); //Launch the data, transaction and realtime handlers into dedicated threads threadFeed = new Thread(_algorithmHandlers.DataFeed.Run) { IsBackground = true, Name = "DataFeed Thread" }; threadTransactions = new Thread(_algorithmHandlers.Transactions.Run) { IsBackground = true, Name = "Transaction Thread" }; threadRealTime = new Thread(_algorithmHandlers.RealTime.Run) { IsBackground = true, Name = "RealTime Thread" }; //Launch the data feed, result sending, and transaction models/handlers in separate threads. threadFeed.Start(); // Data feed pushing data packets into thread bridge; threadTransactions.Start(); // Transaction modeller scanning new order requests threadRealTime.Start(); // RealTime scan time for time based events: // Result manager scanning message queue: (started earlier) _algorithmHandlers.Results.DebugMessage(string.Format("Launching analysis for {0} with LEAN Engine v{1}", job.AlgorithmId, Globals.Version)); try { //Create a new engine isolator class var isolator = new Isolator(); // Execute the Algorithm Code: var complete = isolator.ExecuteWithTimeLimit(_algorithmHandlers.Setup.MaximumRuntime, algorithmManager.TimeLoopWithinLimits, () => { try { //Run Algorithm Job: // -> Using this Data Feed, // -> Send Orders to this TransactionHandler, // -> Send Results to ResultHandler. algorithmManager.Run(job, algorithm, _algorithmHandlers.DataFeed, _algorithmHandlers.Transactions, _algorithmHandlers.Results, _algorithmHandlers.RealTime, _systemHandlers.LeanManager, isolator.CancellationToken); } catch (Exception err) { //Debugging at this level is difficult, stack trace needed. Log.Error(err); algorithm.RunTimeError = err; algorithmManager.SetStatus(AlgorithmStatus.RuntimeError); return; } Log.Trace("Engine.Run(): Exiting Algorithm Manager"); }, job.Controls.RamAllocation); if (!complete) { Log.Error("Engine.Main(): Failed to complete in time: " + _algorithmHandlers.Setup.MaximumRuntime.ToString("F")); throw new Exception("Failed to complete algorithm within " + _algorithmHandlers.Setup.MaximumRuntime.ToString("F") + " seconds. Please make it run faster."); } // Algorithm runtime error: if (algorithm.RunTimeError != null) { HandleAlgorithmError(job, algorithm.RunTimeError); } } catch (Exception err) { //Error running the user algorithm: purge datafeed, send error messages, set algorithm status to failed. HandleAlgorithmError(job, err); } try { var trades = algorithm.TradeBuilder.ClosedTrades; var charts = new Dictionary <string, Chart>(_algorithmHandlers.Results.Charts); var orders = new Dictionary <int, Order>(_algorithmHandlers.Transactions.Orders); var holdings = new Dictionary <string, Holding>(); var banner = new Dictionary <string, string>(); var statisticsResults = new StatisticsResults(); var csvTransactionsFileName = Config.Get("transaction-log"); if (!string.IsNullOrEmpty(csvTransactionsFileName)) { SaveListOfTrades(_algorithmHandlers.Transactions, csvTransactionsFileName); } try { //Generates error when things don't exist (no charting logged, runtime errors in main algo execution) const string strategyEquityKey = "Strategy Equity"; const string equityKey = "Equity"; const string dailyPerformanceKey = "Daily Performance"; const string benchmarkKey = "Benchmark"; // make sure we've taken samples for these series before just blindly requesting them if (charts.ContainsKey(strategyEquityKey) && charts[strategyEquityKey].Series.ContainsKey(equityKey) && charts[strategyEquityKey].Series.ContainsKey(dailyPerformanceKey) && charts.ContainsKey(benchmarkKey) && charts[benchmarkKey].Series.ContainsKey(benchmarkKey) ) { var equity = charts[strategyEquityKey].Series[equityKey].Values; var performance = charts[strategyEquityKey].Series[dailyPerformanceKey].Values; var profitLoss = new SortedDictionary <DateTime, decimal>(algorithm.Transactions.TransactionRecord); var totalTransactions = algorithm.Transactions.GetOrders(x => x.Status.IsFill()).Count(); var benchmark = charts[benchmarkKey].Series[benchmarkKey].Values; statisticsResults = StatisticsBuilder.Generate(trades, profitLoss, equity, performance, benchmark, _algorithmHandlers.Setup.StartingPortfolioValue, algorithm.Portfolio.TotalFees, totalTransactions); //Some users have $0 in their brokerage account / starting cash of $0. Prevent divide by zero errors var netReturn = _algorithmHandlers.Setup.StartingPortfolioValue > 0 ? (algorithm.Portfolio.TotalPortfolioValue - _algorithmHandlers.Setup.StartingPortfolioValue) / _algorithmHandlers.Setup.StartingPortfolioValue : 0; //Add other fixed parameters. banner.Add("Unrealized", "$" + algorithm.Portfolio.TotalUnrealizedProfit.ToString("N2")); banner.Add("Fees", "-$" + algorithm.Portfolio.TotalFees.ToString("N2")); banner.Add("Net Profit", "$" + algorithm.Portfolio.TotalProfit.ToString("N2")); banner.Add("Return", netReturn.ToString("P")); banner.Add("Equity", "$" + algorithm.Portfolio.TotalPortfolioValue.ToString("N2")); } } catch (Exception err) { Log.Error(err, "Error generating statistics packet"); } //Diagnostics Completed, Send Result Packet: var totalSeconds = (DateTime.Now - startTime).TotalSeconds; var dataPoints = algorithmManager.DataPoints + algorithm.HistoryProvider.DataPointCount; _algorithmHandlers.Results.DebugMessage( string.Format("Algorithm Id:({0}) completed in {1} seconds at {2}k data points per second. Processing total of {3} data points.", job.AlgorithmId, totalSeconds.ToString("F2"), ((dataPoints / (double)1000) / totalSeconds).ToString("F0"), dataPoints.ToString("N0"))); _algorithmHandlers.Results.SendFinalResult(job, orders, algorithm.Transactions.TransactionRecord, holdings, algorithm.Portfolio.CashBook, statisticsResults, banner); } catch (Exception err) { Log.Error(err, "Error sending analysis results"); } //Before we return, send terminate commands to close up the threads _algorithmHandlers.Transactions.Exit(); _algorithmHandlers.DataFeed.Exit(); _algorithmHandlers.RealTime.Exit(); } //Close result handler: _algorithmHandlers.Results.Exit(); //Wait for the threads to complete: var ts = Stopwatch.StartNew(); while ((_algorithmHandlers.Results.IsActive || (_algorithmHandlers.Transactions != null && _algorithmHandlers.Transactions.IsActive) || (_algorithmHandlers.DataFeed != null && _algorithmHandlers.DataFeed.IsActive) || (_algorithmHandlers.RealTime != null && _algorithmHandlers.RealTime.IsActive)) && ts.ElapsedMilliseconds < 30 * 1000) { Thread.Sleep(100); Log.Trace("Waiting for threads to exit..."); } //Terminate threads still in active state. if (threadFeed != null && threadFeed.IsAlive) { threadFeed.Abort(); } if (threadTransactions != null && threadTransactions.IsAlive) { threadTransactions.Abort(); } if (threadResults != null && threadResults.IsAlive) { threadResults.Abort(); } if (brokerage != null) { Log.Trace("Engine.Run(): Disconnecting from brokerage..."); brokerage.Disconnect(); brokerage.Dispose(); } if (_algorithmHandlers.Setup != null) { Log.Trace("Engine.Run(): Disposing of setup handler..."); _algorithmHandlers.Setup.Dispose(); } Log.Trace("Engine.Main(): Analysis Completed and Results Posted."); } catch (Exception err) { Log.Error(err, "Error running algorithm"); } finally { //No matter what for live mode; make sure we've set algorithm status in the API for "not running" conditions: if (_liveMode && algorithmManager.State != AlgorithmStatus.Running && algorithmManager.State != AlgorithmStatus.RuntimeError) { _systemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, algorithmManager.State); } _algorithmHandlers.Results.Exit(); _algorithmHandlers.DataFeed.Exit(); _algorithmHandlers.Transactions.Exit(); _algorithmHandlers.RealTime.Exit(); } }