public void Dispose() { if (!_Running) { FlushPendingDisposes(); return; } _Running = false; try { WaitForActiveDraws(); if (_DrawThread != null) { _DrawThread.Dispose(); _DrawThread = null; } FlushPendingDisposes(); } catch (ObjectDisposedException) { } catch (DeviceLostException) { #if !SDL2 } catch (DeviceNotResetException) { #endif } }
public void Should_be_able_to_dispose_failing_worker() { var worker = new TestProcessor(); var target = new WorkerThread<int>("test", TimeSpan.FromDays(1), worker.Process); target.Enqueue(1); target.Dispose(); worker.Logs.Should().Have.SameSequenceAs(new[] { "1" }); }
private void MainExecutionThread() { while (true) { if (disposed) { return; } while (workerThreads.Count < maxThreadCount) { workerThreads.Add(new WorkerThread(checkQueuedWorkItems)); } while (workerThreads.Count > maxThreadCount) { WorkerThread availableThread = workerThreads.FirstOrDefault(x => x.IsAvailable); if (availableThread == null) { break; } workerThreads.Remove(availableThread); availableThread.Dispose(); } lock (workItems) { while (workItems.Count > 0) { if (disposed) { return; } WorkerThread availableThread = workerThreads.FirstOrDefault(x => x.IsAvailable); if (availableThread == null) { break; } var workItem = workItems.Dequeue(); availableThread.ExecuteWorkItem(workItem); } } if (disposed) { return; } checkQueuedWorkItems.WaitOne(); if (disposed) { return; } } }
protected virtual void Dispose(bool disposing) { if (disposing) { if (fWorkerThread != null) { fWorkerThread.Dispose(); } if (fObjectContext != null) { fObjectContext.Dispose(); fObjectContext = null; } } }
/// <summary> /// Unsubscribes from a SubscribeAll for this device /// </summary> public void UnsubscribeAllButtons() { if (AllButtonsMapping == null) { return; } // Stop DeviceWorkerThread if (!AllButtonsMapping.Concurrent && DeviceWorkerThread != null) { DeviceWorkerThread.Dispose(); DeviceWorkerThread = null; } AllButtonsMapping = null; DisableFilterIfNeeded(); }
public void Should_dequeue_all_when_disposing() { var worker = new TestProcessor(); var target = new WorkerThread<int>("test", TimeSpan.FromDays(1), worker.Process); target.Enqueue(1); target.Enqueue(2); target.Enqueue(3); target.Dispose(); worker.Logs.Should().Have.SameSequenceAs(new[] { "1, 2, 3" }); }
public void Dispose_ShouldShutdownThread() { var resetEvent = new AutoResetEvent(false); Action <CancellationToken> callback = (cancellationToken) => { resetEvent.Set(); }; var workerThread = new WorkerThread(callback); workerThread.Dispose(); Thread.Sleep(3); // Give the thread some time to shut down. resetEvent.Reset(); // Thread should be stopped, we should not get another signal. var signalled = resetEvent.WaitOne(3); Assert.IsFalse(signalled); }
//to do something once in handler, though //this code would go in onStart in a windows service. public static void Start() { WorkerThread thread1 = null; WorkerThread thread2 = null; //WorkerTimer thread1 = null; //WorkerTimer thread2 = null; //Console.WriteLine("Start: thread " + Thread.CurrentThread.ManagedThreadId); //watch config FileSystemWatcher watcher = new FileSystemWatcher(); watcher.Path = "../../"; watcher.Filter = "test.xml"; watcher.EnableRaisingEvents = true; //subscribe to changed event. note that this event can be raised a number of times for each save of the file. watcher.Changed += (sender, args) => FileChanged(sender, args); thread1 = new WorkerThread("foo", 10); thread2 = new WorkerThread("bar", 15); //thread1 = new WorkerTimer("foo", 10); //thread2 = new WorkerTimer("bar", 15); while (true) { if (_reload) { //create our two threads. //Console.WriteLine("Start - reload: thread " + Thread.CurrentThread.ManagedThreadId); //wait, to enable other file changed events to pass //Console.WriteLine("Start - waiting: thread " + Thread.CurrentThread.ManagedThreadId); thread1.Dispose(); thread2.Dispose(); Thread.Sleep(3000); //each thread lasts 0.5 seconds, so 3 seconds should be plenty to wait for the //LoadData function to complete. Monitor.Enter(_reloadLock); //GC.Collect(); thread1 = new WorkerThread("foo", 5); thread2 = new WorkerThread("bar", 7); //thread1 = new WorkerTimer("foo", 5); //thread2 = new WorkerTimer("bar", 7); _reload = false; Monitor.Exit(_reloadLock); } } }
public void Dispose_CallbackShouldBeSignalledCancellation() { var controlResetEvent = new AutoResetEvent(false); var resultResetEvent = new AutoResetEvent(false); Action <CancellationToken> callback = (cancellationToken) => { controlResetEvent.Set(); // Signal the test to dispose(). controlResetEvent.WaitOne(100); if (cancellationToken.IsCancellationRequested) { resultResetEvent.Set(); } }; var workerThread = new WorkerThread(callback); controlResetEvent.WaitOne(100); workerThread.Dispose(); controlResetEvent.Set(); // Signal the thread to check the cancellation token. var cancellationRequested = resultResetEvent.WaitOne(100); Assert.IsTrue(cancellationRequested); }
/// <summary> /// Runs a single backtest/live job from the job queue /// </summary> /// <param name="job">The algorithm job to be processed</param> /// <param name="manager">The algorithm manager instance</param> /// <param name="assemblyPath">The path to the algorithm's assembly</param> /// <param name="workerThread">The worker thread instance</param> public void Run(AlgorithmNodePacket job, AlgorithmManager manager, string assemblyPath, WorkerThread workerThread) { var marketHoursDatabaseTask = Task.Run(() => StaticInitializations()); var algorithm = default(IAlgorithm); var algorithmManager = manager; try { //Reset thread holders. var initializeComplete = false; //-> Initialize messaging system SystemHandlers.Notify.SetAuthentication(job); //-> Set the result handler type for this algorithm job, and launch the associated result thread. AlgorithmHandlers.Results.Initialize(job, SystemHandlers.Notify, SystemHandlers.Api, AlgorithmHandlers.Transactions); IBrokerage brokerage = null; DataManager dataManager = null; var synchronizer = _liveMode ? new LiveSynchronizer() : new Synchronizer(); try { // we get the mhdb before creating the algorithm instance, // since the algorithm constructor will use it var marketHoursDatabase = marketHoursDatabaseTask.Result; AlgorithmHandlers.Setup.WorkerThread = workerThread; // Save algorithm to cache, load algorithm instance: algorithm = AlgorithmHandlers.Setup.CreateAlgorithmInstance(job, assemblyPath); // Set algorithm in ILeanManager SystemHandlers.LeanManager.SetAlgorithm(algorithm); // initialize the alphas handler with the algorithm instance AlgorithmHandlers.Alphas.Initialize(job, algorithm, SystemHandlers.Notify, SystemHandlers.Api); // initialize the object store AlgorithmHandlers.ObjectStore.Initialize(algorithm.Name, job.UserId, job.ProjectId, job.UserToken, job.Controls); // notify the user of any errors w/ object store persistence AlgorithmHandlers.ObjectStore.ErrorRaised += (sender, args) => algorithm.Debug($"ObjectStore Persistence Error: {args.Error.Message}"); // Initialize the brokerage IBrokerageFactory factory; brokerage = AlgorithmHandlers.Setup.CreateBrokerage(job, algorithm, out factory); var symbolPropertiesDatabase = SymbolPropertiesDatabase.FromDataFolder(); var registeredTypesProvider = new RegisteredSecurityDataTypesProvider(); var securityService = new SecurityService(algorithm.Portfolio.CashBook, marketHoursDatabase, symbolPropertiesDatabase, algorithm, registeredTypesProvider, new SecurityCacheProvider(algorithm.Portfolio)); algorithm.Securities.SetSecurityService(securityService); dataManager = new DataManager(AlgorithmHandlers.DataFeed, new UniverseSelection( algorithm, securityService), algorithm, algorithm.TimeKeeper, marketHoursDatabase, _liveMode, registeredTypesProvider); AlgorithmHandlers.Results.SetDataManager(dataManager); algorithm.SubscriptionManager.SetDataManager(dataManager); synchronizer.Initialize(algorithm, dataManager); // Initialize the data feed before we initialize so he can intercept added securities/universes via events AlgorithmHandlers.DataFeed.Initialize( algorithm, job, AlgorithmHandlers.Results, AlgorithmHandlers.MapFileProvider, AlgorithmHandlers.FactorFileProvider, AlgorithmHandlers.DataProvider, dataManager, (IDataFeedTimeProvider)synchronizer); // set the order processor on the transaction manager (needs to be done before initializing BrokerageHistoryProvider) algorithm.Transactions.SetOrderProcessor(AlgorithmHandlers.Transactions); algorithm.SetOrderEventProvider(AlgorithmHandlers.Transactions); // set the history provider before setting up the algorithm var historyProvider = GetHistoryProvider(job.HistoryProvider); if (historyProvider is BrokerageHistoryProvider) { (historyProvider as BrokerageHistoryProvider).SetBrokerage(brokerage); } var historyDataCacheProvider = new ZipDataCacheProvider(AlgorithmHandlers.DataProvider, isDataEphemeral: _liveMode); historyProvider.Initialize( new HistoryProviderInitializeParameters( job, SystemHandlers.Api, AlgorithmHandlers.DataProvider, historyDataCacheProvider, AlgorithmHandlers.MapFileProvider, AlgorithmHandlers.FactorFileProvider, progress => { // send progress updates to the result handler only during initialization if (!algorithm.GetLocked() || algorithm.IsWarmingUp) { AlgorithmHandlers.Results.SendStatusUpdate(AlgorithmStatus.History, Invariant($"Processing history {progress}%...")); } }, // disable parallel history requests for live trading parallelHistoryRequestsEnabled: !_liveMode ) ); historyProvider.InvalidConfigurationDetected += (sender, args) => { AlgorithmHandlers.Results.ErrorMessage(args.Message); }; historyProvider.NumericalPrecisionLimited += (sender, args) => { AlgorithmHandlers.Results.DebugMessage(args.Message); }; historyProvider.DownloadFailed += (sender, args) => { AlgorithmHandlers.Results.ErrorMessage(args.Message, args.StackTrace); }; historyProvider.ReaderErrorDetected += (sender, args) => { AlgorithmHandlers.Results.RuntimeError(args.Message, args.StackTrace); }; algorithm.HistoryProvider = historyProvider; // initialize the default brokerage message handler algorithm.BrokerageMessageHandler = factory.CreateBrokerageMessageHandler(algorithm, job, SystemHandlers.Api); //Initialize the internal state of algorithm and job: executes the algorithm.Initialize() method. initializeComplete = AlgorithmHandlers.Setup.Setup(new SetupHandlerParameters(dataManager.UniverseSelection, algorithm, brokerage, job, AlgorithmHandlers.Results, AlgorithmHandlers.Transactions, AlgorithmHandlers.RealTime, AlgorithmHandlers.ObjectStore)); // set this again now that we've actually added securities AlgorithmHandlers.Results.SetAlgorithm(algorithm, AlgorithmHandlers.Setup.StartingPortfolioValue); // alpha handler needs start/end dates to determine sample step sizes AlgorithmHandlers.Alphas.OnAfterAlgorithmInitialized(algorithm); //If there are any reasons it failed, pass these back to the IDE. if (!initializeComplete || algorithm.ErrorMessages.Count > 0 || AlgorithmHandlers.Setup.Errors.Count > 0) { initializeComplete = false; //Get all the error messages: internal in algorithm and external in setup handler. var errorMessage = string.Join(",", algorithm.ErrorMessages); errorMessage += string.Join(",", AlgorithmHandlers.Setup.Errors.Select(e => { var message = e.Message; if (e.InnerException != null) { var err = _exceptionInterpreter.Value.Interpret(e.InnerException, _exceptionInterpreter.Value); message += _exceptionInterpreter.Value.GetExceptionMessageHeader(err); } return(message); })); Log.Error("Engine.Run(): " + errorMessage); AlgorithmHandlers.Results.RuntimeError(errorMessage); SystemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, errorMessage); } } catch (Exception err) { Log.Error(err); var runtimeMessage = "Algorithm.Initialize() Error: " + err.Message + " Stack Trace: " + err; AlgorithmHandlers.Results.RuntimeError(runtimeMessage, err.ToString()); SystemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, runtimeMessage); } // log the job endpoints Log.Trace("JOB HANDLERS: "); Log.Trace(" DataFeed: " + AlgorithmHandlers.DataFeed.GetType().FullName); Log.Trace(" Setup: " + AlgorithmHandlers.Setup.GetType().FullName); Log.Trace(" RealTime: " + AlgorithmHandlers.RealTime.GetType().FullName); Log.Trace(" Results: " + AlgorithmHandlers.Results.GetType().FullName); Log.Trace(" Transactions: " + AlgorithmHandlers.Transactions.GetType().FullName); Log.Trace(" Alpha: " + AlgorithmHandlers.Alphas.GetType().FullName); Log.Trace(" ObjectStore: " + AlgorithmHandlers.ObjectStore.GetType().FullName); if (algorithm?.HistoryProvider != null) { Log.Trace(" History Provider: " + algorithm.HistoryProvider.GetType().FullName); } if (job is LiveNodePacket) { Log.Trace(" Brokerage: " + brokerage?.GetType().FullName); } //-> Using the job + initialization: load the designated handlers: if (initializeComplete) { // notify the LEAN manager that the algorithm is initialized and starting SystemHandlers.LeanManager.OnAlgorithmStart(); //-> Reset the backtest stopwatch; we're now running the algorithm. var startTime = DateTime.Now; //Set algorithm as locked; set it to live mode if we're trading live, and set it to locked for no further updates. algorithm.SetAlgorithmId(job.AlgorithmId); algorithm.SetLocked(); //Load the associated handlers for transaction and realtime events: AlgorithmHandlers.Transactions.Initialize(algorithm, brokerage, AlgorithmHandlers.Results); AlgorithmHandlers.RealTime.Setup(algorithm, job, AlgorithmHandlers.Results, SystemHandlers.Api, algorithmManager.TimeLimit); // wire up the brokerage message handler brokerage.Message += (sender, message) => { algorithm.BrokerageMessageHandler.Handle(message); // fire brokerage message events algorithm.OnBrokerageMessage(message); switch (message.Type) { case BrokerageMessageType.Disconnect: algorithm.OnBrokerageDisconnect(); break; case BrokerageMessageType.Reconnect: algorithm.OnBrokerageReconnect(); break; } }; //Send status to user the algorithm is now executing. AlgorithmHandlers.Results.SendStatusUpdate(AlgorithmStatus.Running); // Result manager scanning message queue: (started earlier) AlgorithmHandlers.Results.DebugMessage( $"Launching analysis for {job.AlgorithmId} with LEAN Engine v{Globals.Version}"); try { //Create a new engine isolator class var isolator = new Isolator(); // Execute the Algorithm Code: var complete = isolator.ExecuteWithTimeLimit(AlgorithmHandlers.Setup.MaximumRuntime, algorithmManager.TimeLimit.IsWithinLimit, () => { try { //Run Algorithm Job: // -> Using this Data Feed, // -> Send Orders to this TransactionHandler, // -> Send Results to ResultHandler. algorithmManager.Run(job, algorithm, synchronizer, AlgorithmHandlers.Transactions, AlgorithmHandlers.Results, AlgorithmHandlers.RealTime, SystemHandlers.LeanManager, AlgorithmHandlers.Alphas, isolator.CancellationToken); } catch (Exception err) { //Debugging at this level is difficult, stack trace needed. Log.Error(err); algorithm.RunTimeError = err; algorithmManager.SetStatus(AlgorithmStatus.RuntimeError); return; } Log.Trace("Engine.Run(): Exiting Algorithm Manager"); }, job.Controls.RamAllocation, workerThread: workerThread); if (!complete) { Log.Error("Engine.Main(): Failed to complete in time: " + AlgorithmHandlers.Setup.MaximumRuntime.ToStringInvariant("F")); throw new Exception("Failed to complete algorithm within " + AlgorithmHandlers.Setup.MaximumRuntime.ToStringInvariant("F") + " seconds. Please make it run faster."); } // Algorithm runtime error: if (algorithm.RunTimeError != null) { HandleAlgorithmError(job, algorithm.RunTimeError); } } catch (Exception err) { //Error running the user algorithm: purge datafeed, send error messages, set algorithm status to failed. algorithm.RunTimeError = err; algorithm.SetStatus(AlgorithmStatus.RuntimeError); HandleAlgorithmError(job, err); } // notify the LEAN manager that the algorithm has finished SystemHandlers.LeanManager.OnAlgorithmEnd(); try { var csvTransactionsFileName = Config.Get("transaction-log"); if (!string.IsNullOrEmpty(csvTransactionsFileName)) { SaveListOfTrades(AlgorithmHandlers.Transactions, csvTransactionsFileName); } if (!_liveMode) { //Diagnostics Completed, Send Result Packet: var totalSeconds = (DateTime.Now - startTime).TotalSeconds; var dataPoints = algorithmManager.DataPoints + algorithm.HistoryProvider.DataPointCount; var kps = dataPoints / (double)1000 / totalSeconds; AlgorithmHandlers.Results.DebugMessage($"Algorithm Id:({job.AlgorithmId}) completed in {totalSeconds:F2} seconds at {kps:F0}k data points per second. Processing total of {dataPoints:N0} data points."); } } catch (Exception err) { Log.Error(err, "Error sending analysis results"); } //Before we return, send terminate commands to close up the threads AlgorithmHandlers.Transactions.Exit(); AlgorithmHandlers.RealTime.Exit(); dataManager?.RemoveAllSubscriptions(); workerThread?.Dispose(); } // Close data feed, alphas. Could be running even if algorithm initialization failed AlgorithmHandlers.DataFeed.Exit(); AlgorithmHandlers.Alphas.Exit(); //Close result handler: AlgorithmHandlers.Results.Exit(); //Wait for the threads to complete: var millisecondInterval = 10; var millisecondTotalWait = 0; while ((AlgorithmHandlers.Results.IsActive || (AlgorithmHandlers.Transactions != null && AlgorithmHandlers.Transactions.IsActive) || (AlgorithmHandlers.DataFeed != null && AlgorithmHandlers.DataFeed.IsActive) || (AlgorithmHandlers.RealTime != null && AlgorithmHandlers.RealTime.IsActive) || (AlgorithmHandlers.Alphas != null && AlgorithmHandlers.Alphas.IsActive)) && millisecondTotalWait < 30 * 1000) { Thread.Sleep(millisecondInterval); if (millisecondTotalWait % (millisecondInterval * 10) == 0) { Log.Trace("Waiting for threads to exit..."); } millisecondTotalWait += millisecondInterval; } if (brokerage != null) { Log.Trace("Engine.Run(): Disconnecting from brokerage..."); brokerage.Disconnect(); brokerage.Dispose(); } if (AlgorithmHandlers.Setup != null) { Log.Trace("Engine.Run(): Disposing of setup handler..."); AlgorithmHandlers.Setup.Dispose(); } Log.Trace("Engine.Main(): Analysis Completed and Results Posted."); } catch (Exception err) { Log.Error(err, "Error running algorithm"); } finally { //No matter what for live mode; make sure we've set algorithm status in the API for "not running" conditions: if (_liveMode && algorithmManager.State != AlgorithmStatus.Running && algorithmManager.State != AlgorithmStatus.RuntimeError) { SystemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, algorithmManager.State); } AlgorithmHandlers.Results.Exit(); AlgorithmHandlers.DataFeed.Exit(); AlgorithmHandlers.Transactions.Exit(); AlgorithmHandlers.RealTime.Exit(); } }
/// <summary> /// Runs a single backtest/live job from the job queue /// </summary> /// <param name="job">The algorithm job to be processed</param> /// <param name="manager"></param> /// <param name="assemblyPath">The path to the algorithm's assembly</param> public void Run(AlgorithmNodePacket job, AlgorithmManager manager, string assemblyPath) { var marketHoursDatabaseTask = Task.Run(() => StaticInitializations()); var algorithm = default(IAlgorithm); var algorithmManager = manager; try { //Reset thread holders. var initializeComplete = false; Thread threadResults = null; Thread threadRealTime = null; Thread threadAlphas = null; WorkerThread workerThread = null; //-> Initialize messaging system SystemHandlers.Notify.SetAuthentication(job); //-> Set the result handler type for this algorithm job, and launch the associated result thread. AlgorithmHandlers.Results.Initialize(job, SystemHandlers.Notify, SystemHandlers.Api, AlgorithmHandlers.Transactions); threadResults = new Thread(AlgorithmHandlers.Results.Run, 0) { IsBackground = true, Name = "Result Thread" }; threadResults.Start(); IBrokerage brokerage = null; DataManager dataManager = null; var synchronizer = _liveMode ? new LiveSynchronizer() : new Synchronizer(); try { // we get the mhdb before creating the algorithm instance, // since the algorithm constructor will use it var marketHoursDatabase = marketHoursDatabaseTask.Result; // start worker thread workerThread = new WorkerThread(); AlgorithmHandlers.Setup.WorkerThread = workerThread; // Save algorithm to cache, load algorithm instance: algorithm = AlgorithmHandlers.Setup.CreateAlgorithmInstance(job, assemblyPath); // Set algorithm in ILeanManager SystemHandlers.LeanManager.SetAlgorithm(algorithm); // initialize the alphas handler with the algorithm instance AlgorithmHandlers.Alphas.Initialize(job, algorithm, SystemHandlers.Notify, SystemHandlers.Api); // Initialize the brokerage IBrokerageFactory factory; brokerage = AlgorithmHandlers.Setup.CreateBrokerage(job, algorithm, out factory); var symbolPropertiesDatabase = SymbolPropertiesDatabase.FromDataFolder(); var securityService = new SecurityService(algorithm.Portfolio.CashBook, marketHoursDatabase, symbolPropertiesDatabase, (ISecurityInitializerProvider)algorithm); algorithm.Securities.SetSecurityService(securityService); dataManager = new DataManager(AlgorithmHandlers.DataFeed, new UniverseSelection( algorithm, securityService), algorithm, algorithm.TimeKeeper, marketHoursDatabase, _liveMode); AlgorithmHandlers.Results.SetDataManager(dataManager); algorithm.SubscriptionManager.SetDataManager(dataManager); synchronizer.Initialize(algorithm, dataManager); // Initialize the data feed before we initialize so he can intercept added securities/universes via events AlgorithmHandlers.DataFeed.Initialize( algorithm, job, AlgorithmHandlers.Results, AlgorithmHandlers.MapFileProvider, AlgorithmHandlers.FactorFileProvider, AlgorithmHandlers.DataProvider, dataManager, (IDataFeedTimeProvider)synchronizer); // set the order processor on the transaction manager (needs to be done before initializing BrokerageHistoryProvider) algorithm.Transactions.SetOrderProcessor(AlgorithmHandlers.Transactions); algorithm.SetOrderEventProvider(AlgorithmHandlers.Transactions); // set the history provider before setting up the algorithm var historyProvider = GetHistoryProvider(job.HistoryProvider); if (historyProvider is BrokerageHistoryProvider) { (historyProvider as BrokerageHistoryProvider).SetBrokerage(brokerage); } var historyDataCacheProvider = new ZipDataCacheProvider(AlgorithmHandlers.DataProvider, isDataEphemeral: _liveMode); historyProvider.Initialize( new HistoryProviderInitializeParameters( job, SystemHandlers.Api, AlgorithmHandlers.DataProvider, historyDataCacheProvider, AlgorithmHandlers.MapFileProvider, AlgorithmHandlers.FactorFileProvider, progress => { // send progress updates to the result handler only during initialization if (!algorithm.GetLocked() || algorithm.IsWarmingUp) { AlgorithmHandlers.Results.SendStatusUpdate(AlgorithmStatus.History, $"Processing history {progress}%..."); } } ) ); historyProvider.InvalidConfigurationDetected += (sender, args) => { AlgorithmHandlers.Results.ErrorMessage(args.Message); }; historyProvider.NumericalPrecisionLimited += (sender, args) => { AlgorithmHandlers.Results.DebugMessage(args.Message); }; historyProvider.DownloadFailed += (sender, args) => { AlgorithmHandlers.Results.ErrorMessage(args.Message, args.StackTrace); }; historyProvider.ReaderErrorDetected += (sender, args) => { AlgorithmHandlers.Results.RuntimeError(args.Message, args.StackTrace); }; algorithm.HistoryProvider = historyProvider; // initialize the default brokerage message handler algorithm.BrokerageMessageHandler = factory.CreateBrokerageMessageHandler(algorithm, job, SystemHandlers.Api); //Initialize the internal state of algorithm and job: executes the algorithm.Initialize() method. initializeComplete = AlgorithmHandlers.Setup.Setup(new SetupHandlerParameters(dataManager.UniverseSelection, algorithm, brokerage, job, AlgorithmHandlers.Results, AlgorithmHandlers.Transactions, AlgorithmHandlers.RealTime)); // set this again now that we've actually added securities AlgorithmHandlers.Results.SetAlgorithm(algorithm, AlgorithmHandlers.Setup.StartingPortfolioValue); // alpha handler needs start/end dates to determine sample step sizes AlgorithmHandlers.Alphas.OnAfterAlgorithmInitialized(algorithm); //If there are any reasons it failed, pass these back to the IDE. if (!initializeComplete || algorithm.ErrorMessages.Count > 0 || AlgorithmHandlers.Setup.Errors.Count > 0) { initializeComplete = false; //Get all the error messages: internal in algorithm and external in setup handler. var errorMessage = string.Join(",", algorithm.ErrorMessages); errorMessage += string.Join(",", AlgorithmHandlers.Setup.Errors.Select(e => { var message = e.Message; if (e.InnerException != null) { var err = _exceptionInterpreter.Value.Interpret(e.InnerException, _exceptionInterpreter.Value); message += _exceptionInterpreter.Value.GetExceptionMessageHeader(err); } return(message); })); Log.Error("Engine.Run(): " + errorMessage); AlgorithmHandlers.Results.RuntimeError(errorMessage); SystemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, errorMessage); } } catch (Exception err) { Log.Error(err); var runtimeMessage = "Algorithm.Initialize() Error: " + err.Message + " Stack Trace: " + err; AlgorithmHandlers.Results.RuntimeError(runtimeMessage, err.ToString()); SystemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, runtimeMessage); } // log the job endpoints Log.Trace("JOB HANDLERS: "); Log.Trace(" DataFeed: " + AlgorithmHandlers.DataFeed.GetType().FullName); Log.Trace(" Setup: " + AlgorithmHandlers.Setup.GetType().FullName); Log.Trace(" RealTime: " + AlgorithmHandlers.RealTime.GetType().FullName); Log.Trace(" Results: " + AlgorithmHandlers.Results.GetType().FullName); Log.Trace(" Transactions: " + AlgorithmHandlers.Transactions.GetType().FullName); Log.Trace(" Alpha: " + AlgorithmHandlers.Alphas.GetType().FullName); if (algorithm?.HistoryProvider != null) { Log.Trace(" History Provider: " + algorithm.HistoryProvider.GetType().FullName); } if (job is LiveNodePacket) { Log.Trace(" Brokerage: " + brokerage?.GetType().FullName); } //-> Using the job + initialization: load the designated handlers: if (initializeComplete) { // notify the LEAN manager that the algorithm is initialized and starting SystemHandlers.LeanManager.OnAlgorithmStart(); //-> Reset the backtest stopwatch; we're now running the algorithm. var startTime = DateTime.Now; //Set algorithm as locked; set it to live mode if we're trading live, and set it to locked for no further updates. algorithm.SetAlgorithmId(job.AlgorithmId); algorithm.SetLocked(); //Load the associated handlers for transaction and realtime events: AlgorithmHandlers.Transactions.Initialize(algorithm, brokerage, AlgorithmHandlers.Results); AlgorithmHandlers.RealTime.Setup(algorithm, job, AlgorithmHandlers.Results, SystemHandlers.Api); // wire up the brokerage message handler brokerage.Message += (sender, message) => { algorithm.BrokerageMessageHandler.Handle(message); // fire brokerage message events algorithm.OnBrokerageMessage(message); switch (message.Type) { case BrokerageMessageType.Disconnect: algorithm.OnBrokerageDisconnect(); break; case BrokerageMessageType.Reconnect: algorithm.OnBrokerageReconnect(); break; } }; //Send status to user the algorithm is now executing. AlgorithmHandlers.Results.SendStatusUpdate(AlgorithmStatus.Running); //Launch the data, transaction and realtime handlers into dedicated threads threadRealTime = new Thread(AlgorithmHandlers.RealTime.Run) { IsBackground = true, Name = "RealTime Thread" }; threadAlphas = new Thread(() => AlgorithmHandlers.Alphas.Run()) { IsBackground = true, Name = "Alpha Thread" }; //Launch the data feed, result sending, and transaction models/handlers in separate threads. threadRealTime.Start(); // RealTime scan time for time based events: threadAlphas.Start(); // Alpha thread for processing algorithm alpha insights // Result manager scanning message queue: (started earlier) AlgorithmHandlers.Results.DebugMessage( $"Launching analysis for {job.AlgorithmId} with LEAN Engine v{Globals.Version}"); try { //Create a new engine isolator class var isolator = new Isolator(); // Execute the Algorithm Code: var complete = isolator.ExecuteWithTimeLimit(AlgorithmHandlers.Setup.MaximumRuntime, algorithmManager.TimeLoopWithinLimits, () => { try { //Run Algorithm Job: // -> Using this Data Feed, // -> Send Orders to this TransactionHandler, // -> Send Results to ResultHandler. algorithmManager.Run(job, algorithm, synchronizer, AlgorithmHandlers.Transactions, AlgorithmHandlers.Results, AlgorithmHandlers.RealTime, SystemHandlers.LeanManager, AlgorithmHandlers.Alphas, isolator.CancellationToken); } catch (Exception err) { //Debugging at this level is difficult, stack trace needed. Log.Error(err); algorithm.RunTimeError = err; algorithmManager.SetStatus(AlgorithmStatus.RuntimeError); return; } Log.Trace("Engine.Run(): Exiting Algorithm Manager"); }, job.Controls.RamAllocation, workerThread: workerThread); if (!complete) { Log.Error("Engine.Main(): Failed to complete in time: " + AlgorithmHandlers.Setup.MaximumRuntime.ToString("F")); throw new Exception("Failed to complete algorithm within " + AlgorithmHandlers.Setup.MaximumRuntime.ToString("F") + " seconds. Please make it run faster."); } // Algorithm runtime error: if (algorithm.RunTimeError != null) { HandleAlgorithmError(job, algorithm.RunTimeError); } } catch (Exception err) { //Error running the user algorithm: purge datafeed, send error messages, set algorithm status to failed. HandleAlgorithmError(job, err); } // notify the LEAN manager that the algorithm has finished SystemHandlers.LeanManager.OnAlgorithmEnd(); try { var trades = algorithm.TradeBuilder.ClosedTrades; var charts = new Dictionary <string, Chart>(AlgorithmHandlers.Results.Charts); var orders = new Dictionary <int, Order>(AlgorithmHandlers.Transactions.Orders); var holdings = new Dictionary <string, Holding>(); var banner = new Dictionary <string, string>(); var statisticsResults = new StatisticsResults(); var csvTransactionsFileName = Config.Get("transaction-log"); if (!string.IsNullOrEmpty(csvTransactionsFileName)) { SaveListOfTrades(AlgorithmHandlers.Transactions, csvTransactionsFileName); } try { //Generates error when things don't exist (no charting logged, runtime errors in main algo execution) const string strategyEquityKey = "Strategy Equity"; const string equityKey = "Equity"; const string dailyPerformanceKey = "Daily Performance"; const string benchmarkKey = "Benchmark"; // make sure we've taken samples for these series before just blindly requesting them if (charts.ContainsKey(strategyEquityKey) && charts[strategyEquityKey].Series.ContainsKey(equityKey) && charts[strategyEquityKey].Series.ContainsKey(dailyPerformanceKey) && charts.ContainsKey(benchmarkKey) && charts[benchmarkKey].Series.ContainsKey(benchmarkKey) ) { var equity = charts[strategyEquityKey].Series[equityKey].Values; var performance = charts[strategyEquityKey].Series[dailyPerformanceKey].Values; var profitLoss = new SortedDictionary <DateTime, decimal>(algorithm.Transactions.TransactionRecord); var totalTransactions = algorithm.Transactions.GetOrders(x => x.Status.IsFill()).Count(); var benchmark = charts[benchmarkKey].Series[benchmarkKey].Values; statisticsResults = StatisticsBuilder.Generate(trades, profitLoss, equity, performance, benchmark, AlgorithmHandlers.Setup.StartingPortfolioValue, algorithm.Portfolio.TotalFees, totalTransactions); //Some users have $0 in their brokerage account / starting cash of $0. Prevent divide by zero errors var netReturn = AlgorithmHandlers.Setup.StartingPortfolioValue > 0 ? (algorithm.Portfolio.TotalPortfolioValue - AlgorithmHandlers.Setup.StartingPortfolioValue) / AlgorithmHandlers.Setup.StartingPortfolioValue : 0; //Add other fixed parameters. banner.Add("Unrealized", "$" + algorithm.Portfolio.TotalUnrealizedProfit.ToString("N2")); banner.Add("Fees", "-$" + algorithm.Portfolio.TotalFees.ToString("N2")); banner.Add("Net Profit", "$" + algorithm.Portfolio.TotalProfit.ToString("N2")); banner.Add("Return", netReturn.ToString("P")); banner.Add("Equity", "$" + algorithm.Portfolio.TotalPortfolioValue.ToString("N2")); } } catch (Exception err) { Log.Error(err, "Error generating statistics packet"); } //Diagnostics Completed, Send Result Packet: var totalSeconds = (DateTime.Now - startTime).TotalSeconds; var dataPoints = algorithmManager.DataPoints + algorithm.HistoryProvider.DataPointCount; if (!_liveMode) { var kps = dataPoints / (double)1000 / totalSeconds; AlgorithmHandlers.Results.DebugMessage($"Algorithm Id:({job.AlgorithmId}) completed in {totalSeconds:F2} seconds at {kps:F0}k data points per second. Processing total of {dataPoints:N0} data points."); } AlgorithmHandlers.Results.SendFinalResult(job, orders, algorithm.Transactions.TransactionRecord, holdings, algorithm.Portfolio.CashBook, statisticsResults, banner); } catch (Exception err) { Log.Error(err, "Error sending analysis results"); } //Before we return, send terminate commands to close up the threads AlgorithmHandlers.Transactions.Exit(); AlgorithmHandlers.DataFeed.Exit(); AlgorithmHandlers.RealTime.Exit(); AlgorithmHandlers.Alphas.Exit(); dataManager?.RemoveAllSubscriptions(); workerThread?.Dispose(); } //Close result handler: AlgorithmHandlers.Results.Exit(); //Wait for the threads to complete: var millisecondInterval = 10; var millisecondTotalWait = 0; while ((AlgorithmHandlers.Results.IsActive || (AlgorithmHandlers.Transactions != null && AlgorithmHandlers.Transactions.IsActive) || (AlgorithmHandlers.DataFeed != null && AlgorithmHandlers.DataFeed.IsActive) || (AlgorithmHandlers.RealTime != null && AlgorithmHandlers.RealTime.IsActive) || (AlgorithmHandlers.Alphas != null && AlgorithmHandlers.Alphas.IsActive)) && millisecondTotalWait < 30 * 1000) { Thread.Sleep(millisecondInterval); if (millisecondTotalWait % (millisecondInterval * 10) == 0) { Log.Trace("Waiting for threads to exit..."); } millisecondTotalWait += millisecondInterval; } //Terminate threads still in active state. if (threadResults != null && threadResults.IsAlive) { threadResults.Abort(); } if (threadAlphas != null && threadAlphas.IsAlive) { threadAlphas.Abort(); } if (brokerage != null) { Log.Trace("Engine.Run(): Disconnecting from brokerage..."); brokerage.Disconnect(); brokerage.Dispose(); } if (AlgorithmHandlers.Setup != null) { Log.Trace("Engine.Run(): Disposing of setup handler..."); AlgorithmHandlers.Setup.Dispose(); } Log.Trace("Engine.Main(): Analysis Completed and Results Posted."); } catch (Exception err) { Log.Error(err, "Error running algorithm"); } finally { //No matter what for live mode; make sure we've set algorithm status in the API for "not running" conditions: if (_liveMode && algorithmManager.State != AlgorithmStatus.Running && algorithmManager.State != AlgorithmStatus.RuntimeError) { SystemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, algorithmManager.State); } AlgorithmHandlers.Results.Exit(); AlgorithmHandlers.DataFeed.Exit(); AlgorithmHandlers.Transactions.Exit(); AlgorithmHandlers.RealTime.Exit(); } }
public void Dispose() { Stop(); FThread.Dispose(); }
public void TearDown() { _workerThread.Dispose(); }
private void WorkerThreadMain(object state) { WorkerThread wt = (WorkerThread)state; if (LOGGER.IsDebugEnabled) { LOGGER.Debug(String.Format("THREADPOOL[{0}]: starting new ThreadPool thread ({1}).", string.IsNullOrEmpty(mName) ? GetHashCode().ToString() : mName, Thread.CurrentThread.Name)); } try { while (!mDisposed && !wt.IsShutdownForce) { if (wt.Task == null) { lock (mLockObject) { if (!mDisposed && !wt.IsShutdownForce) { // nincs leállás if (wt.Task == null) { // nincs kiosztott feladat if (mMaxThreadNumber < mThreadNumber) { // túl sok thread van, nekem nincs feladatom, leállok mInactiveThreads.Remove(wt); break; } else if (mTaskContainerQueue.Count > 0) { // van meló, kérek magamnak wt.Task = mTaskContainerQueue.Dequeue(); mInactiveThreads.Remove(wt); } } } else { // leállítás parancs van érvényben mInactiveThreads.Remove(wt); break; } } } if (wt.Task != null || wt.WakeUpEvent.WaitOne(mShutDownIdleThreadTime)) { // jelzést kaptam vagy osztottak rám melót if (wt.Task != null && !mDisposed && !wt.IsShutdownForce) { Execute(wt); } } else { // nincs jelzésem if (!mDisposed && !wt.IsShutdownForce) { lock (mLockObject) { if (wt.Task == null && mMinThreadNumber > mThreadNumber) { // nincs feladatom, leállás... mInactiveThreads.Remove(wt); break; } } if (wt.Task != null && !mDisposed && !wt.IsShutdownForce) { // közben kaptam feladatot... Execute(wt); } } } } } catch (ThreadAbortException) { } lock (mLockObject) { Interlocked.Decrement(ref mThreadNumber); mInactiveThreads.Remove(wt); mExistingThreads.Remove(wt); if (wt.IsShutdownForce) { // leállítási parancs érvényben, jelzés... wt.ShutdownEvent.Set(); } else { wt.Dispose(); } } if (LOGGER.IsDebugEnabled) { LOGGER.Debug(String.Format("THREADPOOL[{0}]: thread ({1}) has exited.", string.IsNullOrEmpty(mName) ? GetHashCode().ToString() : mName, Thread.CurrentThread.Name)); } }