/// <summary> /// Will first check and add all the required conversion rate securities /// and later will seed an initial value to them. /// </summary> /// <param name="algorithm">The algorithm instance</param> /// <param name="universeSelection">The universe selection instance</param> public static void SetupCurrencyConversions( IAlgorithm algorithm, UniverseSelection universeSelection) { // this is needed to have non-zero currency conversion rates during warmup // will also set the Cash.ConversionRateSecurity universeSelection.EnsureCurrencyDataFeeds(SecurityChanges.None); // now set conversion rates var cashToUpdate = algorithm.Portfolio.CashBook.Values .Where(x => x.ConversionRateSecurity != null && x.ConversionRate == 0) .ToList(); var historyRequestFactory = new HistoryRequestFactory(algorithm); var historyRequests = new List <HistoryRequest>(); foreach (var cash in cashToUpdate) { var configs = algorithm .SubscriptionManager .SubscriptionDataConfigService .GetSubscriptionDataConfigs(cash.ConversionRateSecurity.Symbol, includeInternalConfigs: true); // we need to order and select a specific configuration type // so the conversion rate is deterministic var configToUse = configs.OrderBy(x => x.TickType).First(); var hours = cash.ConversionRateSecurity.Exchange.Hours; var resolution = configs.GetHighestResolution(); var startTime = historyRequestFactory.GetStartTimeAlgoTz( cash.ConversionRateSecurity.Symbol, 10, resolution, hours, configToUse.DataTimeZone); var endTime = algorithm.Time; historyRequests.Add(historyRequestFactory.CreateHistoryRequest( configToUse, startTime, endTime, cash.ConversionRateSecurity.Exchange.Hours, resolution)); } var slices = algorithm.HistoryProvider.GetHistory(historyRequests, algorithm.TimeZone); slices.PushThrough(data => { foreach (var cash in cashToUpdate .Where(x => x.ConversionRateSecurity.Symbol == data.Symbol)) { cash.Update(data); } }); Log.Trace("BaseSetupHandler.SetupCurrencyConversions():" + $"{Environment.NewLine}{algorithm.Portfolio.CashBook}"); }
/// <summary> /// Will first check and add all the required conversion rate securities /// and later will seed an initial value to them. /// </summary> /// <param name="algorithm">The algorithm instance</param> /// <param name="universeSelection">The universe selection instance</param> public static void SetupCurrencyConversions( IAlgorithm algorithm, UniverseSelection universeSelection) { // this is needed to have non-zero currency conversion rates during warmup // will also set the Cash.ConversionRateSecurity universeSelection.EnsureCurrencyDataFeeds(SecurityChanges.None); // now set conversion rates var cashToUpdate = algorithm.Portfolio.CashBook.Values .Where(x => x.ConversionRateSecurity != null && x.ConversionRate == 0) .ToList(); var historyRequestFactory = new HistoryRequestFactory(algorithm); var historyRequests = new List <HistoryRequest>(); foreach (var cash in cashToUpdate) { // if we already added a history request for this security, skip if (historyRequests.Any(x => x.Symbol == cash.ConversionRateSecurity.Symbol)) { continue; } var configs = algorithm .SubscriptionManager .SubscriptionDataConfigService .GetSubscriptionDataConfigs(cash.ConversionRateSecurity.Symbol); var resolution = configs.GetHighestResolution(); var startTime = historyRequestFactory.GetStartTimeAlgoTz( cash.ConversionRateSecurity.Symbol, 1, resolution, cash.ConversionRateSecurity.Exchange.Hours); var endTime = algorithm.Time.RoundDown(resolution.ToTimeSpan()); // we need to order and select a specific configuration type // so the conversion rate is deterministic var configToUse = configs.OrderBy(x => x.TickType).First(); historyRequests.Add(historyRequestFactory.CreateHistoryRequest( configToUse, startTime, endTime, cash.ConversionRateSecurity.Exchange.Hours, resolution)); } var slices = algorithm.HistoryProvider.GetHistory(historyRequests, algorithm.TimeZone); slices.PushThrough(data => { foreach (var cash in cashToUpdate .Where(x => x.ConversionRateSecurity.Symbol == data.Symbol)) { cash.Update(data); } }); }
/// <summary> /// Used to send data updates to algorithm framework models /// </summary> /// <param name="slice">The current data slice</param> public void OnFrameworkData(Slice slice) { if (UtcTime >= UniverseSelection.GetNextRefreshTimeUtc()) { var universes = UniverseSelection.CreateUniverses(this).ToDictionary(u => u.Configuration.Symbol); // remove deselected universes by symbol foreach (var ukvp in UniverseManager) { var universeSymbol = ukvp.Key; var qcUserDefined = UserDefinedUniverse.CreateSymbol(ukvp.Value.SecurityType, ukvp.Value.Market); if (universeSymbol.Equals(qcUserDefined)) { // prevent removal of qc algorithm created user defined universes continue; } Universe universe; if (!universes.TryGetValue(universeSymbol, out universe)) { if (ukvp.Value.DisposeRequested) { UniverseManager.Remove(universeSymbol); } // mark this universe as disposed to remove all child subscriptions ukvp.Value.Dispose(); } } // add newly selected universes foreach (var ukvp in universes) { // note: UniverseManager.Add uses TryAdd, so don't need to worry about duplicates here UniverseManager.Add(ukvp); } } // we only want to run universe selection if there's no data available in the slice if (!slice.HasData) { return; } // insight timestamping handled via InsightsGenerated event handler var insightsEnumerable = Alpha.Update(this, slice); // for performance only call 'ToArray' if not empty enumerable (which is static) var insights = insightsEnumerable == Enumerable.Empty <Insight>() ? new Insight[] { } : insightsEnumerable.ToArray(); // only fire insights generated event if we actually have insights if (insights.Length != 0) { OnInsightsGenerated(insights.Select(InitializeInsightFields)); } ProcessInsights(insights); }
/// <summary> /// Called by setup handlers after Initialize and allows the algorithm a chance to organize /// the data gather in the Initialize method /// </summary> public override void PostInitialize() { CheckModels(); foreach (var universe in UniverseSelection.CreateUniverses(this)) { AddUniverse(universe); } base.PostInitialize(); }
/// <summary> /// Called by setup handlers after Initialize and allows the algorithm a chance to organize /// the data gather in the Initialize method /// </summary> public void FrameworkPostInitialize() { foreach (var universe in UniverseSelection.CreateUniverses(this)) { AddUniverse(universe); } if (DebugMode) { InsightsGenerated += (algorithm, data) => Log($"{Time}: {string.Join(" | ", data.Insights.OrderBy(i => i.Symbol.ToString()))}"); } }
private void TestSubscriptionSynchronizerSpeed(QCAlgorithm algorithm) { algorithm.Initialize(); algorithm.PostInitialize(); // set exchanges to be always open foreach (var kvp in algorithm.Securities) { var security = kvp.Value; security.Exchange = new SecurityExchange(SecurityExchangeHours.AlwaysOpen(security.Exchange.TimeZone)); } var endTimeUtc = algorithm.EndDate.ConvertToUtc(TimeZones.NewYork); var startTimeUtc = algorithm.StartDate.ConvertToUtc(TimeZones.NewYork); var feed = new AlgorithmManagerTests.MockDataFeed(); var universeSelection = new UniverseSelection(feed, algorithm); var synchronizer = new SubscriptionSynchronizer(universeSelection, algorithm.TimeZone, algorithm.Portfolio.CashBook, startTimeUtc); var totalDataPoints = 0; var subscriptions = new List <Subscription>(); foreach (var kvp in algorithm.Securities) { int dataPointCount; subscriptions.Add(CreateSubscription(algorithm, kvp.Value, startTimeUtc, endTimeUtc, out dataPointCount)); totalDataPoints += dataPointCount; } // force JIT synchronizer.Sync(subscriptions); // log what we're doing Console.WriteLine($"Running {subscriptions.Count} subscriptions with a total of {totalDataPoints} data points. Start: {algorithm.StartDate:yyyy-MM-dd} End: {algorithm.EndDate:yyyy-MM-dd}"); var count = 0; DateTime currentTime; var dateTimeMaxValue = DateTime.MaxValue; var stopwatch = Stopwatch.StartNew(); do { var timeSlice = synchronizer.Sync(subscriptions); currentTime = timeSlice.Time; count += timeSlice.DataPointCount; }while (currentTime != dateTimeMaxValue); stopwatch.Stop(); var kps = count / 1000d / stopwatch.Elapsed.TotalSeconds; Console.WriteLine($"Current Time: {currentTime:u} Elapsed time: {(int)stopwatch.Elapsed.TotalSeconds,4}s KPS: {kps,7:.00} COUNT: {count,10}"); }
private void TestSubscriptionSynchronizerSpeed(QCAlgorithm algorithm) { algorithm.Initialize(); // set exchanges to be always open foreach (var kvp in algorithm.Securities) { var security = kvp.Value; security.Exchange = new SecurityExchange(SecurityExchangeHours.AlwaysOpen(security.Exchange.TimeZone)); } var endTimeUtc = algorithm.EndDate.ConvertToUtc(TimeZones.NewYork); var startTimeUtc = algorithm.StartDate.ConvertToUtc(TimeZones.NewYork); var feed = new AlgorithmManagerTests.MockDataFeed(); var universeSelection = new UniverseSelection(feed, algorithm); var synchronizer = new SubscriptionSynchronizer(universeSelection, algorithm.TimeZone, algorithm.Portfolio.CashBook, startTimeUtc); var subscriptions = new List <Subscription>(); foreach (var kvp in algorithm.Securities) { subscriptions.Add(CreateSubscription(algorithm, kvp.Value, startTimeUtc, endTimeUtc)); } var count = 0; double kps = 0; var currentTime = default(DateTime); var stopwatch = new Stopwatch(); var timer = new Timer(_ => { kps = count / 1000d / stopwatch.Elapsed.TotalSeconds; Console.WriteLine($"Current Time: {currentTime:u} Elapsed time: {(int)stopwatch.Elapsed.TotalSeconds,4}s KPS: {kps,7:.00} COUNT: {count,10}"); }); timer.Change(TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(10)); stopwatch.Start(); do { var timeSlice = synchronizer.Sync(subscriptions); currentTime = timeSlice.Time; count += timeSlice.DataPointCount; }while (currentTime < endTimeUtc); stopwatch.Stop(); timer.Dispose(); kps = count / 1000d / stopwatch.Elapsed.TotalSeconds; Console.WriteLine($"Current Time: {currentTime:u} Elapsed time: {(int)stopwatch.Elapsed.TotalSeconds,4}s KPS: {kps,7:.00} COUNT: {count,10}"); }
/// <summary> /// Called by setup handlers after Initialize and allows the algorithm a chance to organize /// the data gather in the Initialize method /// </summary> public void FrameworkPostInitialize() { foreach (var universe in UniverseSelection.CreateUniverses(this)) { // on purpose we don't call 'AddUniverse' here so that these universes don't get registered as user added // this is so that later during 'UniverseSelection.CreateUniverses' we wont remove them from UniverseManager _pendingUniverseAdditions.Add(universe); } if (DebugMode) { InsightsGenerated += (algorithm, data) => Log($"{Time}: {string.Join(" | ", data.Insights.OrderBy(i => i.Symbol.ToString()))}"); } }
/// <summary> /// Creates a new instance /// </summary> /// <param name="universeSelection">The universe selection instance</param> /// <param name="algorithm">Algorithm instance</param> /// <param name="brokerage">New brokerage output instance</param> /// <param name="algorithmNodePacket">Algorithm job task</param> /// <param name="resultHandler">The configured result handler</param> /// <param name="transactionHandler">The configured transaction handler</param> /// <param name="realTimeHandler">The configured real time handler</param> public SetupHandlerParameters(UniverseSelection universeSelection, IAlgorithm algorithm, IBrokerage brokerage, AlgorithmNodePacket algorithmNodePacket, IResultHandler resultHandler, ITransactionHandler transactionHandler, IRealTimeHandler realTimeHandler ) { UniverseSelection = universeSelection; Algorithm = algorithm; Brokerage = brokerage; AlgorithmNodePacket = algorithmNodePacket; ResultHandler = resultHandler; TransactionHandler = transactionHandler; RealTimeHandler = realTimeHandler; }
private void SetupImpl(IDataQueueHandler dataQueueHandler, Synchronizer synchronizer, IDataAggregator dataAggregator) { _dataFeed = new TestableLiveTradingDataFeed(dataQueueHandler ?? new FakeDataQueue(dataAggregator ?? new AggregationManager())); _algorithm = new AlgorithmStub(createDataManager: false); _synchronizer = synchronizer ?? new LiveSynchronizer(); var registeredTypesProvider = new RegisteredSecurityDataTypesProvider(); var securityService = new SecurityService(_algorithm.Portfolio.CashBook, MarketHoursDatabase.FromDataFolder(), SymbolPropertiesDatabase.FromDataFolder(), _algorithm, registeredTypesProvider, new SecurityCacheProvider(_algorithm.Portfolio)); var universeSelection = new UniverseSelection( _algorithm, securityService, new DataPermissionManager(), TestGlobals.DataProvider, Resolution.Second); _dataManager = new DataManager(_dataFeed, universeSelection, _algorithm, new TimeKeeper(DateTime.UtcNow, TimeZones.NewYork), MarketHoursDatabase.FromDataFolder(), true, new RegisteredSecurityDataTypesProvider(), new DataPermissionManager()); _resultHandler = new TestResultHandler(); _synchronizer.Initialize(_algorithm, _dataManager); _dataFeed.Initialize(_algorithm, new LiveNodePacket(), _resultHandler, TestGlobals.MapFileProvider, TestGlobals.FactorFileProvider, TestGlobals.DataProvider, _dataManager, _synchronizer, new DataChannelProvider()); _algorithm.SubscriptionManager.SetDataManager(_dataManager); _algorithm.Securities.SetSecurityService(securityService); _algorithm.SetFinishedWarmingUp(); var backtestingTransactionHandler = new BacktestingTransactionHandler(); backtestingTransactionHandler.Initialize(_algorithm, new PaperBrokerage(_algorithm, new LiveNodePacket()), _resultHandler); _algorithm.Transactions.SetOrderProcessor(backtestingTransactionHandler); _algorithm.PostInitialize(); }
/// <summary> /// Adds a new universe selection model /// </summary> /// <param name="universeSelection">Model defining universes for the algorithm to add</param> public void AddUniverseSelection(IUniverseSelectionModel universeSelection) { if (UniverseSelection.GetType() != typeof(NullUniverseSelectionModel)) { var compositeUniverseSelection = UniverseSelection as CompositeUniverseSelectionModel; if (compositeUniverseSelection != null) { compositeUniverseSelection.AddUniverseSelection(universeSelection); } else { UniverseSelection = new CompositeUniverseSelectionModel(UniverseSelection, universeSelection); } } else { UniverseSelection = universeSelection; } }
public void Setup() { _dataFeed = new TestableLiveTradingDataFeed(new FakeDataQueue()); _algorithm = new AlgorithmStub(createDataManager: false); _synchronizer = new LiveSynchronizer(); var registeredTypesProvider = new RegisteredSecurityDataTypesProvider(); var securityService = new SecurityService(_algorithm.Portfolio.CashBook, MarketHoursDatabase.FromDataFolder(), SymbolPropertiesDatabase.FromDataFolder(), _algorithm, registeredTypesProvider, new SecurityCacheProvider(_algorithm.Portfolio)); var universeSelection = new UniverseSelection( _algorithm, securityService, new DataPermissionManager(), new DefaultDataProvider(), Resolution.Second); _dataManager = new DataManager(_dataFeed, universeSelection, _algorithm, new TimeKeeper(DateTime.UtcNow, TimeZones.NewYork), MarketHoursDatabase.FromDataFolder(), true, new RegisteredSecurityDataTypesProvider(), new DataPermissionManager()); _synchronizer.Initialize(_algorithm, _dataManager); _dataFeed.Initialize(_algorithm, new LiveNodePacket(), new TestResultHandler(), new LocalDiskMapFileProvider(), new LocalDiskFactorFileProvider(), new DefaultDataProvider(), _dataManager, _synchronizer, new DataChannelProvider()); _algorithm.SubscriptionManager.SetDataManager(_dataManager); _algorithm.Securities.SetSecurityService(securityService); _algorithm.SetFinishedWarmingUp(); _algorithm.Transactions.SetOrderProcessor(new FakeOrderProcessor()); }
/// <summary> /// Called by setup handlers after Initialize and allows the algorithm a chance to organize /// the data gather in the Initialize method /// </summary> public void FrameworkPostInitialize() { //Prevents execution in the case of cash brokerage with IExecutionModel and IPortfolioConstructionModel if (PortfolioConstruction.GetType() != typeof(NullPortfolioConstructionModel) && Execution.GetType() != typeof(NullExecutionModel) && BrokerageModel.AccountType == AccountType.Cash) { throw new InvalidOperationException($"Non null {nameof(IExecutionModel)} and {nameof(IPortfolioConstructionModel)} are currently unsuitable for Cash Modeled brokerages (e.g. GDAX) and may result in unexpected trades." + " To prevent possible user error we've restricted them to Margin trading. You can select margin account types with" + $" SetBrokerage( ... AccountType.Margin). Or please set them to {nameof(NullExecutionModel)}, {nameof(NullPortfolioConstructionModel)}"); } foreach (var universe in UniverseSelection.CreateUniverses(this)) { // on purpose we don't call 'AddUniverse' here so that these universes don't get registered as user added // this is so that later during 'UniverseSelection.CreateUniverses' we wont remove them from UniverseManager _pendingUniverseAdditions.Add(universe); } if (DebugMode) { InsightsGenerated += (algorithm, data) => Log($"{Time}: {string.Join(" | ", data.Insights.OrderBy(i => i.Symbol.ToString()))}"); } }
/// <summary> /// Creates a new instance /// </summary> /// <param name="universeSelection">The universe selection instance</param> /// <param name="algorithm">Algorithm instance</param> /// <param name="brokerage">New brokerage output instance</param> /// <param name="algorithmNodePacket">Algorithm job task</param> /// <param name="resultHandler">The configured result handler</param> /// <param name="transactionHandler">The configured transaction handler</param> /// <param name="realTimeHandler">The configured real time handler</param> /// <param name="objectStore">The configured object store</param> /// <param name="dataCacheProvider">The configured data cache provider</param> /// <param name="mapFileProvider">The map file provider</param> public SetupHandlerParameters(UniverseSelection universeSelection, IAlgorithm algorithm, IBrokerage brokerage, AlgorithmNodePacket algorithmNodePacket, IResultHandler resultHandler, ITransactionHandler transactionHandler, IRealTimeHandler realTimeHandler, IObjectStore objectStore, IDataCacheProvider dataCacheProvider, IMapFileProvider mapFileProvider ) { UniverseSelection = universeSelection; Algorithm = algorithm; Brokerage = brokerage; AlgorithmNodePacket = algorithmNodePacket; ResultHandler = resultHandler; TransactionHandler = transactionHandler; RealTimeHandler = realTimeHandler; ObjectStore = objectStore; DataCacheProvider = dataCacheProvider; MapFileProvider = mapFileProvider; }
/// <summary> /// Called by setup handlers after Initialize and allows the algorithm a chance to organize /// the data gather in the Initialize method /// </summary> public override void PostInitialize() { CheckModels(); foreach (var universe in UniverseSelection.CreateUniverses(this)) { AddUniverse(universe); } if (DebugMode) { InsightsGenerated += (algorithm, data) => Log($"{Time}: {string.Join(" | ", data.Insights.OrderBy(i => i.Symbol.ToString()))}"); } // emit warning message about using the framework with cash modelling if (BrokerageModel.AccountType == AccountType.Cash) { Error("These models are currently unsuitable for Cash Modeled brokerages (e.g. GDAX) and may result in unexpected trades." + " To prevent possible user error we've restricted them to Margin trading. You can select margin account types with" + " SetBrokerage( ... AccountType.Margin)"); } base.PostInitialize(); }
/// <summary> /// Used to send data updates to algorithm framework models /// </summary> /// <param name="slice">The current data slice</param> public sealed override void OnFrameworkData(Slice slice) { if (UtcTime >= UniverseSelection.GetNextRefreshTimeUtc()) { var universes = UniverseSelection.CreateUniverses(this).ToDictionary(u => u.Configuration.Symbol); // remove deselected universes by symbol foreach (var ukvp in UniverseManager) { var universeSymbol = ukvp.Key; var qcUserDefined = UserDefinedUniverse.CreateSymbol(ukvp.Value.SecurityType, ukvp.Value.Market); if (universeSymbol.Equals(qcUserDefined)) { // prevent removal of qc algorithm created user defined universes continue; } Universe universe; if (!universes.TryGetValue(universeSymbol, out universe)) { if (ukvp.Value.DisposeRequested) { UniverseManager.Remove(universeSymbol); } // mark this universe as disposed to remove all child subscriptions ukvp.Value.Dispose(); } } // add newly selected universes foreach (var ukvp in universes) { // note: UniverseManager.Add uses TryAdd, so don't need to worry about duplicates here UniverseManager.Add(ukvp); } } // we only want to run universe selection if there's no data available in the slice if (!slice.HasData) { return; } // insight timestamping handled via InsightsGenerated event handler var insights = Alpha.Update(this, slice).ToArray(); // only fire insights generated event if we actually have insights if (insights.Length != 0) { // debug printing of generated insights if (DebugMode) { Log($"{Time}: ALPHA: {string.Join(" | ", insights.Select(i => i.ToString()).OrderBy(i => i))}"); } OnInsightsGenerated(insights); } // construct portfolio targets from insights var targets = PortfolioConstruction.CreateTargets(this, insights).ToArray(); // set security targets w/ those generated via portfolio construction module foreach (var target in targets) { var security = Securities[target.Symbol]; security.Holdings.Target = target; } if (DebugMode) { // debug printing of generated targets if (targets.Length > 0) { Log($"{Time}: PORTFOLIO: {string.Join(" | ", targets.Select(t => t.ToString()).OrderBy(t => t))}"); } } var riskTargetOverrides = RiskManagement.ManageRisk(this, targets).ToArray(); // override security targets w/ those generated via risk management module foreach (var target in riskTargetOverrides) { var security = Securities[target.Symbol]; security.Holdings.Target = target; } if (DebugMode) { // debug printing of generated risk target overrides if (riskTargetOverrides.Length > 0) { Log($"{Time}: RISK: {string.Join(" | ", riskTargetOverrides.Select(t => t.ToString()).OrderBy(t => t))}"); } } // execute on the targets, overriding targets for symbols w/ risk targets var riskAdjustedTargets = riskTargetOverrides.Concat(targets).DistinctBy(pt => pt.Symbol).ToArray(); if (DebugMode) { // only log adjusted targets if we've performed an adjustment if (riskTargetOverrides.Length > 0) { Log($"{Time}: RISK ADJUSTED TARGETS: {string.Join(" | ", riskAdjustedTargets.Select(t => t.ToString()).OrderBy(t => t))}"); } } Execution.Execute(this, riskAdjustedTargets); }
/// <summary> /// Runs a single backtest/live job from the job queue /// </summary> /// <param name="job">The algorithm job to be processed</param> /// <param name="assemblyPath">The path to the algorithm's assembly</param> public void Run(AlgorithmNodePacket job, string assemblyPath) { var algorithm = default(IAlgorithm); var algorithmManager = new AlgorithmManager(_liveMode); //Start monitoring the backtest active status: var statusPing = new StateCheck.Ping(algorithmManager, _systemHandlers.Api, _algorithmHandlers.Results, _systemHandlers.Notify, job); var statusPingThread = new Thread(statusPing.Run); statusPingThread.Start(); try { //Reset thread holders. var initializeComplete = false; Thread threadFeed = null; Thread threadTransactions = null; Thread threadResults = null; Thread threadRealTime = null; //-> Initialize messaging system _systemHandlers.Notify.SetChannel(job.Channel); //-> Set the result handler type for this algorithm job, and launch the associated result thread. _algorithmHandlers.Results.Initialize(job, _systemHandlers.Notify, _systemHandlers.Api, _algorithmHandlers.DataFeed, _algorithmHandlers.Setup, _algorithmHandlers.Transactions); threadResults = new Thread(_algorithmHandlers.Results.Run, 0) { Name = "Result Thread" }; threadResults.Start(); IBrokerage brokerage = null; try { // Save algorithm to cache, load algorithm instance: algorithm = _algorithmHandlers.Setup.CreateAlgorithmInstance(assemblyPath, job.Language); // set the history provider before setting up the algorithm _algorithmHandlers.HistoryProvider.Initialize(job, progress => { // send progress updates to the result handler only during initialization if (!algorithm.GetLocked() || algorithm.IsWarmingUp) { _algorithmHandlers.Results.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.History, string.Format("Processing history {0}%...", progress)); } }); algorithm.HistoryProvider = _algorithmHandlers.HistoryProvider; // initialize the default brokerage message handler algorithm.BrokerageMessageHandler = new DefaultBrokerageMessageHandler(algorithm, job, _algorithmHandlers.Results, _systemHandlers.Api); //Initialize the internal state of algorithm and job: executes the algorithm.Initialize() method. initializeComplete = _algorithmHandlers.Setup.Setup(algorithm, out brokerage, job, _algorithmHandlers.Results, _algorithmHandlers.Transactions, _algorithmHandlers.RealTime); // set this again now that we've actually added securities _algorithmHandlers.Results.SetAlgorithm(algorithm); //If there are any reasons it failed, pass these back to the IDE. if (!initializeComplete || algorithm.ErrorMessages.Count > 0 || _algorithmHandlers.Setup.Errors.Count > 0) { initializeComplete = false; //Get all the error messages: internal in algorithm and external in setup handler. var errorMessage = String.Join(",", algorithm.ErrorMessages); errorMessage += String.Join(",", _algorithmHandlers.Setup.Errors); _algorithmHandlers.Results.RuntimeError(errorMessage); _systemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, errorMessage); } } catch (Exception err) { var runtimeMessage = "Algorithm.Initialize() Error: " + err.Message + " Stack Trace: " + err.StackTrace; _algorithmHandlers.Results.RuntimeError(runtimeMessage, err.StackTrace); _systemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, runtimeMessage); } //-> Using the job + initialization: load the designated handlers: if (initializeComplete) { //-> Reset the backtest stopwatch; we're now running the algorithm. var startTime = DateTime.Now; //Set algorithm as locked; set it to live mode if we're trading live, and set it to locked for no further updates. algorithm.SetAlgorithmId(job.AlgorithmId); algorithm.SetLocked(); //Wire up the universe selection event handler before kicking off the data feed var universeSelection = new UniverseSelection(_algorithmHandlers.DataFeed, algorithm, _liveMode); _algorithmHandlers.DataFeed.UniverseSelection += (sender, args) => universeSelection.ApplyUniverseSelection(args); //Load the associated handlers for data, transaction and realtime events: _algorithmHandlers.DataFeed.Initialize(algorithm, job, _algorithmHandlers.Results); _algorithmHandlers.Transactions.Initialize(algorithm, brokerage, _algorithmHandlers.Results); _algorithmHandlers.RealTime.Setup(algorithm, job, _algorithmHandlers.Results, _systemHandlers.Api); // wire up the brokerage message handler brokerage.Message += (sender, message) => algorithm.BrokerageMessageHandler.Handle(message); //Send status to user the algorithm is now executing. _algorithmHandlers.Results.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.Running); //Launch the data, transaction and realtime handlers into dedicated threads threadFeed = new Thread(_algorithmHandlers.DataFeed.Run) { Name = "DataFeed Thread" }; threadTransactions = new Thread(_algorithmHandlers.Transactions.Run) { Name = "Transaction Thread" }; threadRealTime = new Thread(_algorithmHandlers.RealTime.Run) { Name = "RealTime Thread" }; //Launch the data feed, result sending, and transaction models/handlers in separate threads. threadFeed.Start(); // Data feed pushing data packets into thread bridge; threadTransactions.Start(); // Transaction modeller scanning new order requests threadRealTime.Start(); // RealTime scan time for time based events: // Result manager scanning message queue: (started earlier) _algorithmHandlers.Results.DebugMessage(string.Format("Launching analysis for {0} with LEAN Engine v{1}", job.AlgorithmId, Constants.Version)); try { //Create a new engine isolator class var isolator = new Isolator(); // Execute the Algorithm Code: var complete = isolator.ExecuteWithTimeLimit(_algorithmHandlers.Setup.MaximumRuntime, algorithmManager.TimeLoopWithinLimits, () => { try { //Run Algorithm Job: // -> Using this Data Feed, // -> Send Orders to this TransactionHandler, // -> Send Results to ResultHandler. algorithmManager.Run(job, algorithm, _algorithmHandlers.DataFeed, _algorithmHandlers.Transactions, _algorithmHandlers.Results, _algorithmHandlers.RealTime, isolator.CancellationToken); } catch (Exception err) { //Debugging at this level is difficult, stack trace needed. Log.Error(err); algorithm.RunTimeError = err; algorithmManager.SetStatus(AlgorithmStatus.RuntimeError); return; } Log.Trace("Engine.Run(): Exiting Algorithm Manager"); }, job.RamAllocation); if (!complete) { Log.Error("Engine.Main(): Failed to complete in time: " + _algorithmHandlers.Setup.MaximumRuntime.ToString("F")); throw new Exception("Failed to complete algorithm within " + _algorithmHandlers.Setup.MaximumRuntime.ToString("F") + " seconds. Please make it run faster."); } // Algorithm runtime error: if (algorithm.RunTimeError != null) { throw algorithm.RunTimeError; } } catch (Exception err) { //Error running the user algorithm: purge datafeed, send error messages, set algorithm status to failed. Log.Error("Engine.Run(): Breaking out of parent try-catch: " + err.Message + " " + err.StackTrace); if (_algorithmHandlers.DataFeed != null) { _algorithmHandlers.DataFeed.Exit(); } if (_algorithmHandlers.Results != null) { var message = "Runtime Error: " + err.Message; Log.Trace("Engine.Run(): Sending runtime error to user..."); _algorithmHandlers.Results.LogMessage(message); _algorithmHandlers.Results.RuntimeError(message, err.StackTrace); _systemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, message + " Stack Trace: " + err.StackTrace); } } //Send result data back: this entire code block could be rewritten. // todo: - Split up statistics class, its enormous. // todo: - Make a dedicated Statistics.Benchmark class. // todo: - Move all creation and transmission of statistics out of primary engine loop. // todo: - Statistics.Generate(algorithm, resulthandler, transactionhandler); try { var trades = algorithm.TradeBuilder.ClosedTrades; var charts = new Dictionary <string, Chart>(_algorithmHandlers.Results.Charts); var orders = new Dictionary <int, Order>(_algorithmHandlers.Transactions.Orders); var holdings = new Dictionary <string, Holding>(); var banner = new Dictionary <string, string>(); var statisticsResults = new StatisticsResults(); try { //Generates error when things don't exist (no charting logged, runtime errors in main algo execution) const string strategyEquityKey = "Strategy Equity"; const string equityKey = "Equity"; const string dailyPerformanceKey = "Daily Performance"; const string benchmarkKey = "Benchmark"; // make sure we've taken samples for these series before just blindly requesting them if (charts.ContainsKey(strategyEquityKey) && charts[strategyEquityKey].Series.ContainsKey(equityKey) && charts[strategyEquityKey].Series.ContainsKey(dailyPerformanceKey)) { var equity = charts[strategyEquityKey].Series[equityKey].Values; var performance = charts[strategyEquityKey].Series[dailyPerformanceKey].Values; var profitLoss = new SortedDictionary <DateTime, decimal>(algorithm.Transactions.TransactionRecord); var totalTransactions = algorithm.Transactions.GetOrders(x => x.Status.IsFill()).Count(); var benchmark = charts[benchmarkKey].Series[benchmarkKey].Values; statisticsResults = StatisticsBuilder.Generate(trades, profitLoss, equity, performance, benchmark, _algorithmHandlers.Setup.StartingPortfolioValue, algorithm.Portfolio.TotalFees, totalTransactions); } } catch (Exception err) { Log.Error("Algorithm.Node.Engine(): Error generating statistics packet: " + err.Message); } //Diagnostics Completed, Send Result Packet: var totalSeconds = (DateTime.Now - startTime).TotalSeconds; var dataPoints = algorithmManager.DataPoints + _algorithmHandlers.HistoryProvider.DataPointCount; _algorithmHandlers.Results.DebugMessage( string.Format("Algorithm Id:({0}) completed in {1} seconds at {2}k data points per second. Processing total of {3} data points.", job.AlgorithmId, totalSeconds.ToString("F2"), ((dataPoints / (double)1000) / totalSeconds).ToString("F0"), dataPoints.ToString("N0"))); _algorithmHandlers.Results.SendFinalResult(job, orders, algorithm.Transactions.TransactionRecord, holdings, statisticsResults, banner); } catch (Exception err) { Log.Error("Engine.Main(): Error sending analysis result: " + err.Message + " ST >> " + err.StackTrace); } //Before we return, send terminate commands to close up the threads _algorithmHandlers.Transactions.Exit(); _algorithmHandlers.DataFeed.Exit(); _algorithmHandlers.RealTime.Exit(); } //Close result handler: _algorithmHandlers.Results.Exit(); statusPing.Exit(); //Wait for the threads to complete: var ts = Stopwatch.StartNew(); while ((_algorithmHandlers.Results.IsActive || (_algorithmHandlers.Transactions != null && _algorithmHandlers.Transactions.IsActive) || (_algorithmHandlers.DataFeed != null && _algorithmHandlers.DataFeed.IsActive) || (_algorithmHandlers.RealTime != null && _algorithmHandlers.RealTime.IsActive)) && ts.ElapsedMilliseconds < 30 * 1000) { Thread.Sleep(100); Log.Trace("Waiting for threads to exit..."); } //Terminate threads still in active state. if (threadFeed != null && threadFeed.IsAlive) { threadFeed.Abort(); } if (threadTransactions != null && threadTransactions.IsAlive) { threadTransactions.Abort(); } if (threadResults != null && threadResults.IsAlive) { threadResults.Abort(); } if (statusPingThread != null && statusPingThread.IsAlive) { statusPingThread.Abort(); } if (brokerage != null) { brokerage.Disconnect(); } if (_algorithmHandlers.Setup != null) { _algorithmHandlers.Setup.Dispose(); } Log.Trace("Engine.Main(): Analysis Completed and Results Posted."); } catch (Exception err) { Log.Error("Engine.Main(): Error running algorithm: " + err.Message + " >> " + err.StackTrace); } finally { //No matter what for live mode; make sure we've set algorithm status in the API for "not running" conditions: if (_liveMode && algorithmManager.State != AlgorithmStatus.Running && algorithmManager.State != AlgorithmStatus.RuntimeError) { _systemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, algorithmManager.State); } _algorithmHandlers.Results.Exit(); _algorithmHandlers.DataFeed.Exit(); _algorithmHandlers.Transactions.Exit(); _algorithmHandlers.RealTime.Exit(); } }
/// <summary> /// Launch the algorithm manager to run this strategy /// </summary> /// <param name="job">Algorithm job</param> /// <param name="algorithm">Algorithm instance</param> /// <param name="feed">Datafeed object</param> /// <param name="transactions">Transaction manager object</param> /// <param name="results">Result handler object</param> /// <param name="realtime">Realtime processing object</param> /// <param name="token">Cancellation token</param> /// <remarks>Modify with caution</remarks> public void Run(AlgorithmNodePacket job, IAlgorithm algorithm, IDataFeed feed, ITransactionHandler transactions, IResultHandler results, IRealTimeHandler realtime, CancellationToken token) { //Initialize: _dataPointCount = 0; var portfolioValue = algorithm.Portfolio.TotalPortfolioValue; var backtestMode = (job.Type == PacketType.BacktestNode); var methodInvokers = new Dictionary <Type, MethodInvoker>(); var marginCallFrequency = TimeSpan.FromMinutes(5); var nextMarginCallTime = DateTime.MinValue; var delistingTickets = new List <OrderTicket>(); //Initialize Properties: _algorithmId = job.AlgorithmId; _algorithmState = AlgorithmStatus.Running; _previousTime = algorithm.StartDate.Date; //Create the method accessors to push generic types into algorithm: Find all OnData events: // Algorithm 2.0 data accessors var hasOnDataTradeBars = AddMethodInvoker <TradeBars>(algorithm, methodInvokers); var hasOnDataTicks = AddMethodInvoker <Ticks>(algorithm, methodInvokers); // dividend and split events var hasOnDataDividends = AddMethodInvoker <Dividends>(algorithm, methodInvokers); var hasOnDataSplits = AddMethodInvoker <Splits>(algorithm, methodInvokers); var hasOnDataDelistings = AddMethodInvoker <Delistings>(algorithm, methodInvokers); var hasOnDataSymbolChangedEvents = AddMethodInvoker <SymbolChangedEvents>(algorithm, methodInvokers); // Algorithm 3.0 data accessors var hasOnDataSlice = algorithm.GetType().GetMethods() .Where(x => x.Name == "OnData" && x.GetParameters().Length == 1 && x.GetParameters()[0].ParameterType == typeof(Slice)) .FirstOrDefault(x => x.DeclaringType == algorithm.GetType()) != null; //Go through the subscription types and create invokers to trigger the event handlers for each custom type: foreach (var config in feed.Subscriptions.Select(x => x.Configuration)) { //If type is a tradebar, combine tradebars and ticks into unified array: if (config.Type.Name != "TradeBar" && config.Type.Name != "Tick" && !config.IsInternalFeed) { //Get the matching method for this event handler - e.g. public void OnData(Quandl data) { .. } var genericMethod = (algorithm.GetType()).GetMethod("OnData", new[] { config.Type }); //If we already have this Type-handler then don't add it to invokers again. if (methodInvokers.ContainsKey(config.Type)) { continue; } //If we couldnt find the event handler, let the user know we can't fire that event. if (genericMethod == null && !hasOnDataSlice) { algorithm.RunTimeError = new Exception("Data event handler not found, please create a function matching this template: public void OnData(" + config.Type.Name + " data) { }"); _algorithmState = AlgorithmStatus.RuntimeError; return; } if (genericMethod != null) { methodInvokers.Add(config.Type, genericMethod.DelegateForCallMethod()); } } } // wire up universe selection. it is assumed that the data feed will perform the // required thread synchronization. var universeSelection = new UniverseSelection(feed, algorithm, _liveMode); feed.Fundamental += (sender, args) => { var market = args.Configuration.Market; var localTime = args.DateTimeUtc.ConvertFromUtc(args.Configuration.TimeZone); universeSelection.ApplyUniverseSelection(localTime, market, args.Data.OfType <CoarseFundamental>()); }; //Loop over the queues: get a data collection, then pass them all into relevent methods in the algorithm. Log.Trace("AlgorithmManager.Run(): Begin DataStream - Start: " + algorithm.StartDate + " Stop: " + algorithm.EndDate); foreach (var timeSlice in Stream(job, algorithm, feed, results, token)) { // reset our timer on each loop _currentTimeStepTime = DateTime.UtcNow; //Check this backtest is still running: if (_algorithmState != AlgorithmStatus.Running) { Log.Error(string.Format("AlgorithmManager.Run(): Algorthm state changed to {0} at {1}", _algorithmState, timeSlice.Time)); break; } //Execute with TimeLimit Monitor: if (token.IsCancellationRequested) { Log.Error("AlgorithmManager.Run(): CancellationRequestion at " + timeSlice.Time); return; } var time = timeSlice.Time; _dataPointCount += timeSlice.DataPointCount; //If we're in backtest mode we need to capture the daily performance. We do this here directly //before updating the algorithm state with the new data from this time step, otherwise we'll //produce incorrect samples (they'll take into account this time step's new price values) if (backtestMode) { //On day-change sample equity and daily performance for statistics calculations if (_previousTime.Date != time.Date) { SampleBenchmark(algorithm, results, _previousTime.Date); //Sample the portfolio value over time for chart. results.SampleEquity(_previousTime, Math.Round(algorithm.Portfolio.TotalPortfolioValue, 4)); //Check for divide by zero if (portfolioValue == 0m) { results.SamplePerformance(_previousTime.Date, 0); } else { results.SamplePerformance(_previousTime.Date, Math.Round((algorithm.Portfolio.TotalPortfolioValue - portfolioValue) * 100 / portfolioValue, 10)); } portfolioValue = algorithm.Portfolio.TotalPortfolioValue; } } else { // live mode continously sample the benchmark SampleBenchmark(algorithm, results, time); } //Update algorithm state after capturing performance from previous day //Set the algorithm and real time handler's time algorithm.SetDateTime(time); if (timeSlice.Slice.SymbolChangedEvents.Count != 0) { if (hasOnDataSymbolChangedEvents) { methodInvokers[typeof(SymbolChangedEvents)](algorithm, timeSlice.Slice.SymbolChangedEvents); } foreach (var symbol in timeSlice.Slice.SymbolChangedEvents.Keys) { // cancel all orders for the old symbol foreach (var ticket in transactions.GetOrderTickets(x => x.Status.IsOpen() && x.Symbol == symbol)) { ticket.Cancel("Open order cancelled on symbol changed event"); } } } if (timeSlice.SecurityChanges != SecurityChanges.None) { foreach (var security in timeSlice.SecurityChanges.AddedSecurities) { if (!algorithm.Securities.ContainsKey(security.Symbol)) { // add the new security algorithm.Securities.Add(security); } } } //On each time step push the real time prices to the cashbook so we can have updated conversion rates foreach (var kvp in timeSlice.CashBookUpdateData) { kvp.Key.Update(kvp.Value); } //Update the securities properties: first before calling user code to avoid issues with data foreach (var kvp in timeSlice.SecuritiesUpdateData) { kvp.Key.SetMarketPrice(kvp.Value); // Send market price updates to the TradeBuilder if (kvp.Value != null) { algorithm.TradeBuilder.SetMarketPrice(kvp.Key.Symbol, kvp.Value.Price); } } // fire real time events after we've updated based on the new data realtime.SetTime(timeSlice.Time); // process fill models on the updated data before entering algorithm, applies to all non-market orders transactions.ProcessSynchronousEvents(); if (delistingTickets.Count != 0) { for (int i = 0; i < delistingTickets.Count; i++) { var ticket = delistingTickets[i]; if (ticket.Status == OrderStatus.Filled) { algorithm.Securities.Remove(ticket.Symbol); delistingTickets.RemoveAt(i--); Log.Trace("AlgorithmManager.Run(): Delisted Security removed: " + ticket.Symbol.Permtick); } } } //Check if the user's signalled Quit: loop over data until day changes. if (algorithm.GetQuit()) { _algorithmState = AlgorithmStatus.Quit; Log.Trace("AlgorithmManager.Run(): Algorithm quit requested."); break; } if (algorithm.RunTimeError != null) { _algorithmState = AlgorithmStatus.RuntimeError; Log.Trace(string.Format("AlgorithmManager.Run(): Algorithm encountered a runtime error at {0}. Error: {1}", timeSlice.Time, algorithm.RunTimeError)); break; } // perform margin calls, in live mode we can also use realtime to emit these if (time >= nextMarginCallTime || (_liveMode && nextMarginCallTime > DateTime.Now)) { // determine if there are possible margin call orders to be executed bool issueMarginCallWarning; var marginCallOrders = algorithm.Portfolio.ScanForMarginCall(out issueMarginCallWarning); if (marginCallOrders.Count != 0) { var executingMarginCall = false; try { // tell the algorithm we're about to issue the margin call algorithm.OnMarginCall(marginCallOrders); executingMarginCall = true; // execute the margin call orders var executedTickets = algorithm.Portfolio.MarginCallModel.ExecuteMarginCall(marginCallOrders); foreach (var ticket in executedTickets) { algorithm.Error(string.Format("{0} - Executed MarginCallOrder: {1} - Quantity: {2} @ {3}", algorithm.Time, ticket.Symbol, ticket.Quantity, ticket.AverageFillPrice)); } } catch (Exception err) { algorithm.RunTimeError = err; _algorithmState = AlgorithmStatus.RuntimeError; var locator = executingMarginCall ? "Portfolio.MarginCallModel.ExecuteMarginCall" : "OnMarginCall"; Log.Error(string.Format("AlgorithmManager.Run(): RuntimeError: {0}: ", locator) + err.Message + " STACK >>> " + err.StackTrace); return; } } // we didn't perform a margin call, but got the warning flag back, so issue the warning to the algorithm else if (issueMarginCallWarning) { try { algorithm.OnMarginCallWarning(); } catch (Exception err) { algorithm.RunTimeError = err; _algorithmState = AlgorithmStatus.RuntimeError; Log.Error("AlgorithmManager.Run(): RuntimeError: OnMarginCallWarning: " + err.Message + " STACK >>> " + err.StackTrace); return; } } nextMarginCallTime = time + marginCallFrequency; } // before we call any events, let the algorithm know about universe changes if (timeSlice.SecurityChanges != SecurityChanges.None) { try { algorithm.OnSecuritiesChanged(timeSlice.SecurityChanges); } catch (Exception err) { algorithm.RunTimeError = err; _algorithmState = AlgorithmStatus.RuntimeError; Log.Error("AlgorithmManager.Run(): RuntimeError: OnSecuritiesChanged event: " + err.Message); return; } } // apply dividends foreach (var dividend in timeSlice.Slice.Dividends.Values) { Log.Trace("AlgorithmManager.Run(): Applying Dividend for " + dividend.Symbol, true); algorithm.Portfolio.ApplyDividend(dividend); } // apply splits foreach (var split in timeSlice.Slice.Splits.Values) { try { Log.Trace("AlgorithmManager.Run(): Applying Split for " + split.Symbol, true); algorithm.Portfolio.ApplySplit(split); // apply the split to open orders as well in raw mode, all other modes are split adjusted if (_liveMode || algorithm.Securities[split.Symbol].SubscriptionDataConfig.DataNormalizationMode == DataNormalizationMode.Raw) { // in live mode we always want to have our order match the order at the brokerage, so apply the split to the orders var openOrders = transactions.GetOrderTickets(ticket => ticket.Status.IsOpen() && ticket.Symbol == split.Symbol); algorithm.BrokerageModel.ApplySplit(openOrders.ToList(), split); } } catch (Exception err) { algorithm.RunTimeError = err; _algorithmState = AlgorithmStatus.RuntimeError; Log.Error("AlgorithmManager.Run(): RuntimeError: Split event: " + err.Message); return; } } //Update registered consolidators for this symbol index try { foreach (var kvp in timeSlice.ConsolidatorUpdateData) { var consolidators = kvp.Key.Consolidators; foreach (var dataPoint in kvp.Value) { foreach (var consolidator in consolidators) { consolidator.Update(dataPoint); } } } } catch (Exception err) { algorithm.RunTimeError = err; _algorithmState = AlgorithmStatus.RuntimeError; Log.Error("AlgorithmManager.Run(): RuntimeError: Consolidators update: " + err.Message); return; } // fire custom event handlers foreach (var kvp in timeSlice.CustomData) { MethodInvoker methodInvoker; if (!methodInvokers.TryGetValue(kvp.Key.SubscriptionDataConfig.Type, out methodInvoker)) { continue; } try { foreach (var dataPoint in kvp.Value) { methodInvoker(algorithm, dataPoint); } } catch (Exception err) { algorithm.RunTimeError = err; _algorithmState = AlgorithmStatus.RuntimeError; Log.Error("AlgorithmManager.Run(): RuntimeError: Custom Data: " + err.Message + " STACK >>> " + err.StackTrace); return; } } try { // fire off the dividend and split events before pricing events if (hasOnDataDividends && timeSlice.Slice.Dividends.Count != 0) { methodInvokers[typeof(Dividends)](algorithm, timeSlice.Slice.Dividends); } if (hasOnDataSplits && timeSlice.Slice.Splits.Count != 0) { methodInvokers[typeof(Splits)](algorithm, timeSlice.Slice.Splits); } if (hasOnDataDelistings && timeSlice.Slice.Delistings.Count != 0) { methodInvokers[typeof(Delistings)](algorithm, timeSlice.Slice.Delistings); } } catch (Exception err) { algorithm.RunTimeError = err; _algorithmState = AlgorithmStatus.RuntimeError; Log.Error("AlgorithmManager.Run(): RuntimeError: Dividends/Splits/Delistings: " + err.Message + " STACK >>> " + err.StackTrace); return; } // run the delisting logic after firing delisting events HandleDelistedSymbols(algorithm, timeSlice.Slice.Delistings, delistingTickets); //After we've fired all other events in this second, fire the pricing events: try { if (hasOnDataTradeBars && timeSlice.Slice.Bars.Count > 0) { methodInvokers[typeof(TradeBars)](algorithm, timeSlice.Slice.Bars); } if (hasOnDataTicks && timeSlice.Slice.Ticks.Count > 0) { methodInvokers[typeof(Ticks)](algorithm, timeSlice.Slice.Ticks); } } catch (Exception err) { algorithm.RunTimeError = err; _algorithmState = AlgorithmStatus.RuntimeError; Log.Error("AlgorithmManager.Run(): RuntimeError: New Style Mode: " + err.Message + " STACK >>> " + err.StackTrace); return; } try { if (timeSlice.Slice.Count != 0) { // EVENT HANDLER v3.0 -- all data in a single event algorithm.OnData(timeSlice.Slice); } } catch (Exception err) { algorithm.RunTimeError = err; _algorithmState = AlgorithmStatus.RuntimeError; Log.Error("AlgorithmManager.Run(): RuntimeError: Slice: " + err.Message + " STACK >>> " + err.StackTrace); return; } //If its the historical/paper trading models, wait until market orders have been "filled" // Manually trigger the event handler to prevent thread switch. transactions.ProcessSynchronousEvents(); //Save the previous time for the sample calculations _previousTime = time; // Process any required events of the results handler such as sampling assets, equity, or stock prices. results.ProcessSynchronousEvents(); } // End of ForEach feed.Bridge.GetConsumingEnumerable // stop timing the loops _currentTimeStepTime = DateTime.MinValue; //Stream over:: Send the final packet and fire final events: Log.Trace("AlgorithmManager.Run(): Firing On End Of Algorithm..."); try { algorithm.OnEndOfAlgorithm(); } catch (Exception err) { _algorithmState = AlgorithmStatus.RuntimeError; algorithm.RunTimeError = new Exception("Error running OnEndOfAlgorithm(): " + err.Message, err.InnerException); Log.Error("AlgorithmManager.OnEndOfAlgorithm(): " + err.Message + " STACK >>> " + err.StackTrace); return; } // Process any required events of the results handler such as sampling assets, equity, or stock prices. results.ProcessSynchronousEvents(forceProcess: true); //Liquidate Holdings for Calculations: if (_algorithmState == AlgorithmStatus.Liquidated && _liveMode) { Log.Trace("AlgorithmManager.Run(): Liquidating algorithm holdings..."); algorithm.Liquidate(); results.LogMessage("Algorithm Liquidated"); results.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.Liquidated); } //Manually stopped the algorithm if (_algorithmState == AlgorithmStatus.Stopped) { Log.Trace("AlgorithmManager.Run(): Stopping algorithm..."); results.LogMessage("Algorithm Stopped"); results.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.Stopped); } //Backtest deleted. if (_algorithmState == AlgorithmStatus.Deleted) { Log.Trace("AlgorithmManager.Run(): Deleting algorithm..."); results.DebugMessage("Algorithm Id:(" + job.AlgorithmId + ") Deleted by request."); results.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.Deleted); } //Algorithm finished, send regardless of commands: results.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.Completed); //Take final samples: results.SampleRange(algorithm.GetChartUpdates()); results.SampleEquity(_previousTime, Math.Round(algorithm.Portfolio.TotalPortfolioValue, 4)); SampleBenchmark(algorithm, results, _previousTime); results.SamplePerformance(_previousTime, Math.Round((algorithm.Portfolio.TotalPortfolioValue - portfolioValue) * 100 / portfolioValue, 10)); } // End of Run();
/// <summary> /// Will first check and add all the required conversion rate securities /// and later will seed an initial value to them. /// </summary> /// <param name="algorithm">The algorithm instance</param> /// <param name="universeSelection">The universe selection instance</param> public static void SetupCurrencyConversions( IAlgorithm algorithm, UniverseSelection universeSelection) { // this is needed to have non-zero currency conversion rates during warmup // will also set the Cash.ConversionRateSecurity universeSelection.EnsureCurrencyDataFeeds(SecurityChanges.None); // now set conversion rates var cashToUpdate = algorithm.Portfolio.CashBook.Values .Where(x => x.CurrencyConversion != null && x.ConversionRate == 0) .ToList(); var securitiesToUpdate = cashToUpdate .SelectMany(x => x.CurrencyConversion.ConversionRateSecurities) .Distinct() .ToList(); var historyRequestFactory = new HistoryRequestFactory(algorithm); var historyRequests = new List <HistoryRequest>(); foreach (var security in securitiesToUpdate) { var configs = algorithm .SubscriptionManager .SubscriptionDataConfigService .GetSubscriptionDataConfigs(security.Symbol, includeInternalConfigs: true); // we need to order and select a specific configuration type // so the conversion rate is deterministic var configToUse = configs.OrderBy(x => x.TickType).First(); var hours = security.Exchange.Hours; var resolution = configs.GetHighestResolution(); var startTime = historyRequestFactory.GetStartTimeAlgoTz( security.Symbol, 10, resolution, hours, configToUse.DataTimeZone); var endTime = algorithm.Time; historyRequests.Add(historyRequestFactory.CreateHistoryRequest( configToUse, startTime, endTime, security.Exchange.Hours, resolution)); } // Attempt to get history for these requests and update cash var slices = algorithm.HistoryProvider.GetHistory(historyRequests, algorithm.TimeZone); slices.PushThrough(data => { foreach (var security in securitiesToUpdate.Where(x => x.Symbol == data.Symbol)) { security.SetMarketPrice(data); } }); foreach (var cash in cashToUpdate) { cash.Update(); } // Any remaining unassigned cash will attempt to fall back to a daily resolution history request to resolve var unassignedCash = cashToUpdate.Where(x => x.ConversionRate == 0).ToList(); if (unassignedCash.Any()) { Log.Error( $"Failed to assign conversion rates for the following cash: {string.Join(",", unassignedCash.Select(x => x.Symbol))}." + $" Attempting to request daily resolution history to resolve conversion rate"); var unassignedCashSymbols = unassignedCash .SelectMany(x => x.SecuritySymbols) .ToHashSet(); var replacementHistoryRequests = new List <HistoryRequest>(); foreach (var request in historyRequests.Where(x => unassignedCashSymbols.Contains(x.Symbol) && x.Resolution < Resolution.Daily)) { var newRequest = new HistoryRequest(request.EndTimeUtc.AddDays(-10), request.EndTimeUtc, request.DataType, request.Symbol, Resolution.Daily, request.ExchangeHours, request.DataTimeZone, request.FillForwardResolution, request.IncludeExtendedMarketHours, request.IsCustomData, request.DataNormalizationMode, request.TickType); replacementHistoryRequests.Add(newRequest); } slices = algorithm.HistoryProvider.GetHistory(replacementHistoryRequests, algorithm.TimeZone); slices.PushThrough(data => { foreach (var security in securitiesToUpdate.Where(x => x.Symbol == data.Symbol)) { security.SetMarketPrice(data); } }); foreach (var cash in unassignedCash) { cash.Update(); } } Log.Trace("BaseSetupHandler.SetupCurrencyConversions():" + $"{Environment.NewLine}{algorithm.Portfolio.CashBook}"); }
/// <summary> /// Used to send data updates to algorithm framework models /// </summary> /// <param name="slice">The current data slice</param> public void OnFrameworkData(Slice slice) { if (UtcTime >= UniverseSelection.GetNextRefreshTimeUtc()) { var universes = UniverseSelection.CreateUniverses(this).ToDictionary(u => u.Configuration.Symbol); // remove deselected universes by symbol foreach (var ukvp in UniverseManager) { var universeSymbol = ukvp.Key; if (_userAddedUniverses.Contains(universeSymbol)) { // prevent removal of qc algorithm created user defined universes continue; } if (ukvp.Value.DisposeRequested) { // have to remove in the next loop after the universe is marked as disposed, when 'Dispose()' is called it will trigger universe selection // and deselect all symbols, sending the removed security changes, which are picked up by the AlgorithmManager and tags securities // as non tradable as long as they are not active in any universe (uses UniverseManager.ActiveSecurities) // but they will remain tradable if a position is still being hold since they won't be remove from the UniverseManager // but this last part will not happen if we remove the universe from the UniverseManager right away, since it won't be part of 'UniverseManager'. // And we have to remove the universe even if it's present at 'universes' because that one is another New universe that should get added! // 'UniverseManager' will skip duplicate entries getting added. UniverseManager.Remove(universeSymbol); } Universe universe; if (!universes.TryGetValue(universeSymbol, out universe)) { // mark this universe as disposed to remove all child subscriptions ukvp.Value.Dispose(); } } // add newly selected universes foreach (var ukvp in universes) { // note: UniverseManager.Add uses TryAdd, so don't need to worry about duplicates here UniverseManager.Add(ukvp); } } // we only want to run universe selection if there's no data available in the slice if (!slice.HasData) { return; } // insight timestamping handled via InsightsGenerated event handler var insightsEnumerable = Alpha.Update(this, slice); // for performance only call 'ToArray' if not empty enumerable (which is static) var insights = insightsEnumerable == Enumerable.Empty <Insight>() ? new Insight[] { } : insightsEnumerable.ToArray(); // only fire insights generated event if we actually have insights if (insights.Length != 0) { insights = InitializeInsights(insights); OnInsightsGenerated(insights); } ProcessInsights(insights); }