/// <summary> /// Initializes the data feed for the specified job and algorithm /// </summary> public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler, IMapFileProvider mapFileProvider, IFactorFileProvider factorFileProvider, IDataProvider dataProvider) { _algorithm = algorithm; _resultHandler = resultHandler; _mapFileProvider = mapFileProvider; _factorFileProvider = factorFileProvider; _dataProvider = dataProvider; _subscriptions = new SubscriptionCollection(); _universeSelection = new UniverseSelection(this, algorithm); _cancellationTokenSource = new CancellationTokenSource(); _subscriptionfactory = new SubscriptionDataReaderSubscriptionEnumeratorFactory(_resultHandler, _mapFileProvider, _factorFileProvider, _dataProvider, false, true); IsActive = true; var threadCount = Math.Max(1, Math.Min(4, Environment.ProcessorCount - 3)); _controller = new ParallelRunnerController(threadCount); _controller.Start(_cancellationTokenSource.Token); var ffres = Time.OneMinute; _fillForwardResolution = Ref.Create(() => ffres, res => ffres = res); // wire ourselves up to receive notifications when universes are added/removed algorithm.UniverseManager.CollectionChanged += (sender, args) => { switch (args.Action) { case NotifyCollectionChangedAction.Add: foreach (var universe in args.NewItems.OfType <Universe>()) { var config = universe.Configuration; var start = _algorithm.UtcTime; var marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); var exchangeHours = marketHoursDatabase.GetExchangeHours(config); Security security; if (!_algorithm.Securities.TryGetValue(config.Symbol, out security)) { // create a canonical security object if it doesn't exist security = new Security(exchangeHours, config, _algorithm.Portfolio.CashBook[CashBook.AccountCurrency], SymbolProperties.GetDefault(CashBook.AccountCurrency)); } var end = _algorithm.EndDate.ConvertToUtc(_algorithm.TimeZone); AddSubscription(new SubscriptionRequest(true, universe, security, config, start, end)); } break; case NotifyCollectionChangedAction.Remove: foreach (var universe in args.OldItems.OfType <Universe>()) { RemoveSubscription(universe.Configuration); } break; default: throw new NotImplementedException("The specified action is not implemented: " + args.Action); } }; }
/// <summary> /// Initializes the data feed for the specified job and algorithm /// </summary> public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler, IMapFileProvider mapFileProvider, IFactorFileProvider factorFileProvider, IDataProvider dataProvider, IDataFeedSubscriptionManager subscriptionManager, IDataFeedTimeProvider dataFeedTimeProvider) { _algorithm = algorithm; _resultHandler = resultHandler; _mapFileProvider = mapFileProvider; _factorFileProvider = factorFileProvider; _dataProvider = dataProvider; _subscriptions = subscriptionManager.DataFeedSubscriptions; _universeSelection = subscriptionManager.UniverseSelection; _cancellationTokenSource = new CancellationTokenSource(); _subscriptionFactory = new SubscriptionDataReaderSubscriptionEnumeratorFactory( _resultHandler, _mapFileProvider, _factorFileProvider, _dataProvider, includeAuxiliaryData: true); IsActive = true; var threadCount = Math.Max(1, Math.Min(4, Environment.ProcessorCount - 3)); }
/// <summary> /// Initializes a new instance of the <see cref="SubscriptionSynchronizer"/> class /// </summary> /// <param name="universeSelection">The universe selection instance used to handle universe /// selection subscription output</param> /// <param name="sliceTimeZone">The time zone of the created slice object</param> /// <param name="cashBook">The cash book, used for creating the cash book updates</param> /// <returns>A time slice for the specified frontier time</returns> /// <param name="frontierUtc">The initial UTC frontier time to syncronize at</param> public SubscriptionSynchronizer(UniverseSelection universeSelection, DateTimeZone sliceTimeZone, CashBook cashBook, DateTime frontierUtc) { _frontier = frontierUtc; _universeSelection = universeSelection; _sliceTimeZone = sliceTimeZone; _cashBook = cashBook; }
/// <summary> /// Creates a new instance of the DataManager /// </summary> public DataManager(IDataFeed dataFeed, UniverseSelection universeSelection, IAlgorithmSettings algorithmSettings, ITimeKeeper timeKeeper) { _dataFeed = dataFeed; UniverseSelection = universeSelection; _algorithmSettings = algorithmSettings; AvailableDataTypes = SubscriptionManager.DefaultDataTypes(); _timeKeeper = timeKeeper; }
/// <summary> /// Initializes a new instance of the <see cref="SubscriptionSynchronizer"/> class /// </summary> /// <param name="universeSelection">The universe selection instance used to handle universe /// selection subscription output</param> /// <param name="sliceTimeZone">The time zone of the created slice object</param> /// <param name="cashBook">The cash book, used for creating the cash book updates</param> /// <returns>A time slice for the specified frontier time</returns> /// <param name="timeProvider">The time provider, used to obtain the current frontier UTC value.</param> public SubscriptionSynchronizer(UniverseSelection universeSelection, DateTimeZone sliceTimeZone, CashBook cashBook, ITimeProvider timeProvider) { _timeProvider = timeProvider; _universeSelection = universeSelection; _sliceTimeZone = sliceTimeZone; _cashBook = cashBook; }
/// <summary> /// Initializes the data feed for the specified job and algorithm /// </summary> public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler, IMapFileProvider mapFileProvider, IFactorFileProvider factorFileProvider) { _algorithm = algorithm; _resultHandler = resultHandler; _mapFileProvider = mapFileProvider; _factorFileProvider = factorFileProvider; _subscriptions = new SubscriptionCollection(); _universeSelection = new UniverseSelection(this, algorithm, job.Controls); _cancellationTokenSource = new CancellationTokenSource(); IsActive = true; var threadCount = Math.Max(1, Math.Min(4, Environment.ProcessorCount - 3)); _controller = new ParallelRunnerController(threadCount); _controller.Start(_cancellationTokenSource.Token); var ffres = Time.OneMinute; _fillForwardResolution = Ref.Create(() => ffres, res => ffres = res); // wire ourselves up to receive notifications when universes are added/removed algorithm.UniverseManager.CollectionChanged += (sender, args) => { switch (args.Action) { case NotifyCollectionChangedAction.Add: foreach (var universe in args.NewItems.OfType<Universe>()) { var config = universe.Configuration; var start = _frontierUtc != DateTime.MinValue ? _frontierUtc : _algorithm.StartDate.ConvertToUtc(_algorithm.TimeZone); var marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); var exchangeHours = marketHoursDatabase.GetExchangeHours(config); Security security; if (!_algorithm.Securities.TryGetValue(config.Symbol, out security)) { // create a canonical security object if it doesn't exist security = new Security(exchangeHours, config, _algorithm.Portfolio.CashBook[CashBook.AccountCurrency], SymbolProperties.GetDefault(CashBook.AccountCurrency)); } var end = _algorithm.EndDate.ConvertToUtc(_algorithm.TimeZone); AddSubscription(new SubscriptionRequest(true, universe, security, config, start, end)); } break; case NotifyCollectionChangedAction.Remove: foreach (var universe in args.OldItems.OfType<Universe>()) { RemoveSubscription(universe.Configuration); } break; default: throw new NotImplementedException("The specified action is not implemented: " + args.Action); } }; }
/// <summary> /// Initializes the data feed for the specified job and algorithm /// </summary> public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler, IMapFileProvider mapFileProvider, IFactorFileProvider factorFileProvider) { _algorithm = algorithm; _resultHandler = resultHandler; _mapFileProvider = mapFileProvider; _factorFileProvider = factorFileProvider; _subscriptions = new ConcurrentDictionary <Symbol, Subscription>(); _universeSelection = new UniverseSelection(this, algorithm, job.Controls); _cancellationTokenSource = new CancellationTokenSource(); IsActive = true; var threadCount = Math.Max(1, Math.Min(4, Environment.ProcessorCount - 3)); _controller = new ParallelRunnerController(threadCount); _controller.Start(_cancellationTokenSource.Token); var ffres = Time.OneSecond; _fillForwardResolution = Ref.Create(() => ffres, res => ffres = res); // find the minimum resolution, ignoring ticks ffres = ResolveFillForwardResolution(algorithm); // wire ourselves up to receive notifications when universes are added/removed algorithm.UniverseManager.CollectionChanged += (sender, args) => { switch (args.Action) { case NotifyCollectionChangedAction.Add: foreach (var universe in args.NewItems.OfType <Universe>()) { var start = _frontierUtc != DateTime.MinValue ? _frontierUtc : _algorithm.StartDate.ConvertToUtc(_algorithm.TimeZone); AddUniverseSubscription(universe, start, _algorithm.EndDate.ConvertToUtc(_algorithm.TimeZone)); } break; case NotifyCollectionChangedAction.Remove: foreach (var universe in args.OldItems.OfType <Universe>()) { Subscription subscription; if (_subscriptions.TryGetValue(universe.Configuration.Symbol, out subscription)) { RemoveSubscription(subscription); } } break; default: throw new NotImplementedException("The specified action is not implemented: " + args.Action); } }; }
/// <summary> /// Initializes the data feed for the specified job and algorithm /// </summary> public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler, IMapFileProvider mapFileProvider, IFactorFileProvider factorFileProvider) { _algorithm = algorithm; _resultHandler = resultHandler; _mapFileProvider = mapFileProvider; _factorFileProvider = factorFileProvider; _subscriptions = new ConcurrentDictionary<Symbol, Subscription>(); _universeSelection = new UniverseSelection(this, algorithm, job.Controls); _cancellationTokenSource = new CancellationTokenSource(); IsActive = true; var threadCount = Math.Max(1, Math.Min(4, Environment.ProcessorCount - 3)); _controller = new ParallelRunnerController(threadCount); _controller.Start(_cancellationTokenSource.Token); var ffres = Time.OneSecond; _fillForwardResolution = Ref.Create(() => ffres, res => ffres = res); // find the minimum resolution, ignoring ticks ffres = ResolveFillForwardResolution(algorithm); // wire ourselves up to receive notifications when universes are added/removed algorithm.UniverseManager.CollectionChanged += (sender, args) => { switch (args.Action) { case NotifyCollectionChangedAction.Add: foreach (var universe in args.NewItems.OfType<Universe>()) { var start = _frontierUtc != DateTime.MinValue ? _frontierUtc : _algorithm.StartDate.ConvertToUtc(_algorithm.TimeZone); AddUniverseSubscription(universe, start, _algorithm.EndDate.ConvertToUtc(_algorithm.TimeZone)); } break; case NotifyCollectionChangedAction.Remove: foreach (var universe in args.OldItems.OfType<Universe>()) { Subscription subscription; if (_subscriptions.TryGetValue(universe.Configuration.Symbol, out subscription)) { RemoveSubscription(subscription); } } break; default: throw new NotImplementedException("The specified action is not implemented: " + args.Action); } }; }
/// <summary> /// Initializes the data feed for the specified job and algorithm /// </summary> public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler, IMapFileProvider mapFileProvider, IFactorFileProvider factorFileProvider, IDataProvider dataProvider, IDataFeedSubscriptionManager subscriptionManager, IDataFeedTimeProvider dataFeedTimeProvider) { if (!(job is LiveNodePacket)) { throw new ArgumentException("The LiveTradingDataFeed requires a LiveNodePacket."); } _cancellationTokenSource = new CancellationTokenSource(); _algorithm = algorithm; _job = (LiveNodePacket)job; _timeProvider = dataFeedTimeProvider.TimeProvider; _dataQueueHandler = GetDataQueueHandler(); _dataProvider = dataProvider; _channelProvider = GetDataChannelProvider(); _frontierTimeProvider = dataFeedTimeProvider.FrontierTimeProvider; _customExchange = new BaseDataExchange("CustomDataExchange") { SleepInterval = 10 }; // sleep is controlled on this exchange via the GetNextTicksEnumerator _exchange = new BaseDataExchange("DataQueueExchange") { SleepInterval = 0 }; _exchange.AddEnumerator(DataQueueHandlerSymbol, GetNextTicksEnumerator()); _subscriptions = subscriptionManager.DataFeedSubscriptions; _universeSelection = subscriptionManager.UniverseSelection; // run the exchanges Task.Run(() => _exchange.Start(_cancellationTokenSource.Token)); Task.Run(() => _customExchange.Start(_cancellationTokenSource.Token)); IsActive = true; }
/// <summary> /// Live trading datafeed handler provides a base implementation of a live trading datafeed. Derived types /// need only implement the GetNextTicks() function to return unprocessed ticks from a data source. /// This creates a new data feed with a DataFeedEndpoint of LiveTrading. /// </summary> public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler) { if (!(job is LiveNodePacket)) { throw new ArgumentException("The LiveTradingDataFeed requires a LiveNodePacket."); } _job = (LiveNodePacket)job; _isActive = true; _algorithm = algorithm; _resultHandler = resultHandler; _cancellationTokenSource = new CancellationTokenSource(); _universeSelection = new UniverseSelection(this, algorithm, true); _dataQueue = Composer.Instance.GetExportedValueByTypeName <IDataQueueHandler>(Configuration.Config.Get("data-queue-handler", "LiveDataQueue")); Bridge = new BusyBlockingCollection <TimeSlice>(); _subscriptions = new ConcurrentDictionary <SymbolSecurityType, LiveSubscription>(); var periodStart = DateTime.UtcNow.ConvertFromUtc(algorithm.TimeZone).AddDays(-7); var periodEnd = Time.EndOfTime; foreach (var security in algorithm.Securities.Values) { var subscription = CreateSubscription(algorithm, resultHandler, security, periodStart, periodEnd); _subscriptions.AddOrUpdate(new SymbolSecurityType(subscription), subscription); } // request for data from these symbols var symbols = BuildTypeSymbolList(algorithm.Securities.Values); if (symbols.Any()) { // don't subscribe if there's nothing there, this allows custom data to // work without an IDataQueueHandler implementation by specifying LiveDataQueue // in the configuration, that implementation throws on every method, but we actually // don't need it if we're only doing custom data _dataQueue.Subscribe(_job, symbols); } }
public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler) { if (algorithm.SubscriptionManager.Subscriptions.Count == 0 && algorithm.Universe == null) { throw new Exception("No subscriptions registered and no universe defined."); } _algorithm = algorithm; _resultHandler = resultHandler; _subscriptions = new ConcurrentDictionary <SymbolSecurityType, Subscription>(); _cancellationTokenSource = new CancellationTokenSource(); _universeSelection = new UniverseSelection(this, algorithm, false); IsActive = true; Bridge = new BusyBlockingCollection <TimeSlice>(100); // find the minimum resolution, ignoring ticks _fillForwardResolution = algorithm.SubscriptionManager.Subscriptions .Where(x => x.Resolution != Resolution.Tick) .Select(x => x.Resolution) .DefaultIfEmpty(algorithm.UniverseSettings.Resolution) .Min(); // initialize the original user defined securities foreach (var security in _algorithm.Securities.Values) { var subscription = CreateSubscription(resultHandler, security, algorithm.StartDate, algorithm.EndDate, _fillForwardResolution, true); if (subscription != null) { _subscriptions.AddOrUpdate(new SymbolSecurityType(security), subscription); // prime the pump, run method checks current before move next calls PrimeSubscriptionPump(subscription, true); } } }
/// <summary> /// Execute the primary thread for retrieving stock data. /// 1. Subscribe to the streams requested. /// 2. Build bars or tick data requested, primary loop increment smallest possible. /// </summary> public void Run() { //Initialize: // Set up separate thread to handle stream and building packets: var streamThread = new Thread(StreamStoreConsumer); streamThread.Start(); Thread.Sleep(5); // Wait a little for the other thread to init. // This thread converts data into bars "on" the second - assuring the bars are close as // possible to a second unit tradebar (starting at 0 milliseconds). var realtime = new RealTimeSynchronizedTimer(TimeSpan.FromSeconds(1), utcTriggerTime => { // determine if we're on even time boundaries for data emit var onMinute = utcTriggerTime.Second == 0; var onHour = onMinute && utcTriggerTime.Minute == 0; // Determine if this subscription needs to be archived: var items = new List <KeyValuePair <Security, List <BaseData> > >(); var changes = SecurityChanges.None; var performedUniverseSelection = new HashSet <string>(); foreach (var kvp in _subscriptions) { var subscription = kvp.Value; if (subscription.Configuration.Resolution == Resolution.Tick) { continue; } var localTime = new DateTime(utcTriggerTime.Ticks - subscription.OffsetProvider.GetOffsetTicks(utcTriggerTime)); var onDay = onHour && localTime.Hour == 0; // perform universe selection if requested on day changes (don't perform multiple times per market) if (onDay && _algorithm.Universe != null && performedUniverseSelection.Add(subscription.Configuration.Market)) { var coarse = UniverseSelection.GetCoarseFundamentals(subscription.Configuration.Market, subscription.TimeZone, localTime.Date, true); OnFundamental(FundamentalType.Coarse, utcTriggerTime, subscription.Configuration, coarse.ToList()); } var triggerArchive = false; switch (subscription.Configuration.Resolution) { case Resolution.Second: triggerArchive = true; break; case Resolution.Minute: triggerArchive = onMinute; break; case Resolution.Hour: triggerArchive = onHour; break; case Resolution.Daily: triggerArchive = onDay; break; } if (triggerArchive) { var data = subscription.StreamStore.TriggerArchive(utcTriggerTime); if (data != null) { items.Add(new KeyValuePair <Security, List <BaseData> >(subscription.Security, new List <BaseData> { data })); } } } // don't try to add if we're already cancelling if (_cancellationTokenSource.IsCancellationRequested) { return; } Bridge.Add(TimeSlice.Create(utcTriggerTime, _algorithm.TimeZone, _algorithm.Portfolio.CashBook, items, changes)); }); //Start the realtime sampler above realtime.Start(); while (!_cancellationTokenSource.IsCancellationRequested && !_endOfBridges) { // main work of this class is done in the realtime and stream store consumer threads Thread.Sleep(1000); } //Dispose of the realtime clock. realtime.Stop(); //Stop thread _isActive = false; //Exit Live DataStream Feed: Log.Trace("LiveTradingDataFeed.Run(): Exiting LiveTradingDataFeed Run Method"); }
/// <summary> /// Creates a new instance of the DataManager /// </summary> public DataManager( IDataFeed dataFeed, UniverseSelection universeSelection, IAlgorithm algorithm, ITimeKeeper timeKeeper, MarketHoursDatabase marketHoursDatabase, bool liveMode, IRegisteredSecurityDataTypesProvider registeredTypesProvider, IDataPermissionManager dataPermissionManager) { _dataFeed = dataFeed; UniverseSelection = universeSelection; UniverseSelection.SetDataManager(this); _algorithmSettings = algorithm.Settings; AvailableDataTypes = SubscriptionManager.DefaultDataTypes(); _timeKeeper = timeKeeper; _marketHoursDatabase = marketHoursDatabase; _liveMode = liveMode; _registeredTypesProvider = registeredTypesProvider; _dataPermissionManager = dataPermissionManager; // wire ourselves up to receive notifications when universes are added/removed algorithm.UniverseManager.CollectionChanged += (sender, args) => { switch (args.Action) { case NotifyCollectionChangedAction.Add: foreach (var universe in args.NewItems.OfType <Universe>()) { var config = universe.Configuration; var start = algorithm.UtcTime; var end = algorithm.LiveMode ? Time.EndOfTime : algorithm.EndDate.ConvertToUtc(algorithm.TimeZone); Security security; if (!algorithm.Securities.TryGetValue(config.Symbol, out security)) { // create a canonical security object if it doesn't exist security = new Security( _marketHoursDatabase.GetExchangeHours(config), config, algorithm.Portfolio.CashBook[algorithm.AccountCurrency], SymbolProperties.GetDefault(algorithm.AccountCurrency), algorithm.Portfolio.CashBook, RegisteredSecurityDataTypesProvider.Null, new SecurityCache() ); } AddSubscription( new SubscriptionRequest(true, universe, security, config, start, end)); } break; case NotifyCollectionChangedAction.Remove: foreach (var universe in args.OldItems.OfType <Universe>()) { // removing the subscription will be handled by the SubscriptionSynchronizer // in the next loop as well as executing a UniverseSelection one last time. if (!universe.DisposeRequested) { universe.Dispose(); } } break; default: throw new NotImplementedException("The specified action is not implemented: " + args.Action); } }; }
/// <summary> /// Creates a new instance of the DataManager /// </summary> public DataManager(IDataFeed dataFeed, UniverseSelection universeSelection, IAlgorithmSettings algorithmSettings) { _dataFeed = dataFeed; UniverseSelection = universeSelection; _algorithmSettings = algorithmSettings; }
/// <summary> /// Runs a single backtest/live job from the job queue /// </summary> /// <param name="job">The algorithm job to be processed</param> /// <param name="assemblyPath">The path to the algorithm's assembly</param> public void Run(AlgorithmNodePacket job, string assemblyPath) { var algorithm = default(IAlgorithm); var algorithmManager = new AlgorithmManager(_liveMode); //Start monitoring the backtest active status: var statusPing = new StateCheck.Ping(algorithmManager, _systemHandlers.Api, _algorithmHandlers.Results, _systemHandlers.Notify, job); var statusPingThread = new Thread(statusPing.Run); statusPingThread.Start(); try { //Reset thread holders. var initializeComplete = false; Thread threadFeed = null; Thread threadTransactions = null; Thread threadResults = null; Thread threadRealTime = null; //-> Initialize messaging system _systemHandlers.Notify.SetChannel(job.Channel); //-> Set the result handler type for this algorithm job, and launch the associated result thread. _algorithmHandlers.Results.Initialize(job, _systemHandlers.Notify, _systemHandlers.Api, _algorithmHandlers.DataFeed, _algorithmHandlers.Setup, _algorithmHandlers.Transactions); threadResults = new Thread(_algorithmHandlers.Results.Run, 0) {Name = "Result Thread"}; threadResults.Start(); IBrokerage brokerage = null; try { // Save algorithm to cache, load algorithm instance: algorithm = _algorithmHandlers.Setup.CreateAlgorithmInstance(assemblyPath, job.Language); // set the history provider before setting up the algorithm _algorithmHandlers.HistoryProvider.Initialize(job, progress => { // send progress updates to the result handler only during initialization if (!algorithm.GetLocked() || algorithm.IsWarmingUp) { _algorithmHandlers.Results.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.History, string.Format("Processing history {0}%...", progress)); } }); algorithm.HistoryProvider = _algorithmHandlers.HistoryProvider; // initialize the default brokerage message handler algorithm.BrokerageMessageHandler = new DefaultBrokerageMessageHandler(algorithm, job, _algorithmHandlers.Results, _systemHandlers.Api); //Initialize the internal state of algorithm and job: executes the algorithm.Initialize() method. initializeComplete = _algorithmHandlers.Setup.Setup(algorithm, out brokerage, job, _algorithmHandlers.Results, _algorithmHandlers.Transactions, _algorithmHandlers.RealTime); // set this again now that we've actually added securities _algorithmHandlers.Results.SetAlgorithm(algorithm); //If there are any reasons it failed, pass these back to the IDE. if (!initializeComplete || algorithm.ErrorMessages.Count > 0 || _algorithmHandlers.Setup.Errors.Count > 0) { initializeComplete = false; //Get all the error messages: internal in algorithm and external in setup handler. var errorMessage = String.Join(",", algorithm.ErrorMessages); errorMessage += String.Join(",", _algorithmHandlers.Setup.Errors); Log.Error("Engine.Run(): " + errorMessage); _algorithmHandlers.Results.RuntimeError(errorMessage); _systemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, errorMessage); } } catch (Exception err) { var runtimeMessage = "Algorithm.Initialize() Error: " + err.Message + " Stack Trace: " + err.StackTrace; _algorithmHandlers.Results.RuntimeError(runtimeMessage, err.StackTrace); _systemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, runtimeMessage); } //-> Using the job + initialization: load the designated handlers: if (initializeComplete) { //-> Reset the backtest stopwatch; we're now running the algorithm. var startTime = DateTime.Now; //Set algorithm as locked; set it to live mode if we're trading live, and set it to locked for no further updates. algorithm.SetAlgorithmId(job.AlgorithmId); algorithm.SetLocked(); //Wire up the universe selection event handler before kicking off the data feed var universeSelection = new UniverseSelection(_algorithmHandlers.DataFeed, algorithm, _liveMode); _algorithmHandlers.DataFeed.UniverseSelection += (sender, args) => universeSelection.ApplyUniverseSelection(args); //Load the associated handlers for data, transaction and realtime events: _algorithmHandlers.DataFeed.Initialize(algorithm, job, _algorithmHandlers.Results); _algorithmHandlers.Transactions.Initialize(algorithm, brokerage, _algorithmHandlers.Results); _algorithmHandlers.RealTime.Setup(algorithm, job, _algorithmHandlers.Results, _systemHandlers.Api); // wire up the brokerage message handler brokerage.Message += (sender, message) => algorithm.BrokerageMessageHandler.Handle(message); //Send status to user the algorithm is now executing. _algorithmHandlers.Results.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.Running); //Launch the data, transaction and realtime handlers into dedicated threads threadFeed = new Thread(_algorithmHandlers.DataFeed.Run) {Name = "DataFeed Thread"}; threadTransactions = new Thread(_algorithmHandlers.Transactions.Run) {Name = "Transaction Thread"}; threadRealTime = new Thread(_algorithmHandlers.RealTime.Run) {Name = "RealTime Thread"}; //Launch the data feed, result sending, and transaction models/handlers in separate threads. threadFeed.Start(); // Data feed pushing data packets into thread bridge; threadTransactions.Start(); // Transaction modeller scanning new order requests threadRealTime.Start(); // RealTime scan time for time based events: // Result manager scanning message queue: (started earlier) _algorithmHandlers.Results.DebugMessage(string.Format("Launching analysis for {0} with LEAN Engine v{1}", job.AlgorithmId, Constants.Version)); try { //Create a new engine isolator class var isolator = new Isolator(); // Execute the Algorithm Code: var complete = isolator.ExecuteWithTimeLimit(_algorithmHandlers.Setup.MaximumRuntime, algorithmManager.TimeLoopWithinLimits, () => { try { //Run Algorithm Job: // -> Using this Data Feed, // -> Send Orders to this TransactionHandler, // -> Send Results to ResultHandler. algorithmManager.Run(job, algorithm, _algorithmHandlers.DataFeed, _algorithmHandlers.Transactions, _algorithmHandlers.Results, _algorithmHandlers.RealTime, isolator.CancellationToken); } catch (Exception err) { //Debugging at this level is difficult, stack trace needed. Log.Error(err); algorithm.RunTimeError = err; algorithmManager.SetStatus(AlgorithmStatus.RuntimeError); return; } Log.Trace("Engine.Run(): Exiting Algorithm Manager"); }, job.RamAllocation); if (!complete) { Log.Error("Engine.Main(): Failed to complete in time: " + _algorithmHandlers.Setup.MaximumRuntime.ToString("F")); throw new Exception("Failed to complete algorithm within " + _algorithmHandlers.Setup.MaximumRuntime.ToString("F") + " seconds. Please make it run faster."); } // Algorithm runtime error: if (algorithm.RunTimeError != null) { throw algorithm.RunTimeError; } } catch (Exception err) { //Error running the user algorithm: purge datafeed, send error messages, set algorithm status to failed. Log.Error("Engine.Run(): Breaking out of parent try-catch: " + err.Message + " " + err.StackTrace); if (_algorithmHandlers.DataFeed != null) _algorithmHandlers.DataFeed.Exit(); if (_algorithmHandlers.Results != null) { var message = "Runtime Error: " + err.Message; Log.Trace("Engine.Run(): Sending runtime error to user..."); _algorithmHandlers.Results.LogMessage(message); _algorithmHandlers.Results.RuntimeError(message, err.StackTrace); _systemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, message + " Stack Trace: " + err.StackTrace); } } //Send result data back: this entire code block could be rewritten. // todo: - Split up statistics class, its enormous. // todo: - Make a dedicated Statistics.Benchmark class. // todo: - Move all creation and transmission of statistics out of primary engine loop. // todo: - Statistics.Generate(algorithm, resulthandler, transactionhandler); try { var trades = algorithm.TradeBuilder.ClosedTrades; var charts = new Dictionary<string, Chart>(_algorithmHandlers.Results.Charts); var orders = new Dictionary<int, Order>(_algorithmHandlers.Transactions.Orders); var holdings = new Dictionary<string, Holding>(); var banner = new Dictionary<string, string>(); var statisticsResults = new StatisticsResults(); try { //Generates error when things don't exist (no charting logged, runtime errors in main algo execution) const string strategyEquityKey = "Strategy Equity"; const string equityKey = "Equity"; const string dailyPerformanceKey = "Daily Performance"; const string benchmarkKey = "Benchmark"; // make sure we've taken samples for these series before just blindly requesting them if (charts.ContainsKey(strategyEquityKey) && charts[strategyEquityKey].Series.ContainsKey(equityKey) && charts[strategyEquityKey].Series.ContainsKey(dailyPerformanceKey)) { var equity = charts[strategyEquityKey].Series[equityKey].Values; var performance = charts[strategyEquityKey].Series[dailyPerformanceKey].Values; var profitLoss = new SortedDictionary<DateTime, decimal>(algorithm.Transactions.TransactionRecord); var totalTransactions = algorithm.Transactions.GetOrders(x => x.Status.IsFill()).Count(); var benchmark = charts[benchmarkKey].Series[benchmarkKey].Values; statisticsResults = StatisticsBuilder.Generate(trades, profitLoss, equity, performance, benchmark, _algorithmHandlers.Setup.StartingPortfolioValue, algorithm.Portfolio.TotalFees, totalTransactions); } } catch (Exception err) { Log.Error("Algorithm.Node.Engine(): Error generating statistics packet: " + err.Message); } //Diagnostics Completed, Send Result Packet: var totalSeconds = (DateTime.Now - startTime).TotalSeconds; var dataPoints = algorithmManager.DataPoints + _algorithmHandlers.HistoryProvider.DataPointCount; _algorithmHandlers.Results.DebugMessage( string.Format("Algorithm Id:({0}) completed in {1} seconds at {2}k data points per second. Processing total of {3} data points.", job.AlgorithmId, totalSeconds.ToString("F2"), ((dataPoints/(double) 1000)/totalSeconds).ToString("F0"), dataPoints.ToString("N0"))); _algorithmHandlers.Results.SendFinalResult(job, orders, algorithm.Transactions.TransactionRecord, holdings, statisticsResults, banner); } catch (Exception err) { Log.Error("Engine.Main(): Error sending analysis result: " + err.Message + " ST >> " + err.StackTrace); } //Before we return, send terminate commands to close up the threads _algorithmHandlers.Transactions.Exit(); _algorithmHandlers.DataFeed.Exit(); _algorithmHandlers.RealTime.Exit(); } //Close result handler: _algorithmHandlers.Results.Exit(); statusPing.Exit(); //Wait for the threads to complete: var ts = Stopwatch.StartNew(); while ((_algorithmHandlers.Results.IsActive || (_algorithmHandlers.Transactions != null && _algorithmHandlers.Transactions.IsActive) || (_algorithmHandlers.DataFeed != null && _algorithmHandlers.DataFeed.IsActive) || (_algorithmHandlers.RealTime != null && _algorithmHandlers.RealTime.IsActive)) && ts.ElapsedMilliseconds < 30*1000) { Thread.Sleep(100); Log.Trace("Waiting for threads to exit..."); } //Terminate threads still in active state. if (threadFeed != null && threadFeed.IsAlive) threadFeed.Abort(); if (threadTransactions != null && threadTransactions.IsAlive) threadTransactions.Abort(); if (threadResults != null && threadResults.IsAlive) threadResults.Abort(); if (statusPingThread != null && statusPingThread.IsAlive) statusPingThread.Abort(); if (brokerage != null) { brokerage.Disconnect(); } if (_algorithmHandlers.Setup != null) { _algorithmHandlers.Setup.Dispose(); } Log.Trace("Engine.Main(): Analysis Completed and Results Posted."); } catch (Exception err) { Log.Error("Engine.Main(): Error running algorithm: " + err.Message + " >> " + err.StackTrace); } finally { //No matter what for live mode; make sure we've set algorithm status in the API for "not running" conditions: if (_liveMode && algorithmManager.State != AlgorithmStatus.Running && algorithmManager.State != AlgorithmStatus.RuntimeError) _systemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, algorithmManager.State); _algorithmHandlers.Results.Exit(); _algorithmHandlers.DataFeed.Exit(); _algorithmHandlers.Transactions.Exit(); _algorithmHandlers.RealTime.Exit(); } }
/// <summary> /// Initializes the data feed for the specified job and algorithm /// </summary> public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler, IMapFileProvider mapFileProvider, IFactorFileProvider factorFileProvider) { if (!(job is LiveNodePacket)) { throw new ArgumentException("The LiveTradingDataFeed requires a LiveNodePacket."); } _cancellationTokenSource = new CancellationTokenSource(); _algorithm = algorithm; _job = (LiveNodePacket)job; _resultHandler = resultHandler; _timeProvider = GetTimeProvider(); _dataQueueHandler = GetDataQueueHandler(); _frontierTimeProvider = new ManualTimeProvider(_timeProvider.GetUtcNow()); _customExchange = new BaseDataExchange("CustomDataExchange") { SleepInterval = 10 }; // sleep is controlled on this exchange via the GetNextTicksEnumerator _exchange = new BaseDataExchange("DataQueueExchange") { SleepInterval = 0 }; _exchange.AddEnumerator(DataQueueHandlerSymbol, GetNextTicksEnumerator()); _subscriptions = new ConcurrentDictionary <Symbol, Subscription>(); _bridge = new BusyBlockingCollection <TimeSlice>(); _universeSelection = new UniverseSelection(this, algorithm, job.Controls); // run the exchanges Task.Run(() => _exchange.Start(_cancellationTokenSource.Token)); Task.Run(() => _customExchange.Start(_cancellationTokenSource.Token)); // this value will be modified via calls to AddSubscription/RemoveSubscription var ffres = Time.OneMinute; _fillForwardResolution = Ref.Create(() => ffres, v => ffres = v); // wire ourselves up to receive notifications when universes are added/removed var start = _timeProvider.GetUtcNow(); algorithm.UniverseManager.CollectionChanged += (sender, args) => { switch (args.Action) { case NotifyCollectionChangedAction.Add: foreach (var universe in args.NewItems.OfType <Universe>()) { _subscriptions[universe.Configuration.Symbol] = CreateUniverseSubscription(universe, start, Time.EndOfTime); UpdateFillForwardResolution(); } break; case NotifyCollectionChangedAction.Remove: foreach (var universe in args.OldItems.OfType <Universe>()) { Subscription subscription; if (_subscriptions.TryGetValue(universe.Configuration.Symbol, out subscription)) { RemoveSubscription(subscription.Configuration.Symbol); } } break; default: throw new NotImplementedException("The specified action is not implemented: " + args.Action); } }; }
/// <summary> /// Initializes a new instance of the <see cref="SubscriptionSynchronizer"/> class /// </summary> /// <param name="universeSelection">The universe selection instance used to handle universe /// selection subscription output</param> public SubscriptionSynchronizer(UniverseSelection universeSelection) { _universeSelection = universeSelection; }
/// <summary> /// Initializes the data feed for the specified job and algorithm /// </summary> public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler, IMapFileProvider mapFileProvider, IFactorFileProvider factorFileProvider, IDataProvider dataProvider, IDataFeedSubscriptionManager subscriptionManager) { if (!(job is LiveNodePacket)) { throw new ArgumentException("The LiveTradingDataFeed requires a LiveNodePacket."); } _cancellationTokenSource = new CancellationTokenSource(); _algorithm = algorithm; _job = (LiveNodePacket)job; _resultHandler = resultHandler; _timeProvider = GetTimeProvider(); _dataQueueHandler = GetDataQueueHandler(); _dataProvider = dataProvider; _dataCacheProvider = new SingleEntryDataCacheProvider(dataProvider); _frontierTimeProvider = new ManualTimeProvider(_timeProvider.GetUtcNow()); _customExchange = new BaseDataExchange("CustomDataExchange") { SleepInterval = 10 }; // sleep is controlled on this exchange via the GetNextTicksEnumerator _exchange = new BaseDataExchange("DataQueueExchange") { SleepInterval = 0 }; _exchange.AddEnumerator(DataQueueHandlerSymbol, GetNextTicksEnumerator()); _subscriptions = subscriptionManager.DataFeedSubscriptions; _bridge = new BusyBlockingCollection <TimeSlice>(); _universeSelection = subscriptionManager.UniverseSelection; // run the exchanges Task.Run(() => _exchange.Start(_cancellationTokenSource.Token)); Task.Run(() => _customExchange.Start(_cancellationTokenSource.Token)); // wire ourselves up to receive notifications when universes are added/removed var start = _timeProvider.GetUtcNow(); algorithm.UniverseManager.CollectionChanged += (sender, args) => { switch (args.Action) { case NotifyCollectionChangedAction.Add: foreach (var universe in args.NewItems.OfType <Universe>()) { var config = universe.Configuration; var marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); var exchangeHours = marketHoursDatabase.GetExchangeHours(config); Security security; if (!_algorithm.Securities.TryGetValue(config.Symbol, out security)) { // create a canonical security object security = new Security(exchangeHours, config, _algorithm.Portfolio.CashBook[CashBook.AccountCurrency], SymbolProperties.GetDefault(CashBook.AccountCurrency)); } AddSubscription(new SubscriptionRequest(true, universe, security, config, start, Time.EndOfTime)); } break; case NotifyCollectionChangedAction.Remove: foreach (var universe in args.OldItems.OfType <Universe>()) { RemoveSubscription(universe.Configuration); } break; default: throw new NotImplementedException("The specified action is not implemented: " + args.Action); } }; }
/// <summary> /// Creates a new instance of the DataManager /// </summary> public DataManager(IDataFeed dataFeed, IAlgorithm algorithm) { _dataFeed = dataFeed; UniverseSelection = new UniverseSelection(dataFeed, algorithm); }
/// <summary> /// Live trading datafeed handler provides a base implementation of a live trading datafeed. Derived types /// need only implement the GetNextTicks() function to return unprocessed ticks from a data source. /// This creates a new data feed with a DataFeedEndpoint of LiveTrading. /// </summary> public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler) { if (!(job is LiveNodePacket)) { throw new ArgumentException("The LiveTradingDataFeed requires a LiveNodePacket."); } _job = (LiveNodePacket)job; _isActive = true; _algorithm = algorithm; _resultHandler = resultHandler; _cancellationTokenSource = new CancellationTokenSource(); _universeSelection = new UniverseSelection(this, algorithm, true); _dataQueue = Composer.Instance.GetExportedValueByTypeName<IDataQueueHandler>(Configuration.Config.Get("data-queue-handler", "LiveDataQueue")); Bridge = new BusyBlockingCollection<TimeSlice>(); _subscriptions = new ConcurrentDictionary<SymbolSecurityType, LiveSubscription>(); var periodStart = DateTime.UtcNow.ConvertFromUtc(algorithm.TimeZone).AddDays(-7); var periodEnd = Time.EndOfTime; foreach (var security in algorithm.Securities.Values) { var subscription = CreateSubscription(algorithm, resultHandler, security, periodStart, periodEnd); _subscriptions.AddOrUpdate(new SymbolSecurityType(subscription), subscription); } // request for data from these symbols var symbols = BuildTypeSymbolList(algorithm.Securities.Values); if (symbols.Any()) { // don't subscribe if there's nothing there, this allows custom data to // work without an IDataQueueHandler implementation by specifying LiveDataQueue // in the configuration, that implementation throws on every method, but we actually // don't need it if we're only doing custom data _dataQueue.Subscribe(_job, symbols); } }
/// <summary> /// Initializes the data feed for the specified job and algorithm /// </summary> public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler, IMapFileProvider mapFileProvider, IFactorFileProvider factorFileProvider) { if (!(job is LiveNodePacket)) { throw new ArgumentException("The LiveTradingDataFeed requires a LiveNodePacket."); } _cancellationTokenSource = new CancellationTokenSource(); _algorithm = algorithm; _job = (LiveNodePacket) job; _resultHandler = resultHandler; _timeProvider = GetTimeProvider(); _dataQueueHandler = GetDataQueueHandler(); _frontierTimeProvider = new ManualTimeProvider(_timeProvider.GetUtcNow()); _customExchange = new BaseDataExchange("CustomDataExchange") {SleepInterval = 10}; // sleep is controlled on this exchange via the GetNextTicksEnumerator _exchange = new BaseDataExchange("DataQueueExchange"){SleepInterval = 0}; _exchange.AddEnumerator(DataQueueHandlerSymbol, GetNextTicksEnumerator()); _subscriptions = new ConcurrentDictionary<Symbol, List<Subscription>>(); _bridge = new BusyBlockingCollection<TimeSlice>(); _universeSelection = new UniverseSelection(this, algorithm, job.Controls); // run the exchanges Task.Run(() => _exchange.Start(_cancellationTokenSource.Token)); Task.Run(() => _customExchange.Start(_cancellationTokenSource.Token)); // this value will be modified via calls to AddSubscription/RemoveSubscription var ffres = Time.OneMinute; _fillForwardResolution = Ref.Create(() => ffres, v => ffres = v); // wire ourselves up to receive notifications when universes are added/removed var start = _timeProvider.GetUtcNow(); algorithm.UniverseManager.CollectionChanged += (sender, args) => { switch (args.Action) { case NotifyCollectionChangedAction.Add: foreach (var universe in args.NewItems.OfType<Universe>()) { _subscriptions.Add(universe.Configuration.Symbol, CreateUniverseSubscription(universe, start, Time.EndOfTime)); UpdateFillForwardResolution(); } break; case NotifyCollectionChangedAction.Remove: foreach (var universe in args.OldItems.OfType<Universe>()) { RemoveSubscription(universe.Configuration.Symbol); } break; default: throw new NotImplementedException("The specified action is not implemented: " + args.Action); } }; }
/// <summary> /// Initializes a new instance of the <see cref="SubscriptionSynchronizer"/> class /// </summary> /// <param name="universeSelection">The universe selection instance used to handle universe /// selection subscription output</param> /// <returns>A time slice for the specified frontier time</returns> public SubscriptionSynchronizer(UniverseSelection universeSelection) { _universeSelection = universeSelection; }
public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler) { if (algorithm.SubscriptionManager.Subscriptions.Count == 0 && algorithm.Universe == null) { throw new Exception("No subscriptions registered and no universe defined."); } _algorithm = algorithm; _resultHandler = resultHandler; _subscriptions = new ConcurrentDictionary<SymbolSecurityType, Subscription>(); _cancellationTokenSource = new CancellationTokenSource(); _universeSelection = new UniverseSelection(this, algorithm, false); IsActive = true; Bridge = new BusyBlockingCollection<TimeSlice>(100); // find the minimum resolution, ignoring ticks _fillForwardResolution = algorithm.SubscriptionManager.Subscriptions .Where(x => x.Resolution != Resolution.Tick) .Select(x => x.Resolution) .DefaultIfEmpty(algorithm.UniverseSettings.Resolution) .Min(); // initialize the original user defined securities foreach (var security in _algorithm.Securities.Values) { var subscription = CreateSubscription(resultHandler, security, algorithm.StartDate, algorithm.EndDate, _fillForwardResolution, true); if (subscription != null) { _subscriptions.AddOrUpdate(new SymbolSecurityType(security), subscription); // prime the pump, run method checks current before move next calls PrimeSubscriptionPump(subscription, true); } } }
/// <summary> /// Initializes the data feed for the specified job and algorithm /// </summary> public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler, IMapFileProvider mapFileProvider, IFactorFileProvider factorFileProvider) { if (!(job is LiveNodePacket)) { throw new ArgumentException("The LiveTradingDataFeed requires a LiveNodePacket."); } _cancellationTokenSource = new CancellationTokenSource(); _algorithm = algorithm; _job = (LiveNodePacket) job; _resultHandler = resultHandler; _timeProvider = GetTimeProvider(); _dataQueueHandler = GetDataQueueHandler(); _frontierTimeProvider = new ManualTimeProvider(_timeProvider.GetUtcNow()); _customExchange = new BaseDataExchange("CustomDataExchange") {SleepInterval = 10}; // sleep is controlled on this exchange via the GetNextTicksEnumerator _exchange = new BaseDataExchange("DataQueueExchange"){SleepInterval = 0}; _exchange.AddEnumerator(DataQueueHandlerSymbol, GetNextTicksEnumerator()); _subscriptions = new SubscriptionCollection(); _bridge = new BusyBlockingCollection<TimeSlice>(); _universeSelection = new UniverseSelection(this, algorithm, job.Controls); // run the exchanges Task.Run(() => _exchange.Start(_cancellationTokenSource.Token)); Task.Run(() => _customExchange.Start(_cancellationTokenSource.Token)); // this value will be modified via calls to AddSubscription/RemoveSubscription var ffres = Time.OneMinute; _fillForwardResolution = Ref.Create(() => ffres, v => ffres = v); // wire ourselves up to receive notifications when universes are added/removed var start = _timeProvider.GetUtcNow(); algorithm.UniverseManager.CollectionChanged += (sender, args) => { switch (args.Action) { case NotifyCollectionChangedAction.Add: foreach (var universe in args.NewItems.OfType<Universe>()) { var config = universe.Configuration; var marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); var exchangeHours = marketHoursDatabase.GetExchangeHours(config); Security security; if (!_algorithm.Securities.TryGetValue(config.Symbol, out security)) { // create a canonical security object security = new Security(exchangeHours, config, _algorithm.Portfolio.CashBook[CashBook.AccountCurrency], SymbolProperties.GetDefault(CashBook.AccountCurrency)); } AddSubscription(new SubscriptionRequest(true, universe, security, config, start, Time.EndOfTime)); // Not sure if this is needed but left here because of this: // https://github.com/QuantConnect/Lean/commit/029d70bde6ca83a1eb0c667bb5cc4444bea05678 UpdateFillForwardResolution(); } break; case NotifyCollectionChangedAction.Remove: foreach (var universe in args.OldItems.OfType<Universe>()) { RemoveSubscription(universe.Configuration); } break; default: throw new NotImplementedException("The specified action is not implemented: " + args.Action); } }; }
/// <summary> /// Creates a new instance of the DataManager /// </summary> public DataManager( IDataFeed dataFeed, UniverseSelection universeSelection, IAlgorithm algorithm, ITimeKeeper timeKeeper, MarketHoursDatabase marketHoursDatabase) { _dataFeed = dataFeed; UniverseSelection = universeSelection; UniverseSelection.SetDataManager(this); _algorithmSettings = algorithm.Settings; AvailableDataTypes = SubscriptionManager.DefaultDataTypes(); _timeKeeper = timeKeeper; _marketHoursDatabase = marketHoursDatabase; var liveStart = DateTime.UtcNow; // wire ourselves up to receive notifications when universes are added/removed algorithm.UniverseManager.CollectionChanged += (sender, args) => { switch (args.Action) { case NotifyCollectionChangedAction.Add: foreach (var universe in args.NewItems.OfType <Universe>()) { var config = universe.Configuration; var start = algorithm.LiveMode ? liveStart : algorithm.UtcTime; var end = algorithm.LiveMode ? Time.EndOfTime : algorithm.EndDate.ConvertToUtc(algorithm.TimeZone); Security security; if (!algorithm.Securities.TryGetValue(config.Symbol, out security)) { // create a canonical security object if it doesn't exist security = new Security( _marketHoursDatabase.GetExchangeHours(config), config, algorithm.Portfolio.CashBook[CashBook.AccountCurrency], SymbolProperties.GetDefault(CashBook.AccountCurrency), algorithm.Portfolio.CashBook ); } AddSubscription( new SubscriptionRequest(true, universe, security, config, start, end)); } break; case NotifyCollectionChangedAction.Remove: foreach (var universe in args.OldItems.OfType <Universe>()) { RemoveSubscription(universe.Configuration); } break; default: throw new NotImplementedException("The specified action is not implemented: " + args.Action); } }; }
/// <summary> /// Initializes the data feed for the specified job and algorithm /// </summary> public void Initialize(IAlgorithm algorithm, AlgorithmNodePacket job, IResultHandler resultHandler, IMapFileProvider mapFileProvider, IFactorFileProvider factorFileProvider, IDataFileProvider dataFileProvider) { if (!(job is LiveNodePacket)) { throw new ArgumentException("The LiveTradingDataFeed requires a LiveNodePacket."); } _cancellationTokenSource = new CancellationTokenSource(); _algorithm = algorithm; _job = (LiveNodePacket)job; _resultHandler = resultHandler; _timeProvider = GetTimeProvider(); _dataQueueHandler = GetDataQueueHandler(); _dataFileProvider = dataFileProvider; _frontierTimeProvider = new ManualTimeProvider(_timeProvider.GetUtcNow()); _customExchange = new BaseDataExchange("CustomDataExchange") { SleepInterval = 10 }; // sleep is controlled on this exchange via the GetNextTicksEnumerator _exchange = new BaseDataExchange("DataQueueExchange") { SleepInterval = 0 }; _exchange.AddEnumerator(DataQueueHandlerSymbol, GetNextTicksEnumerator()); _subscriptions = new SubscriptionCollection(); _bridge = new BusyBlockingCollection <TimeSlice>(); _universeSelection = new UniverseSelection(this, algorithm, job.Controls); // run the exchanges Task.Run(() => _exchange.Start(_cancellationTokenSource.Token)); Task.Run(() => _customExchange.Start(_cancellationTokenSource.Token)); // this value will be modified via calls to AddSubscription/RemoveSubscription var ffres = Time.OneMinute; _fillForwardResolution = Ref.Create(() => ffres, v => ffres = v); // wire ourselves up to receive notifications when universes are added/removed var start = _timeProvider.GetUtcNow(); algorithm.UniverseManager.CollectionChanged += (sender, args) => { switch (args.Action) { case NotifyCollectionChangedAction.Add: foreach (var universe in args.NewItems.OfType <Universe>()) { var config = universe.Configuration; var marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); var exchangeHours = marketHoursDatabase.GetExchangeHours(config); Security security; if (!_algorithm.Securities.TryGetValue(config.Symbol, out security)) { // create a canonical security object security = new Security(exchangeHours, config, _algorithm.Portfolio.CashBook[CashBook.AccountCurrency], SymbolProperties.GetDefault(CashBook.AccountCurrency)); } AddSubscription(new SubscriptionRequest(true, universe, security, config, start, Time.EndOfTime)); // Not sure if this is needed but left here because of this: // https://github.com/QuantConnect/Lean/commit/029d70bde6ca83a1eb0c667bb5cc4444bea05678 UpdateFillForwardResolution(); } break; case NotifyCollectionChangedAction.Remove: foreach (var universe in args.OldItems.OfType <Universe>()) { RemoveSubscription(universe.Configuration); } break; default: throw new NotImplementedException("The specified action is not implemented: " + args.Action); } }; }
/// <summary> /// Initializes a new instance of the <see cref="SubscriptionSynchronizer"/> class /// </summary> /// <param name="universeSelection">The universe selection instance used to handle universe /// selection subscription output</param> /// <param name="cashBook">The cash book, used for creating the cash book updates</param> /// <returns>A time slice for the specified frontier time</returns> public SubscriptionSynchronizer(UniverseSelection universeSelection, CashBook cashBook) { _universeSelection = universeSelection; _cashBook = cashBook; }