/// <summary> /// Creates the correct enumerator factory for the given request /// </summary> private ISubscriptionEnumeratorFactory GetEnumeratorFactory(SubscriptionRequest request) { if (request.IsUniverseSubscription) { if (request.Universe is ITimeTriggeredUniverse) { var universe = request.Universe as UserDefinedUniverse; if (universe != null) { // Trigger universe selection when security added/removed after Initialize universe.CollectionChanged += (sender, args) => { var items = args.Action == NotifyCollectionChangedAction.Add ? args.NewItems : args.Action == NotifyCollectionChangedAction.Remove ? args.OldItems : null; if (items == null) { return; } var symbol = items.OfType <Symbol>().FirstOrDefault(); if (symbol == null) { return; } var collection = new BaseDataCollection(_algorithm.UtcTime, symbol); var changes = _universeSelection.ApplyUniverseSelection(universe, _algorithm.UtcTime, collection); _algorithm.OnSecuritiesChanged(changes); }; } return(new TimeTriggeredUniverseSubscriptionEnumeratorFactory(request.Universe as ITimeTriggeredUniverse, MarketHoursDatabase.FromDataFolder())); } if (request.Configuration.Type == typeof(CoarseFundamental)) { return(new BaseDataCollectionSubscriptionEnumeratorFactory()); } if (request.Universe is OptionChainUniverse) { return(new OptionChainUniverseSubscriptionEnumeratorFactory((req, e) => ConfigureEnumerator(req, true, e), _mapFileProvider.Get(request.Security.Symbol.ID.Market), _factorFileProvider)); } if (request.Universe is FuturesChainUniverse) { return(new FuturesChainUniverseSubscriptionEnumeratorFactory((req, e) => ConfigureEnumerator(req, true, e))); } } return(_subscriptionfactory); }
/// <summary> /// Creates the correct enumerator factory for the given request /// </summary> private ISubscriptionEnumeratorFactory GetEnumeratorFactory(SubscriptionRequest request) { if (request.IsUniverseSubscription) { if (request.Universe is UserDefinedUniverse) { // Trigger universe selection when security added/removed after Initialize var universe = (UserDefinedUniverse)request.Universe; universe.CollectionChanged += (sender, args) => { var items = args.Action == NotifyCollectionChangedAction.Add ? args.NewItems : args.Action == NotifyCollectionChangedAction.Remove ? args.OldItems : null; if (items == null || _frontierUtc == DateTime.MinValue) { return; } var symbol = items.OfType <Symbol>().FirstOrDefault(); if (symbol == null) { return; } var collection = new BaseDataCollection(_frontierUtc, symbol); var changes = _universeSelection.ApplyUniverseSelection(universe, _frontierUtc, collection); _algorithm.OnSecuritiesChanged(changes); }; return(new UserDefinedUniverseSubscriptionEnumeratorFactory(request.Universe as UserDefinedUniverse, MarketHoursDatabase.FromDataFolder())); } if (request.Configuration.Type == typeof(CoarseFundamental)) { return(new BaseDataCollectionSubscriptionEnumeratorFactory()); } if (request.Universe is OptionChainUniverse) { return(new OptionChainUniverseSubscriptionEnumeratorFactory((req, e) => ConfigureEnumerator(req, true, e))); } } var mapFileResolver = request.Configuration.SecurityType == SecurityType.Equity ? _mapFileProvider.Get(request.Security.Symbol.ID.Market) : MapFileResolver.Empty; return(new PostCreateConfigureSubscriptionEnumeratorFactory( new SubscriptionDataReaderSubscriptionEnumeratorFactory(_resultHandler, mapFileResolver, _factorFileProvider, _dataFileProvider, false, true), enumerator => ConfigureEnumerator(request, false, enumerator) )); }
/// <summary> /// Syncs the specified subscriptions. The frontier time used for synchronization is /// managed internally and dependent upon previous synchronization operations. /// </summary> /// <param name="subscriptions">The subscriptions to sync</param> public TimeSlice Sync(IEnumerable <Subscription> subscriptions) { var delayedSubscriptionFinished = false; var changes = SecurityChanges.None; var data = new List <DataFeedPacket>(); // NOTE: Tight coupling in UniverseSelection.ApplyUniverseSelection var universeData = new Dictionary <Universe, BaseDataCollection>(); var universeDataForTimeSliceCreate = new Dictionary <Universe, BaseDataCollection>(); _frontierTimeProvider.SetCurrentTimeUtc(_timeProvider.GetUtcNow()); var frontierUtc = _frontierTimeProvider.GetUtcNow(); SecurityChanges newChanges; do { newChanges = SecurityChanges.None; foreach (var subscription in subscriptions) { if (subscription.EndOfStream) { OnSubscriptionFinished(subscription); continue; } // prime if needed if (subscription.Current == null) { if (!subscription.MoveNext()) { OnSubscriptionFinished(subscription); continue; } } var packet = new DataFeedPacket(subscription.Security, subscription.Configuration, subscription.RemovedFromUniverse); while (subscription.Current != null && subscription.Current.EmitTimeUtc <= frontierUtc) { packet.Add(subscription.Current.Data); if (!subscription.MoveNext()) { delayedSubscriptionFinished = true; break; } } if (packet.Count > 0) { // we have new universe data to select based on, store the subscription data until the end if (!subscription.IsUniverseSelectionSubscription) { data.Add(packet); } else { // assume that if the first item is a base data collection then the enumerator handled the aggregation, // otherwise, load all the the data into a new collection instance var packetBaseDataCollection = packet.Data[0] as BaseDataCollection; var packetData = packetBaseDataCollection == null ? packet.Data : packetBaseDataCollection.Data; BaseDataCollection collection; if (universeData.TryGetValue(subscription.Universes.Single(), out collection)) { collection.Data.AddRange(packetData); } else { if (packetBaseDataCollection is OptionChainUniverseDataCollection) { var current = packetBaseDataCollection as OptionChainUniverseDataCollection; collection = new OptionChainUniverseDataCollection(frontierUtc, subscription.Configuration.Symbol, packetData, current?.Underlying); } else if (packetBaseDataCollection is FuturesChainUniverseDataCollection) { collection = new FuturesChainUniverseDataCollection(frontierUtc, subscription.Configuration.Symbol, packetData); } else { collection = new BaseDataCollection(frontierUtc, subscription.Configuration.Symbol, packetData); } universeData[subscription.Universes.Single()] = collection; } } } if (subscription.IsUniverseSelectionSubscription && subscription.Universes.Single().DisposeRequested || delayedSubscriptionFinished) { delayedSubscriptionFinished = false; // we need to do this after all usages of subscription.Universes OnSubscriptionFinished(subscription); } } foreach (var kvp in universeData) { var universe = kvp.Key; var baseDataCollection = kvp.Value; universeDataForTimeSliceCreate[universe] = baseDataCollection; newChanges += _universeSelection.ApplyUniverseSelection(universe, frontierUtc, baseDataCollection); } universeData.Clear(); changes += newChanges; }while (newChanges != SecurityChanges.None); var timeSlice = _timeSliceFactory.Create(frontierUtc, data, changes, universeDataForTimeSliceCreate); return(timeSlice); }
/// <summary> /// Syncs the specifies subscriptions at the frontier time /// </summary> /// <param name="frontier">The time used for syncing, data in the future won't be included in this time slice</param> /// <param name="subscriptions">The subscriptions to sync</param> /// <param name="sliceTimeZone">The time zone of the created slice object</param> /// <param name="cashBook">The cash book, used for creating the cash book updates</param> /// <param name="nextFrontier">The next frontier time as determined by the first piece of data in the future ahead of the frontier. /// This value will equal DateTime.MaxValue when the subscriptions are all finished</param> /// <returns>A time slice for the specified frontier time</returns> public TimeSlice Sync(DateTime frontier, IEnumerable <Subscription> subscriptions, DateTimeZone sliceTimeZone, CashBook cashBook, out DateTime nextFrontier) { var changes = SecurityChanges.None; nextFrontier = DateTime.MaxValue; var earlyBirdTicks = nextFrontier.Ticks; var data = new List <DataFeedPacket>(); var universeData = new Dictionary <Universe, BaseDataCollection>(); SecurityChanges newChanges; do { universeData.Clear(); newChanges = SecurityChanges.None; foreach (var subscription in subscriptions) { if (subscription.EndOfStream) { OnSubscriptionFinished(subscription); continue; } // prime if needed if (subscription.Current == null) { if (!subscription.MoveNext()) { OnSubscriptionFinished(subscription); continue; } } var packet = new DataFeedPacket(subscription.Security, subscription.Configuration); data.Add(packet); var configuration = subscription.Configuration; var offsetProvider = subscription.OffsetProvider; var currentOffsetTicks = offsetProvider.GetOffsetTicks(frontier); while (subscription.Current.EndTime.Ticks - currentOffsetTicks <= frontier.Ticks) { // we want bars rounded using their subscription times, we make a clone // so we don't interfere with the enumerator's internal logic var clone = subscription.Current.Clone(subscription.Current.IsFillForward); clone.Time = clone.Time.ExchangeRoundDown(configuration.Increment, subscription.Security.Exchange.Hours, configuration.ExtendedMarketHours); packet.Add(clone); if (!subscription.MoveNext()) { OnSubscriptionFinished(subscription); break; } } // we have new universe data to select based on, store the subscription data until the end if (subscription.IsUniverseSelectionSubscription && packet.Count > 0) { // assume that if the first item is a base data collection then the enumerator handled the aggregation, // otherwise, load all the the data into a new collection instance var packetBaseDataCollection = packet.Data[0] as BaseDataCollection; var packetData = packetBaseDataCollection == null ? packet.Data : packetBaseDataCollection.Data; BaseDataCollection collection; if (!universeData.TryGetValue(subscription.Universe, out collection)) { if (packetBaseDataCollection is OptionChainUniverseDataCollection) { var current = subscription.Current as OptionChainUniverseDataCollection; var underlying = current != null ? current.Underlying : null; collection = new OptionChainUniverseDataCollection(frontier, subscription.Configuration.Symbol, packetData, underlying); } else { collection = new BaseDataCollection(frontier, subscription.Configuration.Symbol, packetData); } universeData[subscription.Universe] = collection; } else { collection.Data.AddRange(packetData); } } if (subscription.Current != null) { // take the earliest between the next piece of data or the next tz discontinuity earlyBirdTicks = Math.Min(earlyBirdTicks, Math.Min(subscription.Current.EndTime.Ticks - currentOffsetTicks, offsetProvider.GetNextDiscontinuity())); } } foreach (var kvp in universeData) { var universe = kvp.Key; var baseDataCollection = kvp.Value; newChanges += _universeSelection.ApplyUniverseSelection(universe, frontier, baseDataCollection); } changes += newChanges; }while (newChanges != SecurityChanges.None); nextFrontier = new DateTime(Math.Max(earlyBirdTicks, frontier.Ticks), DateTimeKind.Utc); return(TimeSlice.Create(frontier, sliceTimeZone, cashBook, data, changes)); }
/// <summary> /// Syncs the specified subscriptions. The frontier time used for synchronization is /// managed internally and dependent upon previous synchronization operations. /// </summary> /// <param name="subscriptions">The subscriptions to sync</param> /// <param name="cancellationToken">The cancellation token to stop enumeration</param> public IEnumerable <TimeSlice> Sync(IEnumerable <Subscription> subscriptions, CancellationToken cancellationToken) { var delayedSubscriptionFinished = new Queue <Subscription>(); while (!cancellationToken.IsCancellationRequested) { var changes = SecurityChanges.None; var data = new List <DataFeedPacket>(1); // NOTE: Tight coupling in UniverseSelection.ApplyUniverseSelection var universeData = new Dictionary <Universe, BaseDataCollection>(); var universeDataForTimeSliceCreate = new Dictionary <Universe, BaseDataCollection>(); _frontierTimeProvider.SetCurrentTimeUtc(_timeProvider.GetUtcNow()); var frontierUtc = _frontierTimeProvider.GetUtcNow(); SecurityChanges newChanges; do { newChanges = SecurityChanges.None; foreach (var subscription in subscriptions) { if (subscription.EndOfStream) { OnSubscriptionFinished(subscription); continue; } // prime if needed if (subscription.Current == null) { if (!subscription.MoveNext()) { OnSubscriptionFinished(subscription); continue; } } DataFeedPacket packet = null; while (subscription.Current != null && subscription.Current.EmitTimeUtc <= frontierUtc) { if (packet == null) { // for performance, lets be selfish about creating a new instance packet = new DataFeedPacket( subscription.Security, subscription.Configuration, subscription.RemovedFromUniverse ); } packet.Add(subscription.Current.Data); if (!subscription.MoveNext()) { delayedSubscriptionFinished.Enqueue(subscription); break; } } if (packet?.Count > 0) { // we have new universe data to select based on, store the subscription data until the end if (!subscription.IsUniverseSelectionSubscription) { data.Add(packet); } else { // assume that if the first item is a base data collection then the enumerator handled the aggregation, // otherwise, load all the the data into a new collection instance var packetBaseDataCollection = packet.Data[0] as BaseDataCollection; var packetData = packetBaseDataCollection == null ? packet.Data : packetBaseDataCollection.Data; BaseDataCollection collection; if (universeData.TryGetValue(subscription.Universes.Single(), out collection)) { collection.Data.AddRange(packetData); } else { if (packetBaseDataCollection is OptionChainUniverseDataCollection) { var current = packetBaseDataCollection as OptionChainUniverseDataCollection; collection = new OptionChainUniverseDataCollection(frontierUtc, subscription.Configuration.Symbol, packetData, current?.Underlying); } else if (packetBaseDataCollection is FuturesChainUniverseDataCollection) { collection = new FuturesChainUniverseDataCollection(frontierUtc, subscription.Configuration.Symbol, packetData); } else { collection = new BaseDataCollection(frontierUtc, subscription.Configuration.Symbol, packetData); } universeData[subscription.Universes.Single()] = collection; } } } if (subscription.IsUniverseSelectionSubscription && subscription.Universes.Single().DisposeRequested) { // we need to do this after all usages of subscription.Universes OnSubscriptionFinished(subscription); } } if (universeData.Any()) { // if we are going to perform universe selection we emit an empty // time pulse to align algorithm time with current frontier yield return(_timeSliceFactory.CreateTimePulse(frontierUtc)); } foreach (var kvp in universeData) { var universe = kvp.Key; var baseDataCollection = kvp.Value; universeDataForTimeSliceCreate[universe] = baseDataCollection; newChanges += _universeSelection.ApplyUniverseSelection(universe, frontierUtc, baseDataCollection); } universeData.Clear(); changes += newChanges; }while (newChanges != SecurityChanges.None || _universeSelection.AddPendingInternalDataFeeds(frontierUtc)); var timeSlice = _timeSliceFactory.Create(frontierUtc, data, changes, universeDataForTimeSliceCreate); while (delayedSubscriptionFinished.Count > 0) { // these subscriptions added valid data to the packet // we need to trigger OnSubscriptionFinished after we create the TimeSlice // else it will drop the data var subscription = delayedSubscriptionFinished.Dequeue(); OnSubscriptionFinished(subscription); } yield return(timeSlice); } }
/// <summary> /// Runs a single backtest/live job from the job queue /// </summary> /// <param name="job">The algorithm job to be processed</param> /// <param name="assemblyPath">The path to the algorithm's assembly</param> public void Run(AlgorithmNodePacket job, string assemblyPath) { var algorithm = default(IAlgorithm); var algorithmManager = new AlgorithmManager(_liveMode); //Start monitoring the backtest active status: var statusPing = new StateCheck.Ping(algorithmManager, _systemHandlers.Api, _algorithmHandlers.Results, _systemHandlers.Notify, job); var statusPingThread = new Thread(statusPing.Run); statusPingThread.Start(); try { //Reset thread holders. var initializeComplete = false; Thread threadFeed = null; Thread threadTransactions = null; Thread threadResults = null; Thread threadRealTime = null; //-> Initialize messaging system _systemHandlers.Notify.SetChannel(job.Channel); //-> Set the result handler type for this algorithm job, and launch the associated result thread. _algorithmHandlers.Results.Initialize(job, _systemHandlers.Notify, _systemHandlers.Api, _algorithmHandlers.DataFeed, _algorithmHandlers.Setup, _algorithmHandlers.Transactions); threadResults = new Thread(_algorithmHandlers.Results.Run, 0) {Name = "Result Thread"}; threadResults.Start(); IBrokerage brokerage = null; try { // Save algorithm to cache, load algorithm instance: algorithm = _algorithmHandlers.Setup.CreateAlgorithmInstance(assemblyPath, job.Language); // set the history provider before setting up the algorithm _algorithmHandlers.HistoryProvider.Initialize(job, progress => { // send progress updates to the result handler only during initialization if (!algorithm.GetLocked() || algorithm.IsWarmingUp) { _algorithmHandlers.Results.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.History, string.Format("Processing history {0}%...", progress)); } }); algorithm.HistoryProvider = _algorithmHandlers.HistoryProvider; // initialize the default brokerage message handler algorithm.BrokerageMessageHandler = new DefaultBrokerageMessageHandler(algorithm, job, _algorithmHandlers.Results, _systemHandlers.Api); //Initialize the internal state of algorithm and job: executes the algorithm.Initialize() method. initializeComplete = _algorithmHandlers.Setup.Setup(algorithm, out brokerage, job, _algorithmHandlers.Results, _algorithmHandlers.Transactions, _algorithmHandlers.RealTime); // set this again now that we've actually added securities _algorithmHandlers.Results.SetAlgorithm(algorithm); //If there are any reasons it failed, pass these back to the IDE. if (!initializeComplete || algorithm.ErrorMessages.Count > 0 || _algorithmHandlers.Setup.Errors.Count > 0) { initializeComplete = false; //Get all the error messages: internal in algorithm and external in setup handler. var errorMessage = String.Join(",", algorithm.ErrorMessages); errorMessage += String.Join(",", _algorithmHandlers.Setup.Errors); Log.Error("Engine.Run(): " + errorMessage); _algorithmHandlers.Results.RuntimeError(errorMessage); _systemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, errorMessage); } } catch (Exception err) { var runtimeMessage = "Algorithm.Initialize() Error: " + err.Message + " Stack Trace: " + err.StackTrace; _algorithmHandlers.Results.RuntimeError(runtimeMessage, err.StackTrace); _systemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, runtimeMessage); } //-> Using the job + initialization: load the designated handlers: if (initializeComplete) { //-> Reset the backtest stopwatch; we're now running the algorithm. var startTime = DateTime.Now; //Set algorithm as locked; set it to live mode if we're trading live, and set it to locked for no further updates. algorithm.SetAlgorithmId(job.AlgorithmId); algorithm.SetLocked(); //Wire up the universe selection event handler before kicking off the data feed var universeSelection = new UniverseSelection(_algorithmHandlers.DataFeed, algorithm, _liveMode); _algorithmHandlers.DataFeed.UniverseSelection += (sender, args) => universeSelection.ApplyUniverseSelection(args); //Load the associated handlers for data, transaction and realtime events: _algorithmHandlers.DataFeed.Initialize(algorithm, job, _algorithmHandlers.Results); _algorithmHandlers.Transactions.Initialize(algorithm, brokerage, _algorithmHandlers.Results); _algorithmHandlers.RealTime.Setup(algorithm, job, _algorithmHandlers.Results, _systemHandlers.Api); // wire up the brokerage message handler brokerage.Message += (sender, message) => algorithm.BrokerageMessageHandler.Handle(message); //Send status to user the algorithm is now executing. _algorithmHandlers.Results.SendStatusUpdate(job.AlgorithmId, AlgorithmStatus.Running); //Launch the data, transaction and realtime handlers into dedicated threads threadFeed = new Thread(_algorithmHandlers.DataFeed.Run) {Name = "DataFeed Thread"}; threadTransactions = new Thread(_algorithmHandlers.Transactions.Run) {Name = "Transaction Thread"}; threadRealTime = new Thread(_algorithmHandlers.RealTime.Run) {Name = "RealTime Thread"}; //Launch the data feed, result sending, and transaction models/handlers in separate threads. threadFeed.Start(); // Data feed pushing data packets into thread bridge; threadTransactions.Start(); // Transaction modeller scanning new order requests threadRealTime.Start(); // RealTime scan time for time based events: // Result manager scanning message queue: (started earlier) _algorithmHandlers.Results.DebugMessage(string.Format("Launching analysis for {0} with LEAN Engine v{1}", job.AlgorithmId, Constants.Version)); try { //Create a new engine isolator class var isolator = new Isolator(); // Execute the Algorithm Code: var complete = isolator.ExecuteWithTimeLimit(_algorithmHandlers.Setup.MaximumRuntime, algorithmManager.TimeLoopWithinLimits, () => { try { //Run Algorithm Job: // -> Using this Data Feed, // -> Send Orders to this TransactionHandler, // -> Send Results to ResultHandler. algorithmManager.Run(job, algorithm, _algorithmHandlers.DataFeed, _algorithmHandlers.Transactions, _algorithmHandlers.Results, _algorithmHandlers.RealTime, isolator.CancellationToken); } catch (Exception err) { //Debugging at this level is difficult, stack trace needed. Log.Error(err); algorithm.RunTimeError = err; algorithmManager.SetStatus(AlgorithmStatus.RuntimeError); return; } Log.Trace("Engine.Run(): Exiting Algorithm Manager"); }, job.RamAllocation); if (!complete) { Log.Error("Engine.Main(): Failed to complete in time: " + _algorithmHandlers.Setup.MaximumRuntime.ToString("F")); throw new Exception("Failed to complete algorithm within " + _algorithmHandlers.Setup.MaximumRuntime.ToString("F") + " seconds. Please make it run faster."); } // Algorithm runtime error: if (algorithm.RunTimeError != null) { throw algorithm.RunTimeError; } } catch (Exception err) { //Error running the user algorithm: purge datafeed, send error messages, set algorithm status to failed. Log.Error("Engine.Run(): Breaking out of parent try-catch: " + err.Message + " " + err.StackTrace); if (_algorithmHandlers.DataFeed != null) _algorithmHandlers.DataFeed.Exit(); if (_algorithmHandlers.Results != null) { var message = "Runtime Error: " + err.Message; Log.Trace("Engine.Run(): Sending runtime error to user..."); _algorithmHandlers.Results.LogMessage(message); _algorithmHandlers.Results.RuntimeError(message, err.StackTrace); _systemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, message + " Stack Trace: " + err.StackTrace); } } //Send result data back: this entire code block could be rewritten. // todo: - Split up statistics class, its enormous. // todo: - Make a dedicated Statistics.Benchmark class. // todo: - Move all creation and transmission of statistics out of primary engine loop. // todo: - Statistics.Generate(algorithm, resulthandler, transactionhandler); try { var trades = algorithm.TradeBuilder.ClosedTrades; var charts = new Dictionary<string, Chart>(_algorithmHandlers.Results.Charts); var orders = new Dictionary<int, Order>(_algorithmHandlers.Transactions.Orders); var holdings = new Dictionary<string, Holding>(); var banner = new Dictionary<string, string>(); var statisticsResults = new StatisticsResults(); try { //Generates error when things don't exist (no charting logged, runtime errors in main algo execution) const string strategyEquityKey = "Strategy Equity"; const string equityKey = "Equity"; const string dailyPerformanceKey = "Daily Performance"; const string benchmarkKey = "Benchmark"; // make sure we've taken samples for these series before just blindly requesting them if (charts.ContainsKey(strategyEquityKey) && charts[strategyEquityKey].Series.ContainsKey(equityKey) && charts[strategyEquityKey].Series.ContainsKey(dailyPerformanceKey)) { var equity = charts[strategyEquityKey].Series[equityKey].Values; var performance = charts[strategyEquityKey].Series[dailyPerformanceKey].Values; var profitLoss = new SortedDictionary<DateTime, decimal>(algorithm.Transactions.TransactionRecord); var totalTransactions = algorithm.Transactions.GetOrders(x => x.Status.IsFill()).Count(); var benchmark = charts[benchmarkKey].Series[benchmarkKey].Values; statisticsResults = StatisticsBuilder.Generate(trades, profitLoss, equity, performance, benchmark, _algorithmHandlers.Setup.StartingPortfolioValue, algorithm.Portfolio.TotalFees, totalTransactions); } } catch (Exception err) { Log.Error("Algorithm.Node.Engine(): Error generating statistics packet: " + err.Message); } //Diagnostics Completed, Send Result Packet: var totalSeconds = (DateTime.Now - startTime).TotalSeconds; var dataPoints = algorithmManager.DataPoints + _algorithmHandlers.HistoryProvider.DataPointCount; _algorithmHandlers.Results.DebugMessage( string.Format("Algorithm Id:({0}) completed in {1} seconds at {2}k data points per second. Processing total of {3} data points.", job.AlgorithmId, totalSeconds.ToString("F2"), ((dataPoints/(double) 1000)/totalSeconds).ToString("F0"), dataPoints.ToString("N0"))); _algorithmHandlers.Results.SendFinalResult(job, orders, algorithm.Transactions.TransactionRecord, holdings, statisticsResults, banner); } catch (Exception err) { Log.Error("Engine.Main(): Error sending analysis result: " + err.Message + " ST >> " + err.StackTrace); } //Before we return, send terminate commands to close up the threads _algorithmHandlers.Transactions.Exit(); _algorithmHandlers.DataFeed.Exit(); _algorithmHandlers.RealTime.Exit(); } //Close result handler: _algorithmHandlers.Results.Exit(); statusPing.Exit(); //Wait for the threads to complete: var ts = Stopwatch.StartNew(); while ((_algorithmHandlers.Results.IsActive || (_algorithmHandlers.Transactions != null && _algorithmHandlers.Transactions.IsActive) || (_algorithmHandlers.DataFeed != null && _algorithmHandlers.DataFeed.IsActive) || (_algorithmHandlers.RealTime != null && _algorithmHandlers.RealTime.IsActive)) && ts.ElapsedMilliseconds < 30*1000) { Thread.Sleep(100); Log.Trace("Waiting for threads to exit..."); } //Terminate threads still in active state. if (threadFeed != null && threadFeed.IsAlive) threadFeed.Abort(); if (threadTransactions != null && threadTransactions.IsAlive) threadTransactions.Abort(); if (threadResults != null && threadResults.IsAlive) threadResults.Abort(); if (statusPingThread != null && statusPingThread.IsAlive) statusPingThread.Abort(); if (brokerage != null) { brokerage.Disconnect(); } if (_algorithmHandlers.Setup != null) { _algorithmHandlers.Setup.Dispose(); } Log.Trace("Engine.Main(): Analysis Completed and Results Posted."); } catch (Exception err) { Log.Error("Engine.Main(): Error running algorithm: " + err.Message + " >> " + err.StackTrace); } finally { //No matter what for live mode; make sure we've set algorithm status in the API for "not running" conditions: if (_liveMode && algorithmManager.State != AlgorithmStatus.Running && algorithmManager.State != AlgorithmStatus.RuntimeError) _systemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, algorithmManager.State); _algorithmHandlers.Results.Exit(); _algorithmHandlers.DataFeed.Exit(); _algorithmHandlers.Transactions.Exit(); _algorithmHandlers.RealTime.Exit(); } }
/// <summary> /// Syncs the specified subscriptions. The frontier time used for synchronization is /// managed internally and dependent upon previous synchronization operations. /// </summary> /// <param name="subscriptions">The subscriptions to sync</param> public TimeSlice Sync(IEnumerable <Subscription> subscriptions) { long earlyBirdTicks; var changes = SecurityChanges.None; var data = new List <DataFeedPacket>(); // NOTE: Tight coupling in UniverseSelection.ApplyUniverseSelection var universeData = new Dictionary <Universe, BaseDataCollection>(); var universeDataForTimeSliceCreate = new Dictionary <Universe, BaseDataCollection>(); SecurityChanges newChanges; do { earlyBirdTicks = MaxDateTimeTicks; newChanges = SecurityChanges.None; foreach (var subscription in subscriptions) { if (subscription.EndOfStream) { OnSubscriptionFinished(subscription); continue; } // prime if needed if (subscription.Current == null) { if (!subscription.MoveNext()) { OnSubscriptionFinished(subscription); continue; } } var packet = new DataFeedPacket(subscription.Security, subscription.Configuration); while (subscription.Current.EmitTimeUtc <= _frontier) { packet.Add(subscription.Current.Data); if (!subscription.MoveNext()) { OnSubscriptionFinished(subscription); break; } } if (packet.Count > 0) { // we have new universe data to select based on, store the subscription data until the end if (!subscription.IsUniverseSelectionSubscription) { data.Add(packet); } else { // assume that if the first item is a base data collection then the enumerator handled the aggregation, // otherwise, load all the the data into a new collection instance var packetBaseDataCollection = packet.Data[0] as BaseDataCollection; var packetData = packetBaseDataCollection == null ? packet.Data : packetBaseDataCollection.Data; BaseDataCollection collection; if (universeData.TryGetValue(subscription.Universe, out collection)) { collection.Data.AddRange(packetData); } else { if (packetBaseDataCollection is OptionChainUniverseDataCollection) { var current = packetBaseDataCollection as OptionChainUniverseDataCollection; collection = new OptionChainUniverseDataCollection(_frontier, subscription.Configuration.Symbol, packetData, current?.Underlying); } else if (packetBaseDataCollection is FuturesChainUniverseDataCollection) { collection = new FuturesChainUniverseDataCollection(_frontier, subscription.Configuration.Symbol, packetData); } else { collection = new BaseDataCollection(_frontier, subscription.Configuration.Symbol, packetData); } universeData[subscription.Universe] = collection; } } } if (subscription.Current != null) { if (earlyBirdTicks == MaxDateTimeTicks) { earlyBirdTicks = subscription.Current.EmitTimeUtc.Ticks; } else { // take the earliest between the next piece of data or the current earliest bird earlyBirdTicks = Math.Min(earlyBirdTicks, subscription.Current.EmitTimeUtc.Ticks); } } } foreach (var kvp in universeData) { var universe = kvp.Key; var baseDataCollection = kvp.Value; universeDataForTimeSliceCreate[universe] = baseDataCollection; newChanges += _universeSelection.ApplyUniverseSelection(universe, _frontier, baseDataCollection); } changes += newChanges; }while (newChanges != SecurityChanges.None); var timeSlice = TimeSlice.Create(_frontier, _sliceTimeZone, _cashBook, data, changes, universeDataForTimeSliceCreate); // next frontier time _frontier = new DateTime(Math.Max(earlyBirdTicks, _frontier.Ticks), DateTimeKind.Utc); return(timeSlice); }
/// <summary> /// Syncs the specifies subscriptions at the frontier time /// </summary> /// <param name="frontier">The time used for syncing, data in the future won't be included in this time slice</param> /// <param name="subscriptions">The subscriptions to sync</param> /// <param name="sliceTimeZone">The time zone of the created slice object</param> /// <param name="cashBook">The cash book, used for creating the cash book updates</param> /// <param name="nextFrontier">The next frontier time as determined by the first piece of data in the future ahead of the frontier. /// This value will equal DateTime.MaxValue when the subscriptions are all finished</param> /// <returns>A time slice for the specified frontier time</returns> public TimeSlice Sync(DateTime frontier, IEnumerable <Subscription> subscriptions, DateTimeZone sliceTimeZone, CashBook cashBook, out DateTime nextFrontier) { var changes = SecurityChanges.None; nextFrontier = DateTime.MaxValue; var earlyBirdTicks = nextFrontier.Ticks; var data = new List <KeyValuePair <Security, List <BaseData> > >(); SecurityChanges newChanges; do { newChanges = SecurityChanges.None; foreach (var subscription in subscriptions) { if (subscription.EndOfStream) { OnSubscriptionFinished(subscription); continue; } // prime if needed if (subscription.Current == null) { if (!subscription.MoveNext()) { OnSubscriptionFinished(subscription); continue; } } var cache = new KeyValuePair <Security, List <BaseData> >(subscription.Security, new List <BaseData>()); data.Add(cache); var configuration = subscription.Configuration; var offsetProvider = subscription.OffsetProvider; var currentOffsetTicks = offsetProvider.GetOffsetTicks(frontier); while (subscription.Current.EndTime.Ticks - currentOffsetTicks <= frontier.Ticks) { // we want bars rounded using their subscription times, we make a clone // so we don't interfere with the enumerator's internal logic var clone = subscription.Current.Clone(subscription.Current.IsFillForward); clone.Time = clone.Time.ExchangeRoundDown(configuration.Increment, subscription.Security.Exchange.Hours, configuration.ExtendedMarketHours); cache.Value.Add(clone); if (!subscription.MoveNext()) { OnSubscriptionFinished(subscription); break; } } // we have new universe data to select based on if (subscription.IsUniverseSelectionSubscription && cache.Value.Count > 0) { // assume that if the first item is a base data collection then the enumerator handled the aggregation, // otherwise, load all the the data into a new collection instance var collection = cache.Value[0] as BaseDataCollection ?? new BaseDataCollection(frontier, subscription.Configuration.Symbol, cache.Value); newChanges += _universeSelection.ApplyUniverseSelection(subscription.Universe, frontier, collection); } if (subscription.Current != null) { // take the earliest between the next piece of data or the next tz discontinuity earlyBirdTicks = Math.Min(earlyBirdTicks, Math.Min(subscription.Current.EndTime.Ticks - currentOffsetTicks, offsetProvider.GetNextDiscontinuity())); } } changes += newChanges; }while (newChanges != SecurityChanges.None); nextFrontier = new DateTime(Math.Max(earlyBirdTicks, frontier.Ticks), DateTimeKind.Utc); return(TimeSlice.Create(frontier, sliceTimeZone, cashBook, data, changes)); }
/// <summary> /// Main routine for datafeed analysis. /// </summary> /// <remarks>This is a hot-thread and should be kept extremely lean. Modify with caution.</remarks> public void Run() { var universeSelectionMarkets = new List <string> { "usa" }; var frontier = DateTime.MaxValue; try { // don't initialize universe selection if it's not requested if (_algorithm.Universe != null) { // initialize subscriptions used for universe selection foreach (var market in universeSelectionMarkets) { AddSubscriptionForUniverseSelectionMarket(market); } } // compute initial frontier time frontier = GetInitialFrontierTime(); Log.Trace(string.Format("FileSystemDataFeed.Run(): Begin: {0} UTC", frontier)); // continue to loop over each subscription, enqueuing data in time order while (!_cancellationTokenSource.IsCancellationRequested) { var changes = SecurityChanges.None; var earlyBirdTicks = long.MaxValue; var data = new List <KeyValuePair <Security, List <BaseData> > >(); foreach (var subscription in Subscriptions) { if (subscription.EndOfStream) { // skip subscriptions that are finished continue; } var cache = new KeyValuePair <Security, List <BaseData> >(subscription.Security, new List <BaseData>()); data.Add(cache); var currentOffsetTicks = subscription.OffsetProvider.GetOffsetTicks(frontier); while (subscription.Current.EndTime.Ticks - currentOffsetTicks <= frontier.Ticks) { // we want bars rounded using their subscription times, we make a clone // so we don't interfere with the enumerator's internal logic var clone = subscription.Current.Clone(subscription.Current.IsFillForward); clone.Time = clone.Time.RoundDown(subscription.Configuration.Increment); cache.Value.Add(clone); if (!subscription.MoveNext()) { Log.Trace("FileSystemDataFeed.Run(): Finished subscription: " + subscription.Security.Symbol + " at " + frontier + " UTC"); break; } } // we have new universe data to select based on if (subscription.IsFundamentalSubscription && cache.Value.Count > 0) { // always wait for other thread if (!Bridge.Wait(Timeout.Infinite, _cancellationTokenSource.Token)) { break; } changes += _universeSelection.ApplyUniverseSelection(cache.Value[0].EndTime.Date, cache.Value.OfType <CoarseFundamental>()); } if (subscription.Current != null) { earlyBirdTicks = Math.Min(earlyBirdTicks, subscription.Current.EndTime.Ticks - currentOffsetTicks); } } if (earlyBirdTicks == long.MaxValue) { // there's no more data to pull off, we're done break; } // enqueue our next time slice and set the frontier for the next Bridge.Add(TimeSlice.Create(_algorithm, frontier, data, changes), _cancellationTokenSource.Token); // never go backwards in time, so take the max between early birds and the current frontier frontier = new DateTime(Math.Max(earlyBirdTicks, frontier.Ticks), DateTimeKind.Utc); } if (!_cancellationTokenSource.IsCancellationRequested) { Bridge.CompleteAdding(); } } catch (Exception err) { Log.Error("FileSystemDataFeed.Run(): Encountered an error: " + err.Message); if (!_cancellationTokenSource.IsCancellationRequested) { Bridge.CompleteAdding(); _cancellationTokenSource.Cancel(); } } finally { Log.Trace(string.Format("FileSystemDataFeed.Run(): Data Feed Completed at {0} UTC", frontier)); //Close up all streams: foreach (var subscription in Subscriptions) { subscription.Dispose(); } Log.Trace("FileSystemDataFeed.Run(): Ending Thread... "); IsActive = false; } }
/// <summary> /// Execute the primary thread for retrieving stock data. /// 1. Subscribe to the streams requested. /// 2. Build bars or tick data requested, primary loop increment smallest possible. /// </summary> public void Run() { //Initialize: // Set up separate thread to handle stream and building packets: var streamThread = new Thread(StreamStoreConsumer); streamThread.Start(); Thread.Sleep(5); // Wait a little for the other thread to init. // This thread converts data into bars "on" the second - assuring the bars are close as // possible to a second unit tradebar (starting at 0 milliseconds). var realtime = new RealTimeSynchronizedTimer(TimeSpan.FromSeconds(1), utcTriggerTime => { // determine if we're on even time boundaries for data emit var onMinute = utcTriggerTime.Second == 0; var onHour = onMinute && utcTriggerTime.Minute == 0; // Determine if this subscription needs to be archived: var items = new List <KeyValuePair <Security, List <BaseData> > >(); var changes = SecurityChanges.None; var performedUniverseSelection = new HashSet <string>(); foreach (var kvp in _subscriptions) { var subscription = kvp.Value; if (subscription.Configuration.Resolution == Resolution.Tick) { continue; } var localTime = new DateTime(utcTriggerTime.Ticks - subscription.OffsetProvider.GetOffsetTicks(utcTriggerTime)); var onDay = onHour && localTime.Hour == 0; // perform universe selection if requested on day changes (don't perform multiple times per market) if (onDay && _algorithm.Universe != null && !performedUniverseSelection.Contains(subscription.Configuration.Symbol)) { performedUniverseSelection.Add(subscription.Configuration.Symbol); var coarse = UniverseSelection.GetCoarseFundamentals(subscription.Configuration.Market, subscription.TimeZone, localTime.Date, true); changes = _universeSelection.ApplyUniverseSelection(localTime.Date, coarse); } var triggerArchive = false; switch (subscription.Configuration.Resolution) { case Resolution.Second: triggerArchive = true; break; case Resolution.Minute: triggerArchive = onMinute; break; case Resolution.Hour: triggerArchive = onHour; break; case Resolution.Daily: triggerArchive = onDay; break; } if (triggerArchive) { var data = subscription.StreamStore.TriggerArchive(utcTriggerTime); if (data != null) { items.Add(new KeyValuePair <Security, List <BaseData> >(subscription.Security, new List <BaseData> { data })); } } } // don't try to add if we're already cancelling if (_cancellationTokenSource.IsCancellationRequested) { return; } Bridge.Add(TimeSlice.Create(_algorithm, utcTriggerTime, items, changes)); }); //Start the realtime sampler above realtime.Start(); while (!_cancellationTokenSource.IsCancellationRequested && !_endOfBridges) { // main work of this class is done in the realtime and stream store consumer threads Thread.Sleep(1000); } //Dispose of the realtime clock. realtime.Stop(); //Stop thread _isActive = false; //Exit Live DataStream Feed: Log.Trace("LiveTradingDataFeed.Run(): Exiting LiveTradingDataFeed Run Method"); }
/// <summary> /// Syncs the specified subscriptions. The frontier time used for synchronization is /// managed internally and dependent upon previous synchronization operations. /// </summary> /// <param name="subscriptions">The subscriptions to sync</param> /// <param name="cancellationToken">The cancellation token to stop enumeration</param> public IEnumerable <TimeSlice> Sync(IEnumerable <Subscription> subscriptions, CancellationToken cancellationToken) { var delayedSubscriptionFinished = new Queue <Subscription>(); while (!cancellationToken.IsCancellationRequested) { var changes = SecurityChanges.None; var data = new List <DataFeedPacket>(1); // NOTE: Tight coupling in UniverseSelection.ApplyUniverseSelection Dictionary <Universe, BaseDataCollection> universeData = null; // lazy construction for performance var universeDataForTimeSliceCreate = new Dictionary <Universe, BaseDataCollection>(); var frontierUtc = _timeProvider.GetUtcNow(); _frontierTimeProvider.SetCurrentTimeUtc(frontierUtc); SecurityChanges newChanges; do { newChanges = SecurityChanges.None; foreach (var subscription in subscriptions) { if (subscription.EndOfStream) { OnSubscriptionFinished(subscription); continue; } // prime if needed if (subscription.Current == null) { if (!subscription.MoveNext()) { OnSubscriptionFinished(subscription); continue; } } DataFeedPacket packet = null; while (subscription.Current != null && subscription.Current.EmitTimeUtc <= frontierUtc) { if (packet == null) { // for performance, lets be selfish about creating a new instance packet = new DataFeedPacket( subscription.Security, subscription.Configuration, subscription.RemovedFromUniverse ); } // If our subscription is a universe, and we get a delisting event emitted for it, then // the universe itself should be unselected and removed, because the Symbol that the // universe is based on has been delisted. Doing the disposal here allows us to // process the delisting at this point in time before emitting out to the algorithm. // This is very useful for universes that can be delisted, such as ETF constituent // universes (e.g. for ETF constituent universes, since the ETF itself is used to create // the universe Symbol (and set as its underlying), once the ETF is delisted, the // universe should cease to exist, since there are no more constituents of that ETF). if (subscription.IsUniverseSelectionSubscription && subscription.Current.Data is Delisting) { subscription.Universes.Single().Dispose(); } packet.Add(subscription.Current.Data); if (!subscription.MoveNext()) { delayedSubscriptionFinished.Enqueue(subscription); break; } } if (packet?.Count > 0) { // we have new universe data to select based on, store the subscription data until the end if (!subscription.IsUniverseSelectionSubscription) { data.Add(packet); } else { // assume that if the first item is a base data collection then the enumerator handled the aggregation, // otherwise, load all the the data into a new collection instance var packetBaseDataCollection = packet.Data[0] as BaseDataCollection; var packetData = packetBaseDataCollection == null ? packet.Data : packetBaseDataCollection.Data; BaseDataCollection collection; if (universeData != null && universeData.TryGetValue(subscription.Universes.Single(), out collection)) { collection.AddRange(packetData); } else { if (packetBaseDataCollection is OptionChainUniverseDataCollection) { var current = packetBaseDataCollection as OptionChainUniverseDataCollection; collection = new OptionChainUniverseDataCollection(frontierUtc, subscription.Configuration.Symbol, packetData, current?.Underlying); } else if (packetBaseDataCollection is FuturesChainUniverseDataCollection) { collection = new FuturesChainUniverseDataCollection(frontierUtc, subscription.Configuration.Symbol, packetData); } else { collection = new BaseDataCollection(frontierUtc, frontierUtc, subscription.Configuration.Symbol, packetData); } if (universeData == null) { universeData = new Dictionary <Universe, BaseDataCollection>(); } universeData[subscription.Universes.Single()] = collection; } } } if (subscription.IsUniverseSelectionSubscription && subscription.Universes.Single().DisposeRequested) { var universe = subscription.Universes.Single(); // check if a universe selection isn't already scheduled for this disposed universe if (universeData == null || !universeData.ContainsKey(universe)) { if (universeData == null) { universeData = new Dictionary <Universe, BaseDataCollection>(); } // we force trigger one last universe selection for this disposed universe, so it deselects all subscriptions it added universeData[universe] = new BaseDataCollection(frontierUtc, subscription.Configuration.Symbol); } // we need to do this after all usages of subscription.Universes OnSubscriptionFinished(subscription); } } if (universeData != null && universeData.Count > 0) { // if we are going to perform universe selection we emit an empty // time pulse to align algorithm time with current frontier yield return(_timeSliceFactory.CreateTimePulse(frontierUtc)); foreach (var kvp in universeData) { var universe = kvp.Key; var baseDataCollection = kvp.Value; universeDataForTimeSliceCreate[universe] = baseDataCollection; newChanges += _universeSelection.ApplyUniverseSelection(universe, frontierUtc, baseDataCollection); } universeData.Clear(); } changes += newChanges; }while (newChanges != SecurityChanges.None || _universeSelection.AddPendingInternalDataFeeds(frontierUtc)); var timeSlice = _timeSliceFactory.Create(frontierUtc, data, changes, universeDataForTimeSliceCreate); while (delayedSubscriptionFinished.Count > 0) { // these subscriptions added valid data to the packet // we need to trigger OnSubscriptionFinished after we create the TimeSlice // else it will drop the data var subscription = delayedSubscriptionFinished.Dequeue(); OnSubscriptionFinished(subscription); } yield return(timeSlice); } }