/// <summary> /// Initializes the instance of the Synchronizer class /// </summary> public override void Initialize( IAlgorithm algorithm, IDataFeedSubscriptionManager dataFeedSubscriptionManager) { base.Initialize(algorithm, dataFeedSubscriptionManager); // the time provider, is the real time provider _timeProvider = GetTimeProvider(); _frontierTimeProvider = new LiveTimeProvider(realTime: TimeProvider); // the synchronizer will use our '_frontierTimeProvider' which initially during warmup will be using // the base time provider which is the subscription based time provider (like backtesting) // once wawrmup finishes it will start using the realtime provider SubscriptionSynchronizer.SetTimeProvider(_frontierTimeProvider); // attach event handlers to subscriptions dataFeedSubscriptionManager.SubscriptionAdded += (sender, subscription) => { subscription.NewDataAvailable += OnSubscriptionNewDataAvailable; }; dataFeedSubscriptionManager.SubscriptionRemoved += (sender, subscription) => { subscription.NewDataAvailable -= OnSubscriptionNewDataAvailable; }; _realTimeScheduleEventService = new RealTimeScheduleEventService(new RealTimeProvider()); // this schedule event will be our time pulse _realTimeScheduleEventService.NewEvent += (sender, args) => _newLiveDataEmitted.Set(); }
/// <summary> /// Returns an enumerator that iterates through the collection. /// </summary> /// <returns> /// A <see cref="T:System.Collections.Generic.IEnumerator`1"/> that can be used to iterate through the collection. /// </returns> /// <filterpriority>1</filterpriority> public IEnumerator <TimeSlice> GetEnumerator() { // compute initial frontier time _frontierUtc = GetInitialFrontierTime(); Log.Trace(string.Format("FileSystemDataFeed.GetEnumerator(): Begin: {0} UTC", _frontierUtc)); var syncer = new SubscriptionSynchronizer(_universeSelection); syncer.SubscriptionFinished += (sender, subscription) => { if (subscription.EndOfStream && _subscriptions.TryRemove(subscription.Security.Symbol, out subscription)) { Log.Debug(string.Format("FileSystemDataFeed.GetEnumerator(): Finished subscription: {0} at {1} UTC", subscription.Security.Symbol.ID, _frontierUtc)); subscription.Dispose(); } }; while (!_cancellationTokenSource.IsCancellationRequested) { TimeSlice timeSlice; DateTime nextFrontier; try { timeSlice = syncer.Sync(_frontierUtc, Subscriptions, _algorithm.TimeZone, _algorithm.Portfolio.CashBook, out nextFrontier); } catch (Exception err) { Log.Error(err); continue; } // syncer returns MaxValue on failure/end of data if (timeSlice.Time != DateTime.MaxValue) { yield return(timeSlice); // end of data signal if (nextFrontier == DateTime.MaxValue) { break; } _frontierUtc = nextFrontier; } else if (timeSlice.SecurityChanges == SecurityChanges.None) { // there's no more data to pull off, we're done (frontier is max value and no security changes) break; } } //Close up all streams: foreach (var subscription in Subscriptions) { subscription.Dispose(); } Log.Trace(string.Format("FileSystemDataFeed.Run(): Data Feed Completed at {0} UTC", _frontierUtc)); }
/// <summary> /// Returns an enumerator that iterates through the collection. /// </summary> /// <returns> /// A <see cref="T:System.Collections.Generic.IEnumerator`1"/> that can be used to iterate through the collection. /// </returns> /// <filterpriority>1</filterpriority> public IEnumerator <TimeSlice> GetEnumerator() { // compute initial frontier time var frontierUtc = GetInitialFrontierTime(); Log.Trace(string.Format("FileSystemDataFeed.GetEnumerator(): Begin: {0} UTC", frontierUtc)); var syncer = new SubscriptionSynchronizer(_universeSelection, _algorithm.TimeZone, _algorithm.Portfolio.CashBook, frontierUtc); syncer.SubscriptionFinished += (sender, subscription) => { RemoveSubscription(subscription.Configuration); Log.Debug(string.Format("FileSystemDataFeed.GetEnumerator(): Finished subscription: {0} at {1} UTC", subscription.Configuration, _algorithm.UtcTime)); }; while (!_cancellationTokenSource.IsCancellationRequested) { TimeSlice timeSlice; try { timeSlice = syncer.Sync(Subscriptions); } catch (Exception err) { Log.Error(err); // notify the algorithm about the error, so it can be reported to the user _algorithm.RunTimeError = err; _algorithm.Status = AlgorithmStatus.RuntimeError; break; } // syncer returns MaxValue on failure/end of data if (timeSlice.Time != DateTime.MaxValue) { yield return(timeSlice); } else if (timeSlice.SecurityChanges == SecurityChanges.None) { // there's no more data to pull off, we're done (frontier is max value and no security changes) break; } } //Close up all streams: foreach (var subscription in Subscriptions) { subscription.Dispose(); } if (_subscriptionfactory != null) { _subscriptionfactory.Dispose(); } Log.Trace(string.Format("FileSystemDataFeed.Run(): Data Feed Completed at {0} UTC", _algorithm.UtcTime)); }
/// <summary> /// Initializes the instance of the Synchronizer class /// </summary> public virtual void Initialize( IAlgorithm algorithm, IDataFeedSubscriptionManager dataFeedSubscriptionManager) { SubscriptionManager = dataFeedSubscriptionManager; Algorithm = algorithm; SubscriptionSynchronizer = new SubscriptionSynchronizer( SubscriptionManager.UniverseSelection); }
/// <summary> /// Returns an enumerator that iterates through the collection. /// </summary> /// <returns> /// A <see cref="T:System.Collections.Generic.IEnumerator`1"/> that can be used to iterate through the collection. /// </returns> /// <filterpriority>1</filterpriority> public IEnumerator <TimeSlice> GetEnumerator() { // compute initial frontier time var frontierUtc = GetInitialFrontierTime(); Log.Trace(string.Format("FileSystemDataFeed.GetEnumerator(): Begin: {0} UTC", frontierUtc)); var subscriptionFrontierTimeProvider = new SubscriptionFrontierTimeProvider(frontierUtc, _subscriptionManager); var syncer = new SubscriptionSynchronizer(_universeSelection, _algorithm.TimeZone, _algorithm.Portfolio.CashBook, subscriptionFrontierTimeProvider); syncer.SubscriptionFinished += (sender, subscription) => { RemoveSubscription(subscription.Configuration); Log.Debug($"FileSystemDataFeed.SubscriptionFinished(): Finished subscription: {subscription.Configuration} at {_algorithm.UtcTime} UTC"); }; var previousDateTime = DateTime.MaxValue; while (!_cancellationTokenSource.IsCancellationRequested) { TimeSlice timeSlice; try { timeSlice = syncer.Sync(Subscriptions); } catch (Exception err) { Log.Error(err); // notify the algorithm about the error, so it can be reported to the user _algorithm.RunTimeError = err; _algorithm.Status = AlgorithmStatus.RuntimeError; break; } // SubscriptionFrontierTimeProvider will return twice the same time if there are no more subscriptions or if Subscription.Current is null if (timeSlice.Time != previousDateTime) { previousDateTime = timeSlice.Time; yield return(timeSlice); } else if (timeSlice.SecurityChanges == SecurityChanges.None) { // there's no more data to pull off, we're done (frontier is max value and no security changes) break; } } Log.Trace(string.Format("FileSystemDataFeed.GetEnumerator(): Data Feed Completed at {0} UTC", _algorithm.UtcTime)); }
/// <summary> /// Initializes the instance of the Synchronizer class /// </summary> public void Initialize( IAlgorithm algorithm, IDataFeedSubscriptionManager dataFeedSubscriptionManager, bool liveMode) { _subscriptionManager = dataFeedSubscriptionManager; _algorithm = algorithm; _liveMode = liveMode; _subscriptionSynchronizer = new SubscriptionSynchronizer( _subscriptionManager.UniverseSelection); if (_liveMode) { TimeProvider = GetTimeProvider(); _subscriptionSynchronizer.SetTimeProvider(TimeProvider); } }
/// <summary> /// Initializes the instance of the Synchronizer class /// </summary> public override void Initialize( IAlgorithm algorithm, IDataFeedSubscriptionManager dataFeedSubscriptionManager) { base.Initialize(algorithm, dataFeedSubscriptionManager); TimeProvider = GetTimeProvider(); SubscriptionSynchronizer.SetTimeProvider(TimeProvider); // attach event handlers to subscriptions dataFeedSubscriptionManager.SubscriptionAdded += (sender, subscription) => { subscription.NewDataAvailable += OnSubscriptionNewDataAvailable; }; dataFeedSubscriptionManager.SubscriptionRemoved += (sender, subscription) => { subscription.NewDataAvailable -= OnSubscriptionNewDataAvailable; }; }
/// <summary> /// Initializes the instance of the Synchronizer class /// </summary> public void Initialize( IAlgorithm algorithm, IDataFeedSubscriptionManager subscriptionManager, IDataFeed dataFeed, // To be removed, when subscriptionManager is completely in front of the DF bool liveMode, CashBook cashBook) { _subscriptionManager = subscriptionManager; _algorithm = algorithm; _cashBook = cashBook; _liveMode = liveMode; _dataFeed = dataFeed; _subscriptionSynchronizer = new SubscriptionSynchronizer(_subscriptionManager.UniverseSelection, _cashBook); if (_liveMode) { TimeProvider = GetTimeProvider(); _subscriptionSynchronizer.SetTimeProvider(TimeProvider); } }
/// <summary> /// Initializes the instance of the Synchronizer class /// </summary> public override void Initialize( IAlgorithm algorithm, IDataFeedSubscriptionManager dataFeedSubscriptionManager) { base.Initialize(algorithm, dataFeedSubscriptionManager); _timeProvider = GetTimeProvider(); SubscriptionSynchronizer.SetTimeProvider(TimeProvider); // attach event handlers to subscriptions dataFeedSubscriptionManager.SubscriptionAdded += (sender, subscription) => { subscription.NewDataAvailable += OnSubscriptionNewDataAvailable; }; dataFeedSubscriptionManager.SubscriptionRemoved += (sender, subscription) => { subscription.NewDataAvailable -= OnSubscriptionNewDataAvailable; }; _realTimeScheduleEventService = new RealTimeScheduleEventService(new RealTimeProvider()); // this schedule event will be our time pulse _realTimeScheduleEventService.NewEvent += (sender, args) => _newLiveDataEmitted.Set(); }
/// <summary> /// Primary entry point. /// </summary> public void Run() { IsActive = true; // we want to emit to the bridge minimally once a second since the data feed is // the heartbeat of the application, so this value will contain a second after // the last emit time, and if we pass this time, we'll emit even with no data var nextEmit = DateTime.MinValue; var syncer = new SubscriptionSynchronizer(_universeSelection, _algorithm.TimeZone, _algorithm.Portfolio.CashBook, _frontierTimeProvider); syncer.SubscriptionFinished += (sender, subscription) => { RemoveSubscription(subscription.Configuration); Log.Debug($"LiveTradingDataFeed.SubscriptionFinished(): Finished subscription: {subscription.Configuration} at {_algorithm.UtcTime} UTC"); }; try { while (!_cancellationTokenSource.IsCancellationRequested) { // perform sleeps to wake up on the second? _frontierUtc = _timeProvider.GetUtcNow(); _frontierTimeProvider.SetCurrentTime(_frontierUtc); // always wait for other thread to sync up if (!_bridge.WaitHandle.WaitOne(Timeout.Infinite, _cancellationTokenSource.Token)) { break; } var timeSlice = syncer.Sync(Subscriptions); // check for cancellation if (_cancellationTokenSource.IsCancellationRequested) { return; } // emit on data or if we've elapsed a full second since last emit or there are security changes if (timeSlice.SecurityChanges != SecurityChanges.None || timeSlice.Data.Count != 0 || _frontierUtc >= nextEmit) { _bridge.Add(timeSlice, _cancellationTokenSource.Token); // force emitting every second nextEmit = _frontierUtc.RoundDown(Time.OneSecond).Add(Time.OneSecond); } // take a short nap Thread.Sleep(1); } } catch (Exception err) { Log.Error(err); _algorithm.RunTimeError = err; _algorithm.Status = AlgorithmStatus.RuntimeError; // send last empty packet list before terminating, // so the algorithm manager has a chance to detect the runtime error // and exit showing the correct error instead of a timeout nextEmit = _frontierUtc.RoundDown(Time.OneSecond).Add(Time.OneSecond); if (!_cancellationTokenSource.IsCancellationRequested) { _bridge.Add( TimeSlice.Create(nextEmit, _algorithm.TimeZone, _algorithm.Portfolio.CashBook, new List <DataFeedPacket>(), SecurityChanges.None, new Dictionary <Universe, BaseDataCollection>()), _cancellationTokenSource.Token); } } Log.Trace("LiveTradingDataFeed.Run(): Exited thread."); IsActive = false; }
/// <summary> /// Returns an enumerator that iterates through the collection. /// </summary> /// <returns> /// A <see cref="T:System.Collections.Generic.IEnumerator`1"/> that can be used to iterate through the collection. /// </returns> /// <filterpriority>1</filterpriority> public IEnumerator<TimeSlice> GetEnumerator() { // compute initial frontier time _frontierUtc = GetInitialFrontierTime(); Log.Trace(string.Format("FileSystemDataFeed.GetEnumerator(): Begin: {0} UTC", _frontierUtc)); var syncer = new SubscriptionSynchronizer(_universeSelection); syncer.SubscriptionFinished += (sender, subscription) => { RemoveSubscription(subscription.Configuration); Log.Debug(string.Format("FileSystemDataFeed.GetEnumerator(): Finished subscription: {0} at {1} UTC", subscription.Configuration, _frontierUtc)); }; while (!_cancellationTokenSource.IsCancellationRequested) { TimeSlice timeSlice; DateTime nextFrontier; try { timeSlice = syncer.Sync(_frontierUtc, Subscriptions, _algorithm.TimeZone, _algorithm.Portfolio.CashBook, out nextFrontier); } catch (Exception err) { Log.Error(err); continue; } // syncer returns MaxValue on failure/end of data if (timeSlice.Time != DateTime.MaxValue) { yield return timeSlice; // end of data signal if (nextFrontier == DateTime.MaxValue) break; _frontierUtc = nextFrontier; } else if (timeSlice.SecurityChanges == SecurityChanges.None) { // there's no more data to pull off, we're done (frontier is max value and no security changes) break; } } //Close up all streams: foreach (var subscription in Subscriptions) { subscription.Dispose(); } Log.Trace(string.Format("FileSystemDataFeed.Run(): Data Feed Completed at {0} UTC", _frontierUtc)); }
/// <summary> /// Returns an enumerable which provides the data to stream to the algorithm /// </summary> public override IEnumerable <TimeSlice> StreamData(CancellationToken cancellationToken) { PostInitialize(); var shouldSendExtraEmptyPacket = false; var nextEmit = DateTime.MinValue; var lastLoopStart = DateTime.UtcNow; var enumerator = SubscriptionSynchronizer .Sync(SubscriptionManager.DataFeedSubscriptions, cancellationToken) .GetEnumerator(); var previousWasTimePulse = false; while (!cancellationToken.IsCancellationRequested) { var now = DateTime.UtcNow; if (!previousWasTimePulse) { if (!_newLiveDataEmitted.IsSet) { // if we just crossed into the next second let's loop again, we will flush any consolidator bar // else we will wait to be notified by the subscriptions or our scheduled event service every second if (lastLoopStart.Second == now.Second) { _realTimeScheduleEventService.ScheduleEvent(TimeSpan.FromMilliseconds(GetPulseDueTime(now)), now); _newLiveDataEmitted.Wait(); } } _newLiveDataEmitted.Reset(); } lastLoopStart = now; TimeSlice timeSlice; try { if (!enumerator.MoveNext()) { // the enumerator ended break; } timeSlice = enumerator.Current; } catch (Exception err) { Log.Error(err); // notify the algorithm about the error, so it can be reported to the user Algorithm.RunTimeError = err; Algorithm.Status = AlgorithmStatus.RuntimeError; shouldSendExtraEmptyPacket = true; break; } // check for cancellation if (timeSlice == null || cancellationToken.IsCancellationRequested) { break; } var frontierUtc = FrontierTimeProvider.GetUtcNow(); // emit on data or if we've elapsed a full second since last emit or there are security changes if (timeSlice.SecurityChanges != SecurityChanges.None || timeSlice.IsTimePulse || timeSlice.Data.Count != 0 || frontierUtc >= nextEmit) { previousWasTimePulse = timeSlice.IsTimePulse; yield return(timeSlice); // force emitting every second since the data feed is // the heartbeat of the application nextEmit = frontierUtc.RoundDown(Time.OneSecond).Add(Time.OneSecond); } } if (shouldSendExtraEmptyPacket) { // send last empty packet list before terminating, // so the algorithm manager has a chance to detect the runtime error // and exit showing the correct error instead of a timeout nextEmit = FrontierTimeProvider.GetUtcNow().RoundDown(Time.OneSecond); if (!cancellationToken.IsCancellationRequested) { var timeSlice = TimeSliceFactory.Create( nextEmit, new List <DataFeedPacket>(), SecurityChanges.None, new Dictionary <Universe, BaseDataCollection>()); yield return(timeSlice); } } enumerator.DisposeSafely(); Log.Trace("LiveSynchronizer.GetEnumerator(): Exited thread."); }
/// <summary> /// Returns an enumerable which provides the data to stream to the algorithm /// </summary> public override IEnumerable <TimeSlice> StreamData(CancellationToken cancellationToken) { PostInitialize(); var shouldSendExtraEmptyPacket = false; var nextEmit = DateTime.MinValue; while (!cancellationToken.IsCancellationRequested) { _newLiveDataEmitted.WaitOne(TimeSpan.FromMilliseconds(500)); TimeSlice timeSlice; try { timeSlice = SubscriptionSynchronizer.Sync(SubscriptionManager.DataFeedSubscriptions); } catch (Exception err) { Log.Error(err); // notify the algorithm about the error, so it can be reported to the user Algorithm.RunTimeError = err; Algorithm.Status = AlgorithmStatus.RuntimeError; shouldSendExtraEmptyPacket = true; break; } // check for cancellation if (cancellationToken.IsCancellationRequested) { break; } var frontierUtc = FrontierTimeProvider.GetUtcNow(); // emit on data or if we've elapsed a full second since last emit or there are security changes if (timeSlice.SecurityChanges != SecurityChanges.None || timeSlice.Data.Count != 0 || frontierUtc >= nextEmit) { yield return(timeSlice); // force emitting every second since the data feed is // the heartbeat of the application nextEmit = frontierUtc.RoundDown(Time.OneSecond).Add(Time.OneSecond); } } if (shouldSendExtraEmptyPacket) { // send last empty packet list before terminating, // so the algorithm manager has a chance to detect the runtime error // and exit showing the correct error instead of a timeout nextEmit = FrontierTimeProvider.GetUtcNow().RoundDown(Time.OneSecond); if (!cancellationToken.IsCancellationRequested) { var timeSlice = TimeSliceFactory.Create( nextEmit, new List <DataFeedPacket>(), SecurityChanges.None, new Dictionary <Universe, BaseDataCollection>()); yield return(timeSlice); } } Log.Trace("LiveSynchronizer.GetEnumerator(): Exited thread."); }