/// <summary> /// Syncs the specified subscriptions. The frontier time used for synchronization is /// managed internally and dependent upon previous synchronization operations. /// </summary> /// <param name="subscriptions">The subscriptions to sync</param> public TimeSlice Sync(IEnumerable <Subscription> subscriptions) { var delayedSubscriptionFinished = false; var changes = SecurityChanges.None; var data = new List <DataFeedPacket>(); // NOTE: Tight coupling in UniverseSelection.ApplyUniverseSelection var universeData = new Dictionary <Universe, BaseDataCollection>(); var universeDataForTimeSliceCreate = new Dictionary <Universe, BaseDataCollection>(); _frontierTimeProvider.SetCurrentTimeUtc(_timeProvider.GetUtcNow()); var frontierUtc = _frontierTimeProvider.GetUtcNow(); SecurityChanges newChanges; do { newChanges = SecurityChanges.None; foreach (var subscription in subscriptions) { if (subscription.EndOfStream) { OnSubscriptionFinished(subscription); continue; } // prime if needed if (subscription.Current == null) { if (!subscription.MoveNext()) { OnSubscriptionFinished(subscription); continue; } } var packet = new DataFeedPacket(subscription.Security, subscription.Configuration, subscription.RemovedFromUniverse); while (subscription.Current != null && subscription.Current.EmitTimeUtc <= frontierUtc) { packet.Add(subscription.Current.Data); if (!subscription.MoveNext()) { delayedSubscriptionFinished = true; break; } } if (packet.Count > 0) { // we have new universe data to select based on, store the subscription data until the end if (!subscription.IsUniverseSelectionSubscription) { data.Add(packet); } else { // assume that if the first item is a base data collection then the enumerator handled the aggregation, // otherwise, load all the the data into a new collection instance var packetBaseDataCollection = packet.Data[0] as BaseDataCollection; var packetData = packetBaseDataCollection == null ? packet.Data : packetBaseDataCollection.Data; BaseDataCollection collection; if (universeData.TryGetValue(subscription.Universes.Single(), out collection)) { collection.Data.AddRange(packetData); } else { if (packetBaseDataCollection is OptionChainUniverseDataCollection) { var current = packetBaseDataCollection as OptionChainUniverseDataCollection; collection = new OptionChainUniverseDataCollection(frontierUtc, subscription.Configuration.Symbol, packetData, current?.Underlying); } else if (packetBaseDataCollection is FuturesChainUniverseDataCollection) { collection = new FuturesChainUniverseDataCollection(frontierUtc, subscription.Configuration.Symbol, packetData); } else { collection = new BaseDataCollection(frontierUtc, subscription.Configuration.Symbol, packetData); } universeData[subscription.Universes.Single()] = collection; } } } if (subscription.IsUniverseSelectionSubscription && subscription.Universes.Single().DisposeRequested || delayedSubscriptionFinished) { delayedSubscriptionFinished = false; // we need to do this after all usages of subscription.Universes OnSubscriptionFinished(subscription); } } foreach (var kvp in universeData) { var universe = kvp.Key; var baseDataCollection = kvp.Value; universeDataForTimeSliceCreate[universe] = baseDataCollection; newChanges += _universeSelection.ApplyUniverseSelection(universe, frontierUtc, baseDataCollection); } universeData.Clear(); changes += newChanges; }while (newChanges != SecurityChanges.None); var timeSlice = _timeSliceFactory.Create(frontierUtc, data, changes, universeDataForTimeSliceCreate); return(timeSlice); }
/// <summary> /// Syncs the specified subscriptions. The frontier time used for synchronization is /// managed internally and dependent upon previous synchronization operations. /// </summary> /// <param name="subscriptions">The subscriptions to sync</param> /// <param name="cancellationToken">The cancellation token to stop enumeration</param> public IEnumerable <TimeSlice> Sync(IEnumerable <Subscription> subscriptions, CancellationToken cancellationToken) { var delayedSubscriptionFinished = new Queue <Subscription>(); while (!cancellationToken.IsCancellationRequested) { var changes = SecurityChanges.None; var data = new List <DataFeedPacket>(1); // NOTE: Tight coupling in UniverseSelection.ApplyUniverseSelection var universeData = new Dictionary <Universe, BaseDataCollection>(); var universeDataForTimeSliceCreate = new Dictionary <Universe, BaseDataCollection>(); _frontierTimeProvider.SetCurrentTimeUtc(_timeProvider.GetUtcNow()); var frontierUtc = _frontierTimeProvider.GetUtcNow(); SecurityChanges newChanges; do { newChanges = SecurityChanges.None; foreach (var subscription in subscriptions) { if (subscription.EndOfStream) { OnSubscriptionFinished(subscription); continue; } // prime if needed if (subscription.Current == null) { if (!subscription.MoveNext()) { OnSubscriptionFinished(subscription); continue; } } DataFeedPacket packet = null; while (subscription.Current != null && subscription.Current.EmitTimeUtc <= frontierUtc) { if (packet == null) { // for performance, lets be selfish about creating a new instance packet = new DataFeedPacket( subscription.Security, subscription.Configuration, subscription.RemovedFromUniverse ); } packet.Add(subscription.Current.Data); if (!subscription.MoveNext()) { delayedSubscriptionFinished.Enqueue(subscription); break; } } if (packet?.Count > 0) { // we have new universe data to select based on, store the subscription data until the end if (!subscription.IsUniverseSelectionSubscription) { data.Add(packet); } else { // assume that if the first item is a base data collection then the enumerator handled the aggregation, // otherwise, load all the the data into a new collection instance var packetBaseDataCollection = packet.Data[0] as BaseDataCollection; var packetData = packetBaseDataCollection == null ? packet.Data : packetBaseDataCollection.Data; BaseDataCollection collection; if (universeData.TryGetValue(subscription.Universes.Single(), out collection)) { collection.Data.AddRange(packetData); } else { if (packetBaseDataCollection is OptionChainUniverseDataCollection) { var current = packetBaseDataCollection as OptionChainUniverseDataCollection; collection = new OptionChainUniverseDataCollection(frontierUtc, subscription.Configuration.Symbol, packetData, current?.Underlying); } else if (packetBaseDataCollection is FuturesChainUniverseDataCollection) { collection = new FuturesChainUniverseDataCollection(frontierUtc, subscription.Configuration.Symbol, packetData); } else { collection = new BaseDataCollection(frontierUtc, subscription.Configuration.Symbol, packetData); } universeData[subscription.Universes.Single()] = collection; } } } if (subscription.IsUniverseSelectionSubscription && subscription.Universes.Single().DisposeRequested) { // we need to do this after all usages of subscription.Universes OnSubscriptionFinished(subscription); } } if (universeData.Any()) { // if we are going to perform universe selection we emit an empty // time pulse to align algorithm time with current frontier yield return(_timeSliceFactory.CreateTimePulse(frontierUtc)); } foreach (var kvp in universeData) { var universe = kvp.Key; var baseDataCollection = kvp.Value; universeDataForTimeSliceCreate[universe] = baseDataCollection; newChanges += _universeSelection.ApplyUniverseSelection(universe, frontierUtc, baseDataCollection); } universeData.Clear(); changes += newChanges; }while (newChanges != SecurityChanges.None || _universeSelection.AddPendingInternalDataFeeds(frontierUtc)); var timeSlice = _timeSliceFactory.Create(frontierUtc, data, changes, universeDataForTimeSliceCreate); while (delayedSubscriptionFinished.Count > 0) { // these subscriptions added valid data to the packet // we need to trigger OnSubscriptionFinished after we create the TimeSlice // else it will drop the data var subscription = delayedSubscriptionFinished.Dequeue(); OnSubscriptionFinished(subscription); } yield return(timeSlice); } }
/// <summary> /// Returns an enumerable which provides the data to stream to the algorithm /// </summary> public override IEnumerable <TimeSlice> StreamData(CancellationToken cancellationToken) { PostInitialize(); var shouldSendExtraEmptyPacket = false; var nextEmit = DateTime.MinValue; var lastLoopStart = DateTime.UtcNow; var enumerator = SubscriptionSynchronizer .Sync(SubscriptionManager.DataFeedSubscriptions, cancellationToken) .GetEnumerator(); var previousWasTimePulse = false; while (!cancellationToken.IsCancellationRequested) { var now = DateTime.UtcNow; if (!previousWasTimePulse) { if (!_newLiveDataEmitted.IsSet) { // if we just crossed into the next second let's loop again, we will flush any consolidator bar // else we will wait to be notified by the subscriptions or our scheduled event service every second if (lastLoopStart.Second == now.Second) { _realTimeScheduleEventService.ScheduleEvent(TimeSpan.FromMilliseconds(GetPulseDueTime(now)), now); _newLiveDataEmitted.Wait(); } } _newLiveDataEmitted.Reset(); } lastLoopStart = now; TimeSlice timeSlice; try { if (!enumerator.MoveNext()) { // the enumerator ended break; } timeSlice = enumerator.Current; } catch (Exception err) { Log.Error(err); // notify the algorithm about the error, so it can be reported to the user Algorithm.RunTimeError = err; Algorithm.Status = AlgorithmStatus.RuntimeError; shouldSendExtraEmptyPacket = true; break; } // check for cancellation if (timeSlice == null || cancellationToken.IsCancellationRequested) { break; } var frontierUtc = FrontierTimeProvider.GetUtcNow(); // emit on data or if we've elapsed a full second since last emit or there are security changes if (timeSlice.SecurityChanges != SecurityChanges.None || timeSlice.IsTimePulse || timeSlice.Data.Count != 0 || frontierUtc >= nextEmit) { previousWasTimePulse = timeSlice.IsTimePulse; yield return(timeSlice); // force emitting every second since the data feed is // the heartbeat of the application nextEmit = frontierUtc.RoundDown(Time.OneSecond).Add(Time.OneSecond); } } if (shouldSendExtraEmptyPacket) { // send last empty packet list before terminating, // so the algorithm manager has a chance to detect the runtime error // and exit showing the correct error instead of a timeout nextEmit = FrontierTimeProvider.GetUtcNow().RoundDown(Time.OneSecond); if (!cancellationToken.IsCancellationRequested) { var timeSlice = TimeSliceFactory.Create( nextEmit, new List <DataFeedPacket>(), SecurityChanges.None, new Dictionary <Universe, BaseDataCollection>()); yield return(timeSlice); } } enumerator.DisposeSafely(); Log.Trace("LiveSynchronizer.GetEnumerator(): Exited thread."); }
/// <summary> /// Syncs the specified subscriptions. The frontier time used for synchronization is /// managed internally and dependent upon previous synchronization operations. /// </summary> /// <param name="subscriptions">The subscriptions to sync</param> /// <param name="cancellationToken">The cancellation token to stop enumeration</param> public IEnumerable <TimeSlice> Sync(IEnumerable <Subscription> subscriptions, CancellationToken cancellationToken) { var delayedSubscriptionFinished = new Queue <Subscription>(); while (!cancellationToken.IsCancellationRequested) { var changes = SecurityChanges.None; var data = new List <DataFeedPacket>(1); // NOTE: Tight coupling in UniverseSelection.ApplyUniverseSelection Dictionary <Universe, BaseDataCollection> universeData = null; // lazy construction for performance var universeDataForTimeSliceCreate = new Dictionary <Universe, BaseDataCollection>(); var frontierUtc = _timeProvider.GetUtcNow(); _frontierTimeProvider.SetCurrentTimeUtc(frontierUtc); SecurityChanges newChanges; do { newChanges = SecurityChanges.None; foreach (var subscription in subscriptions) { if (subscription.EndOfStream) { OnSubscriptionFinished(subscription); continue; } // prime if needed if (subscription.Current == null) { if (!subscription.MoveNext()) { OnSubscriptionFinished(subscription); continue; } } DataFeedPacket packet = null; while (subscription.Current != null && subscription.Current.EmitTimeUtc <= frontierUtc) { if (packet == null) { // for performance, lets be selfish about creating a new instance packet = new DataFeedPacket( subscription.Security, subscription.Configuration, subscription.RemovedFromUniverse ); } // If our subscription is a universe, and we get a delisting event emitted for it, then // the universe itself should be unselected and removed, because the Symbol that the // universe is based on has been delisted. Doing the disposal here allows us to // process the delisting at this point in time before emitting out to the algorithm. // This is very useful for universes that can be delisted, such as ETF constituent // universes (e.g. for ETF constituent universes, since the ETF itself is used to create // the universe Symbol (and set as its underlying), once the ETF is delisted, the // universe should cease to exist, since there are no more constituents of that ETF). if (subscription.IsUniverseSelectionSubscription && subscription.Current.Data is Delisting) { subscription.Universes.Single().Dispose(); } packet.Add(subscription.Current.Data); if (!subscription.MoveNext()) { delayedSubscriptionFinished.Enqueue(subscription); break; } } if (packet?.Count > 0) { // we have new universe data to select based on, store the subscription data until the end if (!subscription.IsUniverseSelectionSubscription) { data.Add(packet); } else { // assume that if the first item is a base data collection then the enumerator handled the aggregation, // otherwise, load all the the data into a new collection instance var packetBaseDataCollection = packet.Data[0] as BaseDataCollection; var packetData = packetBaseDataCollection == null ? packet.Data : packetBaseDataCollection.Data; BaseDataCollection collection; if (universeData != null && universeData.TryGetValue(subscription.Universes.Single(), out collection)) { collection.AddRange(packetData); } else { if (packetBaseDataCollection is OptionChainUniverseDataCollection) { var current = packetBaseDataCollection as OptionChainUniverseDataCollection; collection = new OptionChainUniverseDataCollection(frontierUtc, subscription.Configuration.Symbol, packetData, current?.Underlying); } else if (packetBaseDataCollection is FuturesChainUniverseDataCollection) { collection = new FuturesChainUniverseDataCollection(frontierUtc, subscription.Configuration.Symbol, packetData); } else { collection = new BaseDataCollection(frontierUtc, frontierUtc, subscription.Configuration.Symbol, packetData); } if (universeData == null) { universeData = new Dictionary <Universe, BaseDataCollection>(); } universeData[subscription.Universes.Single()] = collection; } } } if (subscription.IsUniverseSelectionSubscription && subscription.Universes.Single().DisposeRequested) { var universe = subscription.Universes.Single(); // check if a universe selection isn't already scheduled for this disposed universe if (universeData == null || !universeData.ContainsKey(universe)) { if (universeData == null) { universeData = new Dictionary <Universe, BaseDataCollection>(); } // we force trigger one last universe selection for this disposed universe, so it deselects all subscriptions it added universeData[universe] = new BaseDataCollection(frontierUtc, subscription.Configuration.Symbol); } // we need to do this after all usages of subscription.Universes OnSubscriptionFinished(subscription); } } if (universeData != null && universeData.Count > 0) { // if we are going to perform universe selection we emit an empty // time pulse to align algorithm time with current frontier yield return(_timeSliceFactory.CreateTimePulse(frontierUtc)); foreach (var kvp in universeData) { var universe = kvp.Key; var baseDataCollection = kvp.Value; universeDataForTimeSliceCreate[universe] = baseDataCollection; newChanges += _universeSelection.ApplyUniverseSelection(universe, frontierUtc, baseDataCollection); } universeData.Clear(); } changes += newChanges; }while (newChanges != SecurityChanges.None || _universeSelection.AddPendingInternalDataFeeds(frontierUtc)); var timeSlice = _timeSliceFactory.Create(frontierUtc, data, changes, universeDataForTimeSliceCreate); while (delayedSubscriptionFinished.Count > 0) { // these subscriptions added valid data to the packet // we need to trigger OnSubscriptionFinished after we create the TimeSlice // else it will drop the data var subscription = delayedSubscriptionFinished.Dequeue(); OnSubscriptionFinished(subscription); } yield return(timeSlice); } }
/// <summary> /// Returns an enumerable which provides the data to stream to the algorithm /// </summary> public override IEnumerable <TimeSlice> StreamData(CancellationToken cancellationToken) { PostInitialize(); var shouldSendExtraEmptyPacket = false; var nextEmit = DateTime.MinValue; while (!cancellationToken.IsCancellationRequested) { _newLiveDataEmitted.WaitOne(TimeSpan.FromMilliseconds(500)); TimeSlice timeSlice; try { timeSlice = SubscriptionSynchronizer.Sync(SubscriptionManager.DataFeedSubscriptions); } catch (Exception err) { Log.Error(err); // notify the algorithm about the error, so it can be reported to the user Algorithm.RunTimeError = err; Algorithm.Status = AlgorithmStatus.RuntimeError; shouldSendExtraEmptyPacket = true; break; } // check for cancellation if (cancellationToken.IsCancellationRequested) { break; } var frontierUtc = FrontierTimeProvider.GetUtcNow(); // emit on data or if we've elapsed a full second since last emit or there are security changes if (timeSlice.SecurityChanges != SecurityChanges.None || timeSlice.Data.Count != 0 || frontierUtc >= nextEmit) { yield return(timeSlice); // force emitting every second since the data feed is // the heartbeat of the application nextEmit = frontierUtc.RoundDown(Time.OneSecond).Add(Time.OneSecond); } } if (shouldSendExtraEmptyPacket) { // send last empty packet list before terminating, // so the algorithm manager has a chance to detect the runtime error // and exit showing the correct error instead of a timeout nextEmit = FrontierTimeProvider.GetUtcNow().RoundDown(Time.OneSecond); if (!cancellationToken.IsCancellationRequested) { var timeSlice = TimeSliceFactory.Create( nextEmit, new List <DataFeedPacket>(), SecurityChanges.None, new Dictionary <Universe, BaseDataCollection>()); yield return(timeSlice); } } Log.Trace("LiveSynchronizer.GetEnumerator(): Exited thread."); }
/// <summary> /// Returns an enumerable which provides the data to stream to the algorithm /// </summary> public IEnumerable <TimeSlice> StreamData(CancellationToken cancellationToken) { PostInitialize(); var shouldSendExtraEmptyPacket = false; var nextEmit = DateTime.MinValue; var previousEmitTime = DateTime.MaxValue; while (!cancellationToken.IsCancellationRequested) { TimeSlice timeSlice; try { timeSlice = _subscriptionSynchronizer.Sync(_subscriptionManager.DataFeedSubscriptions); } catch (Exception err) { Log.Error(err); // notify the algorithm about the error, so it can be reported to the user _algorithm.RunTimeError = err; _algorithm.Status = AlgorithmStatus.RuntimeError; shouldSendExtraEmptyPacket = _liveMode; break; } // check for cancellation if (cancellationToken.IsCancellationRequested) { break; } if (_liveMode) { var frontierUtc = FrontierTimeProvider.GetUtcNow(); // emit on data or if we've elapsed a full second since last emit or there are security changes if (timeSlice.SecurityChanges != SecurityChanges.None || timeSlice.Data.Count != 0 || frontierUtc >= nextEmit) { yield return(timeSlice); // force emitting every second since the data feed is // the heartbeat of the application nextEmit = frontierUtc.RoundDown(Time.OneSecond).Add(Time.OneSecond); } // take a short nap Thread.Sleep(1); } else { // SubscriptionFrontierTimeProvider will return twice the same time if there are no more subscriptions or if Subscription.Current is null if (timeSlice.Time != previousEmitTime) { previousEmitTime = timeSlice.Time; yield return(timeSlice); } else if (timeSlice.SecurityChanges == SecurityChanges.None) { // there's no more data to pull off, we're done (frontier is max value and no security changes) break; } } } if (shouldSendExtraEmptyPacket) { // send last empty packet list before terminating, // so the algorithm manager has a chance to detect the runtime error // and exit showing the correct error instead of a timeout nextEmit = FrontierTimeProvider.GetUtcNow().RoundDown(Time.OneSecond); if (!cancellationToken.IsCancellationRequested) { var timeSlice = _timeSliceFactory.Create( nextEmit, new List <DataFeedPacket>(), SecurityChanges.None, new Dictionary <Universe, BaseDataCollection>()); yield return(timeSlice); } } Log.Trace("Synchronizer.GetEnumerator(): Exited thread."); }