/// <summary> /// Syncs the specified subscriptions. The frontier time used for synchronization is /// managed internally and dependent upon previous synchronization operations. /// </summary> /// <param name="subscriptions">The subscriptions to sync</param> /// <param name="cancellationToken">The cancellation token to stop enumeration</param> public IEnumerable <TimeSlice> Sync(IEnumerable <Subscription> subscriptions, CancellationToken cancellationToken) { var delayedSubscriptionFinished = new Queue <Subscription>(); while (!cancellationToken.IsCancellationRequested) { var changes = SecurityChanges.None; var data = new List <DataFeedPacket>(1); // NOTE: Tight coupling in UniverseSelection.ApplyUniverseSelection var universeData = new Dictionary <Universe, BaseDataCollection>(); var universeDataForTimeSliceCreate = new Dictionary <Universe, BaseDataCollection>(); _frontierTimeProvider.SetCurrentTimeUtc(_timeProvider.GetUtcNow()); var frontierUtc = _frontierTimeProvider.GetUtcNow(); SecurityChanges newChanges; do { newChanges = SecurityChanges.None; foreach (var subscription in subscriptions) { if (subscription.EndOfStream) { OnSubscriptionFinished(subscription); continue; } // prime if needed if (subscription.Current == null) { if (!subscription.MoveNext()) { OnSubscriptionFinished(subscription); continue; } } DataFeedPacket packet = null; while (subscription.Current != null && subscription.Current.EmitTimeUtc <= frontierUtc) { if (packet == null) { // for performance, lets be selfish about creating a new instance packet = new DataFeedPacket( subscription.Security, subscription.Configuration, subscription.RemovedFromUniverse ); } packet.Add(subscription.Current.Data); if (!subscription.MoveNext()) { delayedSubscriptionFinished.Enqueue(subscription); break; } } if (packet?.Count > 0) { // we have new universe data to select based on, store the subscription data until the end if (!subscription.IsUniverseSelectionSubscription) { data.Add(packet); } else { // assume that if the first item is a base data collection then the enumerator handled the aggregation, // otherwise, load all the the data into a new collection instance var packetBaseDataCollection = packet.Data[0] as BaseDataCollection; var packetData = packetBaseDataCollection == null ? packet.Data : packetBaseDataCollection.Data; BaseDataCollection collection; if (universeData.TryGetValue(subscription.Universes.Single(), out collection)) { collection.Data.AddRange(packetData); } else { if (packetBaseDataCollection is OptionChainUniverseDataCollection) { var current = packetBaseDataCollection as OptionChainUniverseDataCollection; collection = new OptionChainUniverseDataCollection(frontierUtc, subscription.Configuration.Symbol, packetData, current?.Underlying); } else if (packetBaseDataCollection is FuturesChainUniverseDataCollection) { collection = new FuturesChainUniverseDataCollection(frontierUtc, subscription.Configuration.Symbol, packetData); } else { collection = new BaseDataCollection(frontierUtc, subscription.Configuration.Symbol, packetData); } universeData[subscription.Universes.Single()] = collection; } } } if (subscription.IsUniverseSelectionSubscription && subscription.Universes.Single().DisposeRequested) { // we need to do this after all usages of subscription.Universes OnSubscriptionFinished(subscription); } } if (universeData.Any()) { // if we are going to perform universe selection we emit an empty // time pulse to align algorithm time with current frontier yield return(_timeSliceFactory.CreateTimePulse(frontierUtc)); } foreach (var kvp in universeData) { var universe = kvp.Key; var baseDataCollection = kvp.Value; universeDataForTimeSliceCreate[universe] = baseDataCollection; newChanges += _universeSelection.ApplyUniverseSelection(universe, frontierUtc, baseDataCollection); } universeData.Clear(); changes += newChanges; }while (newChanges != SecurityChanges.None || _universeSelection.AddPendingInternalDataFeeds(frontierUtc)); var timeSlice = _timeSliceFactory.Create(frontierUtc, data, changes, universeDataForTimeSliceCreate); while (delayedSubscriptionFinished.Count > 0) { // these subscriptions added valid data to the packet // we need to trigger OnSubscriptionFinished after we create the TimeSlice // else it will drop the data var subscription = delayedSubscriptionFinished.Dequeue(); OnSubscriptionFinished(subscription); } yield return(timeSlice); } }
/// <summary> /// Syncs the specified subscriptions. The frontier time used for synchronization is /// managed internally and dependent upon previous synchronization operations. /// </summary> /// <param name="subscriptions">The subscriptions to sync</param> /// <param name="cancellationToken">The cancellation token to stop enumeration</param> public IEnumerable <TimeSlice> Sync(IEnumerable <Subscription> subscriptions, CancellationToken cancellationToken) { var delayedSubscriptionFinished = new Queue <Subscription>(); while (!cancellationToken.IsCancellationRequested) { var changes = SecurityChanges.None; var data = new List <DataFeedPacket>(1); // NOTE: Tight coupling in UniverseSelection.ApplyUniverseSelection Dictionary <Universe, BaseDataCollection> universeData = null; // lazy construction for performance var universeDataForTimeSliceCreate = new Dictionary <Universe, BaseDataCollection>(); var frontierUtc = _timeProvider.GetUtcNow(); _frontierTimeProvider.SetCurrentTimeUtc(frontierUtc); SecurityChanges newChanges; do { newChanges = SecurityChanges.None; foreach (var subscription in subscriptions) { if (subscription.EndOfStream) { OnSubscriptionFinished(subscription); continue; } // prime if needed if (subscription.Current == null) { if (!subscription.MoveNext()) { OnSubscriptionFinished(subscription); continue; } } DataFeedPacket packet = null; while (subscription.Current != null && subscription.Current.EmitTimeUtc <= frontierUtc) { if (packet == null) { // for performance, lets be selfish about creating a new instance packet = new DataFeedPacket( subscription.Security, subscription.Configuration, subscription.RemovedFromUniverse ); } // If our subscription is a universe, and we get a delisting event emitted for it, then // the universe itself should be unselected and removed, because the Symbol that the // universe is based on has been delisted. Doing the disposal here allows us to // process the delisting at this point in time before emitting out to the algorithm. // This is very useful for universes that can be delisted, such as ETF constituent // universes (e.g. for ETF constituent universes, since the ETF itself is used to create // the universe Symbol (and set as its underlying), once the ETF is delisted, the // universe should cease to exist, since there are no more constituents of that ETF). if (subscription.IsUniverseSelectionSubscription && subscription.Current.Data is Delisting) { subscription.Universes.Single().Dispose(); } packet.Add(subscription.Current.Data); if (!subscription.MoveNext()) { delayedSubscriptionFinished.Enqueue(subscription); break; } } if (packet?.Count > 0) { // we have new universe data to select based on, store the subscription data until the end if (!subscription.IsUniverseSelectionSubscription) { data.Add(packet); } else { // assume that if the first item is a base data collection then the enumerator handled the aggregation, // otherwise, load all the the data into a new collection instance var packetBaseDataCollection = packet.Data[0] as BaseDataCollection; var packetData = packetBaseDataCollection == null ? packet.Data : packetBaseDataCollection.Data; BaseDataCollection collection; if (universeData != null && universeData.TryGetValue(subscription.Universes.Single(), out collection)) { collection.AddRange(packetData); } else { if (packetBaseDataCollection is OptionChainUniverseDataCollection) { var current = packetBaseDataCollection as OptionChainUniverseDataCollection; collection = new OptionChainUniverseDataCollection(frontierUtc, subscription.Configuration.Symbol, packetData, current?.Underlying); } else if (packetBaseDataCollection is FuturesChainUniverseDataCollection) { collection = new FuturesChainUniverseDataCollection(frontierUtc, subscription.Configuration.Symbol, packetData); } else { collection = new BaseDataCollection(frontierUtc, frontierUtc, subscription.Configuration.Symbol, packetData); } if (universeData == null) { universeData = new Dictionary <Universe, BaseDataCollection>(); } universeData[subscription.Universes.Single()] = collection; } } } if (subscription.IsUniverseSelectionSubscription && subscription.Universes.Single().DisposeRequested) { var universe = subscription.Universes.Single(); // check if a universe selection isn't already scheduled for this disposed universe if (universeData == null || !universeData.ContainsKey(universe)) { if (universeData == null) { universeData = new Dictionary <Universe, BaseDataCollection>(); } // we force trigger one last universe selection for this disposed universe, so it deselects all subscriptions it added universeData[universe] = new BaseDataCollection(frontierUtc, subscription.Configuration.Symbol); } // we need to do this after all usages of subscription.Universes OnSubscriptionFinished(subscription); } } if (universeData != null && universeData.Count > 0) { // if we are going to perform universe selection we emit an empty // time pulse to align algorithm time with current frontier yield return(_timeSliceFactory.CreateTimePulse(frontierUtc)); foreach (var kvp in universeData) { var universe = kvp.Key; var baseDataCollection = kvp.Value; universeDataForTimeSliceCreate[universe] = baseDataCollection; newChanges += _universeSelection.ApplyUniverseSelection(universe, frontierUtc, baseDataCollection); } universeData.Clear(); } changes += newChanges; }while (newChanges != SecurityChanges.None || _universeSelection.AddPendingInternalDataFeeds(frontierUtc)); var timeSlice = _timeSliceFactory.Create(frontierUtc, data, changes, universeDataForTimeSliceCreate); while (delayedSubscriptionFinished.Count > 0) { // these subscriptions added valid data to the packet // we need to trigger OnSubscriptionFinished after we create the TimeSlice // else it will drop the data var subscription = delayedSubscriptionFinished.Dequeue(); OnSubscriptionFinished(subscription); } yield return(timeSlice); } }