/// <summary>
/// Syncs the specified subscriptions. The frontier time used for synchronization is
/// managed internally and dependent upon previous synchronization operations.
/// </summary>
/// <param name="subscriptions">The subscriptions to sync</param>
/// <param name="cancellationToken">The cancellation token to stop enumeration</param>
public IEnumerable <TimeSlice> Sync(IEnumerable <Subscription> subscriptions,
CancellationToken cancellationToken)
{
var delayedSubscriptionFinished = new Queue <Subscription>();
while (!cancellationToken.IsCancellationRequested)
{
var changes = SecurityChanges.None;
var data = new List <DataFeedPacket>(1);
// NOTE: Tight coupling in UniverseSelection.ApplyUniverseSelection
var universeData = new Dictionary <Universe, BaseDataCollection>();
var universeDataForTimeSliceCreate = new Dictionary <Universe, BaseDataCollection>();
_frontierTimeProvider.SetCurrentTimeUtc(_timeProvider.GetUtcNow());
var frontierUtc = _frontierTimeProvider.GetUtcNow();
SecurityChanges newChanges;
do
{
newChanges = SecurityChanges.None;
foreach (var subscription in subscriptions)
{
if (subscription.EndOfStream)
{
OnSubscriptionFinished(subscription);
continue;
}
// prime if needed
if (subscription.Current == null)
{
if (!subscription.MoveNext())
{
OnSubscriptionFinished(subscription);
continue;
}
}
DataFeedPacket packet = null;
while (subscription.Current != null && subscription.Current.EmitTimeUtc <= frontierUtc)
{
if (packet == null)
{
// for performance, lets be selfish about creating a new instance
packet = new DataFeedPacket(
subscription.Security,
subscription.Configuration,
subscription.RemovedFromUniverse
);
}
packet.Add(subscription.Current.Data);
if (!subscription.MoveNext())
{
delayedSubscriptionFinished.Enqueue(subscription);
break;
}
}
if (packet?.Count > 0)
{
// we have new universe data to select based on, store the subscription data until the end
if (!subscription.IsUniverseSelectionSubscription)
{
data.Add(packet);
}
else
{
// assume that if the first item is a base data collection then the enumerator handled the aggregation,
// otherwise, load all the the data into a new collection instance
var packetBaseDataCollection = packet.Data[0] as BaseDataCollection;
var packetData = packetBaseDataCollection == null
? packet.Data
: packetBaseDataCollection.Data;
BaseDataCollection collection;
if (universeData.TryGetValue(subscription.Universes.Single(), out collection))
{
collection.Data.AddRange(packetData);
}
else
{
if (packetBaseDataCollection is OptionChainUniverseDataCollection)
{
var current = packetBaseDataCollection as OptionChainUniverseDataCollection;
collection = new OptionChainUniverseDataCollection(frontierUtc, subscription.Configuration.Symbol, packetData, current?.Underlying);
}
else if (packetBaseDataCollection is FuturesChainUniverseDataCollection)
{
collection = new FuturesChainUniverseDataCollection(frontierUtc, subscription.Configuration.Symbol, packetData);
}
else
{
collection = new BaseDataCollection(frontierUtc, subscription.Configuration.Symbol, packetData);
}
universeData[subscription.Universes.Single()] = collection;
}
}
}
if (subscription.IsUniverseSelectionSubscription &&
subscription.Universes.Single().DisposeRequested)
{
// we need to do this after all usages of subscription.Universes
OnSubscriptionFinished(subscription);
}
}
if (universeData.Any())
{
// if we are going to perform universe selection we emit an empty
// time pulse to align algorithm time with current frontier
yield return(_timeSliceFactory.CreateTimePulse(frontierUtc));
}
foreach (var kvp in universeData)
{
var universe = kvp.Key;
var baseDataCollection = kvp.Value;
universeDataForTimeSliceCreate[universe] = baseDataCollection;
newChanges += _universeSelection.ApplyUniverseSelection(universe, frontierUtc, baseDataCollection);
}
universeData.Clear();
changes += newChanges;
}while (newChanges != SecurityChanges.None ||
_universeSelection.AddPendingInternalDataFeeds(frontierUtc));
var timeSlice = _timeSliceFactory.Create(frontierUtc, data, changes, universeDataForTimeSliceCreate);
while (delayedSubscriptionFinished.Count > 0)
{
// these subscriptions added valid data to the packet
// we need to trigger OnSubscriptionFinished after we create the TimeSlice
// else it will drop the data
var subscription = delayedSubscriptionFinished.Dequeue();
OnSubscriptionFinished(subscription);
}
yield return(timeSlice);
}
}
/// <summary>
/// Syncs the specified subscriptions. The frontier time used for synchronization is
/// managed internally and dependent upon previous synchronization operations.
/// </summary>
/// <param name="subscriptions">The subscriptions to sync</param>
/// <param name="cancellationToken">The cancellation token to stop enumeration</param>
public IEnumerable <TimeSlice> Sync(IEnumerable <Subscription> subscriptions,
CancellationToken cancellationToken)
{
var delayedSubscriptionFinished = new Queue <Subscription>();
while (!cancellationToken.IsCancellationRequested)
{
var changes = SecurityChanges.None;
var data = new List <DataFeedPacket>(1);
// NOTE: Tight coupling in UniverseSelection.ApplyUniverseSelection
Dictionary <Universe, BaseDataCollection> universeData = null; // lazy construction for performance
var universeDataForTimeSliceCreate = new Dictionary <Universe, BaseDataCollection>();
var frontierUtc = _timeProvider.GetUtcNow();
_frontierTimeProvider.SetCurrentTimeUtc(frontierUtc);
SecurityChanges newChanges;
do
{
newChanges = SecurityChanges.None;
foreach (var subscription in subscriptions)
{
if (subscription.EndOfStream)
{
OnSubscriptionFinished(subscription);
continue;
}
// prime if needed
if (subscription.Current == null)
{
if (!subscription.MoveNext())
{
OnSubscriptionFinished(subscription);
continue;
}
}
DataFeedPacket packet = null;
while (subscription.Current != null && subscription.Current.EmitTimeUtc <= frontierUtc)
{
if (packet == null)
{
// for performance, lets be selfish about creating a new instance
packet = new DataFeedPacket(
subscription.Security,
subscription.Configuration,
subscription.RemovedFromUniverse
);
}
// If our subscription is a universe, and we get a delisting event emitted for it, then
// the universe itself should be unselected and removed, because the Symbol that the
// universe is based on has been delisted. Doing the disposal here allows us to
// process the delisting at this point in time before emitting out to the algorithm.
// This is very useful for universes that can be delisted, such as ETF constituent
// universes (e.g. for ETF constituent universes, since the ETF itself is used to create
// the universe Symbol (and set as its underlying), once the ETF is delisted, the
// universe should cease to exist, since there are no more constituents of that ETF).
if (subscription.IsUniverseSelectionSubscription && subscription.Current.Data is Delisting)
{
subscription.Universes.Single().Dispose();
}
packet.Add(subscription.Current.Data);
if (!subscription.MoveNext())
{
delayedSubscriptionFinished.Enqueue(subscription);
break;
}
}
if (packet?.Count > 0)
{
// we have new universe data to select based on, store the subscription data until the end
if (!subscription.IsUniverseSelectionSubscription)
{
data.Add(packet);
}
else
{
// assume that if the first item is a base data collection then the enumerator handled the aggregation,
// otherwise, load all the the data into a new collection instance
var packetBaseDataCollection = packet.Data[0] as BaseDataCollection;
var packetData = packetBaseDataCollection == null
? packet.Data
: packetBaseDataCollection.Data;
BaseDataCollection collection;
if (universeData != null &&
universeData.TryGetValue(subscription.Universes.Single(), out collection))
{
collection.AddRange(packetData);
}
else
{
if (packetBaseDataCollection is OptionChainUniverseDataCollection)
{
var current = packetBaseDataCollection as OptionChainUniverseDataCollection;
collection = new OptionChainUniverseDataCollection(frontierUtc, subscription.Configuration.Symbol, packetData, current?.Underlying);
}
else if (packetBaseDataCollection is FuturesChainUniverseDataCollection)
{
collection = new FuturesChainUniverseDataCollection(frontierUtc, subscription.Configuration.Symbol, packetData);
}
else
{
collection = new BaseDataCollection(frontierUtc, frontierUtc, subscription.Configuration.Symbol, packetData);
}
if (universeData == null)
{
universeData = new Dictionary <Universe, BaseDataCollection>();
}
universeData[subscription.Universes.Single()] = collection;
}
}
}
if (subscription.IsUniverseSelectionSubscription &&
subscription.Universes.Single().DisposeRequested)
{
var universe = subscription.Universes.Single();
// check if a universe selection isn't already scheduled for this disposed universe
if (universeData == null || !universeData.ContainsKey(universe))
{
if (universeData == null)
{
universeData = new Dictionary <Universe, BaseDataCollection>();
}
// we force trigger one last universe selection for this disposed universe, so it deselects all subscriptions it added
universeData[universe] = new BaseDataCollection(frontierUtc, subscription.Configuration.Symbol);
}
// we need to do this after all usages of subscription.Universes
OnSubscriptionFinished(subscription);
}
}
if (universeData != null && universeData.Count > 0)
{
// if we are going to perform universe selection we emit an empty
// time pulse to align algorithm time with current frontier
yield return(_timeSliceFactory.CreateTimePulse(frontierUtc));
foreach (var kvp in universeData)
{
var universe = kvp.Key;
var baseDataCollection = kvp.Value;
universeDataForTimeSliceCreate[universe] = baseDataCollection;
newChanges += _universeSelection.ApplyUniverseSelection(universe, frontierUtc, baseDataCollection);
}
universeData.Clear();
}
changes += newChanges;
}while (newChanges != SecurityChanges.None ||
_universeSelection.AddPendingInternalDataFeeds(frontierUtc));
var timeSlice = _timeSliceFactory.Create(frontierUtc, data, changes, universeDataForTimeSliceCreate);
while (delayedSubscriptionFinished.Count > 0)
{
// these subscriptions added valid data to the packet
// we need to trigger OnSubscriptionFinished after we create the TimeSlice
// else it will drop the data
var subscription = delayedSubscriptionFinished.Dequeue();
OnSubscriptionFinished(subscription);
}
yield return(timeSlice);
}
}
