/// <summary>
/// Syncs the specified subscriptions. The frontier time used for synchronization is
/// managed internally and dependent upon previous synchronization operations.
/// </summary>
/// <param name="subscriptions">The subscriptions to sync</param>
public TimeSlice Sync(IEnumerable <Subscription> subscriptions)
{
var delayedSubscriptionFinished = false;
var changes = SecurityChanges.None;
var data = new List <DataFeedPacket>();
// NOTE: Tight coupling in UniverseSelection.ApplyUniverseSelection
var universeData = new Dictionary <Universe, BaseDataCollection>();
var universeDataForTimeSliceCreate = new Dictionary <Universe, BaseDataCollection>();
_frontierTimeProvider.SetCurrentTimeUtc(_timeProvider.GetUtcNow());
var frontierUtc = _frontierTimeProvider.GetUtcNow();
SecurityChanges newChanges;
do
{
newChanges = SecurityChanges.None;
foreach (var subscription in subscriptions)
{
if (subscription.EndOfStream)
{
OnSubscriptionFinished(subscription);
continue;
}
// prime if needed
if (subscription.Current == null)
{
if (!subscription.MoveNext())
{
OnSubscriptionFinished(subscription);
continue;
}
}
var packet = new DataFeedPacket(subscription.Security, subscription.Configuration, subscription.RemovedFromUniverse);
while (subscription.Current != null && subscription.Current.EmitTimeUtc <= frontierUtc)
{
packet.Add(subscription.Current.Data);
if (!subscription.MoveNext())
{
delayedSubscriptionFinished = true;
break;
}
}
if (packet.Count > 0)
{
// we have new universe data to select based on, store the subscription data until the end
if (!subscription.IsUniverseSelectionSubscription)
{
data.Add(packet);
}
else
{
// assume that if the first item is a base data collection then the enumerator handled the aggregation,
// otherwise, load all the the data into a new collection instance
var packetBaseDataCollection = packet.Data[0] as BaseDataCollection;
var packetData = packetBaseDataCollection == null
? packet.Data
: packetBaseDataCollection.Data;
BaseDataCollection collection;
if (universeData.TryGetValue(subscription.Universes.Single(), out collection))
{
collection.Data.AddRange(packetData);
}
else
{
if (packetBaseDataCollection is OptionChainUniverseDataCollection)
{
var current = packetBaseDataCollection as OptionChainUniverseDataCollection;
collection = new OptionChainUniverseDataCollection(frontierUtc, subscription.Configuration.Symbol, packetData, current?.Underlying);
}
else if (packetBaseDataCollection is FuturesChainUniverseDataCollection)
{
collection = new FuturesChainUniverseDataCollection(frontierUtc, subscription.Configuration.Symbol, packetData);
}
else
{
collection = new BaseDataCollection(frontierUtc, subscription.Configuration.Symbol, packetData);
}
universeData[subscription.Universes.Single()] = collection;
}
}
}
if (subscription.IsUniverseSelectionSubscription &&
subscription.Universes.Single().DisposeRequested ||
delayedSubscriptionFinished)
{
delayedSubscriptionFinished = false;
// we need to do this after all usages of subscription.Universes
OnSubscriptionFinished(subscription);
}
}
foreach (var kvp in universeData)
{
var universe = kvp.Key;
var baseDataCollection = kvp.Value;
universeDataForTimeSliceCreate[universe] = baseDataCollection;
newChanges += _universeSelection.ApplyUniverseSelection(universe, frontierUtc, baseDataCollection);
}
universeData.Clear();
changes += newChanges;
}while (newChanges != SecurityChanges.None);
var timeSlice = _timeSliceFactory.Create(frontierUtc, data, changes, universeDataForTimeSliceCreate);
return(timeSlice);
}
/// <summary>
/// Syncs the specified subscriptions. The frontier time used for synchronization is
/// managed internally and dependent upon previous synchronization operations.
/// </summary>
/// <param name="subscriptions">The subscriptions to sync</param>
/// <param name="cancellationToken">The cancellation token to stop enumeration</param>
public IEnumerable <TimeSlice> Sync(IEnumerable <Subscription> subscriptions,
CancellationToken cancellationToken)
{
var delayedSubscriptionFinished = new Queue <Subscription>();
while (!cancellationToken.IsCancellationRequested)
{
var changes = SecurityChanges.None;
var data = new List <DataFeedPacket>(1);
// NOTE: Tight coupling in UniverseSelection.ApplyUniverseSelection
var universeData = new Dictionary <Universe, BaseDataCollection>();
var universeDataForTimeSliceCreate = new Dictionary <Universe, BaseDataCollection>();
_frontierTimeProvider.SetCurrentTimeUtc(_timeProvider.GetUtcNow());
var frontierUtc = _frontierTimeProvider.GetUtcNow();
SecurityChanges newChanges;
do
{
newChanges = SecurityChanges.None;
foreach (var subscription in subscriptions)
{
if (subscription.EndOfStream)
{
OnSubscriptionFinished(subscription);
continue;
}
// prime if needed
if (subscription.Current == null)
{
if (!subscription.MoveNext())
{
OnSubscriptionFinished(subscription);
continue;
}
}
DataFeedPacket packet = null;
while (subscription.Current != null && subscription.Current.EmitTimeUtc <= frontierUtc)
{
if (packet == null)
{
// for performance, lets be selfish about creating a new instance
packet = new DataFeedPacket(
subscription.Security,
subscription.Configuration,
subscription.RemovedFromUniverse
);
}
packet.Add(subscription.Current.Data);
if (!subscription.MoveNext())
{
delayedSubscriptionFinished.Enqueue(subscription);
break;
}
}
if (packet?.Count > 0)
{
// we have new universe data to select based on, store the subscription data until the end
if (!subscription.IsUniverseSelectionSubscription)
{
data.Add(packet);
}
else
{
// assume that if the first item is a base data collection then the enumerator handled the aggregation,
// otherwise, load all the the data into a new collection instance
var packetBaseDataCollection = packet.Data[0] as BaseDataCollection;
var packetData = packetBaseDataCollection == null
? packet.Data
: packetBaseDataCollection.Data;
BaseDataCollection collection;
if (universeData.TryGetValue(subscription.Universes.Single(), out collection))
{
collection.Data.AddRange(packetData);
}
else
{
if (packetBaseDataCollection is OptionChainUniverseDataCollection)
{
var current = packetBaseDataCollection as OptionChainUniverseDataCollection;
collection = new OptionChainUniverseDataCollection(frontierUtc, subscription.Configuration.Symbol, packetData, current?.Underlying);
}
else if (packetBaseDataCollection is FuturesChainUniverseDataCollection)
{
collection = new FuturesChainUniverseDataCollection(frontierUtc, subscription.Configuration.Symbol, packetData);
}
else
{
collection = new BaseDataCollection(frontierUtc, subscription.Configuration.Symbol, packetData);
}
universeData[subscription.Universes.Single()] = collection;
}
}
}
if (subscription.IsUniverseSelectionSubscription &&
subscription.Universes.Single().DisposeRequested)
{
// we need to do this after all usages of subscription.Universes
OnSubscriptionFinished(subscription);
}
}
if (universeData.Any())
{
// if we are going to perform universe selection we emit an empty
// time pulse to align algorithm time with current frontier
yield return(_timeSliceFactory.CreateTimePulse(frontierUtc));
}
foreach (var kvp in universeData)
{
var universe = kvp.Key;
var baseDataCollection = kvp.Value;
universeDataForTimeSliceCreate[universe] = baseDataCollection;
newChanges += _universeSelection.ApplyUniverseSelection(universe, frontierUtc, baseDataCollection);
}
universeData.Clear();
changes += newChanges;
}while (newChanges != SecurityChanges.None ||
_universeSelection.AddPendingInternalDataFeeds(frontierUtc));
var timeSlice = _timeSliceFactory.Create(frontierUtc, data, changes, universeDataForTimeSliceCreate);
while (delayedSubscriptionFinished.Count > 0)
{
// these subscriptions added valid data to the packet
// we need to trigger OnSubscriptionFinished after we create the TimeSlice
// else it will drop the data
var subscription = delayedSubscriptionFinished.Dequeue();
OnSubscriptionFinished(subscription);
}
yield return(timeSlice);
}
}
/// <summary>
/// Returns an enumerable which provides the data to stream to the algorithm
/// </summary>
public override IEnumerable <TimeSlice> StreamData(CancellationToken cancellationToken)
{
PostInitialize();
var shouldSendExtraEmptyPacket = false;
var nextEmit = DateTime.MinValue;
var lastLoopStart = DateTime.UtcNow;
var enumerator = SubscriptionSynchronizer
.Sync(SubscriptionManager.DataFeedSubscriptions, cancellationToken)
.GetEnumerator();
var previousWasTimePulse = false;
while (!cancellationToken.IsCancellationRequested)
{
var now = DateTime.UtcNow;
if (!previousWasTimePulse)
{
if (!_newLiveDataEmitted.IsSet)
{
// if we just crossed into the next second let's loop again, we will flush any consolidator bar
// else we will wait to be notified by the subscriptions or our scheduled event service every second
if (lastLoopStart.Second == now.Second)
{
_realTimeScheduleEventService.ScheduleEvent(TimeSpan.FromMilliseconds(GetPulseDueTime(now)), now);
_newLiveDataEmitted.Wait();
}
}
_newLiveDataEmitted.Reset();
}
lastLoopStart = now;
TimeSlice timeSlice;
try
{
if (!enumerator.MoveNext())
{
// the enumerator ended
break;
}
timeSlice = enumerator.Current;
}
catch (Exception err)
{
Log.Error(err);
// notify the algorithm about the error, so it can be reported to the user
Algorithm.RunTimeError = err;
Algorithm.Status = AlgorithmStatus.RuntimeError;
shouldSendExtraEmptyPacket = true;
break;
}
// check for cancellation
if (timeSlice == null || cancellationToken.IsCancellationRequested)
{
break;
}
var frontierUtc = FrontierTimeProvider.GetUtcNow();
// emit on data or if we've elapsed a full second since last emit or there are security changes
if (timeSlice.SecurityChanges != SecurityChanges.None ||
timeSlice.IsTimePulse ||
timeSlice.Data.Count != 0 ||
frontierUtc >= nextEmit)
{
previousWasTimePulse = timeSlice.IsTimePulse;
yield return(timeSlice);
// force emitting every second since the data feed is
// the heartbeat of the application
nextEmit = frontierUtc.RoundDown(Time.OneSecond).Add(Time.OneSecond);
}
}
if (shouldSendExtraEmptyPacket)
{
// send last empty packet list before terminating,
// so the algorithm manager has a chance to detect the runtime error
// and exit showing the correct error instead of a timeout
nextEmit = FrontierTimeProvider.GetUtcNow().RoundDown(Time.OneSecond);
if (!cancellationToken.IsCancellationRequested)
{
var timeSlice = TimeSliceFactory.Create(
nextEmit,
new List <DataFeedPacket>(),
SecurityChanges.None,
new Dictionary <Universe, BaseDataCollection>());
yield return(timeSlice);
}
}
enumerator.DisposeSafely();
Log.Trace("LiveSynchronizer.GetEnumerator(): Exited thread.");
}
/// <summary>
/// Syncs the specified subscriptions. The frontier time used for synchronization is
/// managed internally and dependent upon previous synchronization operations.
/// </summary>
/// <param name="subscriptions">The subscriptions to sync</param>
/// <param name="cancellationToken">The cancellation token to stop enumeration</param>
public IEnumerable <TimeSlice> Sync(IEnumerable <Subscription> subscriptions,
CancellationToken cancellationToken)
{
var delayedSubscriptionFinished = new Queue <Subscription>();
while (!cancellationToken.IsCancellationRequested)
{
var changes = SecurityChanges.None;
var data = new List <DataFeedPacket>(1);
// NOTE: Tight coupling in UniverseSelection.ApplyUniverseSelection
Dictionary <Universe, BaseDataCollection> universeData = null; // lazy construction for performance
var universeDataForTimeSliceCreate = new Dictionary <Universe, BaseDataCollection>();
var frontierUtc = _timeProvider.GetUtcNow();
_frontierTimeProvider.SetCurrentTimeUtc(frontierUtc);
SecurityChanges newChanges;
do
{
newChanges = SecurityChanges.None;
foreach (var subscription in subscriptions)
{
if (subscription.EndOfStream)
{
OnSubscriptionFinished(subscription);
continue;
}
// prime if needed
if (subscription.Current == null)
{
if (!subscription.MoveNext())
{
OnSubscriptionFinished(subscription);
continue;
}
}
DataFeedPacket packet = null;
while (subscription.Current != null && subscription.Current.EmitTimeUtc <= frontierUtc)
{
if (packet == null)
{
// for performance, lets be selfish about creating a new instance
packet = new DataFeedPacket(
subscription.Security,
subscription.Configuration,
subscription.RemovedFromUniverse
);
}
// If our subscription is a universe, and we get a delisting event emitted for it, then
// the universe itself should be unselected and removed, because the Symbol that the
// universe is based on has been delisted. Doing the disposal here allows us to
// process the delisting at this point in time before emitting out to the algorithm.
// This is very useful for universes that can be delisted, such as ETF constituent
// universes (e.g. for ETF constituent universes, since the ETF itself is used to create
// the universe Symbol (and set as its underlying), once the ETF is delisted, the
// universe should cease to exist, since there are no more constituents of that ETF).
if (subscription.IsUniverseSelectionSubscription && subscription.Current.Data is Delisting)
{
subscription.Universes.Single().Dispose();
}
packet.Add(subscription.Current.Data);
if (!subscription.MoveNext())
{
delayedSubscriptionFinished.Enqueue(subscription);
break;
}
}
if (packet?.Count > 0)
{
// we have new universe data to select based on, store the subscription data until the end
if (!subscription.IsUniverseSelectionSubscription)
{
data.Add(packet);
}
else
{
// assume that if the first item is a base data collection then the enumerator handled the aggregation,
// otherwise, load all the the data into a new collection instance
var packetBaseDataCollection = packet.Data[0] as BaseDataCollection;
var packetData = packetBaseDataCollection == null
? packet.Data
: packetBaseDataCollection.Data;
BaseDataCollection collection;
if (universeData != null &&
universeData.TryGetValue(subscription.Universes.Single(), out collection))
{
collection.AddRange(packetData);
}
else
{
if (packetBaseDataCollection is OptionChainUniverseDataCollection)
{
var current = packetBaseDataCollection as OptionChainUniverseDataCollection;
collection = new OptionChainUniverseDataCollection(frontierUtc, subscription.Configuration.Symbol, packetData, current?.Underlying);
}
else if (packetBaseDataCollection is FuturesChainUniverseDataCollection)
{
collection = new FuturesChainUniverseDataCollection(frontierUtc, subscription.Configuration.Symbol, packetData);
}
else
{
collection = new BaseDataCollection(frontierUtc, frontierUtc, subscription.Configuration.Symbol, packetData);
}
if (universeData == null)
{
universeData = new Dictionary <Universe, BaseDataCollection>();
}
universeData[subscription.Universes.Single()] = collection;
}
}
}
if (subscription.IsUniverseSelectionSubscription &&
subscription.Universes.Single().DisposeRequested)
{
var universe = subscription.Universes.Single();
// check if a universe selection isn't already scheduled for this disposed universe
if (universeData == null || !universeData.ContainsKey(universe))
{
if (universeData == null)
{
universeData = new Dictionary <Universe, BaseDataCollection>();
}
// we force trigger one last universe selection for this disposed universe, so it deselects all subscriptions it added
universeData[universe] = new BaseDataCollection(frontierUtc, subscription.Configuration.Symbol);
}
// we need to do this after all usages of subscription.Universes
OnSubscriptionFinished(subscription);
}
}
if (universeData != null && universeData.Count > 0)
{
// if we are going to perform universe selection we emit an empty
// time pulse to align algorithm time with current frontier
yield return(_timeSliceFactory.CreateTimePulse(frontierUtc));
foreach (var kvp in universeData)
{
var universe = kvp.Key;
var baseDataCollection = kvp.Value;
universeDataForTimeSliceCreate[universe] = baseDataCollection;
newChanges += _universeSelection.ApplyUniverseSelection(universe, frontierUtc, baseDataCollection);
}
universeData.Clear();
}
changes += newChanges;
}while (newChanges != SecurityChanges.None ||
_universeSelection.AddPendingInternalDataFeeds(frontierUtc));
var timeSlice = _timeSliceFactory.Create(frontierUtc, data, changes, universeDataForTimeSliceCreate);
while (delayedSubscriptionFinished.Count > 0)
{
// these subscriptions added valid data to the packet
// we need to trigger OnSubscriptionFinished after we create the TimeSlice
// else it will drop the data
var subscription = delayedSubscriptionFinished.Dequeue();
OnSubscriptionFinished(subscription);
}
yield return(timeSlice);
}
}
/// <summary>
/// Returns an enumerable which provides the data to stream to the algorithm
/// </summary>
public override IEnumerable <TimeSlice> StreamData(CancellationToken cancellationToken)
{
PostInitialize();
var shouldSendExtraEmptyPacket = false;
var nextEmit = DateTime.MinValue;
while (!cancellationToken.IsCancellationRequested)
{
_newLiveDataEmitted.WaitOne(TimeSpan.FromMilliseconds(500));
TimeSlice timeSlice;
try
{
timeSlice = SubscriptionSynchronizer.Sync(SubscriptionManager.DataFeedSubscriptions);
}
catch (Exception err)
{
Log.Error(err);
// notify the algorithm about the error, so it can be reported to the user
Algorithm.RunTimeError = err;
Algorithm.Status = AlgorithmStatus.RuntimeError;
shouldSendExtraEmptyPacket = true;
break;
}
// check for cancellation
if (cancellationToken.IsCancellationRequested)
{
break;
}
var frontierUtc = FrontierTimeProvider.GetUtcNow();
// emit on data or if we've elapsed a full second since last emit or there are security changes
if (timeSlice.SecurityChanges != SecurityChanges.None ||
timeSlice.Data.Count != 0 ||
frontierUtc >= nextEmit)
{
yield return(timeSlice);
// force emitting every second since the data feed is
// the heartbeat of the application
nextEmit = frontierUtc.RoundDown(Time.OneSecond).Add(Time.OneSecond);
}
}
if (shouldSendExtraEmptyPacket)
{
// send last empty packet list before terminating,
// so the algorithm manager has a chance to detect the runtime error
// and exit showing the correct error instead of a timeout
nextEmit = FrontierTimeProvider.GetUtcNow().RoundDown(Time.OneSecond);
if (!cancellationToken.IsCancellationRequested)
{
var timeSlice = TimeSliceFactory.Create(
nextEmit,
new List <DataFeedPacket>(),
SecurityChanges.None,
new Dictionary <Universe, BaseDataCollection>());
yield return(timeSlice);
}
}
Log.Trace("LiveSynchronizer.GetEnumerator(): Exited thread.");
}
/// <summary>
/// Returns an enumerable which provides the data to stream to the algorithm
/// </summary>
public IEnumerable <TimeSlice> StreamData(CancellationToken cancellationToken)
{
PostInitialize();
var shouldSendExtraEmptyPacket = false;
var nextEmit = DateTime.MinValue;
var previousEmitTime = DateTime.MaxValue;
while (!cancellationToken.IsCancellationRequested)
{
TimeSlice timeSlice;
try
{
timeSlice = _subscriptionSynchronizer.Sync(_subscriptionManager.DataFeedSubscriptions);
}
catch (Exception err)
{
Log.Error(err);
// notify the algorithm about the error, so it can be reported to the user
_algorithm.RunTimeError = err;
_algorithm.Status = AlgorithmStatus.RuntimeError;
shouldSendExtraEmptyPacket = _liveMode;
break;
}
// check for cancellation
if (cancellationToken.IsCancellationRequested)
{
break;
}
if (_liveMode)
{
var frontierUtc = FrontierTimeProvider.GetUtcNow();
// emit on data or if we've elapsed a full second since last emit or there are security changes
if (timeSlice.SecurityChanges != SecurityChanges.None ||
timeSlice.Data.Count != 0 ||
frontierUtc >= nextEmit)
{
yield return(timeSlice);
// force emitting every second since the data feed is
// the heartbeat of the application
nextEmit = frontierUtc.RoundDown(Time.OneSecond).Add(Time.OneSecond);
}
// take a short nap
Thread.Sleep(1);
}
else
{
// SubscriptionFrontierTimeProvider will return twice the same time if there are no more subscriptions or if Subscription.Current is null
if (timeSlice.Time != previousEmitTime)
{
previousEmitTime = timeSlice.Time;
yield return(timeSlice);
}
else if (timeSlice.SecurityChanges == SecurityChanges.None)
{
// there's no more data to pull off, we're done (frontier is max value and no security changes)
break;
}
}
}
if (shouldSendExtraEmptyPacket)
{
// send last empty packet list before terminating,
// so the algorithm manager has a chance to detect the runtime error
// and exit showing the correct error instead of a timeout
nextEmit = FrontierTimeProvider.GetUtcNow().RoundDown(Time.OneSecond);
if (!cancellationToken.IsCancellationRequested)
{
var timeSlice = _timeSliceFactory.Create(
nextEmit,
new List <DataFeedPacket>(),
SecurityChanges.None,
new Dictionary <Universe, BaseDataCollection>());
yield return(timeSlice);
}
}
Log.Trace("Synchronizer.GetEnumerator(): Exited thread.");
}