/// <summary>
/// Initializes the instance of the Synchronizer class
/// </summary>
public override void Initialize(
IAlgorithm algorithm,
IDataFeedSubscriptionManager dataFeedSubscriptionManager)
{
base.Initialize(algorithm, dataFeedSubscriptionManager);
// the time provider, is the real time provider
_timeProvider = GetTimeProvider();
_frontierTimeProvider = new LiveTimeProvider(realTime: TimeProvider);
// the synchronizer will use our '_frontierTimeProvider' which initially during warmup will be using
// the base time provider which is the subscription based time provider (like backtesting)
// once wawrmup finishes it will start using the realtime provider
SubscriptionSynchronizer.SetTimeProvider(_frontierTimeProvider);
// attach event handlers to subscriptions
dataFeedSubscriptionManager.SubscriptionAdded += (sender, subscription) =>
{
subscription.NewDataAvailable += OnSubscriptionNewDataAvailable;
};
dataFeedSubscriptionManager.SubscriptionRemoved += (sender, subscription) =>
{
subscription.NewDataAvailable -= OnSubscriptionNewDataAvailable;
};
_realTimeScheduleEventService = new RealTimeScheduleEventService(new RealTimeProvider());
// this schedule event will be our time pulse
_realTimeScheduleEventService.NewEvent += (sender, args) => _newLiveDataEmitted.Set();
}
/// <summary>
/// Returns an enumerator that iterates through the collection.
/// </summary>
/// <returns>
/// A <see cref="T:System.Collections.Generic.IEnumerator`1"/> that can be used to iterate through the collection.
/// </returns>
/// <filterpriority>1</filterpriority>
public IEnumerator <TimeSlice> GetEnumerator()
{
// compute initial frontier time
_frontierUtc = GetInitialFrontierTime();
Log.Trace(string.Format("FileSystemDataFeed.GetEnumerator(): Begin: {0} UTC", _frontierUtc));
var syncer = new SubscriptionSynchronizer(_universeSelection);
syncer.SubscriptionFinished += (sender, subscription) =>
{
if (subscription.EndOfStream && _subscriptions.TryRemove(subscription.Security.Symbol, out subscription))
{
Log.Debug(string.Format("FileSystemDataFeed.GetEnumerator(): Finished subscription: {0} at {1} UTC", subscription.Security.Symbol.ID, _frontierUtc));
subscription.Dispose();
}
};
while (!_cancellationTokenSource.IsCancellationRequested)
{
TimeSlice timeSlice;
DateTime nextFrontier;
try
{
timeSlice = syncer.Sync(_frontierUtc, Subscriptions, _algorithm.TimeZone, _algorithm.Portfolio.CashBook, out nextFrontier);
}
catch (Exception err)
{
Log.Error(err);
continue;
}
// syncer returns MaxValue on failure/end of data
if (timeSlice.Time != DateTime.MaxValue)
{
yield return(timeSlice);
// end of data signal
if (nextFrontier == DateTime.MaxValue)
{
break;
}
_frontierUtc = nextFrontier;
}
else if (timeSlice.SecurityChanges == SecurityChanges.None)
{
// there's no more data to pull off, we're done (frontier is max value and no security changes)
break;
}
}
//Close up all streams:
foreach (var subscription in Subscriptions)
{
subscription.Dispose();
}
Log.Trace(string.Format("FileSystemDataFeed.Run(): Data Feed Completed at {0} UTC", _frontierUtc));
}
/// <summary>
/// Returns an enumerator that iterates through the collection.
/// </summary>
/// <returns>
/// A <see cref="T:System.Collections.Generic.IEnumerator`1"/> that can be used to iterate through the collection.
/// </returns>
/// <filterpriority>1</filterpriority>
public IEnumerator <TimeSlice> GetEnumerator()
{
// compute initial frontier time
var frontierUtc = GetInitialFrontierTime();
Log.Trace(string.Format("FileSystemDataFeed.GetEnumerator(): Begin: {0} UTC", frontierUtc));
var syncer = new SubscriptionSynchronizer(_universeSelection, _algorithm.TimeZone, _algorithm.Portfolio.CashBook, frontierUtc);
syncer.SubscriptionFinished += (sender, subscription) =>
{
RemoveSubscription(subscription.Configuration);
Log.Debug(string.Format("FileSystemDataFeed.GetEnumerator(): Finished subscription: {0} at {1} UTC", subscription.Configuration, _algorithm.UtcTime));
};
while (!_cancellationTokenSource.IsCancellationRequested)
{
TimeSlice timeSlice;
try
{
timeSlice = syncer.Sync(Subscriptions);
}
catch (Exception err)
{
Log.Error(err);
// notify the algorithm about the error, so it can be reported to the user
_algorithm.RunTimeError = err;
_algorithm.Status = AlgorithmStatus.RuntimeError;
break;
}
// syncer returns MaxValue on failure/end of data
if (timeSlice.Time != DateTime.MaxValue)
{
yield return(timeSlice);
}
else if (timeSlice.SecurityChanges == SecurityChanges.None)
{
// there's no more data to pull off, we're done (frontier is max value and no security changes)
break;
}
}
//Close up all streams:
foreach (var subscription in Subscriptions)
{
subscription.Dispose();
}
if (_subscriptionfactory != null)
{
_subscriptionfactory.Dispose();
}
Log.Trace(string.Format("FileSystemDataFeed.Run(): Data Feed Completed at {0} UTC", _algorithm.UtcTime));
}
/// <summary>
/// Initializes the instance of the Synchronizer class
/// </summary>
public virtual void Initialize(
IAlgorithm algorithm,
IDataFeedSubscriptionManager dataFeedSubscriptionManager)
{
SubscriptionManager = dataFeedSubscriptionManager;
Algorithm = algorithm;
SubscriptionSynchronizer = new SubscriptionSynchronizer(
SubscriptionManager.UniverseSelection);
}
/// <summary>
/// Returns an enumerator that iterates through the collection.
/// </summary>
/// <returns>
/// A <see cref="T:System.Collections.Generic.IEnumerator`1"/> that can be used to iterate through the collection.
/// </returns>
/// <filterpriority>1</filterpriority>
public IEnumerator <TimeSlice> GetEnumerator()
{
// compute initial frontier time
var frontierUtc = GetInitialFrontierTime();
Log.Trace(string.Format("FileSystemDataFeed.GetEnumerator(): Begin: {0} UTC", frontierUtc));
var subscriptionFrontierTimeProvider = new SubscriptionFrontierTimeProvider(frontierUtc, _subscriptionManager);
var syncer = new SubscriptionSynchronizer(_universeSelection, _algorithm.TimeZone, _algorithm.Portfolio.CashBook, subscriptionFrontierTimeProvider);
syncer.SubscriptionFinished += (sender, subscription) =>
{
RemoveSubscription(subscription.Configuration);
Log.Debug($"FileSystemDataFeed.SubscriptionFinished(): Finished subscription: {subscription.Configuration} at {_algorithm.UtcTime} UTC");
};
var previousDateTime = DateTime.MaxValue;
while (!_cancellationTokenSource.IsCancellationRequested)
{
TimeSlice timeSlice;
try
{
timeSlice = syncer.Sync(Subscriptions);
}
catch (Exception err)
{
Log.Error(err);
// notify the algorithm about the error, so it can be reported to the user
_algorithm.RunTimeError = err;
_algorithm.Status = AlgorithmStatus.RuntimeError;
break;
}
// SubscriptionFrontierTimeProvider will return twice the same time if there are no more subscriptions or if Subscription.Current is null
if (timeSlice.Time != previousDateTime)
{
previousDateTime = timeSlice.Time;
yield return(timeSlice);
}
else if (timeSlice.SecurityChanges == SecurityChanges.None)
{
// there's no more data to pull off, we're done (frontier is max value and no security changes)
break;
}
}
Log.Trace(string.Format("FileSystemDataFeed.GetEnumerator(): Data Feed Completed at {0} UTC", _algorithm.UtcTime));
}
/// <summary>
/// Initializes the instance of the Synchronizer class
/// </summary>
public void Initialize(
IAlgorithm algorithm,
IDataFeedSubscriptionManager dataFeedSubscriptionManager,
bool liveMode)
{
_subscriptionManager = dataFeedSubscriptionManager;
_algorithm = algorithm;
_liveMode = liveMode;
_subscriptionSynchronizer = new SubscriptionSynchronizer(
_subscriptionManager.UniverseSelection);
if (_liveMode)
{
TimeProvider = GetTimeProvider();
_subscriptionSynchronizer.SetTimeProvider(TimeProvider);
}
}
/// <summary>
/// Initializes the instance of the Synchronizer class
/// </summary>
public override void Initialize(
IAlgorithm algorithm,
IDataFeedSubscriptionManager dataFeedSubscriptionManager)
{
base.Initialize(algorithm, dataFeedSubscriptionManager);
TimeProvider = GetTimeProvider();
SubscriptionSynchronizer.SetTimeProvider(TimeProvider);
// attach event handlers to subscriptions
dataFeedSubscriptionManager.SubscriptionAdded += (sender, subscription) =>
{
subscription.NewDataAvailable += OnSubscriptionNewDataAvailable;
};
dataFeedSubscriptionManager.SubscriptionRemoved += (sender, subscription) =>
{
subscription.NewDataAvailable -= OnSubscriptionNewDataAvailable;
};
}
/// <summary>
/// Initializes the instance of the Synchronizer class
/// </summary>
public void Initialize(
IAlgorithm algorithm,
IDataFeedSubscriptionManager subscriptionManager,
IDataFeed dataFeed, // To be removed, when subscriptionManager is completely in front of the DF
bool liveMode,
CashBook cashBook)
{
_subscriptionManager = subscriptionManager;
_algorithm = algorithm;
_cashBook = cashBook;
_liveMode = liveMode;
_dataFeed = dataFeed;
_subscriptionSynchronizer = new SubscriptionSynchronizer(_subscriptionManager.UniverseSelection, _cashBook);
if (_liveMode)
{
TimeProvider = GetTimeProvider();
_subscriptionSynchronizer.SetTimeProvider(TimeProvider);
}
}
/// <summary>
/// Initializes the instance of the Synchronizer class
/// </summary>
public override void Initialize(
IAlgorithm algorithm,
IDataFeedSubscriptionManager dataFeedSubscriptionManager)
{
base.Initialize(algorithm, dataFeedSubscriptionManager);
_timeProvider = GetTimeProvider();
SubscriptionSynchronizer.SetTimeProvider(TimeProvider);
// attach event handlers to subscriptions
dataFeedSubscriptionManager.SubscriptionAdded += (sender, subscription) =>
{
subscription.NewDataAvailable += OnSubscriptionNewDataAvailable;
};
dataFeedSubscriptionManager.SubscriptionRemoved += (sender, subscription) =>
{
subscription.NewDataAvailable -= OnSubscriptionNewDataAvailable;
};
_realTimeScheduleEventService = new RealTimeScheduleEventService(new RealTimeProvider());
// this schedule event will be our time pulse
_realTimeScheduleEventService.NewEvent += (sender, args) => _newLiveDataEmitted.Set();
}
/// <summary>
/// Primary entry point.
/// </summary>
public void Run()
{
IsActive = true;
// we want to emit to the bridge minimally once a second since the data feed is
// the heartbeat of the application, so this value will contain a second after
// the last emit time, and if we pass this time, we'll emit even with no data
var nextEmit = DateTime.MinValue;
var syncer = new SubscriptionSynchronizer(_universeSelection, _algorithm.TimeZone, _algorithm.Portfolio.CashBook, _frontierTimeProvider);
syncer.SubscriptionFinished += (sender, subscription) =>
{
RemoveSubscription(subscription.Configuration);
Log.Debug($"LiveTradingDataFeed.SubscriptionFinished(): Finished subscription: {subscription.Configuration} at {_algorithm.UtcTime} UTC");
};
try
{
while (!_cancellationTokenSource.IsCancellationRequested)
{
// perform sleeps to wake up on the second?
_frontierUtc = _timeProvider.GetUtcNow();
_frontierTimeProvider.SetCurrentTime(_frontierUtc);
// always wait for other thread to sync up
if (!_bridge.WaitHandle.WaitOne(Timeout.Infinite, _cancellationTokenSource.Token))
{
break;
}
var timeSlice = syncer.Sync(Subscriptions);
// check for cancellation
if (_cancellationTokenSource.IsCancellationRequested)
{
return;
}
// emit on data or if we've elapsed a full second since last emit or there are security changes
if (timeSlice.SecurityChanges != SecurityChanges.None || timeSlice.Data.Count != 0 || _frontierUtc >= nextEmit)
{
_bridge.Add(timeSlice, _cancellationTokenSource.Token);
// force emitting every second
nextEmit = _frontierUtc.RoundDown(Time.OneSecond).Add(Time.OneSecond);
}
// take a short nap
Thread.Sleep(1);
}
}
catch (Exception err)
{
Log.Error(err);
_algorithm.RunTimeError = err;
_algorithm.Status = AlgorithmStatus.RuntimeError;
// send last empty packet list before terminating,
// so the algorithm manager has a chance to detect the runtime error
// and exit showing the correct error instead of a timeout
nextEmit = _frontierUtc.RoundDown(Time.OneSecond).Add(Time.OneSecond);
if (!_cancellationTokenSource.IsCancellationRequested)
{
_bridge.Add(
TimeSlice.Create(nextEmit, _algorithm.TimeZone, _algorithm.Portfolio.CashBook, new List <DataFeedPacket>(), SecurityChanges.None, new Dictionary <Universe, BaseDataCollection>()),
_cancellationTokenSource.Token);
}
}
Log.Trace("LiveTradingDataFeed.Run(): Exited thread.");
IsActive = false;
}
/// <summary>
/// Returns an enumerator that iterates through the collection.
/// </summary>
/// <returns>
/// A <see cref="T:System.Collections.Generic.IEnumerator`1"/> that can be used to iterate through the collection.
/// </returns>
/// <filterpriority>1</filterpriority>
public IEnumerator<TimeSlice> GetEnumerator()
{
// compute initial frontier time
_frontierUtc = GetInitialFrontierTime();
Log.Trace(string.Format("FileSystemDataFeed.GetEnumerator(): Begin: {0} UTC", _frontierUtc));
var syncer = new SubscriptionSynchronizer(_universeSelection);
syncer.SubscriptionFinished += (sender, subscription) =>
{
RemoveSubscription(subscription.Configuration);
Log.Debug(string.Format("FileSystemDataFeed.GetEnumerator(): Finished subscription: {0} at {1} UTC", subscription.Configuration, _frontierUtc));
};
while (!_cancellationTokenSource.IsCancellationRequested)
{
TimeSlice timeSlice;
DateTime nextFrontier;
try
{
timeSlice = syncer.Sync(_frontierUtc, Subscriptions, _algorithm.TimeZone, _algorithm.Portfolio.CashBook, out nextFrontier);
}
catch (Exception err)
{
Log.Error(err);
continue;
}
// syncer returns MaxValue on failure/end of data
if (timeSlice.Time != DateTime.MaxValue)
{
yield return timeSlice;
// end of data signal
if (nextFrontier == DateTime.MaxValue) break;
_frontierUtc = nextFrontier;
}
else if (timeSlice.SecurityChanges == SecurityChanges.None)
{
// there's no more data to pull off, we're done (frontier is max value and no security changes)
break;
}
}
//Close up all streams:
foreach (var subscription in Subscriptions)
{
subscription.Dispose();
}
Log.Trace(string.Format("FileSystemDataFeed.Run(): Data Feed Completed at {0} UTC", _frontierUtc));
}
/// <summary>
/// Returns an enumerable which provides the data to stream to the algorithm
/// </summary>
public override IEnumerable <TimeSlice> StreamData(CancellationToken cancellationToken)
{
PostInitialize();
var shouldSendExtraEmptyPacket = false;
var nextEmit = DateTime.MinValue;
var lastLoopStart = DateTime.UtcNow;
var enumerator = SubscriptionSynchronizer
.Sync(SubscriptionManager.DataFeedSubscriptions, cancellationToken)
.GetEnumerator();
var previousWasTimePulse = false;
while (!cancellationToken.IsCancellationRequested)
{
var now = DateTime.UtcNow;
if (!previousWasTimePulse)
{
if (!_newLiveDataEmitted.IsSet)
{
// if we just crossed into the next second let's loop again, we will flush any consolidator bar
// else we will wait to be notified by the subscriptions or our scheduled event service every second
if (lastLoopStart.Second == now.Second)
{
_realTimeScheduleEventService.ScheduleEvent(TimeSpan.FromMilliseconds(GetPulseDueTime(now)), now);
_newLiveDataEmitted.Wait();
}
}
_newLiveDataEmitted.Reset();
}
lastLoopStart = now;
TimeSlice timeSlice;
try
{
if (!enumerator.MoveNext())
{
// the enumerator ended
break;
}
timeSlice = enumerator.Current;
}
catch (Exception err)
{
Log.Error(err);
// notify the algorithm about the error, so it can be reported to the user
Algorithm.RunTimeError = err;
Algorithm.Status = AlgorithmStatus.RuntimeError;
shouldSendExtraEmptyPacket = true;
break;
}
// check for cancellation
if (timeSlice == null || cancellationToken.IsCancellationRequested)
{
break;
}
var frontierUtc = FrontierTimeProvider.GetUtcNow();
// emit on data or if we've elapsed a full second since last emit or there are security changes
if (timeSlice.SecurityChanges != SecurityChanges.None ||
timeSlice.IsTimePulse ||
timeSlice.Data.Count != 0 ||
frontierUtc >= nextEmit)
{
previousWasTimePulse = timeSlice.IsTimePulse;
yield return(timeSlice);
// force emitting every second since the data feed is
// the heartbeat of the application
nextEmit = frontierUtc.RoundDown(Time.OneSecond).Add(Time.OneSecond);
}
}
if (shouldSendExtraEmptyPacket)
{
// send last empty packet list before terminating,
// so the algorithm manager has a chance to detect the runtime error
// and exit showing the correct error instead of a timeout
nextEmit = FrontierTimeProvider.GetUtcNow().RoundDown(Time.OneSecond);
if (!cancellationToken.IsCancellationRequested)
{
var timeSlice = TimeSliceFactory.Create(
nextEmit,
new List <DataFeedPacket>(),
SecurityChanges.None,
new Dictionary <Universe, BaseDataCollection>());
yield return(timeSlice);
}
}
enumerator.DisposeSafely();
Log.Trace("LiveSynchronizer.GetEnumerator(): Exited thread.");
}
/// <summary>
/// Returns an enumerable which provides the data to stream to the algorithm
/// </summary>
public override IEnumerable <TimeSlice> StreamData(CancellationToken cancellationToken)
{
PostInitialize();
var shouldSendExtraEmptyPacket = false;
var nextEmit = DateTime.MinValue;
while (!cancellationToken.IsCancellationRequested)
{
_newLiveDataEmitted.WaitOne(TimeSpan.FromMilliseconds(500));
TimeSlice timeSlice;
try
{
timeSlice = SubscriptionSynchronizer.Sync(SubscriptionManager.DataFeedSubscriptions);
}
catch (Exception err)
{
Log.Error(err);
// notify the algorithm about the error, so it can be reported to the user
Algorithm.RunTimeError = err;
Algorithm.Status = AlgorithmStatus.RuntimeError;
shouldSendExtraEmptyPacket = true;
break;
}
// check for cancellation
if (cancellationToken.IsCancellationRequested)
{
break;
}
var frontierUtc = FrontierTimeProvider.GetUtcNow();
// emit on data or if we've elapsed a full second since last emit or there are security changes
if (timeSlice.SecurityChanges != SecurityChanges.None ||
timeSlice.Data.Count != 0 ||
frontierUtc >= nextEmit)
{
yield return(timeSlice);
// force emitting every second since the data feed is
// the heartbeat of the application
nextEmit = frontierUtc.RoundDown(Time.OneSecond).Add(Time.OneSecond);
}
}
if (shouldSendExtraEmptyPacket)
{
// send last empty packet list before terminating,
// so the algorithm manager has a chance to detect the runtime error
// and exit showing the correct error instead of a timeout
nextEmit = FrontierTimeProvider.GetUtcNow().RoundDown(Time.OneSecond);
if (!cancellationToken.IsCancellationRequested)
{
var timeSlice = TimeSliceFactory.Create(
nextEmit,
new List <DataFeedPacket>(),
SecurityChanges.None,
new Dictionary <Universe, BaseDataCollection>());
yield return(timeSlice);
}
}
Log.Trace("LiveSynchronizer.GetEnumerator(): Exited thread.");
}