public void SynchronizesData(int timeOffsetSeconds, int testTimeSeconds)
{
var start = DateTime.UtcNow;
var end = start.AddSeconds(testTimeSeconds);
var time = start;
var tickList1 = Enumerable.Range(0, 10).Select(x => new Tick {
Time = time.AddSeconds(x * 1 + timeOffsetSeconds), Value = x
}).ToList();
var tickList2 = Enumerable.Range(0, 5).Select(x => new Tick {
Time = time.AddSeconds(x * 2 + timeOffsetSeconds), Value = x + 100
}).ToList();
var stream1 = tickList1.GetEnumerator();
var stream2 = tickList2.GetEnumerator();
var count1 = 0;
var count2 = 0;
var previous = DateTime.MinValue;
var synchronizer = new LiveAuxiliaryDataSynchronizingEnumerator(new RealTimeProvider(), DateTimeZone.Utc, stream1, new List <IEnumerator <BaseData> > {
stream2
});
while (synchronizer.MoveNext() && DateTime.UtcNow < end)
{
if (synchronizer.Current != null)
{
if (synchronizer.Current.Value < 100)
{
Assert.AreEqual(count1, synchronizer.Current.Value);
count1++;
}
else
{
Assert.AreEqual(count2 + 100, synchronizer.Current.Value);
count2++;
}
Assert.That(synchronizer.Current.EndTime, Is.GreaterThanOrEqualTo(previous));
previous = synchronizer.Current.EndTime;
Log.Trace($"Data point emitted: {synchronizer.Current.EndTime:O} - {synchronizer.Current}");
}
}
Log.Trace($"Total point count: {count1 + count2}");
Assert.AreEqual(tickList1.Count, count1);
Assert.AreEqual(tickList2.Count, count2);
synchronizer.Dispose();
}
/// <summary>
/// Creates a new subscription for the specified security
/// </summary>
/// <param name="request">The subscription request</param>
/// <returns>A new subscription instance of the specified security</returns>
protected Subscription CreateDataSubscription(SubscriptionRequest request)
{
Subscription subscription = null;
try
{
var localEndTime = request.EndTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone);
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Configuration.ExchangeTimeZone, request.StartTimeUtc, request.EndTimeUtc);
IEnumerator <BaseData> enumerator;
if (!_channelProvider.ShouldStreamSubscription(request.Configuration))
{
if (!Quandl.IsAuthCodeSet)
{
// we're not using the SubscriptionDataReader, so be sure to set the auth token here
Quandl.SetAuthCode(Config.Get("quandl-auth-token"));
}
if (!Tiingo.IsAuthCodeSet)
{
// we're not using the SubscriptionDataReader, so be sure to set the auth token here
Tiingo.SetAuthCode(Config.Get("tiingo-auth-token"));
}
var factory = new LiveCustomDataSubscriptionEnumeratorFactory(_timeProvider);
var enumeratorStack = factory.CreateEnumerator(request, _dataProvider);
_customExchange.AddEnumerator(request.Configuration.Symbol, enumeratorStack);
var enqueable = new EnqueueableEnumerator <BaseData>();
_customExchange.SetDataHandler(request.Configuration.Symbol, data =>
{
enqueable.Enqueue(data);
subscription.OnNewDataAvailable();
});
enumerator = enqueable;
}
else
{
var auxEnumerators = new List <IEnumerator <BaseData> >();
if (LiveAuxiliaryDataEnumerator.TryCreate(request.Configuration, _timeProvider, _dataQueueHandler,
request.Security.Cache, _mapFileProvider, _factorFileProvider, request.StartTimeLocal, out var auxDataEnumator))
{
auxEnumerators.Add(auxDataEnumator);
}
EventHandler handler = (_, _) => subscription?.OnNewDataAvailable();
enumerator = Subscribe(request.Configuration, handler);
if (request.Configuration.EmitSplitsAndDividends())
{
auxEnumerators.Add(Subscribe(new SubscriptionDataConfig(request.Configuration, typeof(Dividend)), handler));
auxEnumerators.Add(Subscribe(new SubscriptionDataConfig(request.Configuration, typeof(Split)), handler));
}
if (auxEnumerators.Count > 0)
{
enumerator = new LiveAuxiliaryDataSynchronizingEnumerator(_timeProvider, request.Configuration.ExchangeTimeZone, enumerator, auxEnumerators);
}
}
if (request.Configuration.FillDataForward)
{
var fillForwardResolution = _subscriptions.UpdateAndGetFillForwardResolution(request.Configuration);
enumerator = new LiveFillForwardEnumerator(_frontierTimeProvider, enumerator, request.Security.Exchange, fillForwardResolution, request.Configuration.ExtendedMarketHours, localEndTime, request.Configuration.Increment, request.Configuration.DataTimeZone);
}
// define market hours and user filters to incoming data
if (request.Configuration.IsFilteredSubscription)
{
enumerator = new SubscriptionFilterEnumerator(enumerator, request.Security, localEndTime, request.Configuration.ExtendedMarketHours, true, request.ExchangeHours);
}
// finally, make our subscriptions aware of the frontier of the data feed, prevents future data from spewing into the feed
enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, timeZoneOffsetProvider);
var subscriptionDataEnumerator = new SubscriptionDataEnumerator(request.Configuration, request.Security.Exchange.Hours, timeZoneOffsetProvider, enumerator, request.IsUniverseSubscription);
subscription = new Subscription(request, subscriptionDataEnumerator, timeZoneOffsetProvider);
}
catch (Exception err)
{
Log.Error(err);
}
return(subscription);
}
/// <summary>
/// Creates a new subscription for the specified security
/// </summary>
/// <param name="request">The subscription request</param>
/// <returns>A new subscription instance of the specified security</returns>
private Subscription CreateDataSubscription(SubscriptionRequest request)
{
Subscription subscription = null;
try
{
var localEndTime = request.EndTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone);
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Configuration.ExchangeTimeZone, request.StartTimeUtc, request.EndTimeUtc);
IEnumerator <BaseData> enumerator = null;
// during warmup we might get requested to add some asset which has already expired in which case the live enumerator will be empty
if (!IsExpired(request.Configuration))
{
if (!_channelProvider.ShouldStreamSubscription(request.Configuration))
{
if (!Tiingo.IsAuthCodeSet)
{
// we're not using the SubscriptionDataReader, so be sure to set the auth token here
Tiingo.SetAuthCode(Config.Get("tiingo-auth-token"));
}
var factory = new LiveCustomDataSubscriptionEnumeratorFactory(_timeProvider);
var enumeratorStack = factory.CreateEnumerator(request, _dataProvider);
var enqueable = new EnqueueableEnumerator <BaseData>();
_customExchange.AddEnumerator(request.Configuration.Symbol, enumeratorStack, handleData: data =>
{
enqueable.Enqueue(data);
subscription?.OnNewDataAvailable();
});
enumerator = enqueable;
}
else
{
var auxEnumerators = new List <IEnumerator <BaseData> >();
if (LiveAuxiliaryDataEnumerator.TryCreate(request.Configuration, _timeProvider, _dataQueueHandler,
request.Security.Cache, _mapFileProvider, _factorFileProvider, request.StartTimeLocal, out var auxDataEnumator))
{
auxEnumerators.Add(auxDataEnumator);
}
EventHandler handler = (_, _) => subscription?.OnNewDataAvailable();
enumerator = Subscribe(request.Configuration, handler);
if (request.Configuration.EmitSplitsAndDividends())
{
auxEnumerators.Add(Subscribe(new SubscriptionDataConfig(request.Configuration, typeof(Dividend)), handler));
auxEnumerators.Add(Subscribe(new SubscriptionDataConfig(request.Configuration, typeof(Split)), handler));
}
if (auxEnumerators.Count > 0)
{
enumerator = new LiveAuxiliaryDataSynchronizingEnumerator(_timeProvider, request.Configuration.ExchangeTimeZone, enumerator, auxEnumerators);
}
}
// scale prices before 'SubscriptionFilterEnumerator' since it updates securities realtime price
// and before fill forwarding so we don't happen to apply twice the factor
if (request.Configuration.PricesShouldBeScaled(liveMode: true))
{
enumerator = new PriceScaleFactorEnumerator(
enumerator,
request.Configuration,
_factorFileProvider,
liveMode: true);
}
if (request.Configuration.FillDataForward)
{
var fillForwardResolution = _subscriptions.UpdateAndGetFillForwardResolution(request.Configuration);
enumerator = new LiveFillForwardEnumerator(_frontierTimeProvider, enumerator, request.Security.Exchange, fillForwardResolution, request.Configuration.ExtendedMarketHours, localEndTime, request.Configuration.Increment, request.Configuration.DataTimeZone);
}
// make our subscriptions aware of the frontier of the data feed, prevents future data from spewing into the feed
enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, timeZoneOffsetProvider);
// define market hours and user filters to incoming data after the frontier enumerator so during warmup we avoid any realtime data making it's way into the securities
if (request.Configuration.IsFilteredSubscription)
{
enumerator = new SubscriptionFilterEnumerator(enumerator, request.Security, localEndTime, request.Configuration.ExtendedMarketHours, true, request.ExchangeHours);
}
}
else
{
enumerator = Enumerable.Empty <BaseData>().GetEnumerator();
}
enumerator = GetWarmupEnumerator(request, enumerator);
var subscriptionDataEnumerator = new SubscriptionDataEnumerator(request.Configuration, request.Security.Exchange.Hours, timeZoneOffsetProvider, enumerator, request.IsUniverseSubscription);
subscription = new Subscription(request, subscriptionDataEnumerator, timeZoneOffsetProvider);
}
catch (Exception err)
{
Log.Error(err);
}
return(subscription);
}
