public void EmitsDelistingEventsBasedOnCurrentTime()
{
var config = new SubscriptionDataConfig(typeof(TradeBar),
Symbols.SPY_C_192_Feb19_2016,
Resolution.Daily,
TimeZones.NewYork,
TimeZones.NewYork,
true,
true,
false);
var delistingDate = config.Symbol.GetDelistingDate();
var time = delistingDate.AddDays(-10);
var cache = new SecurityCache();
cache.AddData(new Tick(DateTime.UtcNow, config.Symbol, 20, 10));
var timeProvider = new ManualTimeProvider(time);
IEnumerator <BaseData> enumerator;
Assert.IsTrue(LiveAuxiliaryDataEnumerator.TryCreate(config, timeProvider, null, cache, TestGlobals.MapFileProvider, TestGlobals.FactorFileProvider, config.Symbol.ID.Date, out enumerator));
Assert.IsTrue(enumerator.MoveNext());
Assert.IsNull(enumerator.Current);
// advance until delisting date, take into account 5 hour offset of NY + TradableDateOffset
timeProvider.Advance(TimeSpan.FromDays(10));
timeProvider.Advance(TimeSpan.FromHours(5));
timeProvider.Advance(Time.LiveAuxiliaryDataOffset);
Assert.IsTrue(enumerator.MoveNext());
Assert.AreEqual(DelistingType.Warning, (enumerator.Current as Delisting).Type);
Assert.AreEqual(config.Symbol, (enumerator.Current as Delisting).Symbol);
Assert.AreEqual(delistingDate, (enumerator.Current as Delisting).Time);
Assert.AreEqual(15, (enumerator.Current as Delisting).Price);
Assert.IsTrue(enumerator.MoveNext());
Assert.IsNull(enumerator.Current);
// when the day ends the delisted event will pass through, respecting the offset
timeProvider.Advance(TimeSpan.FromDays(1));
cache.AddData(new Tick(DateTime.UtcNow, config.Symbol, 40, 20));
Assert.IsTrue(enumerator.MoveNext());
Assert.AreEqual(DelistingType.Delisted, (enumerator.Current as Delisting).Type);
Assert.AreEqual(config.Symbol, (enumerator.Current as Delisting).Symbol);
Assert.AreEqual(delistingDate.AddDays(1), (enumerator.Current as Delisting).Time);
Assert.AreEqual(30, (enumerator.Current as Delisting).Price);
Assert.IsTrue(enumerator.MoveNext());
Assert.IsNull(enumerator.Current);
}
/// <summary>
/// Creates a new subscription for the specified security
/// </summary>
/// <param name="request">The subscription request</param>
/// <returns>A new subscription instance of the specified security</returns>
protected Subscription CreateDataSubscription(SubscriptionRequest request)
{
Subscription subscription = null;
try
{
var localEndTime = request.EndTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone);
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Configuration.ExchangeTimeZone, request.StartTimeUtc, request.EndTimeUtc);
IEnumerator <BaseData> enumerator;
if (!_channelProvider.ShouldStreamSubscription(request.Configuration))
{
if (!Quandl.IsAuthCodeSet)
{
// we're not using the SubscriptionDataReader, so be sure to set the auth token here
Quandl.SetAuthCode(Config.Get("quandl-auth-token"));
}
if (!Tiingo.IsAuthCodeSet)
{
// we're not using the SubscriptionDataReader, so be sure to set the auth token here
Tiingo.SetAuthCode(Config.Get("tiingo-auth-token"));
}
var factory = new LiveCustomDataSubscriptionEnumeratorFactory(_timeProvider);
var enumeratorStack = factory.CreateEnumerator(request, _dataProvider);
_customExchange.AddEnumerator(request.Configuration.Symbol, enumeratorStack);
var enqueable = new EnqueueableEnumerator <BaseData>();
_customExchange.SetDataHandler(request.Configuration.Symbol, data =>
{
enqueable.Enqueue(data);
subscription.OnNewDataAvailable();
});
enumerator = enqueable;
}
else
{
var auxEnumerators = new List <IEnumerator <BaseData> >();
if (LiveAuxiliaryDataEnumerator.TryCreate(request.Configuration, _timeProvider, _dataQueueHandler,
request.Security.Cache, _mapFileProvider, _factorFileProvider, request.StartTimeLocal, out var auxDataEnumator))
{
auxEnumerators.Add(auxDataEnumator);
}
EventHandler handler = (_, _) => subscription?.OnNewDataAvailable();
enumerator = Subscribe(request.Configuration, handler);
if (request.Configuration.EmitSplitsAndDividends())
{
auxEnumerators.Add(Subscribe(new SubscriptionDataConfig(request.Configuration, typeof(Dividend)), handler));
auxEnumerators.Add(Subscribe(new SubscriptionDataConfig(request.Configuration, typeof(Split)), handler));
}
if (auxEnumerators.Count > 0)
{
enumerator = new LiveAuxiliaryDataSynchronizingEnumerator(_timeProvider, request.Configuration.ExchangeTimeZone, enumerator, auxEnumerators);
}
}
if (request.Configuration.FillDataForward)
{
var fillForwardResolution = _subscriptions.UpdateAndGetFillForwardResolution(request.Configuration);
enumerator = new LiveFillForwardEnumerator(_frontierTimeProvider, enumerator, request.Security.Exchange, fillForwardResolution, request.Configuration.ExtendedMarketHours, localEndTime, request.Configuration.Increment, request.Configuration.DataTimeZone);
}
// define market hours and user filters to incoming data
if (request.Configuration.IsFilteredSubscription)
{
enumerator = new SubscriptionFilterEnumerator(enumerator, request.Security, localEndTime, request.Configuration.ExtendedMarketHours, true, request.ExchangeHours);
}
// finally, make our subscriptions aware of the frontier of the data feed, prevents future data from spewing into the feed
enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, timeZoneOffsetProvider);
var subscriptionDataEnumerator = new SubscriptionDataEnumerator(request.Configuration, request.Security.Exchange.Hours, timeZoneOffsetProvider, enumerator, request.IsUniverseSubscription);
subscription = new Subscription(request, subscriptionDataEnumerator, timeZoneOffsetProvider);
}
catch (Exception err)
{
Log.Error(err);
}
return(subscription);
}
/// <summary>
/// Creates a new subscription for the specified security
/// </summary>
/// <param name="request">The subscription request</param>
/// <returns>A new subscription instance of the specified security</returns>
protected Subscription CreateDataSubscription(SubscriptionRequest request)
{
Subscription subscription = null;
try
{
var localEndTime = request.EndTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone);
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc);
IEnumerator <BaseData> enumerator;
if (!_channelProvider.ShouldStreamSubscription(request.Configuration))
{
if (!Quandl.IsAuthCodeSet)
{
// we're not using the SubscriptionDataReader, so be sure to set the auth token here
Quandl.SetAuthCode(Config.Get("quandl-auth-token"));
}
if (!Tiingo.IsAuthCodeSet)
{
// we're not using the SubscriptionDataReader, so be sure to set the auth token here
Tiingo.SetAuthCode(Config.Get("tiingo-auth-token"));
}
if (!USEnergyAPI.IsAuthCodeSet)
{
// we're not using the SubscriptionDataReader, so be sure to set the auth token here
USEnergyAPI.SetAuthCode(Config.Get("us-energy-information-auth-token"));
}
if (!FredApi.IsAuthCodeSet)
{
// we're not using the SubscriptionDataReader, so be sure to set the auth token here
FredApi.SetAuthCode(Config.Get("fred-auth-token"));
}
var factory = new LiveCustomDataSubscriptionEnumeratorFactory(_timeProvider);
var enumeratorStack = factory.CreateEnumerator(request, _dataProvider);
_customExchange.AddEnumerator(request.Configuration.Symbol, enumeratorStack);
var enqueable = new EnqueueableEnumerator <BaseData>();
_customExchange.SetDataHandler(request.Configuration.Symbol, data =>
{
enqueable.Enqueue(data);
subscription.OnNewDataAvailable();
UpdateSubscriptionRealTimePrice(
subscription,
timeZoneOffsetProvider,
request.Security.Exchange.Hours,
data);
});
enumerator = enqueable;
}
else
{
// this enumerator allows the exchange to pump ticks into the 'back' of the enumerator,
// and the time sync loop can pull aggregated trade bars off the front
switch (request.Configuration.Type.Name)
{
case nameof(QuoteBar):
var quoteBarAggregator = new QuoteBarBuilderEnumerator(
request.Configuration.Increment,
request.Security.Exchange.TimeZone,
_timeProvider,
true,
(sender, args) => subscription.OnNewDataAvailable());
_exchange.AddDataHandler(request.Configuration.Symbol, data =>
{
var tick = data as Tick;
if (tick?.TickType == TickType.Quote && !tick.Suspicious)
{
quoteBarAggregator.ProcessData(tick);
UpdateSubscriptionRealTimePrice(
subscription,
timeZoneOffsetProvider,
request.Security.Exchange.Hours,
data);
}
});
enumerator = quoteBarAggregator;
break;
case nameof(TradeBar):
var tradeBarAggregator = new TradeBarBuilderEnumerator(
request.Configuration.Increment,
request.Security.Exchange.TimeZone,
_timeProvider,
true,
(sender, args) => subscription.OnNewDataAvailable());
var auxDataEnumerator = new LiveAuxiliaryDataEnumerator(
request.Security.Exchange.TimeZone,
_timeProvider);
_exchange.AddDataHandler(
request.Configuration.Symbol,
data =>
{
if (data.DataType == MarketDataType.Auxiliary)
{
auxDataEnumerator.Enqueue(data);
subscription.OnNewDataAvailable();
}
else
{
var tick = data as Tick;
if (tick?.TickType == TickType.Trade && !tick.Suspicious)
{
tradeBarAggregator.ProcessData(tick);
UpdateSubscriptionRealTimePrice(
subscription,
timeZoneOffsetProvider,
request.Security.Exchange.Hours,
data);
}
}
});
enumerator = request.Configuration.SecurityType == SecurityType.Equity
? (IEnumerator <BaseData>) new LiveEquityDataSynchronizingEnumerator(_frontierTimeProvider, request.Security.Exchange.TimeZone, auxDataEnumerator, tradeBarAggregator)
: tradeBarAggregator;
break;
case nameof(OpenInterest):
var oiAggregator = new OpenInterestEnumerator(
request.Configuration.Increment,
request.Security.Exchange.TimeZone,
_timeProvider,
true,
(sender, args) => subscription.OnNewDataAvailable());
_exchange.AddDataHandler(request.Configuration.Symbol, data =>
{
var tick = data as Tick;
if (tick?.TickType == TickType.OpenInterest && !tick.Suspicious)
{
oiAggregator.ProcessData(tick);
}
});
enumerator = oiAggregator;
break;
case nameof(Tick):
default:
// tick or streaming custom data subscriptions can pass right through
var tickEnumerator = new EnqueueableEnumerator <BaseData>();
_exchange.AddDataHandler(
request.Configuration.Symbol,
data =>
{
var tick = data as Tick;
if (tick != null)
{
if (tick.TickType == request.Configuration.TickType)
{
tickEnumerator.Enqueue(data);
subscription.OnNewDataAvailable();
if (tick.TickType != TickType.OpenInterest)
{
UpdateSubscriptionRealTimePrice(
subscription,
timeZoneOffsetProvider,
request.Security.Exchange.Hours,
data);
}
}
}
else
{
tickEnumerator.Enqueue(data);
subscription.OnNewDataAvailable();
}
});
enumerator = tickEnumerator;
break;
}
}
if (request.Configuration.FillDataForward)
{
var fillForwardResolution = _subscriptions.UpdateAndGetFillForwardResolution(request.Configuration);
enumerator = new LiveFillForwardEnumerator(_frontierTimeProvider, enumerator, request.Security.Exchange, fillForwardResolution, request.Configuration.ExtendedMarketHours, localEndTime, request.Configuration.Increment, request.Configuration.DataTimeZone, request.StartTimeLocal);
}
// define market hours and user filters to incoming data
if (request.Configuration.IsFilteredSubscription)
{
enumerator = new SubscriptionFilterEnumerator(enumerator, request.Security, localEndTime);
}
// finally, make our subscriptions aware of the frontier of the data feed, prevents future data from spewing into the feed
enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, timeZoneOffsetProvider);
var subscriptionDataEnumerator = new SubscriptionDataEnumerator(request.Configuration, request.Security.Exchange.Hours, timeZoneOffsetProvider, enumerator);
subscription = new Subscription(request, subscriptionDataEnumerator, timeZoneOffsetProvider);
}
catch (Exception err)
{
Log.Error(err);
}
return(subscription);
}
public void EmitsMappingEventsBasedOnCurrentMapFileAndTime(DataMappingMode dataMappingMode, string mappingDate, bool delayed)
{
var config = new SubscriptionDataConfig(typeof(TradeBar),
Symbols.ES_Future_Chain,
Resolution.Daily,
TimeZones.NewYork,
TimeZones.NewYork,
true,
true,
false,
dataMappingMode: dataMappingMode);
var symbolMaps = new List <SubscriptionDataConfig.NewSymbolEventArgs>();
config.NewSymbol += (sender, args) => symbolMaps.Add(args);
var time = new DateTime(2013, 05, 28);
var cache = new SecurityCache();
cache.AddData(new Tick(time, config.Symbol, 20, 10));
var timeProvider = new ManualTimeProvider(time);
var futureTicker1 = "es vhle2yxr5blt";
TestMapFileResolver.MapFile = new MapFile(Futures.Indices.SP500EMini, new []
{
new MapFileRow(Time.BeginningOfTime, Futures.Indices.SP500EMini, Exchange.CME),
new MapFileRow(new DateTime(2013, 06, 01), futureTicker1, Exchange.CME, DataMappingMode.FirstDayMonth),
new MapFileRow(new DateTime(2013, 06, 15), futureTicker1, Exchange.CME, DataMappingMode.OpenInterest),
new MapFileRow(new DateTime(2013, 06, 22), futureTicker1, Exchange.CME, DataMappingMode.LastTradingDay),
});
IEnumerator <BaseData> enumerator;
Assert.IsTrue(LiveAuxiliaryDataEnumerator.TryCreate(config, timeProvider, null, cache, new TestMapFileProvider(), TestGlobals.FactorFileProvider, time, out enumerator));
// get's mapped right away!
Assert.AreEqual(futureTicker1.ToUpper(), config.MappedSymbol);
Assert.AreEqual(1, symbolMaps.Count);
Assert.AreEqual(Symbols.ES_Future_Chain, symbolMaps[0].Old);
Assert.AreEqual(Futures.Indices.SP500EMini, symbolMaps[0].Old.ID.Symbol);
Assert.AreEqual(Symbols.ES_Future_Chain, symbolMaps[0].New);
Assert.AreEqual(futureTicker1.ToUpper(), symbolMaps[0].New.Underlying.ID.ToString());
Assert.IsTrue(enumerator.MoveNext());
Assert.IsNull(enumerator.Current);
var expectedMappingDate = DateTime.ParseExact(mappingDate, DateFormat.EightCharacter, CultureInfo.InvariantCulture);
if (delayed)
{
// we advance to the mapping date, without any new mapFile!
timeProvider.Advance(expectedMappingDate.ConvertToUtc(config.ExchangeTimeZone) - timeProvider.GetUtcNow() + Time.LiveAuxiliaryDataOffset);
}
else
{
// just advance a day to show nothing happens until mapping time
timeProvider.Advance(TimeSpan.FromDays(1));
}
Assert.IsTrue(enumerator.MoveNext());
Assert.IsNull(enumerator.Current);
var futureTicker2 = "es vk2zrh843z7l";
TestMapFileResolver.MapFile = new MapFile(Futures.Indices.SP500EMini, TestMapFileResolver.MapFile.Concat(
new []
{
new MapFileRow(new DateTime(2013, 09, 01), futureTicker2, Exchange.CME, DataMappingMode.FirstDayMonth),
new MapFileRow(new DateTime(2013, 09, 14), futureTicker2, Exchange.CME, DataMappingMode.OpenInterest),
new MapFileRow(new DateTime(2013, 09, 21), futureTicker2, Exchange.CME, DataMappingMode.LastTradingDay),
}));
Assert.IsTrue(enumerator.MoveNext());
Assert.IsNull(enumerator.Current);
if (delayed)
{
// we got a new mapFile! advance the date and expect mapping to have happened
timeProvider.Advance(TimeSpan.FromDays(1));
}
else
{
// we advance to the mapping date
timeProvider.Advance(expectedMappingDate.ConvertToUtc(config.ExchangeTimeZone) - timeProvider.GetUtcNow() + Time.LiveAuxiliaryDataOffset);
}
Assert.IsTrue(enumerator.MoveNext());
Assert.IsNotNull(enumerator.Current);
Assert.AreEqual(2, symbolMaps.Count);
Assert.AreEqual(Symbols.ES_Future_Chain, symbolMaps[1].Old);
Assert.AreEqual(futureTicker1.ToUpper(), symbolMaps[1].Old.Underlying.ID.ToString());
Assert.AreEqual(Symbols.ES_Future_Chain, symbolMaps[1].New);
Assert.AreEqual(futureTicker2.ToUpper(), symbolMaps[1].New.Underlying.ID.ToString());
Assert.AreEqual(futureTicker2.ToUpper(), config.MappedSymbol);
Assert.AreEqual(futureTicker2.ToUpper(), (enumerator.Current as SymbolChangedEvent).NewSymbol);
Assert.AreEqual(futureTicker1.ToUpper(), (enumerator.Current as SymbolChangedEvent).OldSymbol);
Assert.AreEqual(config.Symbol, (enumerator.Current as SymbolChangedEvent).Symbol);
Assert.AreEqual(timeProvider.GetUtcNow().Date, (enumerator.Current as SymbolChangedEvent).Time);
Assert.IsTrue(enumerator.MoveNext());
Assert.IsNull(enumerator.Current);
}
/// <summary>
/// Creates a new subscription for the specified security
/// </summary>
/// <param name="request">The subscription request</param>
/// <returns>A new subscription instance of the specified security</returns>
private Subscription CreateDataSubscription(SubscriptionRequest request)
{
Subscription subscription = null;
try
{
var localEndTime = request.EndTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone);
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Configuration.ExchangeTimeZone, request.StartTimeUtc, request.EndTimeUtc);
IEnumerator <BaseData> enumerator = null;
// during warmup we might get requested to add some asset which has already expired in which case the live enumerator will be empty
if (!IsExpired(request.Configuration))
{
if (!_channelProvider.ShouldStreamSubscription(request.Configuration))
{
if (!Tiingo.IsAuthCodeSet)
{
// we're not using the SubscriptionDataReader, so be sure to set the auth token here
Tiingo.SetAuthCode(Config.Get("tiingo-auth-token"));
}
var factory = new LiveCustomDataSubscriptionEnumeratorFactory(_timeProvider);
var enumeratorStack = factory.CreateEnumerator(request, _dataProvider);
var enqueable = new EnqueueableEnumerator <BaseData>();
_customExchange.AddEnumerator(request.Configuration.Symbol, enumeratorStack, handleData: data =>
{
enqueable.Enqueue(data);
subscription?.OnNewDataAvailable();
});
enumerator = enqueable;
}
else
{
var auxEnumerators = new List <IEnumerator <BaseData> >();
if (LiveAuxiliaryDataEnumerator.TryCreate(request.Configuration, _timeProvider, _dataQueueHandler,
request.Security.Cache, _mapFileProvider, _factorFileProvider, request.StartTimeLocal, out var auxDataEnumator))
{
auxEnumerators.Add(auxDataEnumator);
}
EventHandler handler = (_, _) => subscription?.OnNewDataAvailable();
enumerator = Subscribe(request.Configuration, handler);
if (request.Configuration.EmitSplitsAndDividends())
{
auxEnumerators.Add(Subscribe(new SubscriptionDataConfig(request.Configuration, typeof(Dividend)), handler));
auxEnumerators.Add(Subscribe(new SubscriptionDataConfig(request.Configuration, typeof(Split)), handler));
}
if (auxEnumerators.Count > 0)
{
enumerator = new LiveAuxiliaryDataSynchronizingEnumerator(_timeProvider, request.Configuration.ExchangeTimeZone, enumerator, auxEnumerators);
}
}
// scale prices before 'SubscriptionFilterEnumerator' since it updates securities realtime price
// and before fill forwarding so we don't happen to apply twice the factor
if (request.Configuration.PricesShouldBeScaled(liveMode: true))
{
enumerator = new PriceScaleFactorEnumerator(
enumerator,
request.Configuration,
_factorFileProvider,
liveMode: true);
}
if (request.Configuration.FillDataForward)
{
var fillForwardResolution = _subscriptions.UpdateAndGetFillForwardResolution(request.Configuration);
enumerator = new LiveFillForwardEnumerator(_frontierTimeProvider, enumerator, request.Security.Exchange, fillForwardResolution, request.Configuration.ExtendedMarketHours, localEndTime, request.Configuration.Increment, request.Configuration.DataTimeZone);
}
// make our subscriptions aware of the frontier of the data feed, prevents future data from spewing into the feed
enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, timeZoneOffsetProvider);
// define market hours and user filters to incoming data after the frontier enumerator so during warmup we avoid any realtime data making it's way into the securities
if (request.Configuration.IsFilteredSubscription)
{
enumerator = new SubscriptionFilterEnumerator(enumerator, request.Security, localEndTime, request.Configuration.ExtendedMarketHours, true, request.ExchangeHours);
}
}
else
{
enumerator = Enumerable.Empty <BaseData>().GetEnumerator();
}
enumerator = GetWarmupEnumerator(request, enumerator);
var subscriptionDataEnumerator = new SubscriptionDataEnumerator(request.Configuration, request.Security.Exchange.Hours, timeZoneOffsetProvider, enumerator, request.IsUniverseSubscription);
subscription = new Subscription(request, subscriptionDataEnumerator, timeZoneOffsetProvider);
}
catch (Exception err)
{
Log.Error(err);
}
return(subscription);
}
