public void RefreshesFutureChainUniverseOnDateChange()
{
var startTime = new DateTime(2018, 10, 17, 10, 0, 0);
var timeProvider = new ManualTimeProvider(startTime);
var symbolUniverse = new TestDataQueueUniverseProvider(timeProvider);
var canonicalSymbol = Symbol.Create(Futures.Indices.VIX, SecurityType.Future, Market.CFE, "/VX");
var quoteCurrency = new Cash(Currencies.USD, 0, 1);
var exchangeHours = MarketHoursDatabase.FromDataFolder().GetExchangeHours(Market.CFE, canonicalSymbol, SecurityType.Future);
var config = new SubscriptionDataConfig(
typeof(ZipEntryName),
canonicalSymbol,
Resolution.Minute,
TimeZones.Utc,
TimeZones.Chicago,
true,
false,
false,
false,
TickType.Quote,
false,
DataNormalizationMode.Raw
);
var future = new Future(
canonicalSymbol,
exchangeHours,
quoteCurrency,
SymbolProperties.GetDefault(Currencies.USD),
ErrorCurrencyConverter.Instance,
RegisteredSecurityDataTypesProvider.Null,
new SecurityCache()
);
var universeSettings = new UniverseSettings(Resolution.Minute, 0, true, false, TimeSpan.Zero);
using var universe = new FuturesChainUniverse(future, universeSettings);
var request = new SubscriptionRequest(true, universe, future, config, startTime, Time.EndOfTime);
var enumerator = new DataQueueFuturesChainUniverseDataCollectionEnumerator(request, symbolUniverse, timeProvider);
Assert.IsTrue(enumerator.MoveNext());
Assert.IsNotNull(enumerator.Current);
Assert.AreEqual(1, symbolUniverse.TotalLookupCalls);
var data = enumerator.Current;
Assert.IsNotNull(data);
Assert.AreEqual(1, data.Data.Count);
timeProvider.Advance(Time.OneSecond);
Assert.IsTrue(enumerator.MoveNext());
Assert.IsNull(enumerator.Current);
Assert.AreEqual(1, symbolUniverse.TotalLookupCalls);
timeProvider.Advance(Time.OneMinute);
Assert.IsTrue(enumerator.MoveNext());
Assert.IsNull(enumerator.Current);
Assert.AreEqual(1, symbolUniverse.TotalLookupCalls);
timeProvider.Advance(Time.OneDay);
Assert.IsTrue(enumerator.MoveNext());
Assert.IsNotNull(enumerator.Current);
Assert.AreEqual(2, symbolUniverse.TotalLookupCalls);
data = enumerator.Current;
Assert.IsNotNull(data);
Assert.AreEqual(2, data.Data.Count);
timeProvider.Advance(Time.OneMinute);
Assert.IsTrue(enumerator.MoveNext());
Assert.IsNull(enumerator.Current);
Assert.AreEqual(2, symbolUniverse.TotalLookupCalls);
enumerator.Dispose();
}
/// <summary>
/// Creates a new subscription for universe selection
/// </summary>
/// <param name="request">The subscription request</param>
private Subscription CreateUniverseSubscription(SubscriptionRequest request)
{
Subscription subscription = null;
// TODO : Consider moving the creating of universe subscriptions to a separate, testable class
// grab the relevant exchange hours
var config = request.Universe.Configuration;
var localEndTime = request.EndTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone);
var tzOffsetProvider = new TimeZoneOffsetProvider(request.Configuration.ExchangeTimeZone, request.StartTimeUtc, request.EndTimeUtc);
IEnumerator <BaseData> enumerator = null;
var timeTriggered = request.Universe as ITimeTriggeredUniverse;
if (timeTriggered != null)
{
Log.Trace($"LiveTradingDataFeed.CreateUniverseSubscription(): Creating user defined universe: {config.Symbol.ID}");
// spoof a tick on the requested interval to trigger the universe selection function
var enumeratorFactory = new TimeTriggeredUniverseSubscriptionEnumeratorFactory(timeTriggered, MarketHoursDatabase.FromDataFolder(), _frontierTimeProvider);
enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider);
enumerator = new FrontierAwareEnumerator(enumerator, _timeProvider, tzOffsetProvider);
var enqueueable = new EnqueueableEnumerator <BaseData>();
_customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable));
enumerator = enqueueable;
}
else if (config.Type == typeof(CoarseFundamental) || config.Type == typeof(ETFConstituentData))
{
Log.Trace($"LiveTradingDataFeed.CreateUniverseSubscription(): Creating {config.Type.Name} universe: {config.Symbol.ID}");
// Will try to pull data from the data folder every 10min, file with yesterdays date.
// If lean is started today it will trigger initial coarse universe selection
var factory = new LiveCustomDataSubscriptionEnumeratorFactory(_timeProvider,
// we adjust time to the previous tradable date
time => Time.GetStartTimeForTradeBars(request.Security.Exchange.Hours, time, Time.OneDay, 1, false, config.DataTimeZone),
TimeSpan.FromMinutes(10)
);
var enumeratorStack = factory.CreateEnumerator(request, _dataProvider);
// aggregates each coarse data point into a single BaseDataCollection
var aggregator = new BaseDataCollectionAggregatorEnumerator(enumeratorStack, config.Symbol, true);
var enqueable = new EnqueueableEnumerator <BaseData>();
_customExchange.AddEnumerator(config.Symbol, aggregator, handleData: data =>
{
enqueable.Enqueue(data);
subscription?.OnNewDataAvailable();
});
enumerator = GetConfiguredFrontierAwareEnumerator(enqueable, tzOffsetProvider,
// advance time if before 23pm or after 5am and not on Saturdays
time => time.Hour < 23 && time.Hour > 5 && time.DayOfWeek != DayOfWeek.Saturday);
}
else if (request.Universe is OptionChainUniverse)
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating option chain universe: " + config.Symbol.ID);
Func <SubscriptionRequest, IEnumerator <BaseData> > configure = (subRequest) =>
{
var fillForwardResolution = _subscriptions.UpdateAndGetFillForwardResolution(subRequest.Configuration);
var input = Subscribe(subRequest.Configuration, (sender, args) => subscription?.OnNewDataAvailable());
return(new LiveFillForwardEnumerator(_frontierTimeProvider, input, subRequest.Security.Exchange, fillForwardResolution, subRequest.Configuration.ExtendedMarketHours, localEndTime, subRequest.Configuration.Increment, subRequest.Configuration.DataTimeZone));
};
var symbolUniverse = GetUniverseProvider(SecurityType.Option);
var enumeratorFactory = new OptionChainUniverseSubscriptionEnumeratorFactory(configure, symbolUniverse, _timeProvider);
enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider);
enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, tzOffsetProvider);
}
else if (request.Universe is FuturesChainUniverse)
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating futures chain universe: " + config.Symbol.ID);
var symbolUniverse = GetUniverseProvider(SecurityType.Future);
enumerator = new DataQueueFuturesChainUniverseDataCollectionEnumerator(request, symbolUniverse, _timeProvider);
enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, tzOffsetProvider);
}
else
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating custom universe: " + config.Symbol.ID);
var factory = new LiveCustomDataSubscriptionEnumeratorFactory(_timeProvider);
var enumeratorStack = factory.CreateEnumerator(request, _dataProvider);
enumerator = new BaseDataCollectionAggregatorEnumerator(enumeratorStack, config.Symbol, liveMode: true);
var enqueueable = new EnqueueableEnumerator <BaseData>();
_customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable));
enumerator = enqueueable;
}
enumerator = GetWarmupEnumerator(request, enumerator);
// create the subscription
var subscriptionDataEnumerator = new SubscriptionDataEnumerator(request.Configuration, request.Security.Exchange.Hours, tzOffsetProvider, enumerator, request.IsUniverseSubscription);
subscription = new Subscription(request, subscriptionDataEnumerator, tzOffsetProvider);
// send the subscription for the new symbol through to the data queuehandler
if (_channelProvider.ShouldStreamSubscription(subscription.Configuration))
{
Subscribe(request.Configuration, (sender, args) => subscription?.OnNewDataAvailable());
}
return(subscription);
}
