public void AggregatesUntilNull()
{
var time = new DateTime(2015, 10, 20);
var underlying = Enumerable.Range(0, 5).Select(x => new Tick { Time = time }).ToList();
underlying.AddRange(new Tick[] { null, null, null });
var aggregator = new BaseDataCollectionAggregatorEnumerator(underlying.GetEnumerator(), Symbols.SPY);
Assert.IsTrue(aggregator.MoveNext());
Assert.IsNotNull(aggregator.Current);
Assert.AreEqual(5, aggregator.Current.Data.Count);
}
public void AggregatesUntilTimeChange()
{
var time = new DateTime(2015, 10, 20);
var underlying = Enumerable.Range(0, 5).Select(x => new Tick { Time = time }).ToList();
underlying.AddRange(Enumerable.Range(0, 5).Select(x => new Tick {Time = time.AddSeconds(1)}));
var aggregator = new BaseDataCollectionAggregatorEnumerator(underlying.GetEnumerator(), "underlying");
Assert.IsTrue(aggregator.MoveNext());
Assert.IsNotNull(aggregator.Current);
Assert.AreEqual(5, aggregator.Current.Data.Count);
}
/// <summary>
/// Creates an enumerator for the specified security/configuration
/// </summary>
private IEnumerator <BaseData> CreateSubscriptionEnumerator(Security security,
SubscriptionDataConfig config,
DateTime localStartTime,
DateTime localEndTime,
MapFileResolver mapFileResolver,
IEnumerable <DateTime> tradeableDates,
bool useSubscriptionDataReader,
bool aggregate)
{
IEnumerator <BaseData> enumerator;
if (useSubscriptionDataReader)
{
enumerator = new SubscriptionDataReader(config, localStartTime, localEndTime, _resultHandler, mapFileResolver,
_factorFileProvider, tradeableDates, false);
}
else
{
var sourceFactory = (BaseData)Activator.CreateInstance(config.Type);
enumerator = (from date in tradeableDates
let source = sourceFactory.GetSource(config, date, false)
let factory = SubscriptionDataSourceReader.ForSource(source, config, date, false)
let entriesForDate = factory.Read(source)
from entry in entriesForDate
select entry).GetEnumerator();
}
if (aggregate)
{
enumerator = new BaseDataCollectionAggregatorEnumerator(enumerator, config.Symbol);
}
// optionally apply fill forward logic, but never for tick data
if (config.FillDataForward && config.Resolution != Resolution.Tick)
{
enumerator = new FillForwardEnumerator(enumerator, security.Exchange, _fillForwardResolution,
security.IsExtendedMarketHours, localEndTime, config.Resolution.ToTimeSpan());
}
// optionally apply exchange/user filters
if (config.IsFilteredSubscription)
{
enumerator = SubscriptionFilterEnumerator.WrapForDataFeed(_resultHandler, enumerator, security, localEndTime);
}
return(enumerator);
}
public void AggregatesUntilTimeChange()
{
var time = new DateTime(2015, 10, 20);
var underlying = Enumerable.Range(0, 5).Select(x => new Tick {
Time = time
}).ToList();
underlying.AddRange(Enumerable.Range(0, 5).Select(x => new Tick {
Time = time.AddSeconds(1)
}));
var aggregator = new BaseDataCollectionAggregatorEnumerator(underlying.GetEnumerator(), Symbols.SPY);
Assert.IsTrue(aggregator.MoveNext());
Assert.IsNotNull(aggregator.Current);
Assert.AreEqual(5, aggregator.Current.Data.Count);
}
/// <summary>
/// Configure the enumerator with aggregation/fill-forward/filter behaviors. Returns new instance if re-configured
/// </summary>
protected IEnumerator <BaseData> ConfigureEnumerator(SubscriptionRequest request, bool aggregate, IEnumerator <BaseData> enumerator)
{
if (aggregate)
{
enumerator = new BaseDataCollectionAggregatorEnumerator(enumerator, request.Configuration.Symbol);
}
enumerator = TryAddFillForwardEnumerator(request, enumerator, request.Configuration.FillDataForward);
// optionally apply exchange/user filters
if (request.Configuration.IsFilteredSubscription)
{
enumerator = SubscriptionFilterEnumerator.WrapForDataFeed(_resultHandler, enumerator, request.Security,
request.EndTimeLocal, request.Configuration.ExtendedMarketHours, false, request.ExchangeHours);
}
return(enumerator);
}
/// <summary>
/// Configure the enumerator with aggregation/fill-forward/filter behaviors. Returns new instance if re-configured
/// </summary>
private IEnumerator <BaseData> ConfigureEnumerator(SubscriptionRequest request, bool aggregate, IEnumerator <BaseData> enumerator)
{
if (aggregate)
{
enumerator = new BaseDataCollectionAggregatorEnumerator(enumerator, request.Configuration.Symbol);
}
// optionally apply fill forward logic, but never for tick data
if (request.Configuration.FillDataForward && request.Configuration.Resolution != Resolution.Tick)
{
enumerator = new FillForwardEnumerator(enumerator, request.Security.Exchange, _fillForwardResolution,
request.Security.IsExtendedMarketHours, request.EndTimeLocal, request.Configuration.Resolution.ToTimeSpan());
}
// optionally apply exchange/user filters
if (request.Configuration.IsFilteredSubscription)
{
enumerator = SubscriptionFilterEnumerator.WrapForDataFeed(_resultHandler, enumerator, request.Security, request.EndTimeLocal);
}
return(enumerator);
}
/// <summary>
/// If required will add a new enumerator for the underlying symbol
/// </summary>
protected IEnumerator <BaseData> TryAppendUnderlyingEnumerator(SubscriptionRequest request, IEnumerator <BaseData> parent, Func <SubscriptionRequest, IEnumerator <BaseData> > createEnumerator)
{
if (request.Configuration.Symbol.SecurityType.IsOption() && request.Configuration.Symbol.HasUnderlying)
{
// TODO: creating this subscription request/config is bad
var underlyingRequests = new SubscriptionRequest(request,
isUniverseSubscription: false,
configuration: new SubscriptionDataConfig(request.Configuration, symbol: request.Configuration.Symbol.Underlying, objectType: typeof(TradeBar), tickType: TickType.Trade));
var underlying = createEnumerator(underlyingRequests);
underlying = new FilterEnumerator <BaseData>(underlying, data => data.DataType != MarketDataType.Auxiliary);
parent = new SynchronizingBaseDataEnumerator(parent, underlying);
// we aggregate both underlying and chain data
parent = new BaseDataCollectionAggregatorEnumerator(parent, request.Configuration.Symbol);
// only let through if underlying and chain data present
parent = new FilterEnumerator <BaseData>(parent, data => (data as BaseDataCollection).Underlying != null);
parent = ConfigureEnumerator(request, false, parent);
}
return(parent);
}
/// <summary>
/// Creates a new subscription for universe selection
/// </summary>
/// <param name="request">The subscription request</param>
private Subscription CreateUniverseSubscription(SubscriptionRequest request)
{
Subscription subscription = null;
// TODO : Consider moving the creating of universe subscriptions to a separate, testable class
// grab the relevant exchange hours
var config = request.Universe.Configuration;
var localEndTime = request.EndTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone);
var tzOffsetProvider = new TimeZoneOffsetProvider(request.Configuration.ExchangeTimeZone, request.StartTimeUtc, request.EndTimeUtc);
IEnumerator <BaseData> enumerator = null;
var timeTriggered = request.Universe as ITimeTriggeredUniverse;
if (timeTriggered != null)
{
Log.Trace($"LiveTradingDataFeed.CreateUniverseSubscription(): Creating user defined universe: {config.Symbol.ID}");
// spoof a tick on the requested interval to trigger the universe selection function
var enumeratorFactory = new TimeTriggeredUniverseSubscriptionEnumeratorFactory(timeTriggered, MarketHoursDatabase.FromDataFolder(), _frontierTimeProvider);
enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider);
enumerator = new FrontierAwareEnumerator(enumerator, _timeProvider, tzOffsetProvider);
var enqueueable = new EnqueueableEnumerator <BaseData>();
_customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable));
enumerator = enqueueable;
}
else if (config.Type == typeof(CoarseFundamental) || config.Type == typeof(ETFConstituentData))
{
Log.Trace($"LiveTradingDataFeed.CreateUniverseSubscription(): Creating {config.Type.Name} universe: {config.Symbol.ID}");
// Will try to pull data from the data folder every 10min, file with yesterdays date.
// If lean is started today it will trigger initial coarse universe selection
var factory = new LiveCustomDataSubscriptionEnumeratorFactory(_timeProvider,
// we adjust time to the previous tradable date
time => Time.GetStartTimeForTradeBars(request.Security.Exchange.Hours, time, Time.OneDay, 1, false, config.DataTimeZone),
TimeSpan.FromMinutes(10)
);
var enumeratorStack = factory.CreateEnumerator(request, _dataProvider);
// aggregates each coarse data point into a single BaseDataCollection
var aggregator = new BaseDataCollectionAggregatorEnumerator(enumeratorStack, config.Symbol, true);
_customExchange.AddEnumerator(config.Symbol, aggregator);
var enqueable = new EnqueueableEnumerator <BaseData>();
_customExchange.SetDataHandler(config.Symbol, data =>
{
enqueable.Enqueue(data);
subscription.OnNewDataAvailable();
});
enumerator = GetConfiguredFrontierAwareEnumerator(enqueable, tzOffsetProvider,
// advance time if before 23pm or after 5am and not on Saturdays
time => time.Hour < 23 && time.Hour > 5 && time.DayOfWeek != DayOfWeek.Saturday);
}
else if (request.Universe is OptionChainUniverse)
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating option chain universe: " + config.Symbol.ID);
Func <SubscriptionRequest, IEnumerator <BaseData> > configure = (subRequest) =>
{
var fillForwardResolution = _subscriptions.UpdateAndGetFillForwardResolution(subRequest.Configuration);
var input = Subscribe(subRequest.Configuration, (sender, args) => subscription.OnNewDataAvailable());
return(new LiveFillForwardEnumerator(_frontierTimeProvider, input, subRequest.Security.Exchange, fillForwardResolution, subRequest.Configuration.ExtendedMarketHours, localEndTime, subRequest.Configuration.Increment, subRequest.Configuration.DataTimeZone));
};
var symbolUniverse = _dataQueueHandler as IDataQueueUniverseProvider;
if (symbolUniverse == null)
{
throw new NotSupportedException("The DataQueueHandler does not support Options.");
}
var enumeratorFactory = new OptionChainUniverseSubscriptionEnumeratorFactory(configure, symbolUniverse, _timeProvider);
enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider);
enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, tzOffsetProvider);
}
else if (request.Universe is FuturesChainUniverse)
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating futures chain universe: " + config.Symbol.ID);
var symbolUniverse = _dataQueueHandler as IDataQueueUniverseProvider;
if (symbolUniverse == null)
{
throw new NotSupportedException("The DataQueueHandler does not support Futures.");
}
var enumeratorFactory = new FuturesChainUniverseSubscriptionEnumeratorFactory(symbolUniverse, _timeProvider);
enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider);
enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, tzOffsetProvider);
}
else
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating custom universe: " + config.Symbol.ID);
var factory = new LiveCustomDataSubscriptionEnumeratorFactory(_timeProvider);
var enumeratorStack = factory.CreateEnumerator(request, _dataProvider);
enumerator = new BaseDataCollectionAggregatorEnumerator(enumeratorStack, config.Symbol, liveMode: true);
var enqueueable = new EnqueueableEnumerator <BaseData>();
_customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable));
enumerator = enqueueable;
}
// create the subscription
var subscriptionDataEnumerator = new SubscriptionDataEnumerator(request.Configuration, request.Security.Exchange.Hours, tzOffsetProvider, enumerator, request.IsUniverseSubscription);
subscription = new Subscription(request, subscriptionDataEnumerator, tzOffsetProvider);
// send the subscription for the new symbol through to the data queuehandler
if (_channelProvider.ShouldStreamSubscription(subscription.Configuration))
{
Subscribe(request.Configuration, (sender, args) => subscription.OnNewDataAvailable());
}
return(subscription);
}
/// <summary>
/// Creates a new subscription for universe selection
/// </summary>
/// <param name="request">The subscription request</param>
private Subscription CreateUniverseSubscription(SubscriptionRequest request)
{
Subscription subscription = null;
// TODO : Consider moving the creating of universe subscriptions to a separate, testable class
// grab the relevant exchange hours
var config = request.Universe.Configuration;
var localEndTime = request.EndTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone);
var tzOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc);
IEnumerator <BaseData> enumerator;
var timeTriggered = request.Universe as ITimeTriggeredUniverse;
if (timeTriggered != null)
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating user defined universe: " + config.Symbol.ToString());
// spoof a tick on the requested interval to trigger the universe selection function
var enumeratorFactory = new TimeTriggeredUniverseSubscriptionEnumeratorFactory(timeTriggered, MarketHoursDatabase.FromDataFolder());
enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider);
enumerator = new FrontierAwareEnumerator(enumerator, _timeProvider, tzOffsetProvider);
var enqueueable = new EnqueueableEnumerator <BaseData>();
_customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable));
enumerator = enqueueable;
// Trigger universe selection when security added/removed after Initialize
if (timeTriggered is UserDefinedUniverse)
{
var userDefined = (UserDefinedUniverse)timeTriggered;
userDefined.CollectionChanged += (sender, args) =>
{
var items =
args.Action == NotifyCollectionChangedAction.Add ? args.NewItems :
args.Action == NotifyCollectionChangedAction.Remove ? args.OldItems : null;
var currentFrontierUtcTime = _frontierTimeProvider.GetUtcNow();
if (items == null || currentFrontierUtcTime == DateTime.MinValue)
{
return;
}
var symbol = items.OfType <Symbol>().FirstOrDefault();
if (symbol == null)
{
return;
}
var collection = new BaseDataCollection(currentFrontierUtcTime, symbol);
var changes = _universeSelection.ApplyUniverseSelection(userDefined, currentFrontierUtcTime, collection);
_algorithm.OnSecuritiesChanged(changes);
subscription.OnNewDataAvailable();
};
}
}
else if (config.Type == typeof(CoarseFundamental))
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating coarse universe: " + config.Symbol.ToString());
// we subscribe using a normalized symbol, without a random GUID,
// since the ticker plant will send the coarse data using this symbol
var normalizedSymbol = CoarseFundamental.CreateUniverseSymbol(config.Symbol.ID.Market, false);
// since we're binding to the data queue exchange we'll need to let him
// know that we expect this data
_dataQueueHandler.Subscribe(_job, new[] { normalizedSymbol });
var enqueable = new EnqueueableEnumerator <BaseData>();
// We `AddDataHandler` not `Set` so we can have multiple handlers for the coarse data
_exchange.AddDataHandler(normalizedSymbol, data =>
{
enqueable.Enqueue(data);
subscription.OnNewDataAvailable();
});
enumerator = GetConfiguredFrontierAwareEnumerator(enqueable, tzOffsetProvider,
// advance time if before 23pm or after 5am and not on Saturdays
time => time.Hour < 23 && time.Hour > 5 && time.DayOfWeek != DayOfWeek.Saturday);
}
else if (request.Universe is OptionChainUniverse)
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating option chain universe: " + config.Symbol.ToString());
Func <SubscriptionRequest, IEnumerator <BaseData>, IEnumerator <BaseData> > configure = (subRequest, input) =>
{
// we check if input enumerator is an underlying enumerator. If yes, we subscribe it to the data.
var aggregator = input as TradeBarBuilderEnumerator;
if (aggregator != null)
{
_exchange.SetDataHandler(request.Configuration.Symbol, data =>
{
aggregator.ProcessData((Tick)data);
});
}
var fillForwardResolution = _subscriptions.UpdateAndGetFillForwardResolution(request.Configuration);
return(new LiveFillForwardEnumerator(_frontierTimeProvider, input, request.Security.Exchange, fillForwardResolution, request.Configuration.ExtendedMarketHours, localEndTime, request.Configuration.Increment, request.Configuration.DataTimeZone, request.StartTimeLocal));
};
var symbolUniverse = _dataQueueHandler as IDataQueueUniverseProvider;
if (symbolUniverse == null)
{
throw new NotSupportedException("The DataQueueHandler does not support Options.");
}
var enumeratorFactory = new OptionChainUniverseSubscriptionEnumeratorFactory(configure, symbolUniverse, _timeProvider);
enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider);
enumerator = GetConfiguredFrontierAwareEnumerator(enumerator, tzOffsetProvider,
time => symbolUniverse.CanAdvanceTime(config.SecurityType));
}
else if (request.Universe is FuturesChainUniverse)
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating futures chain universe: " + config.Symbol.ToString());
var symbolUniverse = _dataQueueHandler as IDataQueueUniverseProvider;
if (symbolUniverse == null)
{
throw new NotSupportedException("The DataQueueHandler does not support Futures.");
}
var enumeratorFactory = new FuturesChainUniverseSubscriptionEnumeratorFactory(symbolUniverse, _timeProvider);
enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider);
enumerator = GetConfiguredFrontierAwareEnumerator(enumerator, tzOffsetProvider,
time => symbolUniverse.CanAdvanceTime(config.SecurityType));
}
else
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating custom universe: " + config.Symbol.ToString());
var factory = new LiveCustomDataSubscriptionEnumeratorFactory(_timeProvider);
var enumeratorStack = factory.CreateEnumerator(request, _dataProvider);
enumerator = new BaseDataCollectionAggregatorEnumerator(enumeratorStack, config.Symbol, liveMode: true);
var enqueueable = new EnqueueableEnumerator <BaseData>();
_customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable));
enumerator = enqueueable;
}
// create the subscription
var subscriptionDataEnumerator = new SubscriptionDataEnumerator(request.Configuration, request.Security.Exchange.Hours, tzOffsetProvider, enumerator);
subscription = new Subscription(request, subscriptionDataEnumerator, tzOffsetProvider);
return(subscription);
}
/// <summary>
/// Creates a new subscription for universe selection
/// </summary>
/// <param name="request">The subscription request</param>
private Subscription CreateUniverseSubscription(SubscriptionRequest request)
{
// TODO : Consider moving the creating of universe subscriptions to a separate, testable class
// grab the relevant exchange hours
var config = request.Universe.Configuration;
var localEndTime = request.EndTimeUtc.ConvertFromUtc(request.Security.Exchange.TimeZone);
var tzOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc);
IEnumerator <BaseData> enumerator;
var userDefined = request.Universe as UserDefinedUniverse;
if (userDefined != null)
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating user defined universe: " + config.Symbol.ToString());
// spoof a tick on the requested interval to trigger the universe selection function
var enumeratorFactory = new UserDefinedUniverseSubscriptionEnumeratorFactory(userDefined, MarketHoursDatabase.FromDataFolder());
enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider);
enumerator = new FrontierAwareEnumerator(enumerator, _timeProvider, tzOffsetProvider);
var enqueueable = new EnqueueableEnumerator <BaseData>();
_customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable));
enumerator = enqueueable;
// Trigger universe selection when security added/removed after Initialize
userDefined.CollectionChanged += (sender, args) =>
{
var items =
args.Action == NotifyCollectionChangedAction.Add ? args.NewItems :
args.Action == NotifyCollectionChangedAction.Remove ? args.OldItems : null;
if (items == null || _frontierUtc == DateTime.MinValue)
{
return;
}
var symbol = items.OfType <Symbol>().FirstOrDefault();
if (symbol == null)
{
return;
}
var collection = new BaseDataCollection(_frontierUtc, symbol);
var changes = _universeSelection.ApplyUniverseSelection(userDefined, _frontierUtc, collection);
_algorithm.OnSecuritiesChanged(changes);
};
}
else if (config.Type == typeof(CoarseFundamental))
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating coarse universe: " + config.Symbol.ToString());
// since we're binding to the data queue exchange we'll need to let him
// know that we expect this data
_dataQueueHandler.Subscribe(_job, new[] { request.Security.Symbol });
var enqueable = new EnqueueableEnumerator <BaseData>();
_exchange.SetDataHandler(config.Symbol, data =>
{
enqueable.Enqueue(data);
});
enumerator = enqueable;
}
else if (request.Universe is OptionChainUniverse)
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating option chain universe: " + config.Symbol.ToString());
Func <SubscriptionRequest, IEnumerator <BaseData>, IEnumerator <BaseData> > configure = (subRequest, input) =>
{
// we check if input enumerator is an underlying enumerator. If yes, we subscribe it to the data.
var aggregator = input as TradeBarBuilderEnumerator;
if (aggregator != null)
{
_exchange.SetDataHandler(request.Configuration.Symbol, data =>
{
aggregator.ProcessData((Tick)data);
});
}
var subscriptionConfigs = _subscriptions.Select(x => x.Configuration).Concat(new[] { request.Configuration });
UpdateFillForwardResolution(subscriptionConfigs);
return(new LiveFillForwardEnumerator(_frontierTimeProvider, input, request.Security.Exchange, _fillForwardResolution, request.Configuration.ExtendedMarketHours, localEndTime, request.Configuration.Increment, request.Configuration.DataTimeZone));
};
var symbolUniverse = _dataQueueHandler as IDataQueueUniverseProvider;
if (symbolUniverse == null)
{
throw new NotSupportedException("The DataQueueHandler does not support Options.");
}
var enumeratorFactory = new OptionChainUniverseSubscriptionEnumeratorFactory(configure, symbolUniverse, _timeProvider);
enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider);
enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, tzOffsetProvider);
}
else if (request.Universe is FuturesChainUniverse)
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating futures chain universe: " + config.Symbol.ToString());
var symbolUniverse = _dataQueueHandler as IDataQueueUniverseProvider;
if (symbolUniverse == null)
{
throw new NotSupportedException("The DataQueueHandler does not support Futures.");
}
var enumeratorFactory = new FuturesChainUniverseSubscriptionEnumeratorFactory(symbolUniverse, _timeProvider);
enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider);
enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, tzOffsetProvider);
}
else
{
Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating custom universe: " + config.Symbol.ToString());
var factory = new LiveCustomDataSubscriptionEnumeratorFactory(_timeProvider);
var enumeratorStack = factory.CreateEnumerator(request, _dataProvider);
enumerator = new BaseDataCollectionAggregatorEnumerator(enumeratorStack, config.Symbol);
var enqueueable = new EnqueueableEnumerator <BaseData>();
_customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable));
enumerator = enqueueable;
}
// create the subscription
var subscriptionDataEnumerator = SubscriptionData.Enumerator(request.Configuration, request.Security, tzOffsetProvider, enumerator);
var subscription = new Subscription(request.Universe, request.Security, config, subscriptionDataEnumerator, tzOffsetProvider, request.StartTimeUtc, request.EndTimeUtc, true);
return(subscription);
}
/// <summary>
/// Configure the enumerator with aggregation/fill-forward/filter behaviors. Returns new instance if re-configured
/// </summary>
private IEnumerator<BaseData> ConfigureEnumerator(SubscriptionRequest request, bool aggregate, IEnumerator<BaseData> enumerator)
{
if (aggregate)
{
enumerator = new BaseDataCollectionAggregatorEnumerator(enumerator, request.Configuration.Symbol);
}
// optionally apply fill forward logic, but never for tick data
if (request.Configuration.FillDataForward && request.Configuration.Resolution != Resolution.Tick)
{
enumerator = new FillForwardEnumerator(enumerator, request.Security.Exchange, _fillForwardResolution,
request.Security.IsExtendedMarketHours, request.EndTimeLocal, request.Configuration.Resolution.ToTimeSpan());
}
// optionally apply exchange/user filters
if (request.Configuration.IsFilteredSubscription)
{
enumerator = SubscriptionFilterEnumerator.WrapForDataFeed(_resultHandler, enumerator, request.Security, request.EndTimeLocal);
}
return enumerator;
}
public void DoesNotEmitInvalidData()
{
var startTime = new DateTime(2014, 06, 06, 0, 0, 0);
var endTime = new DateTime(2014, 06, 09, 20, 0, 0);
var canonicalSymbol = Symbol.Create("AAPL", SecurityType.Option, Market.USA, "?AAPL");
var quoteCurrency = new Cash(Currencies.USD, 0, 1);
var exchangeHours = MarketHoursDatabase.FromDataFolder().GetExchangeHours(Market.USA, canonicalSymbol, SecurityType.Option);
var config = new SubscriptionDataConfig(
typeof(ZipEntryName),
canonicalSymbol,
Resolution.Minute,
TimeZones.Utc,
TimeZones.NewYork,
true,
false,
false,
false,
TickType.Quote,
false,
DataNormalizationMode.Raw
);
var option = new Option(
canonicalSymbol,
exchangeHours,
quoteCurrency,
new OptionSymbolProperties(SymbolProperties.GetDefault(Currencies.USD)),
ErrorCurrencyConverter.Instance,
RegisteredSecurityDataTypesProvider.Null,
new SecurityCache()
);
var fillForwardResolution = Ref.CreateReadOnly(() => Resolution.Minute.ToTimeSpan());
Func <SubscriptionRequest, IEnumerator <BaseData>, IEnumerator <BaseData> > underlyingEnumeratorFunc =
(req, input) =>
{
input = new BaseDataCollectionAggregatorEnumerator(input, req.Configuration.Symbol);
return(new FillForwardEnumerator(
input,
option.Exchange,
fillForwardResolution,
false,
endTime,
Resolution.Minute.ToTimeSpan(),
TimeZones.Utc,
startTime));
};
var factory = new OptionChainUniverseSubscriptionEnumeratorFactory(underlyingEnumeratorFunc,
MapFileResolver.Create(Globals.DataFolder, Market.USA),
new LocalDiskFactorFileProvider(new LocalDiskMapFileProvider()));
var request = new SubscriptionRequest(true, null, option, config, startTime, endTime);
var enumerator = factory.CreateEnumerator(request, new DefaultDataProvider());
var emittedCount = 0;
foreach (var data in enumerator.AsEnumerable())
{
emittedCount++;
var optionData = data as OptionChainUniverseDataCollection;
Assert.IsNotNull(optionData);
Assert.IsNotNull(optionData.Underlying);
Assert.AreNotEqual(0, optionData.Data.Count);
}
// 9:30 to 15:59 -> 6.5 hours * 60 => 390 minutes * 2 days = 780
Assert.AreEqual(780, emittedCount);
}
