/// <summary>
/// Creates a subscription to process the history request
/// </summary>
private static Subscription CreateSubscription(HistoryRequest request, IEnumerable <BaseData> history)
{
// data reader expects these values in local times
var start = request.StartTimeUtc.ConvertFromUtc(request.ExchangeHours.TimeZone);
var end = request.EndTimeUtc.ConvertFromUtc(request.ExchangeHours.TimeZone);
var config = new SubscriptionDataConfig(request.DataType,
request.Symbol,
request.Resolution,
request.TimeZone,
request.ExchangeHours.TimeZone,
request.FillForwardResolution.HasValue,
request.IncludeExtendedMarketHours,
false,
request.IsCustomData,
null,
true,
request.DataNormalizationMode
);
var security = new Security(request.ExchangeHours, config, new Cash(CashBook.AccountCurrency, 0, 1m), SymbolProperties.GetDefault(CashBook.AccountCurrency));
var reader = history.GetEnumerator();
// optionally apply fill forward behavior
if (request.FillForwardResolution.HasValue)
{
var readOnlyRef = Ref.CreateReadOnly(() => request.FillForwardResolution.Value.ToTimeSpan());
reader = new FillForwardEnumerator(reader, security.Exchange, readOnlyRef, security.IsExtendedMarketHours, end, config.Increment);
}
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, start, end);
return(new Subscription(null, security, config, reader, timeZoneOffsetProvider, start, end, false));
}
/// <summary>
/// Creates a subscription to process the history request
/// </summary>
private static Subscription CreateSubscription(HistoryRequest request, IEnumerable <BaseData> history)
{
// data reader expects these values in local times
var start = request.StartTimeUtc.ConvertFromUtc(request.ExchangeHours.TimeZone);
var end = request.EndTimeUtc.ConvertFromUtc(request.ExchangeHours.TimeZone);
var config = new SubscriptionDataConfig(request.DataType,
request.Symbol,
request.Resolution,
request.DataTimeZone,
request.ExchangeHours.TimeZone,
request.FillForwardResolution.HasValue,
request.IncludeExtendedMarketHours,
false,
request.IsCustomData,
request.TickType,
true,
request.DataNormalizationMode
);
var security = new Security(
request.ExchangeHours,
config,
new Cash(Currencies.NullCurrency, 0, 1m),
SymbolProperties.GetDefault(Currencies.NullCurrency),
ErrorCurrencyConverter.Instance,
RegisteredSecurityDataTypesProvider.Null,
new SecurityCache()
);
var reader = history.GetEnumerator();
// optionally apply fill forward behavior
if (request.FillForwardResolution.HasValue)
{
// copy forward Bid/Ask bars for QuoteBars
if (request.DataType == typeof(QuoteBar))
{
reader = new QuoteBarFillForwardEnumerator(reader);
}
var readOnlyRef = Ref.CreateReadOnly(() => request.FillForwardResolution.Value.ToTimeSpan());
reader = new FillForwardEnumerator(reader, security.Exchange, readOnlyRef, security.IsExtendedMarketHours, end, config.Increment, config.DataTimeZone, start);
}
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, start, end);
var subscriptionDataEnumerator = new SubscriptionDataEnumerator(config, security.Exchange.Hours, timeZoneOffsetProvider, reader);
var subscriptionRequest = new SubscriptionRequest(false, null, security, config, start, end);
return(new Subscription(subscriptionRequest, subscriptionDataEnumerator, timeZoneOffsetProvider));
}
/// <summary>
/// Initializes a new instance of the <see cref="FillForwardEnumerator"/> class
/// </summary>
/// <param name="enumerator">The source enumerator to be filled forward</param>
/// <param name="exchange">The exchange used to determine when to insert fill forward data</param>
/// <param name="fillForwardResolution">The resolution we'd like to receive data on</param>
/// <param name="isExtendedMarketHours">True to use the exchange's extended market hours, false to use the regular market hours</param>
/// <param name="subscriptionEndTime">The end time of the subscrition, once passing this date the enumerator will stop</param>
/// <param name="dataResolution">The source enumerator's data resolution</param>
public FillForwardEnumerator(IEnumerator <BaseData> enumerator,
SecurityExchange exchange,
TimeSpan fillForwardResolution,
bool isExtendedMarketHours,
DateTime subscriptionEndTime,
TimeSpan dataResolution
)
{
_subscriptionEndTime = subscriptionEndTime;
Exchange = exchange;
_enumerator = enumerator;
_dataResolution = dataResolution;
_fillForwardResolution = Ref.CreateReadOnly(() => fillForwardResolution);
_isExtendedMarketHours = isExtendedMarketHours;
}
public IEnumerable <TimeSlice> StreamData(CancellationToken cancellationToken)
{
var dataFeedPacket = new DataFeedPacket(_algorithm.Securities[_symbol],
_algorithm.SubscriptionManager.Subscriptions.First(s => s.Symbol == _symbol),
new List <BaseData> {
_dividend
}, Ref.CreateReadOnly(() => false));
yield return(_timeSliceFactory.Create(DateTime.UtcNow,
new List <DataFeedPacket> {
dataFeedPacket
},
SecurityChanges.None,
new Dictionary <Universe, BaseDataCollection>()
));
}
public IEnumerator <TimeSlice> GetEnumerator()
{
var dataFeedPacket = new DataFeedPacket(_algorithm.Securities[_symbol],
_algorithm.SubscriptionManager.Subscriptions.First(s => s.Symbol == _symbol),
new List <BaseData> {
_dividend
}, Ref.CreateReadOnly(() => false));
yield return(TimeSlice.Create(DateTime.UtcNow,
TimeZones.NewYork,
_algorithm.Portfolio.CashBook,
new List <DataFeedPacket> {
dataFeedPacket
},
SecurityChanges.None,
new Dictionary <Universe, BaseDataCollection>()
));
}
/// <summary>
/// Initializes a new instance of the <see cref="Subscription"/> class with a universe
/// </summary>
/// <param name="subscriptionRequest">Specified for universe subscriptions</param>
/// <param name="enumerator">The subscription's data source</param>
/// <param name="timeZoneOffsetProvider">The offset provider used to convert data local times to utc</param>
public Subscription(
SubscriptionRequest subscriptionRequest,
IEnumerator <SubscriptionData> enumerator,
TimeZoneOffsetProvider timeZoneOffsetProvider)
{
_subscriptionRequests = new List <SubscriptionRequest> {
subscriptionRequest
};
Security = subscriptionRequest.Security;
_enumerator = enumerator;
IsUniverseSelectionSubscription = subscriptionRequest.IsUniverseSubscription;
Configuration = subscriptionRequest.Configuration;
OffsetProvider = timeZoneOffsetProvider;
UtcStartTime = subscriptionRequest.StartTimeUtc;
UtcEndTime = subscriptionRequest.EndTimeUtc;
RemovedFromUniverse = Ref.CreateReadOnly(() => _removedFromUniverse);
}
/// <summary>
/// Initializes a new instance of the <see cref="Subscription"/> class with a universe
/// </summary>
/// <param name="universe">Specified for universe subscriptions</param>
/// <param name="security">The security this subscription is for</param>
/// <param name="configuration">The subscription configuration that was used to generate the enumerator</param>
/// <param name="enumerator">The subscription's data source</param>
/// <param name="timeZoneOffsetProvider">The offset provider used to convert data local times to utc</param>
/// <param name="utcStartTime">The start time of the subscription</param>
/// <param name="utcEndTime">The end time of the subscription</param>
/// <param name="isUniverseSelectionSubscription">True if this is a subscription for universe selection,
/// that is, the configuration is used to produce the used to perform universe selection, false for a
/// normal data subscription, i.e, SPY</param>
public Subscription(Universe universe,
Security security,
SubscriptionDataConfig configuration,
IEnumerator <SubscriptionData> enumerator,
TimeZoneOffsetProvider timeZoneOffsetProvider,
DateTime utcStartTime,
DateTime utcEndTime,
bool isUniverseSelectionSubscription)
{
Universe = universe;
Security = security;
_enumerator = enumerator;
IsUniverseSelectionSubscription = isUniverseSelectionSubscription;
Configuration = configuration;
OffsetProvider = timeZoneOffsetProvider;
UtcStartTime = utcStartTime;
UtcEndTime = utcEndTime;
RemovedFromUniverse = Ref.CreateReadOnly(() => _removedFromUniverse);
}
/// <summary>
/// Creates a subscription to process the request
/// </summary>
private Subscription CreateSubscription(HistoryRequest request, DateTime startUtc, DateTime endUtc)
{
// data reader expects these values in local times
var startTimeLocal = startUtc.ConvertFromUtc(request.ExchangeHours.TimeZone);
var endTimeLocal = endUtc.ConvertFromUtc(request.ExchangeHours.TimeZone);
var config = new SubscriptionDataConfig(request.DataType,
request.Symbol,
request.Resolution,
request.DataTimeZone,
request.ExchangeHours.TimeZone,
request.FillForwardResolution.HasValue,
request.IncludeExtendedMarketHours,
false,
request.IsCustomData,
request.TickType,
true,
request.DataNormalizationMode
);
_dataPermissionManager.AssertConfiguration(config);
var security = new Security(
request.ExchangeHours,
config,
new Cash(Currencies.NullCurrency, 0, 1m),
SymbolProperties.GetDefault(Currencies.NullCurrency),
ErrorCurrencyConverter.Instance,
RegisteredSecurityDataTypesProvider.Null,
new SecurityCache()
);
var mapFileResolver = MapFileResolver.Empty;
if (config.TickerShouldBeMapped())
{
mapFileResolver = _mapFileProvider.Get(config.Market);
var mapFile = mapFileResolver.ResolveMapFile(config.Symbol.ID.Symbol, config.Symbol.ID.Date);
config.MappedSymbol = mapFile.GetMappedSymbol(startTimeLocal, config.MappedSymbol);
}
// Tradable dates are defined with the data time zone to access the right source
var tradableDates = Time.EachTradeableDayInTimeZone(request.ExchangeHours, startTimeLocal, endTimeLocal, request.DataTimeZone, request.IncludeExtendedMarketHours);
var dataReader = new SubscriptionDataReader(config,
startTimeLocal,
endTimeLocal,
mapFileResolver,
_factorFileProvider,
tradableDates,
false,
_dataCacheProvider
);
dataReader.InvalidConfigurationDetected += (sender, args) => { OnInvalidConfigurationDetected(args); };
dataReader.NumericalPrecisionLimited += (sender, args) => { OnNumericalPrecisionLimited(args); };
dataReader.StartDateLimited += (sender, args) => { OnStartDateLimited(args); };
dataReader.DownloadFailed += (sender, args) => { OnDownloadFailed(args); };
dataReader.ReaderErrorDetected += (sender, args) => { OnReaderErrorDetected(args); };
IEnumerator <BaseData> reader = dataReader;
var intraday = GetIntradayDataEnumerator(dataReader, request);
if (intraday != null)
{
// we optionally concatenate the intraday data enumerator
reader = new ConcatEnumerator(true, reader, intraday);
}
reader = CorporateEventEnumeratorFactory.CreateEnumerators(
reader,
config,
_factorFileProvider,
dataReader,
mapFileResolver,
false,
startTimeLocal);
// optionally apply fill forward behavior
if (request.FillForwardResolution.HasValue)
{
// copy forward Bid/Ask bars for QuoteBars
if (request.DataType == typeof(QuoteBar))
{
reader = new QuoteBarFillForwardEnumerator(reader);
}
var readOnlyRef = Ref.CreateReadOnly(() => request.FillForwardResolution.Value.ToTimeSpan());
reader = new FillForwardEnumerator(reader, security.Exchange, readOnlyRef, request.IncludeExtendedMarketHours, endTimeLocal, config.Increment, config.DataTimeZone, startTimeLocal);
}
// since the SubscriptionDataReader performs an any overlap condition on the trade bar's entire
// range (time->end time) we can end up passing the incorrect data (too far past, possibly future),
// so to combat this we deliberately filter the results from the data reader to fix these cases
// which only apply to non-tick data
reader = new SubscriptionFilterEnumerator(reader, security, endTimeLocal, config.ExtendedMarketHours, false);
reader = new FilterEnumerator <BaseData>(reader, data =>
{
// allow all ticks
if (config.Resolution == Resolution.Tick)
{
return(true);
}
// filter out future data
if (data.EndTime > endTimeLocal)
{
return(false);
}
// filter out data before the start
return(data.EndTime > startTimeLocal);
});
var subscriptionRequest = new SubscriptionRequest(false, null, security, config, request.StartTimeUtc, request.EndTimeUtc);
if (_parallelHistoryRequestsEnabled)
{
return(SubscriptionUtils.CreateAndScheduleWorker(subscriptionRequest, reader, _factorFileProvider, false));
}
return(SubscriptionUtils.Create(subscriptionRequest, reader));
}
/// <summary>
/// Creates a subscription to process the request
/// </summary>
private Subscription CreateSubscription(HistoryRequest request, DateTime start, DateTime end)
{
// data reader expects these values in local times
start = start.ConvertFromUtc(request.ExchangeHours.TimeZone);
end = end.ConvertFromUtc(request.ExchangeHours.TimeZone);
var config = new SubscriptionDataConfig(request.DataType,
request.Symbol,
request.Resolution,
request.DataTimeZone,
request.ExchangeHours.TimeZone,
request.FillForwardResolution.HasValue,
request.IncludeExtendedMarketHours,
false,
request.IsCustomData,
request.TickType,
true,
request.DataNormalizationMode
);
var security = new Security(
request.ExchangeHours,
config,
new Cash(CashBook.AccountCurrency, 0, 1m),
SymbolProperties.GetDefault(CashBook.AccountCurrency),
ErrorCurrencyConverter.Instance
);
var dataReader = new SubscriptionDataReader(config,
start,
end,
config.SecurityType == SecurityType.Equity ? _mapFileProvider.Get(config.Market) : MapFileResolver.Empty,
_factorFileProvider,
Time.EachTradeableDay(request.ExchangeHours, start, end),
false,
_dataCacheProvider,
false
);
dataReader.InvalidConfigurationDetected += (sender, args) => { OnInvalidConfigurationDetected(new InvalidConfigurationDetectedEventArgs(args.Message)); };
dataReader.NumericalPrecisionLimited += (sender, args) => { OnNumericalPrecisionLimited(new NumericalPrecisionLimitedEventArgs(args.Message)); };
dataReader.DownloadFailed += (sender, args) => { OnDownloadFailed(new DownloadFailedEventArgs(args.Message, args.StackTrace)); };
dataReader.ReaderErrorDetected += (sender, args) => { OnReaderErrorDetected(new ReaderErrorDetectedEventArgs(args.Message, args.StackTrace)); };
dataReader.Initialize();
IEnumerator <BaseData> reader = dataReader;
// optionally apply fill forward behavior
if (request.FillForwardResolution.HasValue)
{
// copy forward Bid/Ask bars for QuoteBars
if (request.DataType == typeof(QuoteBar))
{
reader = new QuoteBarFillForwardEnumerator(reader);
}
var readOnlyRef = Ref.CreateReadOnly(() => request.FillForwardResolution.Value.ToTimeSpan());
reader = new FillForwardEnumerator(reader, security.Exchange, readOnlyRef, security.IsExtendedMarketHours, end, config.Increment, config.DataTimeZone);
}
// since the SubscriptionDataReader performs an any overlap condition on the trade bar's entire
// range (time->end time) we can end up passing the incorrect data (too far past, possibly future),
// so to combat this we deliberately filter the results from the data reader to fix these cases
// which only apply to non-tick data
reader = new SubscriptionFilterEnumerator(reader, security, end);
reader = new FilterEnumerator <BaseData>(reader, data =>
{
// allow all ticks
if (config.Resolution == Resolution.Tick)
{
return(true);
}
// filter out future data
if (data.EndTime > end)
{
return(false);
}
// filter out data before the start
return(data.EndTime > start);
});
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, start, end);
var subscriptionDataEnumerator = SubscriptionData.Enumerator(config, security, timeZoneOffsetProvider, reader);
var subscriptionRequest = new SubscriptionRequest(false, null, security, config, start, end);
return(new Subscription(subscriptionRequest, subscriptionDataEnumerator, timeZoneOffsetProvider));
}
public void RefreshesOptionChainUniverseOnDateChange()
{
var startTime = new DateTime(2018, 10, 19, 10, 0, 0);
var timeProvider = new ManualTimeProvider(startTime);
var canonicalSymbol = Symbol.Create("SPY", SecurityType.Option, Market.USA, "?SPY");
var quoteCurrency = new Cash(Currencies.USD, 0, 1);
var exchangeHours = MarketHoursDatabase.FromDataFolder().GetExchangeHours(Market.USA, canonicalSymbol, SecurityType.Option);
var config = new SubscriptionDataConfig(
typeof(ZipEntryName),
canonicalSymbol,
Resolution.Minute,
TimeZones.Utc,
TimeZones.NewYork,
true,
false,
false,
false,
TickType.Quote,
false,
DataNormalizationMode.Raw
);
var option = new Option(
canonicalSymbol,
exchangeHours,
quoteCurrency,
new OptionSymbolProperties(SymbolProperties.GetDefault(Currencies.USD)),
ErrorCurrencyConverter.Instance
);
var fillForwardResolution = Ref.CreateReadOnly(() => Resolution.Minute.ToTimeSpan());
var symbolUniverse = new TestDataQueueUniverseProvider(timeProvider);
TradeBarBuilderEnumerator underlyingEnumerator = null;
Func <SubscriptionRequest, IEnumerator <BaseData>, IEnumerator <BaseData> > underlyingEnumeratorFunc =
(req, input) =>
{
underlyingEnumerator = (TradeBarBuilderEnumerator)input;
return(new LiveFillForwardEnumerator(
timeProvider,
input,
option.Exchange,
fillForwardResolution,
false,
Time.EndOfTime,
Resolution.Minute.ToTimeSpan(),
TimeZones.Utc));
};
var factory = new OptionChainUniverseSubscriptionEnumeratorFactory(underlyingEnumeratorFunc, symbolUniverse, timeProvider);
var universeSettings = new UniverseSettings(Resolution.Minute, 0, true, false, TimeSpan.Zero);
var universe = new OptionChainUniverse(option, universeSettings, true);
var request = new SubscriptionRequest(true, universe, option, config, startTime, Time.EndOfTime);
var enumerator = (DataQueueOptionChainUniverseDataCollectionEnumerator)factory.CreateEnumerator(request, new DefaultDataProvider());
// 2018-10-19 10:00 AM UTC
underlyingEnumerator.ProcessData(new Tick {
Symbol = Symbols.SPY, Value = 280m
});
Assert.IsTrue(enumerator.MoveNext());
// no underlying data available yet
Assert.IsNull(enumerator.Current);
Assert.AreEqual(0, symbolUniverse.TotalLookupCalls);
// 2018-10-19 10:01 AM UTC
timeProvider.Advance(Time.OneMinute);
underlyingEnumerator.ProcessData(new Tick {
Symbol = Symbols.SPY, Value = 280m
});
Assert.IsTrue(enumerator.MoveNext());
Assert.IsNotNull(enumerator.Current);
Assert.AreEqual(1, symbolUniverse.TotalLookupCalls);
var data = enumerator.Current;
Assert.IsNotNull(data);
Assert.AreEqual(1, data.Data.Count);
Assert.IsNotNull(data.Underlying);
// 2018-10-19 10:02 AM UTC
timeProvider.Advance(Time.OneMinute);
underlyingEnumerator.ProcessData(new Tick {
Symbol = Symbols.SPY, Value = 280m
});
Assert.IsTrue(enumerator.MoveNext());
Assert.IsNotNull(enumerator.Current);
Assert.AreEqual(1, symbolUniverse.TotalLookupCalls);
data = enumerator.Current;
Assert.IsNotNull(data);
Assert.AreEqual(1, data.Data.Count);
Assert.IsNotNull(data.Underlying);
// 2018-10-19 10:03 AM UTC
timeProvider.Advance(Time.OneMinute);
underlyingEnumerator.ProcessData(new Tick {
Symbol = Symbols.SPY, Value = 280m
});
Assert.IsTrue(enumerator.MoveNext());
Assert.IsNotNull(enumerator.Current);
Assert.AreEqual(1, symbolUniverse.TotalLookupCalls);
data = enumerator.Current;
Assert.IsNotNull(data);
Assert.AreEqual(1, data.Data.Count);
Assert.IsNotNull(data.Underlying);
// 2018-10-20 10:03 AM UTC
timeProvider.Advance(Time.OneDay);
underlyingEnumerator.ProcessData(new Tick {
Symbol = Symbols.SPY, Value = 280m
});
Assert.IsTrue(enumerator.MoveNext());
Assert.IsNotNull(enumerator.Current);
Assert.AreEqual(2, symbolUniverse.TotalLookupCalls);
data = enumerator.Current;
Assert.IsNotNull(data);
Assert.AreEqual(2, data.Data.Count);
Assert.IsNotNull(data.Underlying);
// 2018-10-20 10:04 AM UTC
timeProvider.Advance(Time.OneMinute);
underlyingEnumerator.ProcessData(new Tick {
Symbol = Symbols.SPY, Value = 280m
});
Assert.IsTrue(enumerator.MoveNext());
Assert.IsNotNull(enumerator.Current);
Assert.AreEqual(2, symbolUniverse.TotalLookupCalls);
data = enumerator.Current;
Assert.IsNotNull(data);
Assert.AreEqual(2, data.Data.Count);
Assert.IsNotNull(data.Underlying);
}
/// <summary>
/// Creates a subscription to process the request
/// </summary>
private Subscription CreateSubscription(HistoryRequest request, DateTime start, DateTime end)
{
// data reader expects these values in local times
start = start.ConvertFromUtc(request.ExchangeHours.TimeZone);
end = end.ConvertFromUtc(request.ExchangeHours.TimeZone);
var config = new SubscriptionDataConfig(request.DataType,
request.Symbol,
request.Resolution,
request.TimeZone,
request.ExchangeHours.TimeZone,
request.FillForwardResolution.HasValue,
request.IncludeExtendedMarketHours,
false,
request.IsCustomData
);
var security = new Security(request.ExchangeHours, config, new Cash(CashBook.AccountCurrency, 0, 1m), SymbolProperties.GetDefault(CashBook.AccountCurrency));
IEnumerator <BaseData> reader = new SubscriptionDataReader(config,
start,
end,
ResultHandlerStub.Instance,
config.SecurityType == SecurityType.Equity ? _mapFileProvider.Get(config.Market) : MapFileResolver.Empty,
_factorFileProvider,
_dataFileProvider,
Time.EachTradeableDay(request.ExchangeHours, start, end),
false,
includeAuxilliaryData: false
);
// optionally apply fill forward behavior
if (request.FillForwardResolution.HasValue)
{
var readOnlyRef = Ref.CreateReadOnly(() => request.FillForwardResolution.Value.ToTimeSpan());
reader = new FillForwardEnumerator(reader, security.Exchange, readOnlyRef, security.IsExtendedMarketHours, end, config.Increment);
}
// since the SubscriptionDataReader performs an any overlap condition on the trade bar's entire
// range (time->end time) we can end up passing the incorrect data (too far past, possibly future),
// so to combat this we deliberately filter the results from the data reader to fix these cases
// which only apply to non-tick data
reader = new SubscriptionFilterEnumerator(reader, security, end);
reader = new FilterEnumerator <BaseData>(reader, data =>
{
// allow all ticks
if (config.Resolution == Resolution.Tick)
{
return(true);
}
// filter out future data
if (data.EndTime > end)
{
return(false);
}
// filter out data before the start
return(data.EndTime > start);
});
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, start, end);
return(new Subscription(null, security, config, reader, timeZoneOffsetProvider, start, end, false));
}
/// <summary>
/// Creates a subscription to process the request
/// </summary>
private Subscription CreateSubscription(HistoryRequest request)
{
var config = request.ToSubscriptionDataConfig();
DataPermissionManager.AssertConfiguration(config, request.StartTimeLocal, request.EndTimeLocal);
var security = new Security(
request.ExchangeHours,
config,
new Cash(Currencies.NullCurrency, 0, 1m),
SymbolProperties.GetDefault(Currencies.NullCurrency),
ErrorCurrencyConverter.Instance,
RegisteredSecurityDataTypesProvider.Null,
new SecurityCache()
);
var dataReader = new SubscriptionDataReader(config,
request,
_mapFileProvider,
_factorFileProvider,
_dataCacheProvider,
_dataProvider
);
dataReader.InvalidConfigurationDetected += (sender, args) => { OnInvalidConfigurationDetected(args); };
dataReader.NumericalPrecisionLimited += (sender, args) => { OnNumericalPrecisionLimited(args); };
dataReader.StartDateLimited += (sender, args) => { OnStartDateLimited(args); };
dataReader.DownloadFailed += (sender, args) => { OnDownloadFailed(args); };
dataReader.ReaderErrorDetected += (sender, args) => { OnReaderErrorDetected(args); };
IEnumerator <BaseData> reader = dataReader;
var intraday = GetIntradayDataEnumerator(dataReader, request);
if (intraday != null)
{
// we optionally concatenate the intraday data enumerator
reader = new ConcatEnumerator(true, reader, intraday);
}
reader = CorporateEventEnumeratorFactory.CreateEnumerators(
reader,
config,
_factorFileProvider,
dataReader,
_mapFileProvider,
request.StartTimeLocal);
// optionally apply fill forward behavior
if (request.FillForwardResolution.HasValue)
{
// copy forward Bid/Ask bars for QuoteBars
if (request.DataType == typeof(QuoteBar))
{
reader = new QuoteBarFillForwardEnumerator(reader);
}
var readOnlyRef = Ref.CreateReadOnly(() => request.FillForwardResolution.Value.ToTimeSpan());
reader = new FillForwardEnumerator(reader, security.Exchange, readOnlyRef, request.IncludeExtendedMarketHours, request.EndTimeLocal, config.Increment, config.DataTimeZone);
}
// since the SubscriptionDataReader performs an any overlap condition on the trade bar's entire
// range (time->end time) we can end up passing the incorrect data (too far past, possibly future),
// so to combat this we deliberately filter the results from the data reader to fix these cases
// which only apply to non-tick data
reader = new SubscriptionFilterEnumerator(reader, security, request.EndTimeLocal, config.ExtendedMarketHours, false, request.ExchangeHours);
reader = new FilterEnumerator <BaseData>(reader, data =>
{
// allow all ticks
if (config.Resolution == Resolution.Tick)
{
return(true);
}
// filter out all aux data
if (data.DataType == MarketDataType.Auxiliary)
{
return(false);
}
// filter out future data
if (data.EndTime > request.EndTimeLocal)
{
return(false);
}
// filter out data before the start
return(data.EndTime > request.StartTimeLocal);
});
var subscriptionRequest = new SubscriptionRequest(false, null, security, config, request.StartTimeUtc, request.EndTimeUtc);
if (_parallelHistoryRequestsEnabled)
{
return(SubscriptionUtils.CreateAndScheduleWorker(subscriptionRequest, reader, _factorFileProvider, false));
}
return(SubscriptionUtils.Create(subscriptionRequest, reader));
}
public void LiveFillForwardEnumeratorDoesNotStall()
{
var now = DateTime.UtcNow;
var timeProvider = new ManualTimeProvider(new DateTime(2020, 5, 21, 9, 40, 0, 100), TimeZones.NewYork);
var enqueueableEnumerator = new EnqueueableEnumerator <BaseData>();
var fillForwardEnumerator = new LiveFillForwardEnumerator(
timeProvider,
enqueueableEnumerator,
new SecurityExchange(MarketHoursDatabase.FromDataFolder()
.ExchangeHoursListing
.First(kvp => kvp.Key.Market == Market.USA && kvp.Key.SecurityType == SecurityType.Equity)
.Value
.ExchangeHours),
Ref.CreateReadOnly(() => Resolution.Minute.ToTimeSpan()),
false,
now,
Resolution.Minute.ToTimeSpan(),
TimeZones.NewYork,
new DateTime(2020, 5, 21)
);
var openingBar = new TradeBar
{
Open = 0.01m,
High = 0.01m,
Low = 0.01m,
Close = 0.01m,
Volume = 1,
EndTime = new DateTime(2020, 5, 21, 9, 40, 0),
Symbol = Symbols.AAPL
};
var secondBar = new TradeBar
{
Open = 1m,
High = 2m,
Low = 1m,
Close = 2m,
Volume = 100,
EndTime = new DateTime(2020, 5, 21, 9, 42, 0),
Symbol = Symbols.AAPL
};
// Enqueue the first point, which will be emitted ASAP.
enqueueableEnumerator.Enqueue(openingBar);
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.NotNull(fillForwardEnumerator.Current);
Assert.AreEqual(openingBar.Open, ((TradeBar)fillForwardEnumerator.Current).Open);
Assert.AreEqual(openingBar.EndTime, fillForwardEnumerator.Current.EndTime);
// Advance the time, we expect a fill-forward bar.
timeProvider.SetCurrentTime(new DateTime(2020, 5, 21, 9, 41, 0, 100));
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward);
Assert.AreEqual(openingBar.Open, ((TradeBar)fillForwardEnumerator.Current).Open);
Assert.AreEqual(openingBar.EndTime.AddMinutes(1), fillForwardEnumerator.Current.EndTime);
// Now we expect data. The secondBar should be fill-forwarded from here on out after the MoveNext
timeProvider.SetCurrentTime(new DateTime(2020, 5, 21, 9, 42, 0, 100));
enqueueableEnumerator.Enqueue(secondBar);
Assert.IsTrue(fillForwardEnumerator.MoveNext());
Assert.IsFalse(fillForwardEnumerator.Current.IsFillForward);
}
public void DoesNotEmitInvalidData()
{
var startTime = new DateTime(2014, 06, 06, 0, 0, 0);
var endTime = new DateTime(2014, 06, 09, 20, 0, 0);
var canonicalSymbol = Symbol.Create("AAPL", SecurityType.Option, Market.USA, "?AAPL");
var quoteCurrency = new Cash(Currencies.USD, 0, 1);
var exchangeHours = MarketHoursDatabase.FromDataFolder().GetExchangeHours(Market.USA, canonicalSymbol, SecurityType.Option);
var config = new SubscriptionDataConfig(
typeof(ZipEntryName),
canonicalSymbol,
Resolution.Minute,
TimeZones.Utc,
TimeZones.NewYork,
true,
false,
false,
false,
TickType.Quote,
false,
DataNormalizationMode.Raw
);
var option = new Option(
canonicalSymbol,
exchangeHours,
quoteCurrency,
new OptionSymbolProperties(SymbolProperties.GetDefault(Currencies.USD)),
ErrorCurrencyConverter.Instance,
RegisteredSecurityDataTypesProvider.Null,
new SecurityCache()
);
var fillForwardResolution = Ref.CreateReadOnly(() => Resolution.Minute.ToTimeSpan());
Func <SubscriptionRequest, IEnumerator <BaseData>, IEnumerator <BaseData> > underlyingEnumeratorFunc =
(req, input) =>
{
input = new BaseDataCollectionAggregatorEnumerator(input, req.Configuration.Symbol);
return(new FillForwardEnumerator(
input,
option.Exchange,
fillForwardResolution,
false,
endTime,
Resolution.Minute.ToTimeSpan(),
TimeZones.Utc,
startTime));
};
var factory = new OptionChainUniverseSubscriptionEnumeratorFactory(underlyingEnumeratorFunc,
MapFileResolver.Create(Globals.DataFolder, Market.USA),
new LocalDiskFactorFileProvider(new LocalDiskMapFileProvider()));
var request = new SubscriptionRequest(true, null, option, config, startTime, endTime);
var enumerator = factory.CreateEnumerator(request, new DefaultDataProvider());
var emittedCount = 0;
foreach (var data in enumerator.AsEnumerable())
{
emittedCount++;
var optionData = data as OptionChainUniverseDataCollection;
Assert.IsNotNull(optionData);
Assert.IsNotNull(optionData.Underlying);
Assert.AreNotEqual(0, optionData.Data.Count);
}
// 9:30 to 15:59 -> 6.5 hours * 60 => 390 minutes * 2 days = 780
Assert.AreEqual(780, emittedCount);
}
