/// <summary>
/// Configure the enumerator with aggregation/fill-forward/filter behaviors. Returns new instance if re-configured
/// </summary>
private IEnumerator <BaseData> ConfigureEnumerator(SubscriptionRequest request, bool aggregate, IEnumerator <BaseData> enumerator)
{
if (aggregate)
{
enumerator = new BaseDataCollectionAggregatorEnumerator(enumerator, request.Configuration.Symbol);
}
// optionally apply fill forward logic, but never for tick data
if (request.Configuration.FillDataForward && request.Configuration.Resolution != Resolution.Tick)
{
// copy forward Bid/Ask bars for QuoteBars
if (request.Configuration.Type == typeof(QuoteBar))
{
enumerator = new QuoteBarFillForwardEnumerator(enumerator);
}
var subscriptionConfigs = _subscriptions.Select(x => x.Configuration).Concat(new[] { request.Configuration });
UpdateFillForwardResolution(subscriptionConfigs);
enumerator = new FillForwardEnumerator(enumerator, request.Security.Exchange, _fillForwardResolution,
request.Security.IsExtendedMarketHours, request.EndTimeLocal, request.Configuration.Resolution.ToTimeSpan(), request.Configuration.DataTimeZone);
}
// optionally apply exchange/user filters
if (request.Configuration.IsFilteredSubscription)
{
enumerator = SubscriptionFilterEnumerator.WrapForDataFeed(_resultHandler, enumerator, request.Security, request.EndTimeLocal);
}
return(enumerator);
}
public void FillsForwardBidAskBars()
{
var bar1 = new QuoteBar
{
Bid = new Bar(3m, 4m, 1m, 2m),
Ask = new Bar(3.1m, 4.1m, 1.1m, 2.1m),
};
var bar2 = new QuoteBar
{
Bid = null,
Ask = null,
};
var data = new[] { bar1, bar2 }.ToList();
var enumerator = data.GetEnumerator();
var fillForwardEnumerator = new QuoteBarFillForwardEnumerator(enumerator);
// 9:31
Assert.IsTrue(fillForwardEnumerator.MoveNext());
var quoteBar1 = (QuoteBar)fillForwardEnumerator.Current;
Assert.AreSame(bar1.Bid, quoteBar1.Bid);
Assert.AreSame(bar1.Ask, quoteBar1.Ask);
// 9:32
Assert.IsTrue(fillForwardEnumerator.MoveNext());
var quoteBar2 = (QuoteBar)fillForwardEnumerator.Current;
Assert.AreSame(quoteBar1.Bid, quoteBar2.Bid);
Assert.AreSame(quoteBar1.Ask, quoteBar2.Ask);
fillForwardEnumerator.Dispose();
}
/// <summary>
/// Creates a subscription to process the history request
/// </summary>
private static Subscription CreateSubscription(HistoryRequest request, IEnumerable <BaseData> history)
{
// data reader expects these values in local times
var start = request.StartTimeUtc.ConvertFromUtc(request.ExchangeHours.TimeZone);
var end = request.EndTimeUtc.ConvertFromUtc(request.ExchangeHours.TimeZone);
var config = new SubscriptionDataConfig(request.DataType,
request.Symbol,
request.Resolution,
request.DataTimeZone,
request.ExchangeHours.TimeZone,
request.FillForwardResolution.HasValue,
request.IncludeExtendedMarketHours,
false,
request.IsCustomData,
request.TickType,
true,
request.DataNormalizationMode
);
var security = new Security(
request.ExchangeHours,
config,
new Cash(Currencies.NullCurrency, 0, 1m),
SymbolProperties.GetDefault(Currencies.NullCurrency),
ErrorCurrencyConverter.Instance,
RegisteredSecurityDataTypesProvider.Null,
new SecurityCache()
);
var reader = history.GetEnumerator();
// optionally apply fill forward behavior
if (request.FillForwardResolution.HasValue)
{
// copy forward Bid/Ask bars for QuoteBars
if (request.DataType == typeof(QuoteBar))
{
reader = new QuoteBarFillForwardEnumerator(reader);
}
var readOnlyRef = Ref.CreateReadOnly(() => request.FillForwardResolution.Value.ToTimeSpan());
reader = new FillForwardEnumerator(reader, security.Exchange, readOnlyRef, security.IsExtendedMarketHours, end, config.Increment, config.DataTimeZone, start);
}
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, start, end);
var subscriptionDataEnumerator = new SubscriptionDataEnumerator(config, security.Exchange.Hours, timeZoneOffsetProvider, reader);
var subscriptionRequest = new SubscriptionRequest(false, null, security, config, start, end);
return(new Subscription(subscriptionRequest, subscriptionDataEnumerator, timeZoneOffsetProvider));
}
/// <summary>
/// Will add a fill forward enumerator if requested
/// </summary>
protected IEnumerator <BaseData> TryAddFillForwardEnumerator(SubscriptionRequest request, IEnumerator <BaseData> enumerator, bool fillForward)
{
// optionally apply fill forward logic, but never for tick data
if (fillForward && request.Configuration.Resolution != Resolution.Tick)
{
// copy forward Bid/Ask bars for QuoteBars
if (request.Configuration.Type == typeof(QuoteBar))
{
enumerator = new QuoteBarFillForwardEnumerator(enumerator);
}
var fillForwardResolution = _subscriptions.UpdateAndGetFillForwardResolution(request.Configuration);
enumerator = new FillForwardEnumerator(enumerator, request.Security.Exchange, fillForwardResolution,
request.Configuration.ExtendedMarketHours, request.EndTimeLocal, request.Configuration.Resolution.ToTimeSpan(), request.Configuration.DataTimeZone);
}
return(enumerator);
}
/// <summary>
/// Creates a subscription to process the request
/// </summary>
private Subscription CreateSubscription(HistoryRequest request, DateTime startUtc, DateTime endUtc)
{
// data reader expects these values in local times
var startTimeLocal = startUtc.ConvertFromUtc(request.ExchangeHours.TimeZone);
var endTimeLocal = endUtc.ConvertFromUtc(request.ExchangeHours.TimeZone);
var config = new SubscriptionDataConfig(request.DataType,
request.Symbol,
request.Resolution,
request.DataTimeZone,
request.ExchangeHours.TimeZone,
request.FillForwardResolution.HasValue,
request.IncludeExtendedMarketHours,
false,
request.IsCustomData,
request.TickType,
true,
request.DataNormalizationMode
);
_dataPermissionManager.AssertConfiguration(config);
var security = new Security(
request.ExchangeHours,
config,
new Cash(Currencies.NullCurrency, 0, 1m),
SymbolProperties.GetDefault(Currencies.NullCurrency),
ErrorCurrencyConverter.Instance,
RegisteredSecurityDataTypesProvider.Null,
new SecurityCache()
);
var mapFileResolver = MapFileResolver.Empty;
if (config.TickerShouldBeMapped())
{
mapFileResolver = _mapFileProvider.Get(config.Market);
var mapFile = mapFileResolver.ResolveMapFile(config.Symbol.ID.Symbol, config.Symbol.ID.Date);
config.MappedSymbol = mapFile.GetMappedSymbol(startTimeLocal, config.MappedSymbol);
}
// Tradable dates are defined with the data time zone to access the right source
var tradableDates = Time.EachTradeableDayInTimeZone(request.ExchangeHours, startTimeLocal, endTimeLocal, request.DataTimeZone, request.IncludeExtendedMarketHours);
var dataReader = new SubscriptionDataReader(config,
startTimeLocal,
endTimeLocal,
mapFileResolver,
_factorFileProvider,
tradableDates,
false,
_dataCacheProvider
);
dataReader.InvalidConfigurationDetected += (sender, args) => { OnInvalidConfigurationDetected(args); };
dataReader.NumericalPrecisionLimited += (sender, args) => { OnNumericalPrecisionLimited(args); };
dataReader.StartDateLimited += (sender, args) => { OnStartDateLimited(args); };
dataReader.DownloadFailed += (sender, args) => { OnDownloadFailed(args); };
dataReader.ReaderErrorDetected += (sender, args) => { OnReaderErrorDetected(args); };
IEnumerator <BaseData> reader = dataReader;
var intraday = GetIntradayDataEnumerator(dataReader, request);
if (intraday != null)
{
// we optionally concatenate the intraday data enumerator
reader = new ConcatEnumerator(true, reader, intraday);
}
reader = CorporateEventEnumeratorFactory.CreateEnumerators(
reader,
config,
_factorFileProvider,
dataReader,
mapFileResolver,
false,
startTimeLocal);
// optionally apply fill forward behavior
if (request.FillForwardResolution.HasValue)
{
// copy forward Bid/Ask bars for QuoteBars
if (request.DataType == typeof(QuoteBar))
{
reader = new QuoteBarFillForwardEnumerator(reader);
}
var readOnlyRef = Ref.CreateReadOnly(() => request.FillForwardResolution.Value.ToTimeSpan());
reader = new FillForwardEnumerator(reader, security.Exchange, readOnlyRef, request.IncludeExtendedMarketHours, endTimeLocal, config.Increment, config.DataTimeZone, startTimeLocal);
}
// since the SubscriptionDataReader performs an any overlap condition on the trade bar's entire
// range (time->end time) we can end up passing the incorrect data (too far past, possibly future),
// so to combat this we deliberately filter the results from the data reader to fix these cases
// which only apply to non-tick data
reader = new SubscriptionFilterEnumerator(reader, security, endTimeLocal, config.ExtendedMarketHours, false);
reader = new FilterEnumerator <BaseData>(reader, data =>
{
// allow all ticks
if (config.Resolution == Resolution.Tick)
{
return(true);
}
// filter out future data
if (data.EndTime > endTimeLocal)
{
return(false);
}
// filter out data before the start
return(data.EndTime > startTimeLocal);
});
var subscriptionRequest = new SubscriptionRequest(false, null, security, config, request.StartTimeUtc, request.EndTimeUtc);
if (_parallelHistoryRequestsEnabled)
{
return(SubscriptionUtils.CreateAndScheduleWorker(subscriptionRequest, reader, _factorFileProvider, false));
}
return(SubscriptionUtils.Create(subscriptionRequest, reader));
}
/// <summary>
/// Creates a subscription to process the request
/// </summary>
private Subscription CreateSubscription(HistoryRequest request, DateTime start, DateTime end)
{
// data reader expects these values in local times
start = start.ConvertFromUtc(request.ExchangeHours.TimeZone);
end = end.ConvertFromUtc(request.ExchangeHours.TimeZone);
var config = new SubscriptionDataConfig(request.DataType,
request.Symbol,
request.Resolution,
request.DataTimeZone,
request.ExchangeHours.TimeZone,
request.FillForwardResolution.HasValue,
request.IncludeExtendedMarketHours,
false,
request.IsCustomData,
request.TickType,
true,
request.DataNormalizationMode
);
var security = new Security(
request.ExchangeHours,
config,
new Cash(CashBook.AccountCurrency, 0, 1m),
SymbolProperties.GetDefault(CashBook.AccountCurrency),
ErrorCurrencyConverter.Instance
);
var dataReader = new SubscriptionDataReader(config,
start,
end,
config.SecurityType == SecurityType.Equity ? _mapFileProvider.Get(config.Market) : MapFileResolver.Empty,
_factorFileProvider,
Time.EachTradeableDay(request.ExchangeHours, start, end),
false,
_dataCacheProvider,
false
);
dataReader.InvalidConfigurationDetected += (sender, args) => { OnInvalidConfigurationDetected(new InvalidConfigurationDetectedEventArgs(args.Message)); };
dataReader.NumericalPrecisionLimited += (sender, args) => { OnNumericalPrecisionLimited(new NumericalPrecisionLimitedEventArgs(args.Message)); };
dataReader.DownloadFailed += (sender, args) => { OnDownloadFailed(new DownloadFailedEventArgs(args.Message, args.StackTrace)); };
dataReader.ReaderErrorDetected += (sender, args) => { OnReaderErrorDetected(new ReaderErrorDetectedEventArgs(args.Message, args.StackTrace)); };
dataReader.Initialize();
IEnumerator <BaseData> reader = dataReader;
// optionally apply fill forward behavior
if (request.FillForwardResolution.HasValue)
{
// copy forward Bid/Ask bars for QuoteBars
if (request.DataType == typeof(QuoteBar))
{
reader = new QuoteBarFillForwardEnumerator(reader);
}
var readOnlyRef = Ref.CreateReadOnly(() => request.FillForwardResolution.Value.ToTimeSpan());
reader = new FillForwardEnumerator(reader, security.Exchange, readOnlyRef, security.IsExtendedMarketHours, end, config.Increment, config.DataTimeZone);
}
// since the SubscriptionDataReader performs an any overlap condition on the trade bar's entire
// range (time->end time) we can end up passing the incorrect data (too far past, possibly future),
// so to combat this we deliberately filter the results from the data reader to fix these cases
// which only apply to non-tick data
reader = new SubscriptionFilterEnumerator(reader, security, end);
reader = new FilterEnumerator <BaseData>(reader, data =>
{
// allow all ticks
if (config.Resolution == Resolution.Tick)
{
return(true);
}
// filter out future data
if (data.EndTime > end)
{
return(false);
}
// filter out data before the start
return(data.EndTime > start);
});
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, start, end);
var subscriptionDataEnumerator = SubscriptionData.Enumerator(config, security, timeZoneOffsetProvider, reader);
var subscriptionRequest = new SubscriptionRequest(false, null, security, config, start, end);
return(new Subscription(subscriptionRequest, subscriptionDataEnumerator, timeZoneOffsetProvider));
}
/// <summary>
/// Creates a subscription to process the request
/// </summary>
private Subscription CreateSubscription(HistoryRequest request)
{
var config = request.ToSubscriptionDataConfig();
DataPermissionManager.AssertConfiguration(config, request.StartTimeLocal, request.EndTimeLocal);
var security = new Security(
request.ExchangeHours,
config,
new Cash(Currencies.NullCurrency, 0, 1m),
SymbolProperties.GetDefault(Currencies.NullCurrency),
ErrorCurrencyConverter.Instance,
RegisteredSecurityDataTypesProvider.Null,
new SecurityCache()
);
var dataReader = new SubscriptionDataReader(config,
request,
_mapFileProvider,
_factorFileProvider,
_dataCacheProvider,
_dataProvider
);
dataReader.InvalidConfigurationDetected += (sender, args) => { OnInvalidConfigurationDetected(args); };
dataReader.NumericalPrecisionLimited += (sender, args) => { OnNumericalPrecisionLimited(args); };
dataReader.StartDateLimited += (sender, args) => { OnStartDateLimited(args); };
dataReader.DownloadFailed += (sender, args) => { OnDownloadFailed(args); };
dataReader.ReaderErrorDetected += (sender, args) => { OnReaderErrorDetected(args); };
IEnumerator <BaseData> reader = dataReader;
var intraday = GetIntradayDataEnumerator(dataReader, request);
if (intraday != null)
{
// we optionally concatenate the intraday data enumerator
reader = new ConcatEnumerator(true, reader, intraday);
}
reader = CorporateEventEnumeratorFactory.CreateEnumerators(
reader,
config,
_factorFileProvider,
dataReader,
_mapFileProvider,
request.StartTimeLocal);
// optionally apply fill forward behavior
if (request.FillForwardResolution.HasValue)
{
// copy forward Bid/Ask bars for QuoteBars
if (request.DataType == typeof(QuoteBar))
{
reader = new QuoteBarFillForwardEnumerator(reader);
}
var readOnlyRef = Ref.CreateReadOnly(() => request.FillForwardResolution.Value.ToTimeSpan());
reader = new FillForwardEnumerator(reader, security.Exchange, readOnlyRef, request.IncludeExtendedMarketHours, request.EndTimeLocal, config.Increment, config.DataTimeZone);
}
// since the SubscriptionDataReader performs an any overlap condition on the trade bar's entire
// range (time->end time) we can end up passing the incorrect data (too far past, possibly future),
// so to combat this we deliberately filter the results from the data reader to fix these cases
// which only apply to non-tick data
reader = new SubscriptionFilterEnumerator(reader, security, request.EndTimeLocal, config.ExtendedMarketHours, false, request.ExchangeHours);
reader = new FilterEnumerator <BaseData>(reader, data =>
{
// allow all ticks
if (config.Resolution == Resolution.Tick)
{
return(true);
}
// filter out all aux data
if (data.DataType == MarketDataType.Auxiliary)
{
return(false);
}
// filter out future data
if (data.EndTime > request.EndTimeLocal)
{
return(false);
}
// filter out data before the start
return(data.EndTime > request.StartTimeLocal);
});
var subscriptionRequest = new SubscriptionRequest(false, null, security, config, request.StartTimeUtc, request.EndTimeUtc);
if (_parallelHistoryRequestsEnabled)
{
return(SubscriptionUtils.CreateAndScheduleWorker(subscriptionRequest, reader, _factorFileProvider, false));
}
return(SubscriptionUtils.Create(subscriptionRequest, reader));
}
