private static MapFile GetMapFileToUse(
SubscriptionDataConfig config,
MapFileResolver mapFileResolver)
{
var mapFileToUse = new MapFile(config.Symbol.Value, new List<MapFileRow>());
// load up the map and factor files for equities, options, and custom data
if (config.TickerShouldBeMapped())
{
try
{
var mapFile = mapFileResolver.ResolveMapFile(config.Symbol, config.Type);
// only take the resolved map file if it has data, otherwise we'll use the empty one we defined above
if (mapFile.Any())
{
mapFileToUse = mapFile;
}
}
catch (Exception err)
{
Log.Error(err, "CorporateEventEnumeratorFactory.GetMapFileToUse():" +
" Map File: " + config.Symbol.ID + ": ");
}
}
return mapFileToUse;
}
/// <summary>
/// Helper method to create a new instance.
/// Knows which security types should create one and determines the appropriate delisting event provider to use
/// </summary>
public static bool TryCreate(SubscriptionDataConfig dataConfig, ITimeProvider timeProvider, IDataQueueHandler dataQueueHandler,
SecurityCache securityCache, IMapFileProvider mapFileProvider, IFactorFileProvider fileProvider, DateTime startTime,
out IEnumerator <BaseData> enumerator)
{
enumerator = null;
var securityType = dataConfig.SecurityType;
if (securityType.IsOption() || securityType == SecurityType.Future || securityType == SecurityType.Equity)
{
var providers = new List <ITradableDateEventProvider>
{
securityType == SecurityType.Equity
? new LiveDataBasedDelistingEventProvider(dataConfig, dataQueueHandler)
: new DelistingEventProvider()
};
if (dataConfig.TickerShouldBeMapped())
{
providers.Add(new LiveMappingEventProvider());
}
enumerator = new LiveAuxiliaryDataEnumerator(dataConfig, fileProvider, mapFileProvider,
providers.ToArray(), startTime, timeProvider, securityCache);
}
return(enumerator != null);
}
/// <summary>
/// Helper method to resolve the mapping file to use.
/// </summary>
/// <remarks>This method is aware of the data type being added for <see cref="SecurityType.Base"/>
/// to the <see cref="SecurityIdentifier.Symbol"/> value</remarks>
/// <param name="mapFileProvider">The map file provider</param>
/// <param name="dataConfig">The configuration to fetch the map file for</param>
/// <returns>The mapping file to use</returns>
public static MapFile ResolveMapFile(this IMapFileProvider mapFileProvider, SubscriptionDataConfig dataConfig)
{
var resolver = MapFileResolver.Empty;
if (dataConfig.TickerShouldBeMapped())
{
resolver = mapFileProvider.Get(AuxiliaryDataKey.Create(dataConfig.Symbol));
}
return(resolver.ResolveMapFile(dataConfig.Symbol, dataConfig.Type.Name, dataConfig.DataMappingMode));
}
/// <summary>
/// Creates a new <see cref="AuxiliaryDataEnumerator"/> that will hold the
/// corporate event providers
/// </summary>
/// <param name="rawDataEnumerator">The underlying raw data enumerator</param>
/// <param name="config">The <see cref="SubscriptionDataConfig"/></param>
/// <param name="factorFileProvider">Used for getting factor files</param>
/// <param name="tradableDayNotifier">Tradable dates provider</param>
/// <param name="mapFileProvider">The <see cref="MapFile"/> provider to use</param>
/// <param name="startTime">Start date for the data request</param>
/// <param name="enablePriceScaling">Applies price factor</param>
/// <returns>The new auxiliary data enumerator</returns>
public static IEnumerator <BaseData> CreateEnumerators(
IEnumerator <BaseData> rawDataEnumerator,
SubscriptionDataConfig config,
IFactorFileProvider factorFileProvider,
ITradableDatesNotifier tradableDayNotifier,
IMapFileProvider mapFileProvider,
DateTime startTime,
bool enablePriceScaling = true)
{
var tradableEventProviders = new List <ITradableDateEventProvider>();
if (config.EmitSplitsAndDividends())
{
tradableEventProviders.Add(new SplitEventProvider());
tradableEventProviders.Add(new DividendEventProvider());
}
if (config.TickerShouldBeMapped())
{
tradableEventProviders.Add(new MappingEventProvider());
}
tradableEventProviders.Add(new DelistingEventProvider());
var enumerator = new AuxiliaryDataEnumerator(
config,
factorFileProvider,
mapFileProvider,
tradableEventProviders.ToArray(),
tradableDayNotifier,
startTime);
// avoid price scaling for backtesting; calculate it directly in worker
// and allow subscription to extract the the data depending on config data mode
var dataEnumerator = rawDataEnumerator;
if (enablePriceScaling && config.PricesShouldBeScaled())
{
dataEnumerator = new PriceScaleFactorEnumerator(
rawDataEnumerator,
config,
factorFileProvider);
}
return(new SynchronizingBaseDataEnumerator(dataEnumerator, enumerator));
}
/// <summary>
/// Creates a subscription to process the request
/// </summary>
private Subscription CreateSubscription(HistoryRequest request, DateTime startUtc, DateTime endUtc)
{
// data reader expects these values in local times
var startTimeLocal = startUtc.ConvertFromUtc(request.ExchangeHours.TimeZone);
var endTimeLocal = endUtc.ConvertFromUtc(request.ExchangeHours.TimeZone);
var config = new SubscriptionDataConfig(request.DataType,
request.Symbol,
request.Resolution,
request.DataTimeZone,
request.ExchangeHours.TimeZone,
request.FillForwardResolution.HasValue,
request.IncludeExtendedMarketHours,
false,
request.IsCustomData,
request.TickType,
true,
request.DataNormalizationMode
);
_dataPermissionManager.AssertConfiguration(config);
var security = new Security(
request.ExchangeHours,
config,
new Cash(Currencies.NullCurrency, 0, 1m),
SymbolProperties.GetDefault(Currencies.NullCurrency),
ErrorCurrencyConverter.Instance,
RegisteredSecurityDataTypesProvider.Null,
new SecurityCache()
);
var mapFileResolver = MapFileResolver.Empty;
if (config.TickerShouldBeMapped())
{
mapFileResolver = _mapFileProvider.Get(config.Market);
var mapFile = mapFileResolver.ResolveMapFile(config.Symbol.ID.Symbol, config.Symbol.ID.Date);
config.MappedSymbol = mapFile.GetMappedSymbol(startTimeLocal, config.MappedSymbol);
}
// Tradable dates are defined with the data time zone to access the right source
var tradableDates = Time.EachTradeableDayInTimeZone(request.ExchangeHours, startTimeLocal, endTimeLocal, request.DataTimeZone, request.IncludeExtendedMarketHours);
var dataReader = new SubscriptionDataReader(config,
startTimeLocal,
endTimeLocal,
mapFileResolver,
_factorFileProvider,
tradableDates,
false,
_dataCacheProvider
);
dataReader.InvalidConfigurationDetected += (sender, args) => { OnInvalidConfigurationDetected(args); };
dataReader.NumericalPrecisionLimited += (sender, args) => { OnNumericalPrecisionLimited(args); };
dataReader.StartDateLimited += (sender, args) => { OnStartDateLimited(args); };
dataReader.DownloadFailed += (sender, args) => { OnDownloadFailed(args); };
dataReader.ReaderErrorDetected += (sender, args) => { OnReaderErrorDetected(args); };
IEnumerator <BaseData> reader = dataReader;
var intraday = GetIntradayDataEnumerator(dataReader, request);
if (intraday != null)
{
// we optionally concatenate the intraday data enumerator
reader = new ConcatEnumerator(true, reader, intraday);
}
reader = CorporateEventEnumeratorFactory.CreateEnumerators(
reader,
config,
_factorFileProvider,
dataReader,
mapFileResolver,
false,
startTimeLocal);
// optionally apply fill forward behavior
if (request.FillForwardResolution.HasValue)
{
// copy forward Bid/Ask bars for QuoteBars
if (request.DataType == typeof(QuoteBar))
{
reader = new QuoteBarFillForwardEnumerator(reader);
}
var readOnlyRef = Ref.CreateReadOnly(() => request.FillForwardResolution.Value.ToTimeSpan());
reader = new FillForwardEnumerator(reader, security.Exchange, readOnlyRef, request.IncludeExtendedMarketHours, endTimeLocal, config.Increment, config.DataTimeZone, startTimeLocal);
}
// since the SubscriptionDataReader performs an any overlap condition on the trade bar's entire
// range (time->end time) we can end up passing the incorrect data (too far past, possibly future),
// so to combat this we deliberately filter the results from the data reader to fix these cases
// which only apply to non-tick data
reader = new SubscriptionFilterEnumerator(reader, security, endTimeLocal, config.ExtendedMarketHours, false);
reader = new FilterEnumerator <BaseData>(reader, data =>
{
// allow all ticks
if (config.Resolution == Resolution.Tick)
{
return(true);
}
// filter out future data
if (data.EndTime > endTimeLocal)
{
return(false);
}
// filter out data before the start
return(data.EndTime > startTimeLocal);
});
var subscriptionRequest = new SubscriptionRequest(false, null, security, config, request.StartTimeUtc, request.EndTimeUtc);
if (_parallelHistoryRequestsEnabled)
{
return(SubscriptionUtils.CreateAndScheduleWorker(subscriptionRequest, reader, _factorFileProvider, false));
}
return(SubscriptionUtils.Create(subscriptionRequest, reader));
}
/// <summary>
/// Initializes the <see cref="SubscriptionDataReader"/> instance
/// </summary>
/// <remarks>Should be called after all consumers of <see cref="NewTradableDate"/> event are set,
/// since it will produce events.</remarks>
public void Initialize()
{
if (_initialized)
{
return;
}
//Save the type of data we'll be getting from the source.
try
{
_dataFactory = _config.Type.GetBaseDataInstance();
}
catch (ArgumentException exception)
{
OnInvalidConfigurationDetected(new InvalidConfigurationDetectedEventArgs(exception.Message));
_endOfStream = true;
return;
}
//If its quandl set the access token in data factory:
var quandl = _dataFactory as Quandl;
if (quandl != null)
{
if (!Quandl.IsAuthCodeSet)
{
Quandl.SetAuthCode(Config.Get("quandl-auth-token"));
}
}
// If Tiingo data, set the access token in data factory
var tiingo = _dataFactory as TiingoDailyData;
if (tiingo != null)
{
if (!Tiingo.IsAuthCodeSet)
{
Tiingo.SetAuthCode(Config.Get("tiingo-auth-token"));
}
}
// If USEnergyInformation data, set the access token in data factory
var energyInformation = _dataFactory as USEnergyInformation;
if (energyInformation != null)
{
if (!USEnergyInformation.IsAuthCodeSet)
{
USEnergyInformation.SetAuthCode(Config.Get("us-energy-information-auth-token"));
}
}
_factorFile = new FactorFile(_config.Symbol.Value, new List <FactorFileRow>());
_mapFile = new MapFile(_config.Symbol.Value, new List <MapFileRow>());
// load up the map files for equities, options, and custom data if it supports it.
// Only load up factor files for equities
if (_config.TickerShouldBeMapped())
{
try
{
var mapFile = _mapFileResolver.ResolveMapFile(_config.Symbol, _config.Type);
// only take the resolved map file if it has data, otherwise we'll use the empty one we defined above
if (mapFile.Any())
{
_mapFile = mapFile;
}
if (!_config.IsCustomData && _config.SecurityType != SecurityType.Option)
{
var factorFile = _factorFileProvider.Get(_config.Symbol);
_hasScaleFactors = factorFile != null;
if (_hasScaleFactors)
{
_factorFile = factorFile;
// if factor file has minimum date, update start period if before minimum date
if (!_isLiveMode && _factorFile != null && _factorFile.FactorFileMinimumDate.HasValue)
{
if (_periodStart < _factorFile.FactorFileMinimumDate.Value)
{
_periodStart = _factorFile.FactorFileMinimumDate.Value;
OnNumericalPrecisionLimited(
new NumericalPrecisionLimitedEventArgs(
$"Data for symbol {_config.Symbol.Value} has been limited due to numerical precision issues in the factor file. " +
$"The starting date has been set to {_factorFile.FactorFileMinimumDate.Value.ToShortDateString()}."));
}
}
}
}
}
catch (Exception err)
{
Log.Error(err, "Fetching Price/Map Factors: " + _config.Symbol.ID + ": ");
}
}
// Estimate delisting date.
switch (_config.Symbol.ID.SecurityType)
{
case SecurityType.Future:
_delistingDate = _config.Symbol.ID.Date;
break;
case SecurityType.Option:
_delistingDate = OptionSymbol.GetLastDayOfTrading(_config.Symbol);
break;
default:
_delistingDate = _mapFile.DelistingDate;
break;
}
// adding a day so we stop at EOD
_delistingDate = _delistingDate.AddDays(1);
_subscriptionFactoryEnumerator = ResolveDataEnumerator(true);
_initialized = true;
}
/// <summary>
/// Creates a subscription to process the request
/// </summary>
private Subscription CreateSubscription(HistoryRequest request, DateTime start, DateTime end)
{
// data reader expects these values in local times
start = start.ConvertFromUtc(request.ExchangeHours.TimeZone);
end = end.ConvertFromUtc(request.ExchangeHours.TimeZone);
var config = new SubscriptionDataConfig(request.DataType,
request.Symbol,
request.Resolution,
request.DataTimeZone,
request.ExchangeHours.TimeZone,
request.FillForwardResolution.HasValue,
request.IncludeExtendedMarketHours,
false,
request.IsCustomData,
request.TickType,
true,
request.DataNormalizationMode
);
var security = new Security(
request.ExchangeHours,
config,
new Cash(Currencies.NullCurrency, 0, 1m),
SymbolProperties.GetDefault(Currencies.NullCurrency),
ErrorCurrencyConverter.Instance,
RegisteredSecurityDataTypesProvider.Null
);
var mapFileResolver = MapFileResolver.Empty;
if (config.TickerShouldBeMapped())
{
mapFileResolver = _mapFileProvider.Get(config.Market);
var mapFile = mapFileResolver.ResolveMapFile(config.Symbol.ID.Symbol, config.Symbol.ID.Date);
config.MappedSymbol = mapFile.GetMappedSymbol(start, config.MappedSymbol);
}
var dataReader = new SubscriptionDataReader(config,
start,
end,
mapFileResolver,
_factorFileProvider,
Time.EachTradeableDay(request.ExchangeHours, start, end),
false,
_dataCacheProvider
);
dataReader.InvalidConfigurationDetected += (sender, args) => { OnInvalidConfigurationDetected(new InvalidConfigurationDetectedEventArgs(args.Message)); };
dataReader.NumericalPrecisionLimited += (sender, args) => { OnNumericalPrecisionLimited(new NumericalPrecisionLimitedEventArgs(args.Message)); };
dataReader.DownloadFailed += (sender, args) => { OnDownloadFailed(new DownloadFailedEventArgs(args.Message, args.StackTrace)); };
dataReader.ReaderErrorDetected += (sender, args) => { OnReaderErrorDetected(new ReaderErrorDetectedEventArgs(args.Message, args.StackTrace)); };
var reader = CorporateEventEnumeratorFactory.CreateEnumerators(
dataReader,
config,
_factorFileProvider,
dataReader,
mapFileResolver,
false);
// has to be initialized after adding all the enumerators since it will execute a MoveNext
dataReader.Initialize();
// optionally apply fill forward behavior
if (request.FillForwardResolution.HasValue)
{
// copy forward Bid/Ask bars for QuoteBars
if (request.DataType == typeof(QuoteBar))
{
reader = new QuoteBarFillForwardEnumerator(reader);
}
var readOnlyRef = Ref.CreateReadOnly(() => request.FillForwardResolution.Value.ToTimeSpan());
reader = new FillForwardEnumerator(reader, security.Exchange, readOnlyRef, security.IsExtendedMarketHours, end, config.Increment, config.DataTimeZone, start);
}
// since the SubscriptionDataReader performs an any overlap condition on the trade bar's entire
// range (time->end time) we can end up passing the incorrect data (too far past, possibly future),
// so to combat this we deliberately filter the results from the data reader to fix these cases
// which only apply to non-tick data
reader = new SubscriptionFilterEnumerator(reader, security, end);
reader = new FilterEnumerator <BaseData>(reader, data =>
{
// allow all ticks
if (config.Resolution == Resolution.Tick)
{
return(true);
}
// filter out future data
if (data.EndTime > end)
{
return(false);
}
// filter out data before the start
return(data.EndTime > start);
});
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, start, end);
var subscriptionDataEnumerator = new SubscriptionDataEnumerator(config, security.Exchange.Hours, timeZoneOffsetProvider, reader);
var subscriptionRequest = new SubscriptionRequest(false, null, security, config, start, end);
return(new Subscription(subscriptionRequest, subscriptionDataEnumerator, timeZoneOffsetProvider));
}
