/// <summary>
/// Creates a <see cref="SubscriptionDataReader"/> to read the specified request
/// </summary>
/// <param name="request">The subscription request to be read</param>
/// <param name="dataProvider">Provider used to get data when it is not present on disk</param>
/// <returns>An enumerator reading the subscription request</returns>
public IEnumerator <BaseData> CreateEnumerator(SubscriptionRequest request, IDataProvider dataProvider)
{
var mapFileResolver = request.Configuration.TickerShouldBeMapped()
? _mapFileProvider.Get(request.Security.Symbol.ID.Market)
: MapFileResolver.Empty;
var dataReader = new SubscriptionDataReader(request.Configuration,
request.StartTimeLocal,
request.EndTimeLocal,
mapFileResolver,
_factorFileProvider,
_tradableDaysProvider(request),
_isLiveMode,
_zipDataCacheProvider
);
dataReader.InvalidConfigurationDetected += (sender, args) => { _resultHandler.ErrorMessage(args.Message); };
dataReader.NumericalPrecisionLimited += (sender, args) => { _resultHandler.DebugMessage(args.Message); };
dataReader.StartDateLimited += (sender, args) => { _resultHandler.DebugMessage(args.Message); };
dataReader.DownloadFailed += (sender, args) => { _resultHandler.ErrorMessage(args.Message, args.StackTrace); };
dataReader.ReaderErrorDetected += (sender, args) => { _resultHandler.RuntimeError(args.Message, args.StackTrace); };
var result = CorporateEventEnumeratorFactory.CreateEnumerators(
dataReader,
request.Configuration,
_factorFileProvider,
dataReader,
mapFileResolver,
_includeAuxiliaryData,
request.StartTimeLocal,
_enablePriceScaling);
return(result);
}
/// <summary>
/// Gets a <see cref="FactorFile"/> instance for the specified symbol, or null if not found
/// </summary>
/// <param name="symbol">The security's symbol whose factor file we seek</param>
/// <returns>The resolved factor file, or null if not found</returns>
public FactorFile Get(Symbol symbol)
{
FactorFile factorFile;
if (_cache.TryGetValue(symbol, out factorFile))
{
return(factorFile);
}
var market = symbol.ID.Market;
// we first need to resolve the map file to get a permtick, that's how the factor files are stored
var mapFileResolver = _mapFileProvider.Get(market);
if (mapFileResolver == null)
{
return(GetFactorFile(symbol, symbol.Value, market));
}
var mapFile = mapFileResolver.ResolveMapFile(symbol.ID.Symbol, symbol.ID.Date);
if (mapFile.IsNullOrEmpty())
{
return(GetFactorFile(symbol, symbol.Value, market));
}
return(GetFactorFile(symbol, mapFile.Permtick, market));
}
private Subscription CreateSubscription(Universe universe, IResultHandler resultHandler, Security security, DateTime startTimeUtc, DateTime endTimeUtc, IReadOnlyRef <TimeSpan> fillForwardResolution)
{
var config = security.SubscriptionDataConfig;
var localStartTime = startTimeUtc.ConvertFromUtc(security.Exchange.TimeZone);
var localEndTime = endTimeUtc.ConvertFromUtc(security.Exchange.TimeZone);
var tradeableDates = Time.EachTradeableDay(security, localStartTime, localEndTime);
// ReSharper disable once PossibleMultipleEnumeration
if (!tradeableDates.Any())
{
_algorithm.Error(string.Format("No data loaded for {0} because there were no tradeable dates for this security.", security.Symbol));
return(null);
}
// get the map file resolver for this market
var mapFileResolver = _mapFileProvider.Get(config.Market);
// ReSharper disable once PossibleMultipleEnumeration
IEnumerator <BaseData> enumerator = new SubscriptionDataReader(config, localStartTime, localEndTime, resultHandler, mapFileResolver, tradeableDates, false);
// optionally apply fill forward logic, but never for tick data
if (config.FillDataForward && config.Resolution != Resolution.Tick)
{
enumerator = new FillForwardEnumerator(enumerator, security.Exchange, fillForwardResolution,
security.IsExtendedMarketHours, localEndTime, config.Resolution.ToTimeSpan());
}
// finally apply exchange/user filters
enumerator = SubscriptionFilterEnumerator.WrapForDataFeed(resultHandler, enumerator, security, localEndTime);
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, startTimeUtc, endTimeUtc);
var subscription = new Subscription(universe, security, enumerator, timeZoneOffsetProvider, startTimeUtc, endTimeUtc, false);
return(subscription);
}
public static MapFile GetMapFile(IMapFileProvider mapFileProvider, string market, string symbol)
{
var resolver = mapFileProvider.Get(market);
var mapFile = resolver.ResolveMapFile(symbol, DateTime.Today);
return(mapFile);
}
/// <summary>
/// Creates a <see cref="SubscriptionDataReader"/> to read the specified request
/// </summary>
/// <param name="request">The subscription request to be read</param>
/// <param name="dataProvider">Provider used to get data when it is not present on disk</param>
/// <returns>An enumerator reading the subscription request</returns>
public IEnumerator <BaseData> CreateEnumerator(SubscriptionRequest request, IDataProvider dataProvider)
{
var mapFileResolver = request.Configuration.SecurityType == SecurityType.Equity ||
request.Configuration.SecurityType == SecurityType.Option
? _mapFileProvider.Get(request.Security.Symbol.ID.Market)
: MapFileResolver.Empty;
var dataReader = new SubscriptionDataReader(request.Configuration,
request.StartTimeLocal,
request.EndTimeLocal,
mapFileResolver,
_factorFileProvider,
_tradableDaysProvider(request),
_isLiveMode,
_zipDataCacheProvider,
_includeAuxiliaryData
);
dataReader.InvalidConfigurationDetected += (sender, args) => { _resultHandler.ErrorMessage(args.Message); };
dataReader.NumericalPrecisionLimited += (sender, args) => { _resultHandler.DebugMessage(args.Message); };
dataReader.DownloadFailed += (sender, args) => { _resultHandler.ErrorMessage(args.Message, args.StackTrace); };
dataReader.ReaderErrorDetected += (sender, args) => { _resultHandler.RuntimeError(args.Message, args.StackTrace); };
dataReader.Initialize();
return(dataReader);
}
/// <summary>
/// Converts a SecurityDefinition to a <see cref="Symbol" />
/// </summary>
/// <param name="securityDefinition">Security definition</param>
/// <param name="tradingDate">
/// The date that the stock was being traded. This is used to resolve
/// the ticker that the stock was trading under on this date.
/// </param>
/// <returns>Symbol if matching Lean Symbol was found on the trading date, null otherwise</returns>
private Symbol SecurityDefinitionToSymbol(SecurityDefinition securityDefinition, DateTime tradingDate)
{
if (securityDefinition == null)
{
return(null);
}
var market = securityDefinition.SecurityIdentifier.Market;
var mapFileResolver = _mapFileProvider.Get(market);
// Get the first ticker the symbol traded under, and then lookup the
// trading date to get the ticker on the trading date.
var mapFile = mapFileResolver
.ResolveMapFile(securityDefinition.SecurityIdentifier.Symbol, securityDefinition.SecurityIdentifier.Date);
// The mapped ticker will be null if the map file is null or there's
// no entry found for the given trading date.
var mappedTicker = mapFile?.GetMappedSymbol(tradingDate, null);
// If we're null, then try again; get the last entry of the map file and use
// it as the Symbol we return to the caller.
mappedTicker ??= mapFile?
.LastOrDefault()?
.MappedSymbol;
return(string.IsNullOrWhiteSpace(mappedTicker)
? null
: new Symbol(securityDefinition.SecurityIdentifier, mappedTicker));
}
/// <summary>
/// Creates a <see cref="SubscriptionDataReader"/> to read the specified request
/// </summary>
/// <param name="request">The subscription request to be read</param>
/// <param name="dataProvider">Provider used to get data when it is not present on disk</param>
/// <returns>An enumerator reading the subscription request</returns>
public IEnumerator <BaseData> CreateEnumerator(SubscriptionRequest request, IDataProvider dataProvider)
{
var mapFileResolver = request.Configuration.SecurityType == SecurityType.Equity ||
request.Configuration.SecurityType == SecurityType.Option
? _mapFileProvider.Get(request.Security.Symbol.ID.Market)
: MapFileResolver.Empty;
var dataReader = new SubscriptionDataReader(request.Configuration,
request.StartTimeLocal,
request.EndTimeLocal,
mapFileResolver,
_factorFileProvider,
_tradableDaysProvider(request),
_isLiveMode,
_zipDataCacheProvider
);
dataReader.InvalidConfigurationDetected += (sender, args) => { _resultHandler.ErrorMessage(args.Message); };
dataReader.NumericalPrecisionLimited += (sender, args) => { _resultHandler.DebugMessage(args.Message); };
dataReader.DownloadFailed += (sender, args) => { _resultHandler.ErrorMessage(args.Message, args.StackTrace); };
dataReader.ReaderErrorDetected += (sender, args) => { _resultHandler.RuntimeError(args.Message, args.StackTrace); };
var enumerator = CorporateEventEnumeratorFactory.CreateEnumerators(
request.Configuration,
_factorFileProvider,
dataReader,
mapFileResolver,
_includeAuxiliaryData);
// has to be initialized after adding all the enumerators since it will execute a MoveNext
dataReader.Initialize();
return(new SynchronizingEnumerator(dataReader, enumerator));
}
/// <summary>
/// Gets a <see cref="FactorFile"/> instance for the specified symbol, or null if not found
/// </summary>
/// <param name="symbol">The security's symbol whose factor file we seek</param>
/// <returns>The resolved factor file, or null if not found</returns>
public IFactorProvider Get(Symbol symbol)
{
symbol = symbol.GetFactorFileSymbol();
IFactorProvider factorFile;
if (_cache.TryGetValue(symbol, out factorFile))
{
return(factorFile);
}
// we first need to resolve the map file to get a permtick, that's how the factor files are stored
var mapFileResolver = _mapFileProvider.Get(AuxiliaryDataKey.Create(symbol));
if (mapFileResolver == null)
{
return(GetFactorFile(symbol, symbol.Value));
}
var mapFile = mapFileResolver.ResolveMapFile(symbol);
if (mapFile.IsNullOrEmpty())
{
return(GetFactorFile(symbol, symbol.Value));
}
return(GetFactorFile(symbol, mapFile.Permtick));
}
private Subscription CreateSubscription(Universe universe, Security security, SubscriptionDataConfig config, DateTime startTimeUtc, DateTime endTimeUtc)
{
var localStartTime = startTimeUtc.ConvertFromUtc(security.Exchange.TimeZone);
var localEndTime = endTimeUtc.ConvertFromUtc(security.Exchange.TimeZone);
var tradeableDates = Time.EachTradeableDayInTimeZone(security.Exchange.Hours, localStartTime, localEndTime, config.DataTimeZone, config.ExtendedMarketHours);
// ReSharper disable once PossibleMultipleEnumeration
if (!tradeableDates.Any())
{
_algorithm.Error(string.Format("No data loaded for {0} because there were no tradeable dates for this security.", security.Symbol));
return(null);
}
// get the map file resolver for this market
var mapFileResolver = MapFileResolver.Empty;
if (config.SecurityType == SecurityType.Equity)
{
mapFileResolver = _mapFileProvider.Get(config.Market);
}
// ReSharper disable once PossibleMultipleEnumeration
var enumerator = CreateSubscriptionEnumerator(security, config, localStartTime, localEndTime, mapFileResolver, tradeableDates);
var enqueueable = new EnqueueableEnumerator <BaseData>(true);
// add this enumerator to our exchange
ScheduleEnumerator(enumerator, enqueueable, GetLowerThreshold(config.Resolution), GetUpperThreshold(config.Resolution));
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(security.Exchange.TimeZone, startTimeUtc, endTimeUtc);
var subscription = new Subscription(universe, security, enqueueable, timeZoneOffsetProvider, startTimeUtc, endTimeUtc, false);
return(subscription);
}
/// <summary>
/// Helper method to resolve the mapping file to use.
/// </summary>
/// <remarks>This method is aware of the data type being added for <see cref="SecurityType.Base"/>
/// to the <see cref="SecurityIdentifier.Symbol"/> value</remarks>
/// <param name="mapFileProvider">The map file provider</param>
/// <param name="dataConfig">The configuration to fetch the map file for</param>
/// <returns>The mapping file to use</returns>
public static MapFile ResolveMapFile(this IMapFileProvider mapFileProvider, SubscriptionDataConfig dataConfig)
{
var resolver = MapFileResolver.Empty;
if (dataConfig.TickerShouldBeMapped())
{
resolver = mapFileProvider.Get(AuxiliaryDataKey.Create(dataConfig.Symbol));
}
return(resolver.ResolveMapFile(dataConfig.Symbol, dataConfig.Type.Name, dataConfig.DataMappingMode));
}
/// <summary>
/// Resolves the first ticker/date of the security represented by <paramref name="tickerToday"/>
/// </summary>
/// <param name="mapFileProvider">The IMapFileProvider instance used for resolving map files</param>
/// <param name="tickerToday">The security's ticker as it trades today</param>
/// <param name="market">The market the security exists in</param>
/// <param name="mappingResolveDate">The date to use to resolve the map file. Default value is <see cref="DateTime.Today"/></param>
/// <returns>The security's first ticker/date if mapping data available, otherwise, the provided ticker and DefaultDate are returned</returns>
private static Tuple <string, DateTime> GetFirstTickerAndDate(IMapFileProvider mapFileProvider, string tickerToday, string market, DateTime?mappingResolveDate = null)
{
var resolver = mapFileProvider.Get(market);
var mapFile = resolver.ResolveMapFile(tickerToday, mappingResolveDate ?? DateTime.Today);
// if we have mapping data, use the first ticker/date from there, otherwise use provided ticker and DefaultDate
return(mapFile.Any()
? Tuple.Create(mapFile.FirstTicker, mapFile.FirstDate)
: Tuple.Create(tickerToday, DefaultDate));
}
/// <summary>
/// Creates the correct enumerator factory for the given request
/// </summary>
private ISubscriptionEnumeratorFactory GetEnumeratorFactory(SubscriptionRequest request)
{
if (request.IsUniverseSubscription)
{
if (request.Universe is UserDefinedUniverse)
{
// Trigger universe selection when security added/removed after Initialize
var universe = (UserDefinedUniverse)request.Universe;
universe.CollectionChanged += (sender, args) =>
{
var items =
args.Action == NotifyCollectionChangedAction.Add ? args.NewItems :
args.Action == NotifyCollectionChangedAction.Remove ? args.OldItems : null;
if (items == null || _frontierUtc == DateTime.MinValue)
{
return;
}
var symbol = items.OfType <Symbol>().FirstOrDefault();
if (symbol == null)
{
return;
}
var collection = new BaseDataCollection(_frontierUtc, symbol);
var changes = _universeSelection.ApplyUniverseSelection(universe, _frontierUtc, collection);
_algorithm.OnSecuritiesChanged(changes);
};
return(new UserDefinedUniverseSubscriptionEnumeratorFactory(request.Universe as UserDefinedUniverse, MarketHoursDatabase.FromDataFolder()));
}
if (request.Configuration.Type == typeof(CoarseFundamental))
{
return(new BaseDataCollectionSubscriptionEnumeratorFactory());
}
if (request.Configuration.Type == typeof(FineFundamental))
{
return(new FineFundamentalSubscriptionEnumeratorFactory());
}
if (request.Universe is OptionChainUniverse)
{
return(new OptionChainUniverseSubscriptionEnumeratorFactory((req, e) => ConfigureEnumerator(req, true, e)));
}
}
var mapFileResolver = request.Configuration.SecurityType == SecurityType.Equity
? _mapFileProvider.Get(request.Security.Symbol.ID.Market)
: MapFileResolver.Empty;
return(new PostCreateConfigureSubscriptionEnumeratorFactory(
new SubscriptionDataReaderSubscriptionEnumeratorFactory(_resultHandler, mapFileResolver, _factorFileProvider, false, true),
enumerator => ConfigureEnumerator(request, false, enumerator)
));
}
private string GetMappedTicker(Symbol symbol)
{
var ticker = symbol.ID.Symbol;
if (symbol.ID.SecurityType == SecurityType.Equity)
{
var mapFile = _mapFileProvider.Get(AuxiliaryDataKey.Create(symbol)).ResolveMapFile(symbol);
ticker = mapFile.GetMappedSymbol(DateTime.UtcNow, symbol.Value);
}
return(ticker);
}
private string GetMappedTicker(Symbol symbol)
{
var ticker = symbol.Value;
if (symbol.ID.SecurityType == SecurityType.Equity)
{
var mapFile = _mapFileProvider.Get(symbol.ID.Market).ResolveMapFile(symbol.ID.Symbol, symbol.ID.Date);
ticker = mapFile.GetMappedSymbol(DateTime.UtcNow, symbol.Value);
}
return(ticker);
}
/// <summary>
/// Creates the correct enumerator factory for the given request
/// </summary>
private ISubscriptionEnumeratorFactory GetEnumeratorFactory(SubscriptionRequest request)
{
if (request.IsUniverseSubscription)
{
if (request.Universe is ITimeTriggeredUniverse)
{
var universe = request.Universe as UserDefinedUniverse;
if (universe != null)
{
// Trigger universe selection when security added/removed after Initialize
universe.CollectionChanged += (sender, args) =>
{
var items =
args.Action == NotifyCollectionChangedAction.Add ? args.NewItems :
args.Action == NotifyCollectionChangedAction.Remove ? args.OldItems : null;
if (items == null)
{
return;
}
var symbol = items.OfType <Symbol>().FirstOrDefault();
if (symbol == null)
{
return;
}
var collection = new BaseDataCollection(_algorithm.UtcTime, symbol);
var changes = _universeSelection.ApplyUniverseSelection(universe, _algorithm.UtcTime, collection);
_algorithm.OnSecuritiesChanged(changes);
};
}
return(new TimeTriggeredUniverseSubscriptionEnumeratorFactory(request.Universe as ITimeTriggeredUniverse, MarketHoursDatabase.FromDataFolder()));
}
if (request.Configuration.Type == typeof(CoarseFundamental))
{
return(new BaseDataCollectionSubscriptionEnumeratorFactory());
}
if (request.Universe is OptionChainUniverse)
{
return(new OptionChainUniverseSubscriptionEnumeratorFactory((req, e) => ConfigureEnumerator(req, true, e),
_mapFileProvider.Get(request.Security.Symbol.ID.Market), _factorFileProvider));
}
if (request.Universe is FuturesChainUniverse)
{
return(new FuturesChainUniverseSubscriptionEnumeratorFactory((req, e) => ConfigureEnumerator(req, true, e)));
}
}
return(_subscriptionfactory);
}
/// <summary>
/// Gets the primary exchange for a given security identifier
/// </summary>
/// <param name="securityIdentifier">The security identifier to get the primary exchange for</param>
/// <returns>Returns the primary exchange or null if not found</returns>
public string GetPrimaryExchange(SecurityIdentifier securityIdentifier)
{
PrimaryExchange primaryExchange;
if (!_primaryExchangeBySid.TryGetValue(securityIdentifier, out primaryExchange))
{
var mapFile = _mapFileProvider.Get(securityIdentifier.Market).ResolveMapFile(securityIdentifier.Symbol, securityIdentifier.Date);
if (mapFile != null && mapFile.Any())
{
primaryExchange = mapFile.Last().PrimaryExchange;
}
_primaryExchangeBySid[securityIdentifier] = primaryExchange;
}
return(primaryExchange.ToString());
}
/// <summary>
/// Creates a <see cref="SubscriptionDataReader"/> to read the specified request
/// </summary>
/// <param name="request">The subscription request to be read</param>
/// <param name="dataProvider">Provider used to get data when it is not present on disk</param>
/// <returns>An enumerator reading the subscription request</returns>
public IEnumerator <BaseData> CreateEnumerator(SubscriptionRequest request, IDataProvider dataProvider)
{
var mapFileResolver = request.Configuration.TickerShouldBeMapped()
? _mapFileProvider.Get(request.Security.Symbol.ID.Market)
: MapFileResolver.Empty;
var dataReader = new SubscriptionDataReader(request.Configuration,
request.StartTimeLocal,
request.EndTimeLocal,
mapFileResolver,
_factorFileProvider,
_tradableDaysProvider(request),
_isLiveMode,
_zipDataCacheProvider,
dataProvider
);
dataReader.InvalidConfigurationDetected += (sender, args) => { _resultHandler.ErrorMessage(args.Message); };
dataReader.StartDateLimited += (sender, args) =>
{
// Queue this warning into our dictionary to report on dispose
if (_startDateLimitedWarnings.Count <= _startDateLimitedWarningsMaxCount)
{
_startDateLimitedWarnings.TryAdd(args.Symbol, args.Message);
}
};
dataReader.DownloadFailed += (sender, args) => { _resultHandler.ErrorMessage(args.Message, args.StackTrace); };
dataReader.ReaderErrorDetected += (sender, args) => { _resultHandler.RuntimeError(args.Message, args.StackTrace); };
dataReader.NumericalPrecisionLimited += (sender, args) =>
{
// Set a hard limit to keep this warning list from getting unnecessarily large
if (_numericalPrecisionLimitedWarnings.Count <= _numericalPrecisionLimitedWarningsMaxCount)
{
_numericalPrecisionLimitedWarnings.TryAdd(args.Symbol, args.Message);
}
};
var result = CorporateEventEnumeratorFactory.CreateEnumerators(
dataReader,
request.Configuration,
_factorFileProvider,
dataReader,
mapFileResolver,
request.StartTimeLocal,
_enablePriceScaling);
return(result);
}
/// Hydrate the <see cref="_factorFiles"/> from the latest zipped factor file on disk
private void HydrateFactorFileFromLatestZip(string market)
{
if (market != QuantConnect.Market.USA.ToLowerInvariant())
{
// don't explode for other markets which request factor files and we don't have
return;
}
// start the search with yesterday, today's file will be available tomorrow
var todayNewYork = DateTime.UtcNow.ConvertFromUtc(TimeZones.NewYork).Date;
var date = todayNewYork.AddDays(-1);
var count = 0;
do
{
var zipFileName = $"equity/{market}/factor_files/factor_files_{date:yyyyMMdd}.zip";
var factorFilePath = Path.Combine(Globals.DataFolder, zipFileName);
// Fetch a stream for our zip from our data provider
var stream = _dataProvider.Fetch(factorFilePath);
// If the file was found we can read the file
if (stream != null)
{
var mapFileResolver = _mapFileProvider.Get(market);
foreach (var keyValuePair in FactorFileZipHelper.ReadFactorFileZip(stream, mapFileResolver, market))
{
// we merge with existing, this will allow to hold multiple markets
_factorFiles[keyValuePair.Key] = keyValuePair.Value;
}
stream.DisposeSafely();
Log.Trace($"LocalZipFactorFileProvider.Get({market}): Fetched factor files for: {date.ToShortDateString()} NY");
return;
}
// Otherwise we will search back another day
Log.Debug($"LocalZipFactorFileProvider.Get(): No factor file found for date {date.ToShortDateString()}");
// prevent infinite recursion if something is wrong
if (count++ > 7)
{
throw new InvalidOperationException($"LocalZipFactorFileProvider.Get(): Could not find any factor files going all the way back to {date}");
}
date = date.AddDays(-1);
}while (true);
}
/// <summary>
/// Converts a Lean symbol instance to an InteractiveBrokers symbol
/// </summary>
/// <param name="symbol">A Lean symbol instance</param>
/// <returns>The InteractiveBrokers symbol</returns>
public string GetBrokerageSymbol(Symbol symbol)
{
if (string.IsNullOrWhiteSpace(symbol?.Value))
{
throw new ArgumentException("Invalid symbol: " + (symbol == null ? "null" : symbol.ToString()));
}
if (symbol.SecurityType != SecurityType.Equity)
{
throw new ArgumentException($"Invalid security type: {symbol.SecurityType}");
}
var mapFile = _mapFileProvider.Get(symbol.ID.Market).ResolveMapFile(symbol.ID.Symbol, symbol.ID.Date);
return(mapFile.GetMappedSymbol(DateTime.UtcNow, symbol.Value));
}
/// <summary>
/// Gets the primary exchange for a given security identifier
/// </summary>
/// <param name="securityIdentifier">The security identifier to get the primary exchange for</param>
/// <returns>Returns the primary exchange or null if not found</returns>
public Exchange GetPrimaryExchange(SecurityIdentifier securityIdentifier)
{
Exchange primaryExchange;
if (!_primaryExchangeBySid.TryGetValue(securityIdentifier, out primaryExchange))
{
var mapFile = _mapFileProvider.Get(AuxiliaryDataKey.Create(securityIdentifier))
.ResolveMapFile(securityIdentifier.Symbol, securityIdentifier.Date);
if (mapFile != null && mapFile.Any())
{
primaryExchange = mapFile.Last().PrimaryExchange;
}
_primaryExchangeBySid[securityIdentifier] = primaryExchange;
}
return(primaryExchange);
}
/// <summary>
/// Gets the list of option contracts for a given underlying symbol
/// </summary>
/// <param name="symbol">The underlying symbol</param>
/// <param name="date">The date for which to request the option chain (only used in backtesting)</param>
/// <returns>The list of option contracts</returns>
public virtual IEnumerable <Symbol> GetOptionContractList(Symbol symbol, DateTime date)
{
if (!symbol.SecurityType.HasOptions())
{
if (symbol.SecurityType.IsOption() && symbol.Underlying != null)
{
// be user friendly and take the underlying
symbol = symbol.Underlying;
}
else
{
throw new NotSupportedException($"BacktestingOptionChainProvider.GetOptionContractList(): " +
$"{nameof(SecurityType.Equity)}, {nameof(SecurityType.Future)}, or {nameof(SecurityType.Index)} is expected but was {symbol.SecurityType}");
}
}
// Resolve any mapping before requesting option contract list for equities
// Needs to be done in order for the data file key to be accurate
Symbol mappedSymbol;
if (symbol.RequiresMapping())
{
var mapFileResolver = _mapFileProvider.Get(AuxiliaryDataKey.Create(symbol));
var mapFile = mapFileResolver.ResolveMapFile(symbol);
var ticker = mapFile.GetMappedSymbol(date, symbol.Value);
mappedSymbol = symbol.UpdateMappedSymbol(ticker);
}
else
{
mappedSymbol = symbol;
}
// create a canonical option symbol for the given underlying
var canonicalSymbol = Symbol.CreateOption(
mappedSymbol,
mappedSymbol.ID.Market,
mappedSymbol.SecurityType.DefaultOptionStyle(),
default(OptionRight),
0,
SecurityIdentifier.DefaultDate);
return(GetSymbols(canonicalSymbol, date));
}
/// <summary>
/// Helper overload that will search the mapfiles to resolve the first date. This implementation
/// uses the configured <see cref="IMapFileProvider"/> via the <see cref="Composer.Instance"/>
/// </summary>
/// <param name="symbol">The symbol as it is known today</param>
/// <param name="market">The market</param>
/// <param name="mapSymbol">Specifies if symbol should be mapped using map file provider</param>
/// <param name="mapFileProvider">Specifies the IMapFileProvider to use for resolving symbols, specify null to load from Composer</param>
/// <returns>A new <see cref="SecurityIdentifier"/> representing the specified symbol today</returns>
public static SecurityIdentifier GenerateEquity(string symbol, string market, bool mapSymbol = true, IMapFileProvider mapFileProvider = null)
{
if (mapSymbol)
{
mapFileProvider = mapFileProvider ?? Composer.Instance.GetExportedValueByTypeName <IMapFileProvider>(MapFileProviderTypeName);
var resolver = mapFileProvider.Get(market);
var mapFile = resolver.ResolveMapFile(symbol, DateTime.Today);
var firstDate = mapFile.FirstDate;
if (mapFile.Any())
{
symbol = mapFile.OrderBy(x => x.Date).First().MappedSymbol;
}
return(GenerateEquity(firstDate, symbol, market));
}
else
{
return(GenerateEquity(DefaultDate, symbol, market));
}
}
/// <summary>
/// Adds a new subscription for universe selection
/// </summary>
/// <param name="request">The subscription request</param>
private Subscription CreateUniverseSubscription(SubscriptionRequest request)
{
ISubscriptionEnumeratorFactory factory = _subscriptionFactory;
if (request.Universe is ITimeTriggeredUniverse)
{
factory = new TimeTriggeredUniverseSubscriptionEnumeratorFactory(request.Universe as ITimeTriggeredUniverse,
MarketHoursDatabase.FromDataFolder(),
_timeProvider);
if (request.Universe is UserDefinedUniverse)
{
// for user defined universe we do not use a worker task, since calls to AddData can happen in any moment
// and we have to be able to inject selection data points into the enumerator
return(SubscriptionUtils.Create(request, factory.CreateEnumerator(request, _dataProvider)));
}
}
if (request.Configuration.Type == typeof(CoarseFundamental))
{
factory = new BaseDataCollectionSubscriptionEnumeratorFactory();
}
if (request.Universe is OptionChainUniverse)
{
factory = new OptionChainUniverseSubscriptionEnumeratorFactory((req) =>
{
var mapFileResolver = req.Security.Symbol.SecurityType == SecurityType.Option
? _mapFileProvider.Get(req.Configuration.Market)
: null;
var underlyingFactory = new BaseDataSubscriptionEnumeratorFactory(false, mapFileResolver, _factorFileProvider);
return(ConfigureEnumerator(req, true, underlyingFactory.CreateEnumerator(req, _dataProvider)));
});
}
if (request.Universe is FuturesChainUniverse)
{
factory = new FuturesChainUniverseSubscriptionEnumeratorFactory((req, e) => ConfigureEnumerator(req, true, e));
}
// define our data enumerator
var enumerator = factory.CreateEnumerator(request, _dataProvider);
return(SubscriptionUtils.CreateAndScheduleWorker(request, enumerator, _factorFileProvider, true));
}
/// Hydrate the <see cref="_factorFiles"/> from the latest zipped factor file on disk
private void HydrateFactorFileFromLatestZip(AuxiliaryDataKey key)
{
var market = key.Market;
// start the search with yesterday, today's file will be available tomorrow
var todayNewYork = DateTime.UtcNow.ConvertFromUtc(TimeZones.NewYork).Date;
var date = todayNewYork.AddDays(-1);
var count = 0;
do
{
var factorFilePath = FactorFileZipHelper.GetFactorFileZipFileName(market, date, key.SecurityType);
// Fetch a stream for our zip from our data provider
var stream = _dataProvider.Fetch(factorFilePath);
// If the file was found we can read the file
if (stream != null)
{
var mapFileResolver = _mapFileProvider.Get(key);
foreach (var keyValuePair in FactorFileZipHelper.ReadFactorFileZip(stream, mapFileResolver, market, key.SecurityType))
{
// we merge with existing, this will allow to hold multiple markets
_factorFiles[keyValuePair.Key] = keyValuePair.Value;
}
stream.DisposeSafely();
Log.Trace($"LocalZipFactorFileProvider.Get({market}): Fetched factor files for: {date.ToShortDateString()} NY");
return;
}
// Otherwise we will search back another day
Log.Debug($"LocalZipFactorFileProvider.Get(): No factor file found for date {date.ToShortDateString()}");
// prevent infinite recursion if something is wrong
if (count++ > 7)
{
throw new InvalidOperationException($"LocalZipFactorFileProvider.Get(): Could not find any factor files going all the way back to {date}");
}
date = date.AddDays(-1);
}while (true);
}
/// <summary>
/// Helper method to create a new instance.
/// Knows which security types should create one and determines the appropriate delisting event provider to use
/// </summary>
public static bool TryCreate(SubscriptionDataConfig dataConfig, ITimeProvider timeProvider, IDataQueueHandler dataQueueHandler,
SecurityCache securityCache, IMapFileProvider mapFileProvider, out IEnumerator <BaseData> enumerator)
{
enumerator = null;
var securityType = dataConfig.SecurityType;
if (securityType == SecurityType.FutureOption || securityType == SecurityType.Future ||
securityType == SecurityType.Option || securityType == SecurityType.Equity)
{
var mapfile = mapFileProvider.Get(dataConfig.Symbol.ID.Market).ResolveMapFile(dataConfig.Symbol, dataConfig.Type);
var delistingEventProvider = new DelistingEventProvider();
if (securityType == SecurityType.Equity)
{
delistingEventProvider = new LiveDataBasedDelistingEventProvider(dataConfig, dataQueueHandler);
}
enumerator = new LiveDelistingEventProviderEnumerator(timeProvider, dataConfig, securityCache, delistingEventProvider, mapfile);
return(true);
}
return(false);
}
/// <summary>
/// Creates a <see cref="SubscriptionDataReader"/> to read the specified request
/// </summary>
/// <param name="request">The subscription request to be read</param>
/// <param name="dataProvider">Provider used to get data when it is not present on disk</param>
/// <returns>An enumerator reading the subscription request</returns>
public IEnumerator <BaseData> CreateEnumerator(SubscriptionRequest request, IDataProvider dataProvider)
{
var mapFileResolver = request.Configuration.SecurityType == SecurityType.Equity ||
request.Configuration.SecurityType == SecurityType.Option
? _mapFileProvider.Get(request.Security.Symbol.ID.Market)
: MapFileResolver.Empty;
return(new SubscriptionDataReader(request.Configuration,
request.StartTimeLocal,
request.EndTimeLocal,
_resultHandler,
mapFileResolver,
_factorFileProvider,
_dataProvider,
_tradableDaysProvider(request),
_isLiveMode,
_zipDataCacheProvider,
_includeAuxiliaryData
));
}
public FactorFile Get(Symbol symbol)
{
FactorFile factorFile;
if (_cache.TryGetValue(symbol, out factorFile))
{
return(factorFile);
}
var market = symbol.ID.Market;
var mapFileResolver = _mapFileProvider.Get(market);
if (mapFileResolver == null)
{
return(GetFactorFile(symbol, symbol.Value, market));
}
var mapFile = mapFileResolver.ResolveMapFile(symbol.ID.Symbol, symbol.ID.Date);
if (mapFile.IsNullOrEmpty())
{
return(GetFactorFile(symbol, symbol.Value, market));
}
return(GetFactorFile(symbol, mapFile.Permtick, market));
}
/// <summary>
/// Runs this instance.
/// </summary>
/// <returns></returns>
public bool Run()
{
var startTime = DateTime.UtcNow;
var success = true;
Log.Trace($"CoarseUniverseGeneratorProgram.ProcessDailyFolder(): Processing: {_dailyDataFolder.FullName}");
var symbolsProcessed = 0;
var filesRead = 0;
var dailyFilesNotFound = 0;
var coarseFilesGenerated = 0;
var mapFileResolver = _mapFileProvider.Get(_market);
var blackListedTickers = new HashSet <string>();
if (_blackListedTickersFile.Exists)
{
blackListedTickers = File.ReadAllLines(_blackListedTickersFile.FullName).ToHashSet();
}
var marketFolder = _dailyDataFolder.Parent;
var fineFundamentalFolder = new DirectoryInfo(Path.Combine(marketFolder.FullName, "fundamental", "fine"));
if (!fineFundamentalFolder.Exists)
{
Log.Error($"CoarseUniverseGenerator.Run(): FAIL, Fine Fundamental folder not found at {fineFundamentalFolder}! ");
return(false);
}
var securityIdentifierContexts = PopulateSidContex(mapFileResolver, blackListedTickers);
var dailyPricesByTicker = new ConcurrentDictionary <string, List <TradeBar> >();
var outputCoarseContent = new ConcurrentDictionary <DateTime, List <string> >();
var parallelOptions = new ParallelOptions {
MaxDegreeOfParallelism = Environment.ProcessorCount / 2
};
try
{
Parallel.ForEach(securityIdentifierContexts, parallelOptions, sidContext =>
{
var symbol = new Symbol(sidContext.SID, sidContext.LastTicker);
var symbolCount = Interlocked.Increment(ref symbolsProcessed);
Log.Debug($"CoarseUniverseGeneratorProgram.Run(): Processing {symbol}");
var factorFile = _factorFileProvider.Get(symbol);
// Populate dailyPricesByTicker with all daily data by ticker for all tickers of this security.
foreach (var ticker in sidContext.Tickers)
{
var dailyFile = new FileInfo(Path.Combine(_dailyDataFolder.FullName, $"{ticker}.zip"));
if (!dailyFile.Exists)
{
Log.Error($"CoarseUniverseGeneratorProgram.Run(): {dailyFile} not found!");
Interlocked.Increment(ref dailyFilesNotFound);
continue;
}
if (!dailyPricesByTicker.ContainsKey(ticker))
{
dailyPricesByTicker.AddOrUpdate(ticker, ParseDailyFile(dailyFile));
Interlocked.Increment(ref filesRead);
}
}
// Look for daily data for each ticker of the actual security
for (int mapFileRowIndex = sidContext.MapFileRows.Length - 1; mapFileRowIndex >= 1; mapFileRowIndex--)
{
var ticker = sidContext.MapFileRows[mapFileRowIndex].Item2.ToLowerInvariant();
var endDate = sidContext.MapFileRows[mapFileRowIndex].Item1;
var startDate = sidContext.MapFileRows[mapFileRowIndex - 1].Item1;
List <TradeBar> tickerDailyData;
if (!dailyPricesByTicker.TryGetValue(ticker, out tickerDailyData))
{
Log.Error($"CoarseUniverseGeneratorProgram.Run(): Daily data for ticker {ticker.ToUpperInvariant()} not found!");
continue;
}
var tickerFineFundamentalFolder = Path.Combine(fineFundamentalFolder.FullName, ticker);
var fineAvailableDates = Enumerable.Empty <DateTime>();
if (Directory.Exists(tickerFineFundamentalFolder))
{
fineAvailableDates = Directory.GetFiles(tickerFineFundamentalFolder, "*.zip")
.Select(f => DateTime.ParseExact(Path.GetFileNameWithoutExtension(f), DateFormat.EightCharacter, CultureInfo.InvariantCulture))
.ToList();
}
// Get daily data only for the time the ticker was
foreach (var tradeBar in tickerDailyData.Where(tb => tb.Time >= startDate && tb.Time <= endDate))
{
var coarseRow = GenerateFactorFileRow(ticker, sidContext, factorFile, tradeBar, fineAvailableDates, fineFundamentalFolder);
outputCoarseContent.AddOrUpdate(tradeBar.Time,
new List <string> {
coarseRow
},
(time, list) =>
{
lock (list)
{
list.Add(coarseRow);
return(list);
}
});
}
}
if (symbolCount % 1000 == 0)
{
var elapsed = DateTime.UtcNow - startTime;
Log.Trace($"CoarseUniverseGeneratorProgram.Run(): Processed {symbolCount} in {elapsed:g} at {symbolCount / elapsed.TotalMinutes:F2} symbols/minute ");
}
});
_destinationFolder.Create();
var startWriting = DateTime.UtcNow;
Parallel.ForEach(outputCoarseContent, coarseByDate =>
{
var filename = $"{coarseByDate.Key.ToString(DateFormat.EightCharacter, CultureInfo.InvariantCulture)}.csv";
var filePath = Path.Combine(_destinationFolder.FullName, filename);
Log.Debug($"CoarseUniverseGeneratorProgram.Run(): Saving {filename} with {coarseByDate.Value.Count} entries.");
File.WriteAllLines(filePath, coarseByDate.Value.OrderBy(cr => cr));
var filesCount = Interlocked.Increment(ref coarseFilesGenerated);
if (filesCount % 1000 == 0)
{
var elapsed = DateTime.UtcNow - startWriting;
Log.Trace($"CoarseUniverseGeneratorProgram.Run(): Processed {filesCount} in {elapsed:g} at {filesCount / elapsed.TotalSeconds:F2} files/second ");
}
});
Log.Trace($"\n\nTotal of {coarseFilesGenerated} coarse files generated in {DateTime.UtcNow - startTime:g}:\n" +
$"\t => {filesRead} daily data files read.\n");
}
catch (Exception e)
{
Log.Error(e, $"CoarseUniverseGeneratorProgram.Run(): FAILED!");
success = false;
}
return(success);
}
/// <summary>
/// Gets the list of option contracts for a given underlying symbol
/// </summary>
/// <param name="underlyingSymbol">The underlying symbol</param>
/// <param name="date">The date for which to request the option chain (only used in backtesting)</param>
/// <returns>The list of option contracts</returns>
public IEnumerable <Symbol> GetOptionContractList(Symbol underlyingSymbol, DateTime date)
{
if (!underlyingSymbol.SecurityType.HasOptions())
{
throw new NotSupportedException($"BacktestingOptionChainProvider.GetOptionContractList(): SecurityType.Equity, SecurityType.Future, or SecurityType.Index is expected but was {underlyingSymbol.SecurityType}");
}
// Resolve any mapping before requesting option contract list for equities
// Needs to be done in order for the data file key to be accurate
Symbol mappedSymbol;
if (underlyingSymbol.SecurityType.RequiresMapping())
{
var mapFileResolver = _mapFileProvider.Get(underlyingSymbol.ID.Market);
var mapFile = mapFileResolver.ResolveMapFile(underlyingSymbol.ID.Symbol, date);
var ticker = mapFile.GetMappedSymbol(date, underlyingSymbol.Value);
mappedSymbol = underlyingSymbol.UpdateMappedSymbol(ticker);
}
else
{
mappedSymbol = underlyingSymbol;
}
// build the option contract list from the open interest zip file entry names
// create a canonical option symbol for the given underlying
var canonicalSymbol = Symbol.CreateOption(
mappedSymbol,
mappedSymbol.ID.Market,
mappedSymbol.SecurityType.DefaultOptionStyle(),
default(OptionRight),
0,
SecurityIdentifier.DefaultDate);
var zipFileName = string.Empty;
Stream stream = null;
// In order of trust-worthiness of containing the complete option chain, OpenInterest is guaranteed
// to have the complete option chain. Quotes come after open-interest
// because it's also likely to contain the option chain. Trades may be
// missing portions of the option chain, so we resort to it last.
foreach (var tickType in new[] { TickType.OpenInterest, TickType.Quote, TickType.Trade })
{
// build the zip file name and fetch it with our provider
zipFileName = LeanData.GenerateZipFilePath(Globals.DataFolder, canonicalSymbol, date, Resolution.Minute, tickType);
stream = _dataProvider.Fetch(zipFileName);
if (stream != null)
{
break;
}
}
if (stream == null)
{
Log.Trace($"BacktestingOptionChainProvider.GetOptionContractList(): File not found: {zipFileName}");
yield break;
}
// generate and return the contract symbol for each zip entry
var zipEntryNames = Compression.GetZipEntryFileNames(stream);
foreach (var zipEntryName in zipEntryNames)
{
yield return(LeanData.ReadSymbolFromZipEntry(canonicalSymbol, Resolution.Minute, zipEntryName));
}
stream.DisposeSafely();
}