/// <summary>
/// Gets a <see cref="FactorFile"/> instance for the specified symbol, or null if not found
/// </summary>
/// <param name="symbol">The security's symbol whose factor file we seek</param>
/// <returns>The resolved factor file, or null if not found</returns>
public IFactorProvider Get(Symbol symbol)
{
symbol = symbol.GetFactorFileSymbol();
IFactorProvider factorFile;
if (_cache.TryGetValue(symbol, out factorFile))
{
return(factorFile);
}
// we first need to resolve the map file to get a permtick, that's how the factor files are stored
var mapFileResolver = _mapFileProvider.Get(AuxiliaryDataKey.Create(symbol));
if (mapFileResolver == null)
{
return(GetFactorFile(symbol, symbol.Value));
}
var mapFile = mapFileResolver.ResolveMapFile(symbol);
if (mapFile.IsNullOrEmpty())
{
return(GetFactorFile(symbol, symbol.Value));
}
return(GetFactorFile(symbol, mapFile.Permtick));
}
/// <summary>
/// Helper method to resolve the mapping file to use.
/// </summary>
/// <remarks>This method is aware of the data type being added for <see cref="SecurityType.Base"/>
/// to the <see cref="SecurityIdentifier.Symbol"/> value</remarks>
/// <param name="mapFileProvider">The map file provider</param>
/// <param name="dataConfig">The configuration to fetch the map file for</param>
/// <returns>The mapping file to use</returns>
public static MapFile ResolveMapFile(this IMapFileProvider mapFileProvider, SubscriptionDataConfig dataConfig)
{
var resolver = MapFileResolver.Empty;
if (dataConfig.TickerShouldBeMapped())
{
resolver = mapFileProvider.Get(AuxiliaryDataKey.Create(dataConfig.Symbol));
}
return(resolver.ResolveMapFile(dataConfig.Symbol, dataConfig.Type.Name, dataConfig.DataMappingMode));
}
/// <summary>
/// Gets a <see cref="MapFileResolver"/> representing all the map files for the specified market
/// </summary>
/// <param name="auxiliaryDataKey">Key used to fetch a map file resolver. Specifying market and security type</param>
/// <returns>A <see cref="MapFileResolver"/> containing all map files for the specified market</returns>
public MapFileResolver Get(AuxiliaryDataKey auxiliaryDataKey)
{
MapFileResolver result;
// we use a lock so that only 1 thread loads the map file resolver while the rest wait
// else we could have multiple threads loading the map file resolver at the same time!
lock (_lock)
{
if (!_cache.TryGetValue(auxiliaryDataKey, out result))
{
_cache[auxiliaryDataKey] = result = GetMapFileResolver(auxiliaryDataKey);
}
}
return(result);
}
/// <summary>
/// Gets the primary exchange for a given security identifier
/// </summary>
/// <param name="securityIdentifier">The security identifier to get the primary exchange for</param>
/// <returns>Returns the primary exchange or null if not found</returns>
public Exchange GetPrimaryExchange(SecurityIdentifier securityIdentifier)
{
Exchange primaryExchange;
if (!_primaryExchangeBySid.TryGetValue(securityIdentifier, out primaryExchange))
{
var mapFile = _mapFileProvider.Get(AuxiliaryDataKey.Create(securityIdentifier))
.ResolveMapFile(securityIdentifier.Symbol, securityIdentifier.Date);
if (mapFile != null && mapFile.Any())
{
primaryExchange = mapFile.Last().PrimaryExchange;
}
_primaryExchangeBySid[securityIdentifier] = primaryExchange;
}
return(primaryExchange);
}
/// Hydrate the <see cref="_factorFiles"/> from the latest zipped factor file on disk
private void HydrateFactorFileFromLatestZip(AuxiliaryDataKey key)
{
var market = key.Market;
// start the search with yesterday, today's file will be available tomorrow
var todayNewYork = DateTime.UtcNow.ConvertFromUtc(TimeZones.NewYork).Date;
var date = todayNewYork.AddDays(-1);
var count = 0;
do
{
var factorFilePath = FactorFileZipHelper.GetFactorFileZipFileName(market, date, key.SecurityType);
// Fetch a stream for our zip from our data provider
var stream = _dataProvider.Fetch(factorFilePath);
// If the file was found we can read the file
if (stream != null)
{
var mapFileResolver = _mapFileProvider.Get(key);
foreach (var keyValuePair in FactorFileZipHelper.ReadFactorFileZip(stream, mapFileResolver, market, key.SecurityType))
{
// we merge with existing, this will allow to hold multiple markets
_factorFiles[keyValuePair.Key] = keyValuePair.Value;
}
stream.DisposeSafely();
Log.Trace($"LocalZipFactorFileProvider.Get({market}): Fetched factor files for: {date.ToShortDateString()} NY");
return;
}
// Otherwise we will search back another day
Log.Debug($"LocalZipFactorFileProvider.Get(): No factor file found for date {date.ToShortDateString()}");
// prevent infinite recursion if something is wrong
if (count++ > 7)
{
throw new InvalidOperationException($"LocalZipFactorFileProvider.Get(): Could not find any factor files going all the way back to {date}");
}
date = date.AddDays(-1);
}while (true);
}
private MapFileResolver GetMapFileResolver(AuxiliaryDataKey key)
{
var securityType = key.SecurityType;
var market = key.Market;
var mapFileDirectory = MapFile.GetMapFilePath(market, securityType);
if (!Directory.Exists(mapFileDirectory))
{
// only write this message once per application instance
if (Interlocked.CompareExchange(ref _wroteTraceStatement, 1, 0) == 0)
{
Log.Error($"LocalDiskMapFileProvider.GetMapFileResolver({market}): " +
$"The specified directory does not exist: {mapFileDirectory}"
);
}
return(MapFileResolver.Empty);
}
return(new MapFileResolver(MapFile.GetMapFiles(mapFileDirectory, market, securityType, _dataProvider)));
}
/// <summary>
/// Gets a <see cref="FactorFile"/> instance for the specified symbol, or null if not found
/// </summary>
/// <param name="symbol">The security's symbol whose factor file we seek</param>
/// <returns>The resolved factor file, or null if not found</returns>
public IFactorProvider Get(Symbol symbol)
{
symbol = symbol.GetFactorFileSymbol();
var key = AuxiliaryDataKey.Create(symbol);
lock (_lock)
{
if (!_seededMarket.ContainsKey(key))
{
HydrateFactorFileFromLatestZip(key);
_seededMarket[key] = true;
}
IFactorProvider factorFile;
if (!_factorFiles.TryGetValue(symbol, out factorFile))
{
// Could not find factor file for symbol
Log.Error($"LocalZipFactorFileProvider.Get({symbol}): No factor file found.");
_factorFiles[symbol] = factorFile = symbol.GetEmptyFactorFile();
}
return(factorFile);
}
}
private MapFileResolver GetMapFileResolver(AuxiliaryDataKey auxiliaryDataKey)
{
var market = auxiliaryDataKey.Market;
var timestamp = DateTime.UtcNow.ConvertFromUtc(TimeZones.NewYork);
var todayNewYork = timestamp.Date;
var yesterdayNewYork = todayNewYork.AddDays(-1);
// start the search with yesterday, today's file will be available tomorrow
var count = 0;
var date = yesterdayNewYork;
do
{
var zipFileName = MapFileZipHelper.GetMapFileZipFileName(market, date, auxiliaryDataKey.SecurityType);
// Fetch a stream for our zip from our data provider
var stream = _dataProvider.Fetch(zipFileName);
// If we found a file we can read it
if (stream != null)
{
Log.Trace("LocalZipMapFileProvider.Get({0}): Fetched map files for: {1} NY", market, date.ToShortDateString());
var result = new MapFileResolver(MapFileZipHelper.ReadMapFileZip(stream, market, auxiliaryDataKey.SecurityType));
stream.DisposeSafely();
return(result);
}
// prevent infinite recursion if something is wrong
if (count++ > 30)
{
throw new InvalidOperationException($"LocalZipMapFileProvider couldn't find any map files going all the way back to {date}");
}
date = date.AddDays(-1);
}while (true);
}
/// <summary>
/// Gets a <see cref="MapFileResolver"/> representing all the map
/// files for the specified market
/// </summary>
/// <param name="auxiliaryDataKey">Key used to fetch a map file resolver. Specifying market and security type</param>
/// <returns>A <see cref="MapFileRow"/> containing all map files for the specified market</returns>
public MapFileResolver Get(AuxiliaryDataKey auxiliaryDataKey)
{
return(_cache.GetOrAdd(auxiliaryDataKey, GetMapFileResolver));
}