/// <summary> /// Gets a <see cref="FactorFile"/> instance for the specified symbol, or null if not found /// </summary> /// <param name="symbol">The security's symbol whose factor file we seek</param> /// <returns>The resolved factor file, or null if not found</returns> public IFactorProvider Get(Symbol symbol) { symbol = symbol.GetFactorFileSymbol(); IFactorProvider factorFile; if (_cache.TryGetValue(symbol, out factorFile)) { return(factorFile); } // we first need to resolve the map file to get a permtick, that's how the factor files are stored var mapFileResolver = _mapFileProvider.Get(AuxiliaryDataKey.Create(symbol)); if (mapFileResolver == null) { return(GetFactorFile(symbol, symbol.Value)); } var mapFile = mapFileResolver.ResolveMapFile(symbol); if (mapFile.IsNullOrEmpty()) { return(GetFactorFile(symbol, symbol.Value)); } return(GetFactorFile(symbol, mapFile.Permtick)); }
/// <summary> /// Helper method to resolve the mapping file to use. /// </summary> /// <remarks>This method is aware of the data type being added for <see cref="SecurityType.Base"/> /// to the <see cref="SecurityIdentifier.Symbol"/> value</remarks> /// <param name="mapFileProvider">The map file provider</param> /// <param name="dataConfig">The configuration to fetch the map file for</param> /// <returns>The mapping file to use</returns> public static MapFile ResolveMapFile(this IMapFileProvider mapFileProvider, SubscriptionDataConfig dataConfig) { var resolver = MapFileResolver.Empty; if (dataConfig.TickerShouldBeMapped()) { resolver = mapFileProvider.Get(AuxiliaryDataKey.Create(dataConfig.Symbol)); } return(resolver.ResolveMapFile(dataConfig.Symbol, dataConfig.Type.Name, dataConfig.DataMappingMode)); }
/// <summary> /// Gets a <see cref="MapFileResolver"/> representing all the map files for the specified market /// </summary> /// <param name="auxiliaryDataKey">Key used to fetch a map file resolver. Specifying market and security type</param> /// <returns>A <see cref="MapFileResolver"/> containing all map files for the specified market</returns> public MapFileResolver Get(AuxiliaryDataKey auxiliaryDataKey) { MapFileResolver result; // we use a lock so that only 1 thread loads the map file resolver while the rest wait // else we could have multiple threads loading the map file resolver at the same time! lock (_lock) { if (!_cache.TryGetValue(auxiliaryDataKey, out result)) { _cache[auxiliaryDataKey] = result = GetMapFileResolver(auxiliaryDataKey); } } return(result); }
/// <summary> /// Gets the primary exchange for a given security identifier /// </summary> /// <param name="securityIdentifier">The security identifier to get the primary exchange for</param> /// <returns>Returns the primary exchange or null if not found</returns> public Exchange GetPrimaryExchange(SecurityIdentifier securityIdentifier) { Exchange primaryExchange; if (!_primaryExchangeBySid.TryGetValue(securityIdentifier, out primaryExchange)) { var mapFile = _mapFileProvider.Get(AuxiliaryDataKey.Create(securityIdentifier)) .ResolveMapFile(securityIdentifier.Symbol, securityIdentifier.Date); if (mapFile != null && mapFile.Any()) { primaryExchange = mapFile.Last().PrimaryExchange; } _primaryExchangeBySid[securityIdentifier] = primaryExchange; } return(primaryExchange); }
/// Hydrate the <see cref="_factorFiles"/> from the latest zipped factor file on disk private void HydrateFactorFileFromLatestZip(AuxiliaryDataKey key) { var market = key.Market; // start the search with yesterday, today's file will be available tomorrow var todayNewYork = DateTime.UtcNow.ConvertFromUtc(TimeZones.NewYork).Date; var date = todayNewYork.AddDays(-1); var count = 0; do { var factorFilePath = FactorFileZipHelper.GetFactorFileZipFileName(market, date, key.SecurityType); // Fetch a stream for our zip from our data provider var stream = _dataProvider.Fetch(factorFilePath); // If the file was found we can read the file if (stream != null) { var mapFileResolver = _mapFileProvider.Get(key); foreach (var keyValuePair in FactorFileZipHelper.ReadFactorFileZip(stream, mapFileResolver, market, key.SecurityType)) { // we merge with existing, this will allow to hold multiple markets _factorFiles[keyValuePair.Key] = keyValuePair.Value; } stream.DisposeSafely(); Log.Trace($"LocalZipFactorFileProvider.Get({market}): Fetched factor files for: {date.ToShortDateString()} NY"); return; } // Otherwise we will search back another day Log.Debug($"LocalZipFactorFileProvider.Get(): No factor file found for date {date.ToShortDateString()}"); // prevent infinite recursion if something is wrong if (count++ > 7) { throw new InvalidOperationException($"LocalZipFactorFileProvider.Get(): Could not find any factor files going all the way back to {date}"); } date = date.AddDays(-1); }while (true); }
private MapFileResolver GetMapFileResolver(AuxiliaryDataKey key) { var securityType = key.SecurityType; var market = key.Market; var mapFileDirectory = MapFile.GetMapFilePath(market, securityType); if (!Directory.Exists(mapFileDirectory)) { // only write this message once per application instance if (Interlocked.CompareExchange(ref _wroteTraceStatement, 1, 0) == 0) { Log.Error($"LocalDiskMapFileProvider.GetMapFileResolver({market}): " + $"The specified directory does not exist: {mapFileDirectory}" ); } return(MapFileResolver.Empty); } return(new MapFileResolver(MapFile.GetMapFiles(mapFileDirectory, market, securityType, _dataProvider))); }
/// <summary> /// Gets a <see cref="FactorFile"/> instance for the specified symbol, or null if not found /// </summary> /// <param name="symbol">The security's symbol whose factor file we seek</param> /// <returns>The resolved factor file, or null if not found</returns> public IFactorProvider Get(Symbol symbol) { symbol = symbol.GetFactorFileSymbol(); var key = AuxiliaryDataKey.Create(symbol); lock (_lock) { if (!_seededMarket.ContainsKey(key)) { HydrateFactorFileFromLatestZip(key); _seededMarket[key] = true; } IFactorProvider factorFile; if (!_factorFiles.TryGetValue(symbol, out factorFile)) { // Could not find factor file for symbol Log.Error($"LocalZipFactorFileProvider.Get({symbol}): No factor file found."); _factorFiles[symbol] = factorFile = symbol.GetEmptyFactorFile(); } return(factorFile); } }
private MapFileResolver GetMapFileResolver(AuxiliaryDataKey auxiliaryDataKey) { var market = auxiliaryDataKey.Market; var timestamp = DateTime.UtcNow.ConvertFromUtc(TimeZones.NewYork); var todayNewYork = timestamp.Date; var yesterdayNewYork = todayNewYork.AddDays(-1); // start the search with yesterday, today's file will be available tomorrow var count = 0; var date = yesterdayNewYork; do { var zipFileName = MapFileZipHelper.GetMapFileZipFileName(market, date, auxiliaryDataKey.SecurityType); // Fetch a stream for our zip from our data provider var stream = _dataProvider.Fetch(zipFileName); // If we found a file we can read it if (stream != null) { Log.Trace("LocalZipMapFileProvider.Get({0}): Fetched map files for: {1} NY", market, date.ToShortDateString()); var result = new MapFileResolver(MapFileZipHelper.ReadMapFileZip(stream, market, auxiliaryDataKey.SecurityType)); stream.DisposeSafely(); return(result); } // prevent infinite recursion if something is wrong if (count++ > 30) { throw new InvalidOperationException($"LocalZipMapFileProvider couldn't find any map files going all the way back to {date}"); } date = date.AddDays(-1); }while (true); }
/// <summary> /// Gets a <see cref="MapFileResolver"/> representing all the map /// files for the specified market /// </summary> /// <param name="auxiliaryDataKey">Key used to fetch a map file resolver. Specifying market and security type</param> /// <returns>A <see cref="MapFileRow"/> containing all map files for the specified market</returns> public MapFileResolver Get(AuxiliaryDataKey auxiliaryDataKey) { return(_cache.GetOrAdd(auxiliaryDataKey, GetMapFileResolver)); }