/// <summary> /// Helper method to resolve the mapping file to use. /// </summary> /// <remarks>This method is aware of the data type being added for <see cref="SecurityType.Base"/> /// to the <see cref="SecurityIdentifier.Symbol"/> value</remarks> /// <param name="mapFileResolver">The map file resolver</param> /// <param name="symbol">The symbol that we want to map</param> /// <param name="dataType">The string data type name if any</param> /// <param name="mappingMode">The mapping mode to use if any</param> /// <returns>The mapping file to use</returns> public static MapFile ResolveMapFile(this MapFileResolver mapFileResolver, Symbol symbol, string dataType = null, DataMappingMode?mappingMode = null) { // Load the symbol and date to complete the mapFile checks in one statement var symbolID = symbol.HasUnderlying ? symbol.Underlying.ID.Symbol : symbol.ID.Symbol; if (dataType == null && symbol.SecurityType == SecurityType.Base) { symbol.ID.Symbol.TryGetCustomDataType(out dataType); } symbolID = symbol.SecurityType == SecurityType.Base && dataType != null?symbolID.RemoveFromEnd($".{dataType}") : symbolID; MapFile result; if (ReferenceEquals(mapFileResolver, MapFileResolver.Empty)) { result = mapFileResolver.ResolveMapFile(symbol.Value, Time.BeginningOfTime); } else { var date = symbol.HasUnderlying ? symbol.Underlying.ID.Date : symbol.ID.Date; result = mapFileResolver.ResolveMapFile(symbolID, date); } result.DataMappingMode = mappingMode; return(result); }
/// <summary> /// Helper method to resolve the mapping file to use. /// </summary> /// <remarks>This method is aware of the data type being added for <see cref="SecurityType.Base"/> /// to the <see cref="SecurityIdentifier.Symbol"/> value</remarks> /// <param name="mapFileResolver">The map file resolver</param> /// <param name="symbol">The symbol that we want to map</param> /// <param name="dataType">The configuration data type <see cref="SubscriptionDataConfig.Type"/></param> /// <returns>The mapping file to use</returns> public static MapFile ResolveMapFile(this MapFileResolver mapFileResolver, Symbol symbol, Type dataType) { // Load the symbol and date to complete the mapFile checks in one statement var symbolID = symbol.HasUnderlying ? symbol.Underlying.ID.Symbol : symbol.ID.Symbol; var date = symbol.HasUnderlying ? symbol.Underlying.ID.Date : symbol.ID.Date; return(mapFileResolver.ResolveMapFile( symbol.SecurityType == SecurityType.Base ? symbolID.RemoveFromEnd($".{dataType.Name}") : symbolID, date)); }
/// <summary> /// Helper method to resolve the mapping file to use. /// </summary> /// <remarks>This method is aware of the data type being added for <see cref="SecurityType.Base"/> /// to the <see cref="SecurityIdentifier.Symbol"/> value</remarks> /// <param name="mapFileResolver">The map file resolver</param> /// <param name="symbol">The symbol that we want to map</param> /// <param name="dataType">The string data type name if any</param> /// <returns>The mapping file to use</returns> public static MapFile ResolveMapFile(this MapFileResolver mapFileResolver, Symbol symbol, string dataType = null) { // Load the symbol and date to complete the mapFile checks in one statement var symbolID = symbol.HasUnderlying ? symbol.Underlying.ID.Symbol : symbol.ID.Symbol; var date = symbol.HasUnderlying ? symbol.Underlying.ID.Date : symbol.ID.Date; if (dataType == null && symbol.SecurityType == SecurityType.Base) { symbol.ID.Symbol.TryGetCustomDataType(out dataType); } return(mapFileResolver.ResolveMapFile( symbol.SecurityType == SecurityType.Base && dataType != null ? symbolID.RemoveFromEnd($".{dataType}") : symbolID, date)); }
private MapFileResolver GetMapFileResolver(string market) { if (market != QuantConnect.Market.USA.ToLowerInvariant()) { // don't explode for other markets which request map files and we don't have return(MapFileResolver.Empty); } var timestamp = DateTime.UtcNow.ConvertFromUtc(TimeZones.NewYork); var todayNewYork = timestamp.Date; var yesterdayNewYork = todayNewYork.AddDays(-1); // start the search with yesterday, today's file will be available tomorrow var count = 0; var date = yesterdayNewYork; do { var zipFileName = Path.Combine(Globals.DataFolder, MapFileZipHelper.GetMapFileZipFileName(market, date)); // Fetch a stream for our zip from our data provider var stream = _dataProvider.Fetch(zipFileName); // If we found a file we can read it if (stream != null) { Log.Trace("LocalZipMapFileProvider.Get({0}): Fetched map files for: {1} NY", market, date.ToShortDateString()); var result = new MapFileResolver(MapFileZipHelper.ReadMapFileZip(stream)); stream.DisposeSafely(); return(result); } // prevent infinite recursion if something is wrong if (count++ > 30) { throw new InvalidOperationException($"LocalZipMapFileProvider couldn't find any map files going all the way back to {date}"); } date = date.AddDays(-1); }while (true); }
private MapFileResolver GetMapFileResolver(AuxiliaryDataKey auxiliaryDataKey) { var market = auxiliaryDataKey.Market; var timestamp = DateTime.UtcNow.ConvertFromUtc(TimeZones.NewYork); var todayNewYork = timestamp.Date; var yesterdayNewYork = todayNewYork.AddDays(-1); // start the search with yesterday, today's file will be available tomorrow var count = 0; var date = yesterdayNewYork; do { var zipFileName = MapFileZipHelper.GetMapFileZipFileName(market, date, auxiliaryDataKey.SecurityType); // Fetch a stream for our zip from our data provider var stream = _dataProvider.Fetch(zipFileName); // If we found a file we can read it if (stream != null) { Log.Trace("LocalZipMapFileProvider.Get({0}): Fetched map files for: {1} NY", market, date.ToShortDateString()); var result = new MapFileResolver(MapFileZipHelper.ReadMapFileZip(stream, market, auxiliaryDataKey.SecurityType)); stream.DisposeSafely(); return(result); } // prevent infinite recursion if something is wrong if (count++ > 30) { throw new InvalidOperationException($"LocalZipMapFileProvider couldn't find any map files going all the way back to {date}"); } date = date.AddDays(-1); }while (true); }
/// <summary> /// Reads the zip bytes as text and parses as FactorFileRows to create FactorFiles /// </summary> public static IEnumerable <KeyValuePair <Symbol, IFactorProvider> > ReadFactorFileZip(Stream file, MapFileResolver mapFileResolver, string market, SecurityType securityType) { if (file == null || file.Length == 0) { return(new Dictionary <Symbol, IFactorProvider>()); } var keyValuePairs = ( from kvp in Compression.Unzip(file) let filename = kvp.Key let lines = kvp.Value let factorFile = PriceScalingExtensions.SafeRead(Path.GetFileNameWithoutExtension(filename), lines, securityType) let mapFile = mapFileResolver.GetByPermtick(factorFile.Permtick) where mapFile != null select new KeyValuePair <Symbol, IFactorProvider>(GetSymbol(mapFile, market, securityType), factorFile) ); return(keyValuePairs); }
/// <summary> /// Reads the zip bytes as text and parses as FactorFileRows to create FactorFiles /// </summary> public static IEnumerable <KeyValuePair <Symbol, FactorFile> > ReadFactorFileZip(Stream file, MapFileResolver mapFileResolver, string market) { if (file == null || file.Length == 0) { return(new Dictionary <Symbol, FactorFile>()); } var keyValuePairs = ( from kvp in Compression.Unzip(file) let filename = kvp.Key let lines = kvp.Value let factorFile = SafeRead(filename, lines) let mapFile = mapFileResolver.GetByPermtick(factorFile.Permtick) where mapFile != null let sid = SecurityIdentifier.GenerateEquity(mapFile.FirstDate, mapFile.FirstTicker, market) let symbol = new Symbol(sid, mapFile.Permtick) select new KeyValuePair <Symbol, FactorFile>(symbol, factorFile) ); return(keyValuePairs); }
/// <summary> /// Helper method to resolve the mapping file to use. /// </summary> /// <remarks>This method is aware of the data type being added for <see cref="SecurityType.Base"/> /// to the <see cref="SecurityIdentifier.Symbol"/> value</remarks> /// <param name="mapFileResolver">The map file resolver</param> /// <param name="symbol">The symbol that we want to map</param> /// <param name="dataType">The configuration data type <see cref="SubscriptionDataConfig.Type"/></param> /// <returns>The mapping file to use</returns> public static MapFile ResolveMapFile(this MapFileResolver mapFileResolver, Symbol symbol, Type dataType) { return(mapFileResolver.ResolveMapFile(symbol, dataType.Name)); }