/// Hydrate the <see cref="_factorFiles"/> from the latest zipped factor file on disk private void HydrateFactorFileFromLatestZip(string market) { if (market != QuantConnect.Market.USA.ToLowerInvariant()) { // don't explode for other markets which request factor files and we don't have return; } // start the search with yesterday, today's file will be available tomorrow var todayNewYork = DateTime.UtcNow.ConvertFromUtc(TimeZones.NewYork).Date; var date = todayNewYork.AddDays(-1); var count = 0; do { var zipFileName = $"equity/{market}/factor_files/factor_files_{date:yyyyMMdd}.zip"; var factorFilePath = Path.Combine(Globals.DataFolder, zipFileName); // Fetch a stream for our zip from our data provider var stream = _dataProvider.Fetch(factorFilePath); // If the file was found we can read the file if (stream != null) { var mapFileResolver = _mapFileProvider.Get(market); foreach (var keyValuePair in FactorFileZipHelper.ReadFactorFileZip(stream, mapFileResolver, market)) { // we merge with existing, this will allow to hold multiple markets _factorFiles[keyValuePair.Key] = keyValuePair.Value; } stream.DisposeSafely(); Log.Trace($"LocalZipFactorFileProvider.Get({market}): Fetched factor files for: {date.ToShortDateString()} NY"); return; } // Otherwise we will search back another day Log.Debug($"LocalZipFactorFileProvider.Get(): No factor file found for date {date.ToShortDateString()}"); // prevent infinite recursion if something is wrong if (count++ > 7) { throw new InvalidOperationException($"LocalZipFactorFileProvider.Get(): Could not find any factor files going all the way back to {date}"); } date = date.AddDays(-1); }while (true); }
/// Hydrate the <see cref="_factorFiles"/> from the latest zipped factor file on disk private void HydrateFactorFileFromLatestZip(AuxiliaryDataKey key) { var market = key.Market; // start the search with yesterday, today's file will be available tomorrow var todayNewYork = DateTime.UtcNow.ConvertFromUtc(TimeZones.NewYork).Date; var date = todayNewYork.AddDays(-1); var count = 0; do { var factorFilePath = FactorFileZipHelper.GetFactorFileZipFileName(market, date, key.SecurityType); // Fetch a stream for our zip from our data provider var stream = _dataProvider.Fetch(factorFilePath); // If the file was found we can read the file if (stream != null) { var mapFileResolver = _mapFileProvider.Get(key); foreach (var keyValuePair in FactorFileZipHelper.ReadFactorFileZip(stream, mapFileResolver, market, key.SecurityType)) { // we merge with existing, this will allow to hold multiple markets _factorFiles[keyValuePair.Key] = keyValuePair.Value; } stream.DisposeSafely(); Log.Trace($"LocalZipFactorFileProvider.Get({market}): Fetched factor files for: {date.ToShortDateString()} NY"); return; } // Otherwise we will search back another day Log.Debug($"LocalZipFactorFileProvider.Get(): No factor file found for date {date.ToShortDateString()}"); // prevent infinite recursion if something is wrong if (count++ > 7) { throw new InvalidOperationException($"LocalZipFactorFileProvider.Get(): Could not find any factor files going all the way back to {date}"); } date = date.AddDays(-1); }while (true); }