/// Hydrate the <see cref="_factorFiles"/> from the latest zipped factor file on disk
private void HydrateFactorFileFromLatestZip(string market)
{
if (market != QuantConnect.Market.USA.ToLowerInvariant())
{
// don't explode for other markets which request factor files and we don't have
return;
}
// start the search with yesterday, today's file will be available tomorrow
var todayNewYork = DateTime.UtcNow.ConvertFromUtc(TimeZones.NewYork).Date;
var date = todayNewYork.AddDays(-1);
var count = 0;
do
{
var zipFileName = $"equity/{market}/factor_files/factor_files_{date:yyyyMMdd}.zip";
var factorFilePath = Path.Combine(Globals.DataFolder, zipFileName);
// Fetch a stream for our zip from our data provider
var stream = _dataProvider.Fetch(factorFilePath);
// If the file was found we can read the file
if (stream != null)
{
var mapFileResolver = _mapFileProvider.Get(market);
foreach (var keyValuePair in FactorFileZipHelper.ReadFactorFileZip(stream, mapFileResolver, market))
{
// we merge with existing, this will allow to hold multiple markets
_factorFiles[keyValuePair.Key] = keyValuePair.Value;
}
stream.DisposeSafely();
Log.Trace($"LocalZipFactorFileProvider.Get({market}): Fetched factor files for: {date.ToShortDateString()} NY");
return;
}
// Otherwise we will search back another day
Log.Debug($"LocalZipFactorFileProvider.Get(): No factor file found for date {date.ToShortDateString()}");
// prevent infinite recursion if something is wrong
if (count++ > 7)
{
throw new InvalidOperationException($"LocalZipFactorFileProvider.Get(): Could not find any factor files going all the way back to {date}");
}
date = date.AddDays(-1);
}while (true);
}
/// Hydrate the <see cref="_factorFiles"/> from the latest zipped factor file on disk
private void HydrateFactorFileFromLatestZip(AuxiliaryDataKey key)
{
var market = key.Market;
// start the search with yesterday, today's file will be available tomorrow
var todayNewYork = DateTime.UtcNow.ConvertFromUtc(TimeZones.NewYork).Date;
var date = todayNewYork.AddDays(-1);
var count = 0;
do
{
var factorFilePath = FactorFileZipHelper.GetFactorFileZipFileName(market, date, key.SecurityType);
// Fetch a stream for our zip from our data provider
var stream = _dataProvider.Fetch(factorFilePath);
// If the file was found we can read the file
if (stream != null)
{
var mapFileResolver = _mapFileProvider.Get(key);
foreach (var keyValuePair in FactorFileZipHelper.ReadFactorFileZip(stream, mapFileResolver, market, key.SecurityType))
{
// we merge with existing, this will allow to hold multiple markets
_factorFiles[keyValuePair.Key] = keyValuePair.Value;
}
stream.DisposeSafely();
Log.Trace($"LocalZipFactorFileProvider.Get({market}): Fetched factor files for: {date.ToShortDateString()} NY");
return;
}
// Otherwise we will search back another day
Log.Debug($"LocalZipFactorFileProvider.Get(): No factor file found for date {date.ToShortDateString()}");
// prevent infinite recursion if something is wrong
if (count++ > 7)
{
throw new InvalidOperationException($"LocalZipFactorFileProvider.Get(): Could not find any factor files going all the way back to {date}");
}
date = date.AddDays(-1);
}while (true);
}