/// <summary>
/// Main filter to determine if this file needs to be downloaded
/// </summary>
/// <param name="filePath">File we are looking at</param>
/// <returns>True if should download</returns>
protected override bool NeedToDownload(string filePath)
{
// Ignore null and fine fundamental data requests
if (filePath == null || filePath.Contains("fine", StringComparison.InvariantCultureIgnoreCase) && filePath.Contains("fundamental", StringComparison.InvariantCultureIgnoreCase))
{
return(false);
}
// Some security types can't be downloaded, lets attempt to extract that information
if (LeanData.TryParseSecurityType(filePath, out SecurityType securityType) && _unsupportedSecurityType.Contains(securityType))
{
if (!_invalidSecurityTypeLog)
{
// let's log this once. Will still use any existing data on disk
_invalidSecurityTypeLog = true;
Log.Error($"ApiDataProvider(): does not support security types: {string.Join(", ", _unsupportedSecurityType)}");
}
return(false);
}
// Only download if it doesn't exist or is out of date.
// Files are only "out of date" for non date based files (hour, daily, margins, etc.) because this data is stored all in one file
var shouldDownload = !File.Exists(filePath) || filePath.IsOutOfDate();
// Final check; If we want to download and the request requires equity data we need to be sure they are subscribed to map and factor files
if (shouldDownload && (securityType == SecurityType.Equity || securityType == SecurityType.Option || IsEquitiesAux(filePath)))
{
CheckMapFactorFileSubscription();
}
return(shouldDownload);
}
/// <summary>
/// Main filter to determine if this file needs to be downloaded
/// </summary>
/// <param name="filePath">File we are looking at</param>
/// <returns>True if should download</returns>
protected override bool NeedToDownload(string filePath)
{
// Ignore null and fine fundamental data requests
if (filePath == null || filePath.Contains("fine", StringComparison.InvariantCultureIgnoreCase) && filePath.Contains("fundamental", StringComparison.InvariantCultureIgnoreCase))
{
return(false);
}
// Some security types can't be downloaded, lets attempt to extract that information
if (LeanData.TryParseSecurityType(filePath, out SecurityType securityType, out var market) && _unsupportedSecurityType.Contains(securityType))
{
// we do support future auxiliary data (map and factor files)
if (securityType != SecurityType.Future || !IsAuxiliaryData(filePath))
{
if (!_invalidSecurityTypeLog)
{
// let's log this once. Will still use any existing data on disk
_invalidSecurityTypeLog = true;
Log.Error($"ApiDataProvider(): does not support security types: {string.Join(", ", _unsupportedSecurityType)}");
}
return(false);
}
}
// Only download if it doesn't exist or is out of date.
// Files are only "out of date" for non date based files (hour, daily, margins, etc.) because this data is stored all in one file
var shouldDownload = !File.Exists(filePath) || filePath.IsOutOfDate();
if (shouldDownload)
{
if (securityType == SecurityType.Future)
{
if (!_subscribedToFutureMapAndFactorFiles)
{
throw new ArgumentException("ApiDataProvider(): Must be subscribed to map and factor files to use the ApiDataProvider " +
"to download Future auxiliary data from QuantConnect. " +
"Please visit https://www.quantconnect.com/datasets/quantconnect-us-futures-security-master for details.");
}
}
// Final check; If we want to download and the request requires equity data we need to be sure they are subscribed to map and factor files
else if (!_subscribedToUsaEquityMapAndFactorFiles && market.Equals(Market.USA, StringComparison.InvariantCultureIgnoreCase) &&
(securityType == SecurityType.Equity || securityType == SecurityType.Option || IsAuxiliaryData(filePath)))
{
throw new ArgumentException("ApiDataProvider(): Must be subscribed to map and factor files to use the ApiDataProvider " +
"to download Equity data from QuantConnect. " +
"Please visit https://www.quantconnect.com/datasets/quantconnect-security-master for details.");
}
else if (!_subscribedToIndiaEquityMapAndFactorFiles && market.Equals(Market.India, StringComparison.InvariantCultureIgnoreCase) &&
(securityType == SecurityType.Equity || securityType == SecurityType.Option || IsAuxiliaryData(filePath)))
{
throw new ArgumentException("ApiDataProvider(): Must be subscribed to map and factor files to use the ApiDataProvider " +
"to download India data from QuantConnect. " +
"Please visit https://www.quantconnect.com/datasets/truedata-india-equity-security-master for details.");
}
}
return(shouldDownload);
}
