/// <summary>
/// Creates an enumerator to read the specified request
/// </summary>
/// <param name="request">The subscription request to be read</param>
/// <param name="dataProvider">Provider used to get data when it is not present on disk</param>
/// <returns>An enumerator reading the subscription request</returns>
public IEnumerator <BaseData> CreateEnumerator(SubscriptionRequest request, IDataProvider dataProvider)
{
using (var dataCacheProvider = new SingleEntryDataCacheProvider(dataProvider))
{
var tradableDays = _tradableDaysProvider(request);
var fineFundamental = new FineFundamental();
var fineFundamentalConfiguration = new SubscriptionDataConfig(request.Configuration, typeof(FineFundamental), request.Security.Symbol);
foreach (var date in tradableDays)
{
var fineFundamentalSource = GetSource(fineFundamental, fineFundamentalConfiguration, date);
var fineFundamentalFactory = SubscriptionDataSourceReader.ForSource(fineFundamentalSource, dataCacheProvider, fineFundamentalConfiguration, date, _isLiveMode);
var fineFundamentalForDate = (FineFundamental)fineFundamentalFactory.Read(fineFundamentalSource).FirstOrDefault();
yield return(new FineFundamental
{
DataType = MarketDataType.Auxiliary,
Symbol = request.Configuration.Symbol,
Time = date,
CompanyReference = fineFundamentalForDate != null ? fineFundamentalForDate.CompanyReference : new CompanyReference(),
SecurityReference = fineFundamentalForDate != null ? fineFundamentalForDate.SecurityReference : new SecurityReference(),
FinancialStatements = fineFundamentalForDate != null ? fineFundamentalForDate.FinancialStatements : new FinancialStatements(),
EarningReports = fineFundamentalForDate != null ? fineFundamentalForDate.EarningReports : new EarningReports(),
OperationRatios = fineFundamentalForDate != null ? fineFundamentalForDate.OperationRatios : new OperationRatios(),
EarningRatios = fineFundamentalForDate != null ? fineFundamentalForDate.EarningRatios : new EarningRatios(),
ValuationRatios = fineFundamentalForDate != null ? fineFundamentalForDate.ValuationRatios : new ValuationRatios(),
AssetClassification = fineFundamentalForDate != null ? fineFundamentalForDate.AssetClassification : new AssetClassification(),
CompanyProfile = fineFundamentalForDate != null ? fineFundamentalForDate.CompanyProfile : new CompanyProfile()
});
}
}
}
private SubscriptionDataSource DailyBackwardsLoop(FineFundamental fine, SubscriptionDataConfig config, DateTime date, SubscriptionDataSource source)
{
var path = Path.GetDirectoryName(source.Source) ?? string.Empty;
if (string.IsNullOrEmpty(path) || !Directory.Exists(path))
{
// directory does not exist
return(source);
}
// loop back in time, for 10 days, until we find an existing file
var count = 10;
do
{
// get previous date
date = date.AddDays(-1);
// get file name for this date
source = fine.GetSource(config, date, _isLiveMode);
if (File.Exists(source.Source))
{
break;
}
}while (--count > 0);
return(count == 0 ? null : source);
}
/// <summary>
/// Creates an enumerator to read the specified request
/// </summary>
/// <param name="request">The subscription request to be read</param>
/// <returns>An enumerator reading the subscription request</returns>
public IEnumerator <BaseData> CreateEnumerator(SubscriptionRequest request)
{
var tradableDays = _tradableDaysProvider(request);
var fineFundamental = new FineFundamental();
var fineFundamentalConfiguration = new SubscriptionDataConfig(request.Configuration, typeof(FineFundamental), request.Security.Symbol);
return((
from date in tradableDays
let fineFundamentalSource = GetSource(fineFundamental, fineFundamentalConfiguration, date)
let fineFundamentalFactory = SubscriptionDataSourceReader.ForSource(fineFundamentalSource, fineFundamentalConfiguration, date, false)
let fineFundamentalForDate = (FineFundamental)fineFundamentalFactory.Read(fineFundamentalSource).FirstOrDefault()
select new FineFundamental
{
DataType = MarketDataType.Auxiliary,
Symbol = request.Configuration.Symbol,
Time = date,
CompanyReference = fineFundamentalForDate != null ? fineFundamentalForDate.CompanyReference : new CompanyReference(),
SecurityReference = fineFundamentalForDate != null ? fineFundamentalForDate.SecurityReference : new SecurityReference(),
FinancialStatements = fineFundamentalForDate != null ? fineFundamentalForDate.FinancialStatements : new FinancialStatements(),
EarningReports = fineFundamentalForDate != null ? fineFundamentalForDate.EarningReports : new EarningReports(),
OperationRatios = fineFundamentalForDate != null ? fineFundamentalForDate.OperationRatios : new OperationRatios(),
EarningRatios = fineFundamentalForDate != null ? fineFundamentalForDate.EarningRatios : new EarningRatios(),
ValuationRatios = fineFundamentalForDate != null ? fineFundamentalForDate.ValuationRatios : new ValuationRatios()
}
).GetEnumerator());
}
public void ZeroMarketCapForDefaultObject()
{
var fine = new FineFundamental();
fine.EarningReports.BasicAverageShares = null;
Assert.AreEqual(0, fine.Price);
Assert.IsNull(fine.EarningReports.BasicAverageShares);
Assert.AreEqual(0, fine.MarketCap);
}
/// <summary>
/// Sets values for non existing periods from a previous instance
/// </summary>
/// <remarks>Used to fill-forward values from previous dates</remarks>
/// <param name="previous">The previous instance</param>
public void UpdateValues(FineFundamental previous)
{
CompanyReference.UpdateValues(previous.CompanyReference);
SecurityReference.UpdateValues(previous.SecurityReference);
FinancialStatements.UpdateValues(previous.FinancialStatements);
EarningReports.UpdateValues(previous.EarningReports);
OperationRatios.UpdateValues(previous.OperationRatios);
EarningRatios.UpdateValues(previous.EarningRatios);
ValuationRatios.UpdateValues(previous.ValuationRatios);
}
/// <summary>
/// Sets values for non existing periods from a previous instance
/// </summary>
/// <remarks>Used to fill-forward values from previous dates</remarks>
/// <param name="previous">The previous instance</param>
public void UpdateValues(FineFundamental previous)
{
CompanyReference.UpdateValues(previous.CompanyReference);
SecurityReference.UpdateValues(previous.SecurityReference);
FinancialStatements.UpdateValues(previous.FinancialStatements);
EarningReports.UpdateValues(previous.EarningReports);
OperationRatios.UpdateValues(previous.OperationRatios);
EarningRatios.UpdateValues(previous.EarningRatios);
ValuationRatios.UpdateValues(previous.ValuationRatios);
}
/// <summary>
/// Sets values for non existing periods from a previous instance
/// </summary>
/// <remarks>Used to fill-forward values from previous dates</remarks>
/// <param name="previous">The previous instance</param>
public void UpdateValues(FineFundamental previous)
{
if (previous == null) return;
if (CompanyReference != null) CompanyReference.UpdateValues(previous.CompanyReference);
if (SecurityReference != null) SecurityReference.UpdateValues(previous.SecurityReference);
if (FinancialStatements != null) FinancialStatements.UpdateValues(previous.FinancialStatements);
if (EarningReports != null) EarningReports.UpdateValues(previous.EarningReports);
if (OperationRatios != null) OperationRatios.UpdateValues(previous.OperationRatios);
if (EarningRatios != null) EarningRatios.UpdateValues(previous.EarningRatios);
if (ValuationRatios != null) ValuationRatios.UpdateValues(previous.ValuationRatios);
}
public void ZeroMarketCapForDefaultBasicAverageShares()
{
var fine = new FineFundamental
{
Symbol = Symbols.AAPL,
EndTime = DateTime.Now,
Value = 267.18m,
};
fine.EarningReports.BasicAverageShares = null;
Assert.AreEqual(267.18m, fine.Price);
Assert.IsNull(fine.EarningReports.BasicAverageShares);
Assert.AreEqual(0, fine.MarketCap);
}
/// <summary>
/// Applies updated values from <paramref name="update"/> to this instance
/// </summary>
/// <remarks>Used to apply data updates to the current instance. This WILL overwrite existing values. Default update values are ignored.</remarks>
/// <param name="update">The next data update for this instance</param>
public void UpdateValues(FineFundamental update)
{
if (update == null)
{
return;
}
CompanyReference?.UpdateValues(update.CompanyReference);
SecurityReference?.UpdateValues(update.SecurityReference);
FinancialStatements?.UpdateValues(update.FinancialStatements);
EarningReports?.UpdateValues(update.EarningReports);
OperationRatios?.UpdateValues(update.OperationRatios);
EarningRatios?.UpdateValues(update.EarningRatios);
ValuationRatios?.UpdateValues(update.ValuationRatios);
CompanyProfile?.UpdateValues(update.CompanyProfile);
AssetClassification?.UpdateValues(update.AssetClassification);
}
/// <summary>
/// Sets values for non existing periods from a previous instance
/// </summary>
/// <remarks>Used to fill-forward values from previous dates</remarks>
/// <param name="previous">The previous instance</param>
public void UpdateValues(FineFundamental previous)
{
if (previous == null)
{
return;
}
if (CompanyReference != null)
{
CompanyReference.UpdateValues(previous.CompanyReference);
}
if (SecurityReference != null)
{
SecurityReference.UpdateValues(previous.SecurityReference);
}
if (FinancialStatements != null)
{
FinancialStatements.UpdateValues(previous.FinancialStatements);
}
if (EarningReports != null)
{
EarningReports.UpdateValues(previous.EarningReports);
}
if (OperationRatios != null)
{
OperationRatios.UpdateValues(previous.OperationRatios);
}
if (EarningRatios != null)
{
EarningRatios.UpdateValues(previous.EarningRatios);
}
if (ValuationRatios != null)
{
ValuationRatios.UpdateValues(previous.ValuationRatios);
}
if (CompanyProfile != null)
{
CompanyProfile.UpdateValues(previous.CompanyProfile);
}
if (AssetClassification != null)
{
AssetClassification.UpdateValues(previous.AssetClassification);
}
}
public void ComputesMarketCapCorrectly()
{
var fine = new FineFundamental
{
Symbol = Symbols.AAPL,
EndTime = DateTime.Now,
Value = 267.18m,
CompanyReference = new CompanyReference
{
CountryId = "USA",
PrimaryExchangeID = "NYS",
IndustryTemplateCode = "N"
},
SecurityReference = new SecurityReference
{
IPODate = new DateTime(1980, 12, 12)
},
ValuationRatios = new ValuationRatios
{
PERatio = 22.476871m
},
CompanyProfile = new CompanyProfile
{
MarketCap = 119985488298
},
EarningReports = new EarningReports
{
BasicAverageShares = new BasicAverageShares
{
ThreeMonths = 449081100m
},
BasicEPS = new BasicEPS
{
TwelveMonths = 11.97m
}
}
};
Assert.AreEqual(119985488298m, fine.MarketCap);
}
/// <summary>
/// Returns a SubscriptionDataSource for the FineFundamental class,
/// returning data from a previous date if not available for the requested date
/// </summary>
private SubscriptionDataSource GetSource(FineFundamental fine, SubscriptionDataConfig config, DateTime date)
{
var source = fine.GetSource(config, date, _isLiveMode);
var fileName = date.ToString("yyyyMMdd");
if (!File.Exists(source.Source))
{
if (_lastUsedFileName == null)
{
// find first file date
var path = Path.GetDirectoryName(source.Source) ?? string.Empty;
if (string.IsNullOrEmpty(path) || !Directory.Exists(path))
{
return(source);
}
var firstFileName = Path.GetFileNameWithoutExtension(Directory.GetFiles(path, "*.zip").OrderBy(x => x).First());
var firstDate = DateTime.ParseExact(firstFileName, "yyyyMMdd", CultureInfo.InvariantCulture);
// requested date before first date, return current invalid source anyway
if (date < firstDate)
{
return(source);
}
// requested date after first date, save date of first existing file
_lastUsedFileName = firstFileName;
// loop back in time until we find an existing file
while (string.CompareOrdinal(fileName, _lastUsedFileName) > 0)
{
// get previous date
date = date.AddDays(-1);
// get file name for this date
source = fine.GetSource(config, date, _isLiveMode);
fileName = Path.GetFileNameWithoutExtension(source.Source);
if (!File.Exists(source.Source))
{
continue;
}
// we found the file, save its name and return the source
_lastUsedFileName = fileName;
break;
}
}
else
{
// return source for last existing file date
date = DateTime.ParseExact(_lastUsedFileName, "yyyyMMdd", CultureInfo.InvariantCulture);
source = fine.GetSource(config, date, _isLiveMode);
}
}
else
{
_lastUsedFileName = fileName;
}
return(source);
}
/// <summary>
/// Initializes a new instance of the <see cref="FineFundamentalUniverse"/> class
/// </summary>
/// <param name="universeSettings">The settings used for new subscriptions generated by this universe</param>
/// <param name="selector">Returns the symbols that should be included in the universe</param>
public FineFundamentalUniverse(UniverseSettings universeSettings, Func <IEnumerable <FineFundamental>, IEnumerable <Symbol> > selector)
: base(CreateConfiguration(FineFundamental.CreateUniverseSymbol(QuantConnect.Market.USA)))
{
UniverseSettings = universeSettings;
_selector = selector;
}
/// <summary>
/// Returns a SubscriptionDataSource for the FineFundamental class,
/// returning data from a previous date if not available for the requested date
/// </summary>
private SubscriptionDataSource GetSource(FineFundamental fine, SubscriptionDataConfig config, DateTime date)
{
var source = fine.GetSource(config, date, _isLiveMode);
if (File.Exists(source.Source))
{
return(source);
}
if (_isLiveMode)
{
var result = DailyBackwardsLoop(fine, config, date, source);
// if we didn't fine any file we just fallback into listing the directory
if (result != null)
{
return(result);
}
}
var cacheKey = config.Symbol.Value.ToLowerInvariant().GetHashCode();
List <DateTime> availableDates;
// only use cache in backtest, since in live mode new fine files are added
// we still didn't load available fine dates for this symbol
if (_isLiveMode || !FineFilesCache.TryGetValue(cacheKey, out availableDates))
{
try
{
var path = Path.GetDirectoryName(source.Source) ?? string.Empty;
availableDates = Directory.GetFiles(path, "*.zip")
.Select(
filePath =>
{
try
{
return(DateTime.ParseExact(
Path.GetFileNameWithoutExtension(filePath),
"yyyyMMdd",
CultureInfo.InvariantCulture
));
}
catch
{
// just in case...
return(DateTime.MaxValue);
}
}
)
.Where(time => time != DateTime.MaxValue)
.OrderBy(x => x)
.ToList();
}
catch
{
// directory doesn't exist or path is null
if (!_isLiveMode)
{
// only add to cache if not live mode
FineFilesCache[cacheKey] = new List <DateTime>();
}
return(source);
}
if (!_isLiveMode)
{
// only add to cache if not live mode
FineFilesCache[cacheKey] = availableDates;
}
}
// requested date before first date, return null source
if (availableDates.Count == 0 || date < availableDates[0])
{
return(source);
}
for (var i = availableDates.Count - 1; i >= 0; i--)
{
// we iterate backwards ^ and find the first data point before 'date'
if (availableDates[i] <= date)
{
return(fine.GetSource(config, availableDates[i], _isLiveMode));
}
}
return(source);
}
/// <summary>
/// Returns a SubscriptionDataSource for the FineFundamental class,
/// returning data from a previous date if not available for the requested date
/// </summary>
private SubscriptionDataSource GetSource(FineFundamental fine, SubscriptionDataConfig config, DateTime date)
{
var source = fine.GetSource(config, date, _isLiveMode);
if (!File.Exists(source.Source))
{
if (_lastUsedDate == default(DateTime))
{
// find first file date
List <string> availableFiles;
try
{
availableFiles = Directory.GetFiles(Path.GetDirectoryName(source.Source),
"*.zip").OrderBy(x => x).ToList();
}
catch
{
// argument null exception or directory doesn't exist
return(source);
}
var fileName = Path.GetFileNameWithoutExtension(source.Source);
// no files or requested date before first date, return null source
if (availableFiles.Count == 0 ||
string.CompareOrdinal(fileName,
Path.GetFileNameWithoutExtension(availableFiles[0])) < 0)
{
return(source);
}
// take advantage of the order and avoid parsing and comparing all files to get the date
var index = availableFiles.BinarySearch(fileName, new FileNameComparer());
if (index < 0)
{
// if negative returns the complement of the index of the next element that is larger than Date
// so subtract 1 to get the previous item which is less than Date
index = ~index - 1;
}
var current = DateTime.ParseExact(
Path.GetFileNameWithoutExtension(availableFiles[index]),
"yyyyMMdd",
CultureInfo.InvariantCulture);
if (current <= date)
{
// we found the file, save its date and return the source
_lastUsedDate = current;
return(fine.GetSource(config, current, _isLiveMode));
}
// this shouldn't ever happen so throw
throw new InvalidOperationException("FineFundamentalSubscriptionEnumeratorFactory.GetSource(): " +
$"failed to resolve source for {config.Symbol} on {date}");
}
else
{
// return source for last existing file date
source = fine.GetSource(config, _lastUsedDate, _isLiveMode);
}
}
else
{
_lastUsedDate = date;
}
return(source);
}