public IndexInfo GetIndexInfo()
{
var directory = _openIndexModel.Directory;
IndexReader indexReader = null;
try
{
indexReader = IndexReader.Open(directory, _openIndexModel.ReadOnly);
var v = IndexGate.GetIndexFormat(directory);
return(new IndexInfo
{
IndexDetails = IndexGate.GetFormatDetails(v),
FieldCount = indexReader.GetFieldNames(IndexReader.FieldOption.ALL).Count,
//LastModified = dir.LastWriteTime, // IndexReader.LastModified(directory),
Version = indexReader.GetVersion().ToString("x"),
DocumentCount = indexReader.NumDocs(),
HasDeletions = indexReader.HasDeletions(),
DeletionCount = indexReader.NumDeletedDocs(),
Optimized = indexReader.IsOptimized(),
IndexPath = Path.GetFullPath(_openIndexModel.Path),
TermCount = 0
});
}
finally
{
if (indexReader != null)
{
indexReader.Close();
}
}
}
/// <summary>
/// Runs the instance of the object.
/// </summary>
/// <returns>True if process all downloads successfully</returns>
public override bool Run()
{
var stopwatch = Stopwatch.StartNew();
try
{
var companies = GetCompanies().Result;
Log.Trace($"EstimizeEstimateDataDownloader.Run(): Start processing {companies.Count} companies");
var tasks = new List <Task>();
foreach (var company in companies)
{
var ticker = company.Ticker;
// Include tickers that are "defunct".
// Remove the tag because it cannot be part of the API endpoint
if (ticker.IndexOf("defunct", StringComparison.OrdinalIgnoreCase) > 0)
{
var length = ticker.IndexOf('-');
ticker = ticker.Substring(0, length).Trim();
}
// Makes sure we don't overrun Estimize rate limits accidentally
IndexGate.WaitToProceed();
tasks.Add(
HttpRequester($"/companies/{ticker}/estimates")
.ContinueWith(
y =>
{
if (y.IsFaulted)
{
Log.Error($"EstimizeEstimateDataDownloader.Run(): Failed to get data for {company}");
return;
}
SaveContentToZipFile(ticker, y.Result);
}
)
);
}
Task.WaitAll(tasks.ToArray());
}
catch (Exception e)
{
Log.Error(e);
return(false);
}
Log.Trace($"EstimizeEstimateDataDownloader.Run(): Finished in {stopwatch.Elapsed}");
return(true);
}
/// <summary>
/// Runs the instance of the object.
/// </summary>
/// <returns>True if process all downloads successfully</returns>
public override bool Run()
{
var stopwatch = Stopwatch.StartNew();
try
{
var companies = GetCompanies().Result.DistinctBy(x => x.Ticker).ToList();
var count = companies.Count;
var currentPercent = 0.05;
var percent = 0.05;
var i = 0;
Log.Trace($"EstimizeReleaseDataDownloader.Run(): Start processing {count} companies");
var tasks = new List <Task>();
foreach (var company in companies)
{
// Makes sure we don't overrun Estimize rate limits accidentally
IndexGate.WaitToProceed();
// Include tickers that are "defunct".
// Remove the tag because it cannot be part of the API endpoint.
// This is separate from the NormalizeTicker(...) method since
// we don't convert tickers with `-`s into the format we can successfully
// index mapfiles with.
var estimizeTicker = company.Ticker;
string ticker;
if (!TryNormalizeDefunctTicker(estimizeTicker, out ticker))
{
Log.Error($"EstimizeReleaseDataDownloader(): Defunct ticker {estimizeTicker} is unable to be parsed. Continuing...");
continue;
}
// Begin processing ticker with a normalized value
Log.Trace($"EstimizeReleaseDataDownloader.Run(): Processing {ticker}");
tasks.Add(
HttpRequester($"/companies/{ticker}/releases")
.ContinueWith(
y =>
{
i++;
if (y.IsFaulted)
{
Log.Error($"EstimizeReleaseDataDownloader.Run(): Failed to get data for {company}");
return;
}
var result = y.Result;
if (string.IsNullOrEmpty(result))
{
// We've already logged inside HttpRequester
return;
}
// Just like TradingEconomics, we only want the events that already occured
// instead of having "forecasts" that will change in the future taint our
// data and make backtests non-deterministic. We want to have
// consistency with our data in live trading historical requests as well
var releases = JsonConvert.DeserializeObject <List <EstimizeRelease> >(result, JsonSerializerSettings)
.Where(x => x.Eps != null)
.GroupBy(x =>
{
var normalizedTicker = NormalizeTicker(ticker);
var releaseDate = x.ReleaseDate;
try
{
var mapFile = _mapFileResolver.ResolveMapFile(normalizedTicker, releaseDate);
var oldTicker = normalizedTicker;
var newTicker = normalizedTicker;
// Ensure we're writing to the correct historical ticker
if (!mapFile.Any())
{
Log.Trace($"EstimizeReleaseDataDownloader.Run(): Failed to find map file for: {newTicker} - on: {releaseDate}");
return(string.Empty);
}
newTicker = mapFile.GetMappedSymbol(releaseDate);
if (string.IsNullOrWhiteSpace(newTicker))
{
Log.Trace($"EstimizeReleaseDataDownloader.Run(): Failed to find mapping for null new ticker. Old ticker: {oldTicker} - on: {releaseDate}");
return(string.Empty);
}
if (oldTicker != newTicker)
{
Log.Trace($"EstimizeReleaseDataDownloader.Run(): Remapped from {oldTicker} to {newTicker} for {releaseDate}");
}
return(newTicker);
}
// We get a failure inside the map file constructor rarely. It tries
// to access the last element of an empty list. Maybe this is a bug?
catch (InvalidOperationException e)
{
Log.Error(e, $"EstimizeReleaseDataDownloader.Run(): Failed to load map file for: {normalizedTicker} - on: {releaseDate}");
return(string.Empty);
}
})
.Where(x => !string.IsNullOrEmpty(x.Key));
foreach (var kvp in releases)
{
var csvContents = kvp.Select(x => $"{x.ReleaseDate.ToUniversalTime():yyyyMMdd HH:mm:ss},{x.Id},{x.FiscalYear},{x.FiscalQuarter},{x.Eps},{x.Revenue},{x.ConsensusEpsEstimate},{x.ConsensusRevenueEstimate},{x.WallStreetEpsEstimate},{x.WallStreetRevenueEstimate},{x.ConsensusWeightedEpsEstimate},{x.ConsensusWeightedRevenueEstimate}");
SaveContentToFile(_destinationFolder, kvp.Key, csvContents);
}
var percentDone = i / count;
if (percentDone >= currentPercent)
{
Log.Trace($"EstimizeEstimateDataDownloader.Run(): {percentDone:P2} complete");
currentPercent += percent;
}
}
)
);
}
Task.WaitAll(tasks.ToArray());
}
catch (Exception e)
{
Log.Error(e);
return(false);
}
Log.Trace($"EstimizeReleaseDataDownloader.Run(): Finished in {stopwatch.Elapsed}");
return(true);
}
/// <summary>
/// Runs the instance of the object.
/// </summary>
/// <returns>True if process all downloads successfully</returns>
public override bool Run()
{
try
{
if (_releaseFiles.Count == 0)
{
Log.Trace($"EstimizeConsensusDataDownloader.Run(): No files found. Please run EstimizeEstimateDataDownloader first");
return(false);
}
var processedConsensusDirectory = _processedDataDirectory == null
? null
: new DirectoryInfo(
Path.Combine(
_processedDataDirectory.FullName,
"alternative",
"estimize",
"consensus"));
var utcNow = DateTime.UtcNow;
foreach (var releaseFileInfoGroup in _releaseFiles.GroupBy(x => x.Name))
{
var stopwatch = Stopwatch.StartNew();
var tasks = new List <Task <List <EstimizeConsensus> > >();
var ticker = Path.GetFileNameWithoutExtension(releaseFileInfoGroup.Key);
if (_processTickers != null && !_processTickers.Contains(ticker, StringComparer.InvariantCultureIgnoreCase))
{
Log.Trace($"EstimizeConsensusDataDownloader.Run(): Skipping {ticker} since it is not in the list of predefined tickers");
continue;
}
var finalPath = Path.Combine(_destinationFolder, $"{ticker}.csv");
var processedConsensusFile = processedConsensusDirectory == null
? null
: Path.Combine(processedConsensusDirectory.FullName, $"{ticker}.csv");
var existingConsensus = (File.Exists(finalPath) ? File.ReadAllLines(finalPath) : new string[] { })
.Concat(processedConsensusFile != null && File.Exists(processedConsensusFile)
? File.ReadAllLines(processedConsensusFile)
: new string[] { })
.Distinct()
.Select(x => new EstimizeConsensus(x))
.ToList();
// We don't need to apply any sort of mapfile transformations to the ticker
// since we've already applied mapping to the release file ticker
var existingReleases = new List <EstimizeRelease>();
foreach (var releaseFile in releaseFileInfoGroup)
{
var releasesParsed = File.ReadAllLines(releaseFile.FullName)
.Where(x => !string.IsNullOrWhiteSpace(x))
.Select(x => new EstimizeRelease(x));
existingReleases = existingReleases.Concat(releasesParsed).ToList();
}
existingReleases = existingReleases
.DistinctBy(x => x.Id)
.ToList();
foreach (var release in existingReleases)
{
// We detect duplicates by checking for release IDs that match consensus IDs
// in consensus files and ensuring that no more updates will be published to
// consensus data by making sure the release has been made public
if ((utcNow - release.ReleaseDate).TotalDays > 1 && existingConsensus.Any(x => x.Id == release.Id))
{
Log.Trace($"EstimizeConsensusDataDownloader.Run(): Duplicate entry found for ID {release.Id} in {finalPath} on: {release.ReleaseDate}");
continue;
}
Log.Trace($"EstimizeConsensusDataDownloader.Run(): Earnings release: {release.ReleaseDate:yyyy-MM-dd} - Parsing Estimate {release.Id} for: {ticker}");
// Makes sure we don't overrun Estimize rate limits accidentally
IndexGate.WaitToProceed();
tasks.Add(
HttpRequester($"/releases/{release.Id}/consensus")
.ContinueWith(
x =>
{
var result = x.Result;
if (string.IsNullOrEmpty(result))
{
return(new List <EstimizeConsensus>());
}
var jObject = JObject.Parse(result);
var list = new List <EstimizeConsensus>();
list.AddRange(Unpack(release, Source.WallStreet, Type.Eps, jObject));
list.AddRange(Unpack(release, Source.WallStreet, Type.Revenue, jObject));
list.AddRange(Unpack(release, Source.Estimize, Type.Eps, jObject));
list.AddRange(Unpack(release, Source.Estimize, Type.Revenue, jObject));
return(list);
}
)
);
}
Task.WaitAll(tasks.ToArray());
var csvContents = tasks.SelectMany(x => x.Result)
.OrderBy(x => x.UpdatedAt)
.Select(x => $"{x.UpdatedAt.ToUniversalTime():yyyyMMdd HH:mm:ss},{x.Id},{x.Source},{x.Type},{x.Mean},{x.High},{x.Low},{x.StandardDeviation},{x.FiscalYear},{x.FiscalQuarter},{x.Count}");
SaveContentToFile(_destinationFolder, ticker, csvContents);
Log.Trace($"EstimizeConsensusDataDownloader.Run(): EstimizeConsensus files for {ticker} created : {stopwatch.Elapsed}");
}
}
catch (Exception e)
{
Log.Error(e, "EstimizeConsensusDataDownloader.Run(): Failure in consensus download");
return(false);
}
return(true);
}
/// <summary>
/// Runs the instance of the object.
/// </summary>
/// <returns>True if process all downloads successfully</returns>
public override bool Run()
{
var stopwatch = Stopwatch.StartNew();
try
{
var companies = GetCompanies().Result.DistinctBy(x => x.Ticker).ToList();
var count = companies.Count;
var currentPercent = 0.05;
var percent = 0.05;
var i = 0;
Log.Trace($"EstimizeEstimateDataDownloader.Run(): Start processing {count} companies");
var tasks = new List <Task>();
foreach (var company in companies)
{
var ticker = company.Ticker;
// Include tickers that are "defunct".
// Remove the tag because it cannot be part of the API endpoint
if (ticker.IndexOf("defunct", StringComparison.OrdinalIgnoreCase) > 0)
{
var length = ticker.IndexOf('-');
ticker = ticker.Substring(0, length).Trim();
}
Log.Trace($"EstimizeEstimateDataDownloader.Run(): Processing {ticker}");
try
{
// Makes sure we don't overrun Estimize rate limits accidentally
IndexGate.WaitToProceed();
}
// This is super super rare, but it failures in RateGate (RG) can still happen nonetheless. Let's not
// rely on RG operating successfully all the time so that if RG fails, our download process doesn't fail
catch (ArgumentOutOfRangeException e)
{
Log.Error(e, $"EstimizeEstimateDataDownloader.Run(): RateGate failed. Sleeping for 110 milliseconds with Thread.Sleep()");
Thread.Sleep(110);
}
tasks.Add(
HttpRequester($"/companies/{ticker}/estimates")
.ContinueWith(
y =>
{
i++;
if (y.IsFaulted)
{
Log.Error($"EstimizeEstimateDataDownloader.Run(): Failed to get data for {company}");
return;
}
var result = y.Result;
if (string.IsNullOrEmpty(result))
{
// We've already logged inside HttpRequester
return;
}
var estimates = JsonConvert.DeserializeObject <List <EstimizeEstimate> >(result)
.GroupBy(estimate =>
{
var oldTicker = ticker;
var newTicker = ticker;
var createdAt = estimate.CreatedAt;
try
{
var mapFile = _mapFileResolver.ResolveMapFile(ticker, createdAt);
// Ensure we're writing to the correct historical ticker
if (!mapFile.Any())
{
Log.Trace($"EstimizeEstimateDataDownloader.Run(): Failed to find map file for: {newTicker} - on: {createdAt}");
return(string.Empty);
}
newTicker = mapFile.GetMappedSymbol(createdAt);
if (string.IsNullOrWhiteSpace(newTicker))
{
Log.Trace($"EstimizeEstimateDataDownloader.Run(): New ticker is null. Old ticker: {oldTicker} - on: {createdAt}");
return(string.Empty);
}
if (oldTicker != newTicker)
{
Log.Trace($"EstimizeEstimateDataDonwloader.Run(): Remapping {oldTicker} to {newTicker}");
}
}
// We get a failure inside the map file constructor rarely. It tries
// to access the last element of an empty list. Maybe this is a bug?
catch (InvalidOperationException e)
{
Log.Error(e, $"EstimizeEstimateDataDownloader.Run(): Failed to load map file for: {oldTicker} - on {createdAt}");
return(string.Empty);
}
return(newTicker);
})
.Where(kvp => !string.IsNullOrEmpty(kvp.Key));
foreach (var kvp in estimates)
{
var csvContents = kvp.Select(x => $"{x.CreatedAt.ToUniversalTime():yyyyMMdd HH:mm:ss},{x.Id},{x.AnalystId},{x.UserName},{x.FiscalYear},{x.FiscalQuarter},{x.Eps},{x.Revenue},{x.Flagged.ToString().ToLower()}");
SaveContentToFile(_destinationFolder, kvp.Key, csvContents);
}
var percentageDone = i / count;
if (percentageDone >= currentPercent)
{
Log.Trace($"EstimizeEstimateDataDownloader.Run(): {percentageDone:P2} complete");
currentPercent += percent;
}
}
)
);
}
Task.WaitAll(tasks.ToArray());
}
catch (Exception e)
{
Log.Error(e);
return(false);
}
Log.Trace($"EstimizeEstimateDataDownloader.Run(): Finished in {stopwatch.Elapsed}");
return(true);
}
/// <summary>
/// Runs the instance of the object.
/// </summary>
/// <returns>True if process all downloads successfully</returns>
public override bool Run()
{
var stopwatch = Stopwatch.StartNew();
try
{
var companies = GetCompanies().Result.DistinctBy(x => x.Ticker).ToList();
var count = companies.Count;
var currentPercent = 0.05;
var percent = 0.05;
var i = 0;
Log.Trace($"QuiverCongressDataDownloader.Run(): Start processing {count.ToStringInvariant()} companies");
var tasks = new List <Task>();
foreach (var company in companies)
{
// Include tickers that are "defunct".
// Remove the tag because it cannot be part of the API endpoint.
// This is separate from the NormalizeTicker(...) method since
// we don't convert tickers with `-`s into the format we can successfully
// index mapfiles with.
var quiverTicker = company.Ticker;
string ticker;
if (!TryNormalizeDefunctTicker(quiverTicker, out ticker))
{
Log.Error($"QuiverCongressDataDownloader(): Defunct ticker {quiverTicker} is unable to be parsed. Continuing...");
continue;
}
// Begin processing ticker with a normalized value
Log.Trace($"QuiverCongressionalDataDownloader.Run(): Processing {ticker}");
// Makes sure we don't overrun Quiver rate limits accidentally
IndexGate.WaitToProceed();
tasks.Add(
HttpRequester($"historical/congresstrading/{ticker}")
.ContinueWith(
y =>
{
i++;
if (y.IsFaulted)
{
Log.Error($"QuiverCongressDataDownloader.Run(): Failed to get data for {company}");
return;
}
var result = y.Result;
if (string.IsNullOrEmpty(result))
{
// We've already logged inside HttpRequester
return;
}
Log.Trace(result);
var followers = JsonConvert.DeserializeObject <List <QuiverCongress> >(result, JsonSerializerSettings);
foreach (var kvp in followers)
{
var csvContents = new string[] {
$"{kvp.ReportDate.ToStringInvariant("yyyyMMdd")}," +
$"{kvp.TransactionDate.ToStringInvariant("yyyyMMdd")}," +
$"{kvp.Representative}," +
$"{kvp.Transaction}," +
$"{kvp.Amount}," +
$"{kvp.House}"
};
SaveContentToFile(_destinationFolder, ticker, csvContents);
}
var percentageDone = i / count;
if (percentageDone >= currentPercent)
{
Log.Trace($"QuiverCongressDataDownloader.Run(): {percentageDone.ToStringInvariant("P2")} complete");
currentPercent += percent;
}
}
)
);
}
Task.WaitAll(tasks.ToArray());
}
catch (Exception e)
{
Log.Error(e);
return(false);
}
Log.Trace($"QuiverCongressDataDownloader.Run(): Finished in {stopwatch.Elapsed.ToStringInvariant(null)}");
return(true);
}
/// <summary>
/// Runs the instance of the object.
/// </summary>
/// <returns>True if process all downloads successfully</returns>
public override bool Run()
{
var stopwatch = Stopwatch.StartNew();
try
{
var companies = GetCompanies().Result.DistinctBy(x => x.Ticker).ToList();
var count = companies.Count;
var currentPercent = 0.05;
var percent = 0.05;
var i = 0;
Log.Trace($"EstimizeEstimateDataDownloader.Run(): Start processing {count.ToStringInvariant()} companies");
var tasks = new List <Task>();
foreach (var company in companies)
{
// Include tickers that are "defunct".
// Remove the tag because it cannot be part of the API endpoint.
// This is separate from the NormalizeTicker(...) method since
// we don't convert tickers with `-`s into the format we can successfully
// index mapfiles with.
var estimizeTicker = company.Ticker;
string ticker;
if (!TryNormalizeDefunctTicker(estimizeTicker, out ticker))
{
Log.Error($"EstimizeEstimateDataDownloader(): Defunct ticker {estimizeTicker} is unable to be parsed. Continuing...");
continue;
}
if (_processTickers != null && !_processTickers.Contains(ticker, StringComparer.InvariantCultureIgnoreCase))
{
Log.Trace($"EstimizeEstimateDataDownloader.Run(): Skipping {ticker} since it is not in the list of predefined tickers");
continue;
}
// Begin processing ticker with a normalized value
Log.Trace($"EstimizeEstimateDataDownloader.Run(): Processing {ticker}");
// Makes sure we don't overrun Estimize rate limits accidentally
IndexGate.WaitToProceed();
tasks.Add(
HttpRequester($"/companies/{ticker}/estimates")
.ContinueWith(
y =>
{
i++;
if (y.IsFaulted)
{
Log.Error($"EstimizeEstimateDataDownloader.Run(): Failed to get data for {company}");
return;
}
var result = y.Result;
if (string.IsNullOrEmpty(result))
{
// We've already logged inside HttpRequester
return;
}
var estimates = JsonConvert.DeserializeObject <List <EstimizeEstimate> >(result, JsonSerializerSettings)
.GroupBy(estimate =>
{
var normalizedTicker = NormalizeTicker(ticker);
var oldTicker = normalizedTicker;
var newTicker = normalizedTicker;
var createdAt = estimate.CreatedAt;
try
{
var mapFile = _mapFileResolver.ResolveMapFile(normalizedTicker, createdAt);
// Ensure we're writing to the correct historical ticker
if (!mapFile.Any())
{
Log.Trace($"EstimizeEstimateDataDownloader.Run(): Failed to find map file for: {newTicker} - on: {createdAt}");
return(string.Empty);
}
newTicker = mapFile.GetMappedSymbol(createdAt);
if (string.IsNullOrWhiteSpace(newTicker))
{
Log.Trace($"EstimizeEstimateDataDownloader.Run(): New ticker is null. Old ticker: {oldTicker} - on: {createdAt.ToStringInvariant()}");
return(string.Empty);
}
if (!string.Equals(oldTicker, newTicker, StringComparison.InvariantCultureIgnoreCase))
{
Log.Trace($"EstimizeEstimateDataDownloader.Run(): Remapping {oldTicker} to {newTicker}");
}
}
// We get a failure inside the map file constructor rarely. It tries
// to access the last element of an empty list. Maybe this is a bug?
catch (InvalidOperationException e)
{
Log.Error(e, $"EstimizeEstimateDataDownloader.Run(): Failed to load map file for: {oldTicker} - on {createdAt}");
return(string.Empty);
}
return(newTicker);
})
.Where(kvp => !string.IsNullOrEmpty(kvp.Key));
foreach (var kvp in estimates)
{
var csvContents = kvp.Select(x =>
$"{x.CreatedAt.ToStringInvariant("yyyyMMdd HH:mm:ss")}," +
$"{x.Id}," +
$"{x.AnalystId}," +
$"{x.UserName}," +
$"{x.FiscalYear.ToStringInvariant()}," +
$"{x.FiscalQuarter.ToStringInvariant()}," +
$"{x.Eps.ToStringInvariant()}," +
$"{x.Revenue.ToStringInvariant()}," +
$"{x.Flagged.ToStringInvariant().ToLowerInvariant()}"
);
SaveContentToFile(_destinationFolder, kvp.Key, csvContents);
}
var percentageDone = i / count;
if (percentageDone >= currentPercent)
{
Log.Trace($"EstimizeEstimateDataDownloader.Run(): {percentageDone.ToStringInvariant("P2")} complete");
currentPercent += percent;
}
}
)
);
}
Task.WaitAll(tasks.ToArray());
}
catch (Exception e)
{
Log.Error(e);
return(false);
}
Log.Trace($"EstimizeEstimateDataDownloader.Run(): Finished in {stopwatch.Elapsed.ToStringInvariant(null)}");
return(true);
}
/// <summary>
/// Runs the instance of the object.
/// </summary>
/// <returns>True if process all downloads successfully</returns>
public override bool Run()
{
var stopwatch = Stopwatch.StartNew();
try
{
if (_zipFiles.Length == 0)
{
Log.Trace($"EstimizeConsensusDataDownloader.Run(): No files found. Please run EstimizeEstimateDataDownloader first");
return(false);
}
foreach (var zipFile in _zipFiles)
{
var tasks = new List <Task <List <EstimizeConsensus> > >();
foreach (var kvp in Compression.Unzip(zipFile))
{
var content = kvp.Value.SingleOrDefault();
var estimates = JsonConvert.DeserializeObject <List <EstimizeEstimate> >(content);
foreach (var estimate in estimates)
{
// Makes sure we don't overrun Estimize rate limits accidentally
IndexGate.WaitToProceed();
tasks.Add(
HttpRequester($"/releases/{estimate.Id}/consensus")
.ContinueWith(
x =>
{
var jObject = JObject.Parse(x.Result);
var list = new List <EstimizeConsensus>();
list.AddRange(Unpack(estimate, Source.WallStreet, Type.Eps, jObject));
list.AddRange(Unpack(estimate, Source.WallStreet, Type.Revenue, jObject));
list.AddRange(Unpack(estimate, Source.Estimize, Type.Eps, jObject));
list.AddRange(Unpack(estimate, Source.Estimize, Type.Revenue, jObject));
return(list);
}
)
);
}
}
Task.WaitAll(tasks.ToArray());
var ticker = Path.GetFileNameWithoutExtension(zipFile) ?? string.Empty;
Directory.CreateDirectory(Path.Combine(_destinationFolder, ticker));
var consensuses = tasks.SelectMany(x => x.Result);
foreach (var kvp in consensuses.GroupBy(x => x.UpdatedAt.Date))
{
var contents = JsonConvert.SerializeObject(kvp.OrderBy(x => x.UpdatedAt));
SaveContentToZipFile(ticker, $"{kvp.Key:yyyyMMdd}", contents);
}
Log.Trace($"EstimizeConsensusDataDownloader.Run(): EstimizeConsensus files for {ticker} created : {stopwatch.Elapsed}");
}
}
catch (Exception e)
{
Log.Error(e);
return(false);
}
Log.Trace($"EstimizeConsensusDataDownloader.Run(): Finished in {stopwatch.Elapsed}");
return(true);
}
/// <summary>
/// Runs the instance of the object.
/// </summary>
/// <returns>True if process all downloads successfully</returns>
public override bool Run()
{
var stopwatch = Stopwatch.StartNew();
try
{
var companies = GetCompanies().Result.DistinctBy(x => x.Ticker).ToList();
var count = companies.Count;
var currentPercent = 0.05;
var percent = 0.05;
var i = 0;
Log.Trace($"EstimizeReleaseDataDownloader.Run(): Start processing {count} companies");
var tasks = new List <Task>();
foreach (var company in companies)
{
try
{
// Makes sure we don't overrun Estimize rate limits accidentally
IndexGate.WaitToProceed();
}
// This is super super rare, but it failures in RateGate (RG) can still happen nonetheless. Let's not
// rely on RG operating successfully all the time so that if RG fails, our download process doesn't fail
catch (ArgumentOutOfRangeException e)
{
Log.Error(e, $"EstimizeReleaseDataDownloader.Run(): RateGate failed. Sleeping for 110 milliseconds with Thread.Sleep()");
Thread.Sleep(110);
}
var ticker = company.Ticker;
if (ticker.IndexOf("defunct", StringComparison.OrdinalIgnoreCase) > 0)
{
var length = ticker.IndexOf('-');
ticker = ticker.Substring(0, length).Trim();
}
Log.Trace($"EstimizeReleaseDataDownloader.Run(): Processing {ticker}");
tasks.Add(
HttpRequester($"/companies/{ticker}/releases")
.ContinueWith(
y =>
{
i++;
if (y.IsFaulted)
{
Log.Error($"EstimizeReleaseDataDownloader.Run(): Failed to get data for {company}");
return;
}
var result = y.Result;
if (string.IsNullOrEmpty(result))
{
// We've already logged inside HttpRequester
return;
}
// Just like TradingEconomics, we only want the events that already occured
// instead of having "forecasts" that will change in the future taint our
// data and make backtests non-deterministic. We want to have
// consistency with our data in live trading historical requests as well
var releases = JsonConvert.DeserializeObject <List <EstimizeRelease> >(result)
.Where(x => x.Eps != null)
.GroupBy(x =>
{
var releaseDate = x.ReleaseDate;
try
{
var mapFile = _mapFileResolver.ResolveMapFile(ticker, releaseDate);
var oldTicker = ticker;
var newTicker = ticker;
// Ensure we're writing to the correct historical ticker
if (!mapFile.Any())
{
Log.Trace($"EstimizeReleaseDataDownloader.Run(): Failed to find map file for: {newTicker} - on: {releaseDate}");
return(string.Empty);
}
newTicker = mapFile.GetMappedSymbol(releaseDate);
if (string.IsNullOrWhiteSpace(newTicker))
{
Log.Trace($"EstimizeReleaseDataDownloader.Run(): Failed to find mapping for null new ticker. Old ticker: {oldTicker} - on: {releaseDate}");
return(string.Empty);
}
if (oldTicker != newTicker)
{
Log.Trace($"EstimizeReleaseDataDownloader.Run(): Remapped from {oldTicker} to {newTicker} for {releaseDate}");
}
return(newTicker);
}
// We get a failure inside the map file constructor rarely. It tries
// to access the last element of an empty list. Maybe this is a bug?
catch (InvalidOperationException e)
{
Log.Error(e, $"EstimizeReleaseDataDownloader.Run(): Failed to load map file for: {ticker} - on: {releaseDate}");
return(string.Empty);
}
})
.Where(x => !string.IsNullOrEmpty(x.Key));
foreach (var kvp in releases)
{
var csvContents = kvp.Select(x => $"{x.ReleaseDate.ToUniversalTime():yyyyMMdd HH:mm:ss},{x.Id},{x.FiscalYear},{x.FiscalQuarter},{x.Eps},{x.Revenue},{x.ConsensusEpsEstimate},{x.ConsensusRevenueEstimate},{x.WallStreetEpsEstimate},{x.WallStreetRevenueEstimate},{x.ConsensusWeightedEpsEstimate},{x.ConsensusWeightedRevenueEstimate}");
SaveContentToFile(_destinationFolder, kvp.Key, csvContents);
}
var percentDone = i / count;
if (percentDone >= currentPercent)
{
Log.Trace($"EstimizeEstimateDataDownloader.Run(): {percentDone:P2} complete");
currentPercent += percent;
}
}
)
);
}
Task.WaitAll(tasks.ToArray());
}
catch (Exception e)
{
Log.Error(e);
return(false);
}
Log.Trace($"EstimizeReleaseDataDownloader.Run(): Finished in {stopwatch.Elapsed}");
return(true);
}
/// <summary>
/// Runs the instance of the object.
/// </summary>
/// <returns>True if process all downloads successfully</returns>
public override bool Run()
{
try
{
if (_releaseFiles.Length == 0)
{
Log.Trace($"EstimizeConsensusDataDownloader.Run(): No files found. Please run EstimizeEstimateDataDownloader first");
return(false);
}
var utcNow = DateTime.UtcNow;
foreach (var releaseFile in _releaseFiles)
{
Log.Trace($"EstimizeConsensusDataDownloader.Run(): Processing release file: {releaseFile}");
var stopwatch = Stopwatch.StartNew();
var tasks = new List <Task <List <EstimizeConsensus> > >();
// We don't need to apply any sort of mapfile transformations to the ticker
// since we've already applied mapping to the release file ticker
var ticker = Path.GetFileNameWithoutExtension(releaseFile) ?? string.Empty;
var releases = File.ReadAllLines(releaseFile).Where(x => !string.IsNullOrWhiteSpace(x)).Select(x => new EstimizeRelease(x));
var finalPath = Path.Combine(_destinationFolder, $"{ticker}.csv");
var existingConsensus = (File.Exists(finalPath) ? File.ReadAllLines(finalPath) : new string[] { })
.Select(x => new EstimizeConsensus(x));
foreach (var release in releases)
{
// We detect duplicates by checking for release IDs that match consensus IDs
// in consensus files and ensuring that no more updates will be published to
// consensus data by making sure the release has been made public
if ((utcNow - release.ReleaseDate).TotalDays > 1 && existingConsensus.Where(x => x.Id == release.Id).Any())
{
Log.Trace($"EstimizeConsensusDataDownloader.Run(): Duplicate entry found for ID {release.Id} in {finalPath} on: {release.ReleaseDate}");
continue;
}
Log.Trace($"EstimizeConsensusDataDownloader.Run(): Earnings release: {release.ReleaseDate:yyyy-MM-dd} - Parsing Estimate {release.Id} for: {ticker}");
try
{
// Makes sure we don't overrun Estimize rate limits accidentally
IndexGate.WaitToProceed();
}
// This is super super rare, but it failures in RateGate (RG) can still happen nonetheless. Let's not
// rely on RG operating successfully all the time so that if RG fails, our download process doesn't fail
catch (ArgumentOutOfRangeException e)
{
Log.Error(e, $"EstimizeConsensusDataDownloader.Run(): RateGate failed. Sleeping for 110 milliseconds with Thread.Sleep()");
Thread.Sleep(110);
}
tasks.Add(
HttpRequester($"/releases/{release.Id}/consensus")
.ContinueWith(
x =>
{
var result = x.Result;
if (string.IsNullOrEmpty(result))
{
return(new List <EstimizeConsensus>());
}
var jObject = JObject.Parse(result);
var list = new List <EstimizeConsensus>();
list.AddRange(Unpack(release, Source.WallStreet, Type.Eps, jObject));
list.AddRange(Unpack(release, Source.WallStreet, Type.Revenue, jObject));
list.AddRange(Unpack(release, Source.Estimize, Type.Eps, jObject));
list.AddRange(Unpack(release, Source.Estimize, Type.Revenue, jObject));
return(list);
}
)
);
}
Task.WaitAll(tasks.ToArray());
var csvContents = tasks.SelectMany(x => x.Result)
.OrderBy(x => x.UpdatedAt)
.Select(x => $"{x.UpdatedAt.ToUniversalTime():yyyyMMdd HH:mm:ss},{x.Id},{x.Source},{x.Type},{x.Mean},{x.High},{x.Low},{x.StandardDeviation},{x.FiscalYear},{x.FiscalQuarter},{x.Count}");
SaveContentToFile(_destinationFolder, ticker, csvContents);
Log.Trace($"EstimizeConsensusDataDownloader.Run(): EstimizeConsensus files for {ticker} created : {stopwatch.Elapsed}");
}
}
catch (Exception e)
{
Log.Error(e, "EstimizeConsensusDataDownloader.Run(): Failure in consensus download");
return(false);
}
return(true);
}
