private static void FinalizeRun(Task <FileAnalysisBatchResult>[] tasks, AtratinusConfiguration config, uint enumeratedFiles, uint amountFiles, AlteryxResult alteryxResult)
{
Task.WaitAll(tasks);
List <Supervised> trainingData = new List <Supervised>();
List <InvestmentActivity> investments = new List <InvestmentActivity>();
var report = new FullQualityReport();
for (int c = 0; c < tasks.Length; c++)
{
trainingData.AddRange(tasks[c].Result.TrainingData);
investments.AddRange(tasks[c].Result.Accessions);
report.Merge(tasks[c].Result.QualityReport);
}
Helper.SaveAccessionsAsCSV(investments, config.OutputFolder, false);
FileHelper.SaveTrainingData(trainingData, config.OutputFolder);
report.BuildAndSaveReport(config.OutputFolder, enumeratedFiles, amountFiles, alteryxResult);
}
static bool TakeInvestment(QualityReport report, AtratinusConfiguration config)
{
if (!config.TakeFilesBeforeSECReform && !report.SubmittedAfterSECReform)
{
return(false);
}
if (report.Quality == QualityLevel.T_TIER)
{
return(false);
}
if (!report.UsefulSubmissionType)
{
return(false);
}
return(true);
}
private static FileAnalysisBatchResult AnalyzeSliceOfFiles(int start, int limit, IReadOnlyList <string> files, IReadOnlyDictionary <string, InvestmentActivity> originAccessions,
IReadOnlyDictionary <string, int> supervised, InvestorHashTableSet investors, AtratinusConfiguration config)
{
IList <InvestmentActivity> investments = new List <InvestmentActivity>();
IList <Supervised> trainingData = new List <Supervised>();
var report = new FullQualityReport();
for (int fileIndex = start; fileIndex < limit; fileIndex++)
{
var analysis = AnalyzeFile(files[fileIndex], originAccessions, supervised, investors);
var fileReport = QualityGate.Measure(analysis.Investment, config);
report.ConsiderQualityReport(files[fileIndex], fileReport);
if (!fileReport.ShouldBeConsidered)
{
continue;
}
if (analysis.Supervised != null)
{
trainingData.Add(analysis.Supervised);
}
investments.Add(analysis.Investment);
}
return(new FileAnalysisBatchResult()
{
Accessions = investments, TrainingData = trainingData, QualityReport = report
});
}
internal static QualityReport Measure(InvestmentActivity investment, AtratinusConfiguration config)
{
var report = new QualityReport();
ushort badStates = 0;
if (!DataValidators.IsValidAccessionNumber(investment.AccessionNumber))
{
report.Invalidations.Add("AccessionNumber");
report.Quality = QualityLevel.T_TIER;
investment.DataQualityLevel = QualityLevel.T_TIER.ToString();
}
if (!DataValidators.IsValidFilingDate(investment.FilingDate))
{
badStates++;
report.Invalidations.Add("FilingDate");
}
else
{
var year = Convert.ToInt32(investment.FilingDate.Substring(0, 4));
var month = Convert.ToInt32(investment.FilingDate.Substring(4, 2));
var day = Convert.ToInt32(investment.FilingDate.Substring(6, 2));
if (new DateTime(year, month, day) < new DateTime(1998, 2, 17))
{
report.SubmittedAfterSECReform = false;
}
else
{
report.SubmittedAfterSECReform = true;
}
}
if (!DataValidators.IsValidPurposeOfTransaction(investment.PurposeOfTransaction))
{
badStates++;
report.Invalidations.Add("PurposeOfTransaction");
}
if (!DataValidators.IsValidTypeOfReportingPerson(investment.TypeOfReportingPerson))
{
badStates++;
report.Invalidations.Add("TypeOfReportingPerson");
}
if (!investment.SubmissionType.HasValue)
{
badStates++;
report.Invalidations.Add("SubmissionType");
report.UsefulSubmissionType = false;
}
else
{
if (investment.SubmissionType.Value == Core.Enums.SubmissionType.SC_13D)
{
report.UsefulSubmissionType = true;
}
else
{
report.UsefulSubmissionType = false;
}
}
if (report.Quality == QualityLevel.T_TIER)
{
return(report);
}
switch (badStates)
{
case 0:
{
investment.DataQualityLevel = QualityLevel.S_TIER.ToString();
report.Quality = QualityLevel.S_TIER;
break;
}
case 1:
{
investment.DataQualityLevel = QualityLevel.A_TIER.ToString();
report.Quality = QualityLevel.A_TIER;
break;
}
case 2:
{
investment.DataQualityLevel = QualityLevel.B_TIER.ToString();
report.Quality = QualityLevel.B_TIER;
break;
}
case 3:
{
investment.DataQualityLevel = QualityLevel.C_TIER.ToString();
report.Quality = QualityLevel.C_TIER;
break;
}
default:
{
investment.DataQualityLevel = QualityLevel.T_TIER.ToString();
report.Quality = QualityLevel.T_TIER;
break;
}
}
report.ShouldBeConsidered = TakeInvestment(report, config);
return(report);
}