private static FileAnalysisBatchResult AnalyzeSliceOfFiles(int start, int limit, IReadOnlyList <string> files, IReadOnlyDictionary <string, InvestmentActivity> originAccessions,
IReadOnlyDictionary <string, int> supervised, InvestorHashTableSet investors, AtratinusConfiguration config)
{
IList <InvestmentActivity> investments = new List <InvestmentActivity>();
IList <Supervised> trainingData = new List <Supervised>();
var report = new FullQualityReport();
for (int fileIndex = start; fileIndex < limit; fileIndex++)
{
var analysis = AnalyzeFile(files[fileIndex], originAccessions, supervised, investors);
var fileReport = QualityGate.Measure(analysis.Investment, config);
report.ConsiderQualityReport(files[fileIndex], fileReport);
if (!fileReport.ShouldBeConsidered)
{
continue;
}
if (analysis.Supervised != null)
{
trainingData.Add(analysis.Supervised);
}
investments.Add(analysis.Investment);
}
return(new FileAnalysisBatchResult()
{
Accessions = investments, TrainingData = trainingData, QualityReport = report
});
}
private static void FinalizeRun(Task <FileAnalysisBatchResult>[] tasks, AtratinusConfiguration config, uint enumeratedFiles, uint amountFiles, AlteryxResult alteryxResult)
{
Task.WaitAll(tasks);
List <Supervised> trainingData = new List <Supervised>();
List <InvestmentActivity> investments = new List <InvestmentActivity>();
var report = new FullQualityReport();
for (int c = 0; c < tasks.Length; c++)
{
trainingData.AddRange(tasks[c].Result.TrainingData);
investments.AddRange(tasks[c].Result.Accessions);
report.Merge(tasks[c].Result.QualityReport);
}
Helper.SaveAccessionsAsCSV(investments, config.OutputFolder, false);
FileHelper.SaveTrainingData(trainingData, config.OutputFolder);
report.BuildAndSaveReport(config.OutputFolder, enumeratedFiles, amountFiles, alteryxResult);
}
