public static void MakeReport(classificationReportCollection reportCollection, classificationReportCollectionSettings settings, aceAuthorNotation appInfo, ILogBuilder log, classificationReportStyleDefinition style, classificationReportDataComplexContext context)
{
DataTableTypeExtended <classificationReportExpanded> table = reportCollection.MakeOverviewTable(context, reportCollection.name, reportCollection.description);
table.SetTitle(reportCollection.name);
var statDataTable = table.GetReportAndSave(reportCollection.rootFolder, appInfo);
log.log("Report [" + table.TableName + "] created at " + statDataTable.lastFilePath);
// context.cumulative_tables.Add(table);
var layers = reportCollection.GetSpaceLayers(style);
foreach (var pair in layers)
{
var reportSpace = classificationReportSpace.BuildReportSpace(pair.Value, reportCollection.datasetName, settings.SELECT_REPORT_NAME_PARTS, style, pair.Key);
if (!context.report_spaces.ContainsKey(reportSpace.name))
{
context.report_spaces.Add(reportSpace.name, new List <classificationReportSpace>());
context.comparative_tables.Add(reportSpace.name, new List <DataTable>());
context.comparative_narrow_tables.Add(reportSpace.name, new List <DataTable>());
}
context.report_spaces[reportSpace.name].Add(reportSpace);
System.Data.DataTable comparative_table = reportSpace.ConstructTable("comparative_" + reportCollection.name + "_" + reportSpace.name, reportCollection.description);
context.comparative_tables[reportSpace.name].Add(comparative_table);
comparative_table.AddExtra("Group path: " + reportCollection.rootFolder.path);
comparative_table.GetReportAndSave(reportCollection.rootFolder, appInfo);
System.Data.DataTable comparative_table_small = reportSpace.ConstructTable("comparative_" + reportCollection.name + "_" + reportSpace.name + "_small", reportCollection.description, classificationReportTableMode.onlyBasic);
context.comparative_narrow_tables[reportSpace.name].Add(comparative_table_small);
var styleFS = style.CloneViaXML();
styleFS.valueToUse = classificationReportStyleDefinition.GetFS(); //new reportExpandedDataPair(classificationReportStyleDefinition.VALUE_FS, "Selected Features", "Number of features actually selected");
reportSpace = classificationReportSpace.BuildReportSpace(pair.Value, reportCollection.datasetName, settings.SELECT_REPORT_NAME_PARTS, styleFS, pair.Key);
reportSpace.ConstructTable("featureSelected_" + reportCollection.name + "_" + reportSpace.name, reportCollection.description).GetReportAndSave(reportCollection.rootFolder, appInfo);
}
// return comparative_table;
}
public void MakeReports(experimentExecutionContext context, folderNode folder)
{
meanClassifierReport = new DocumentSetCaseCollectionReport(extractor.name);
aceDictionary2D <IWebPostClassifier, kFoldValidationCase, DocumentSetCaseCollectionReport> tempStructure = new aceDictionary2D <IWebPostClassifier, kFoldValidationCase, DocumentSetCaseCollectionReport>();
DSCCReports firstCase = null;
List <IWebPostClassifier> classifiers = new List <IWebPostClassifier>();
foreach (var kFoldCasePair in this)
{
if (firstCase == null)
{
firstCase = kFoldCasePair.Value;
}
foreach (var pair in kFoldCasePair.Value.avgReports)
{
tempStructure[pair.Key, kFoldCasePair.Key] = pair.Value;
if (!classifiers.Contains(pair.Key))
{
classifiers.Add(pair.Key);
}
}
}
// DataSet dataSet = new DataSet(context.setup.name);
// <---------- CREATING AVERAGE TABLE -----------------------------------------------------
var tpAvgMacro = new DataTableTypeExtended <DocumentSetCaseCollectionReport>(context.setup.name + " summary", "Cross k-fold averages measures, fold-level measures are computed by macro-average method");
var tpAvgMicro = new DataTableTypeExtended <DocumentSetCaseCollectionReport>(context.setup.name + " summary", "Cross k-fold averages measures, fold-level measures are computed by micro-average method");
List <DocumentSetCaseCollectionReport> macroaverages = new List <DocumentSetCaseCollectionReport>();
DataTableTypeExtended <DocumentSetCaseCollectionReport> EMperKFolds = new DataTableTypeExtended <DocumentSetCaseCollectionReport>(extractor.name + "_allReports");
foreach (IWebPostClassifier classifier in classifiers)
{
// < ---- report on each classifier
context.logger.log("-- producing report about [" + classifier.name + "]");
//objectTable<DocumentSetCaseCollectionReport> tp = new objectTable<DocumentSetCaseCollectionReport>(nameof(DocumentSetCaseCollectionReport.Name), classifier + "_sum");
DocumentSetCaseCollectionReport avg = new DocumentSetCaseCollectionReport(classifier.name + " macro-averaging, k-fold avg. ");
DocumentSetCaseCollectionReport rep_eval = new DocumentSetCaseCollectionReport(classifier.name + " micro-averaging, k-fold avg.");
rep_eval.Classifier = classifier.name;
classificationEvalMetricSet metrics = new classificationEvalMetricSet();
classificationEval eval = new classificationEval();
//eval = metrics[classifier.name];
Int32 c = 0;
foreach (KeyValuePair <kFoldValidationCase, DSCCReports> kFoldCasePair in this)
{
DocumentSetCaseCollectionReport rep = kFoldCasePair.Value.avgReports[classifier];
kFoldValidationCase vCase = kFoldCasePair.Key;
classificationEvalMetricSet met = rep.GetSetMetrics();
if (met != null)
{
foreach (IDocumentSetClass cl in context.classes.GetClasses())
{
eval = eval + met[cl.name];
}
}
rep.Name = classifier.name + "_" + vCase.name;
avg.AddValues(rep);
EMperKFolds.AddRow(rep);
c++;
}
rep_eval.AddValues(metrics, classificationMetricComputation.microAveraging);
avg.Classifier = classifier.name;
avg.DivideValues(c);
// <<< detecting the best performed classifier in all evaluation folds
if (avg.F1measure > highestF1Value)
{
highestF1Value = avg.F1measure;
topClassifierReport = avg;
}
meanClassifierReport.AddValues(avg);
// -----------------
EMperKFolds.AddRow(avg);
tpAvgMacro.AddRow(avg);
macroaverages.Add(avg);
if (DOMAKE_MICROaverage)
{
tpAvgMicro.AddRow(rep_eval);
}
// tp.Add(rep_eval);
if (context.tools.operation.DoMakeReportForEachClassifier)
{
DataTable cTable = EMperKFolds;
cTable.SetTitle($"{classifier.name} report");
cTable.SetDescription("Summary " + context.setup.validationSetup.k + "-fold validation report for [" + classifier.name + "]");
cTable.SetAdditionalInfoEntry("FV Extractor", extractor.name);
cTable.SetAdditionalInfoEntry("Classifier", classifier.name);
cTable.SetAdditionalInfoEntry("Class name", classifier.GetType().Name);
cTable.SetAdditionalInfoEntry("Correct", rep_eval.Correct);
cTable.SetAdditionalInfoEntry("Wrong", rep_eval.Wrong);
//cTable.SetAdditionalInfoEntry("Precision", rep_eval.Precision);
//cTable.SetAdditionalInfoEntry("Recall", rep_eval.Recall);
//cTable.SetAdditionalInfoEntry("F1", rep_eval.F1measure);
cTable.SetAdditionalInfoEntry("True Positives", metrics[classifier.name].truePositives);
cTable.SetAdditionalInfoEntry("False Negatives", metrics[classifier.name].falseNegatives);
cTable.SetAdditionalInfoEntry("False Positives", metrics[classifier.name].falsePositives);
cTable.AddExtra("Classifier: " + classifier.name + " [" + classifier.GetType().Name + "]");
var info = classifier.DescribeSelf();
info.ForEach(x => cTable.AddExtra(x));
cTable.AddExtra("-----------------------------------------------------------------------");
cTable.AddExtra("Precision, Recall and F1 measures expressed in this table are computed by macroaveraging shema");
// output.CopyRowsFrom(cTable);
cTable.GetReportAndSave(folder, appManager.AppInfo, extractor.name + "_classifier_" + classifier.name);
// dataSet.AddTable(cTable);
}
}
rangeFinderForDataTable rangerMacro = new rangeFinderForDataTable(tpAvgMacro, "Name");
meanClassifierReport.DivideValues(classifiers.Count);
if (macroaverages.Count > 0)
{
Double maxF1 = macroaverages.Max(x => x.F1measure);
Double minF1 = macroaverages.Min(x => x.F1measure);
List <String> minCaseNames = macroaverages.Where(x => x.F1measure == minF1).Select(x => x.Name).ToList();
List <String> maxCaseNames = macroaverages.Where(x => x.F1measure == maxF1).Select(x => x.Name).ToList();
var style = EMperKFolds.GetRowMetaSet().SetStyleForRowsWithValue <String>(DataRowInReportTypeEnum.dataHighlightA, nameof(DocumentSetCaseCollectionReport.Name), maxCaseNames);
EMperKFolds.GetRowMetaSet().AddUnit(style);
// style = tpAvgMacro.GetRowMetaSet().SetStyleForRowsWithValue<String>(DataRowInReportTypeEnum.dataHighlightC, nameof(DocumentSetCaseCollectionReport.Name), minCaseNames);
tpAvgMacro.SetAdditionalInfoEntry("FV Extractor", extractor.name);
if (DOMAKE_MICROaverage)
{
tpAvgMicro.SetAdditionalInfoEntry("FV Extractor", extractor.name);
}
List <String> averageNames = macroaverages.Select(x => x.Name).ToList();
var avg_style = EMperKFolds.GetRowMetaSet().SetStyleForRowsWithValue <String>(DataRowInReportTypeEnum.dataHighlightC, nameof(DocumentSetCaseCollectionReport.Name), averageNames);
foreach (var x in averageNames)
{
avg_style.AddMatch(x);
}
}
// ::: ------------------------------------------------------------------------------------------------- ::: --------------------------------------------------------------------- ::: //
tpAvgMacro.SetTitle($"{extractor.name} - macroaverage report");
if (DOMAKE_MICROaverage)
{
tpAvgMicro.SetTitle($"{extractor.name} - microaverage report");
}
tpAvgMacro.AddExtra("Complete report on " + context.setup.validationSetup.k + "-fold validation FVE [" + extractor.name + "]");
tpAvgMacro.AddExtra("Fold-level P, R and F1 measures are computed by macroaveraging method, values here are cross k-fold means.");
if (DOMAKE_MICROaverage)
{
tpAvgMicro.AddExtra("Complete " + context.setup.validationSetup.k + "-fold validation report for FVE [" + extractor.name + "]");
}
if (DOMAKE_MICROaverage)
{
tpAvgMicro.AddExtra("Fold-level P, R and F1 measures are computed by microaveraging method, values here are cross k-fold means.");
}
context.AddExperimentInfo(tpAvgMacro);
if (DOMAKE_MICROaverage)
{
context.AddExperimentInfo(tpAvgMicro);
}
tpAvgMacro.AddExtra(extractor.description);
if (extractor is semanticFVExtractor)
{
semanticFVExtractor semExtractor = (semanticFVExtractor)extractor;
semExtractor.termTableConstructor.DescribeSelf().ForEach(x => tpAvgMacro.AddExtra(x));
semExtractor.CloudConstructor.DescribeSelf().ForEach(x => tpAvgMacro.AddExtra(x));
semExtractor.termTableConstructor.DescribeSelf().ForEach(x => tpAvgMicro.AddExtra(x));
semExtractor.CloudConstructor.DescribeSelf().ForEach(x => tpAvgMicro.AddExtra(x));
}
context.logger.log("-- producing summary reports on [" + extractor.name + "]");
rangerMacro.AddRangeRows("Macroaverage ", tpAvgMacro, true,
imbSCI.Core.math.aggregation.dataPointAggregationType.min | imbSCI.Core.math.aggregation.dataPointAggregationType.max
| imbSCI.Core.math.aggregation.dataPointAggregationType.avg
| imbSCI.Core.math.aggregation.dataPointAggregationType.stdev);
tpAvgMacro.GetReportAndSave(folder, appManager.AppInfo, extractor.name + "_macroaverage_report", true, true);
EMperKFolds.AddExtra("The table shows average measures for each fold --- rows marked with colored background show averages for all folds, per classifier.");
EMperKFolds.GetReportAndSave(folder, appManager.AppInfo, extractor.name + "_allFolds", true, true);
if (DOMAKE_MICROaverage)
{
tpAvgMicro.GetReportAndSave(folder, appManager.AppInfo, extractor.name + "_microaverage_report", true, true);
}
//dataSet.GetReportVersion().serializeDataSet(extractor.name + "_classifiers_MultiSheetSummary", folder, imbSCI.Data.enums.reporting.dataTableExportEnum.excel, appManager.AppInfo);
}
