public WebClassifierResultSet(DocumentSetClasses _setClassCollection, experimentExecutionContext _context, IFVExtractorSettings _extractorSettings)
{
setClassCollection = _setClassCollection;
featureVectors = new WebSiteClassifierResult(_setClassCollection, _extractorSettings);
items.Add(INPUTSET_NAME, featureVectors);
List <IWebPostClassifier> classifiers = new List <IWebPostClassifier>();
for (int i = 0; i < _context.setup.classifiers.Count; i++)
{
classifiers.Add(_context.setup.classifiers[i]);
}
Thread.Sleep(100);
if (classifiers.Count < _context.setup.classifiers.Count)
{
aceLog.log("::: MULTITHREADING --- CLASSIFIERS COUNT MISTMATCHED --- AUTOCORRECTION APPLIED :::");
for (int i = classifiers.Count - 1; i < _context.setup.classifiers.Count; i++)
{
classifiers.Add(_context.setup.classifiers[i]);
}
}
foreach (var cs in classifiers)
{
items.Add(cs.name, new WebSiteClassifierResult(_setClassCollection, _extractorSettings));
}
}
public experimentReport(experimentExecutionContext _context)
{
context = _context;
experiment = context.setup;
StartTime = DateTime.Now.ToLongTimeString();
start = DateTime.Now;
}
public void connectContext(experimentExecutionContext _context, IWebFVExtractor _fve)
{
context = _context;
extractor = _fve;
caseFolder = folder.Add("cases", "Cases", "Directory with serialized DocumentSetCase knowledge, shared among k-fold validation folds for faster execution.");
foreach (kFoldValidationCase setCase in items)
{
setCase.context = context;
setCase.extractor = _fve;
setCase.trainingCases.connectContext(this, setCase);
setCase.evaluationCases.connectContext(this, setCase);
}
knowledgeLibrary = new WebFVExtractorKnowledgeLibrary(this);
}
public DataTable GetAverageTable(experimentExecutionContext context)
{
objectTable <DocumentSetCaseCollectionReport> tp = new objectTable <DocumentSetCaseCollectionReport>(nameof(DocumentSetCaseCollectionReport.Name), parent.validationCase.name + "_avg");
foreach (var pair in avgReports)
{
tp.Add(pair.Value);
}
DataTable output = tp.GetDataTable();
String foldName = parent.validationCase.name;
parent.validationCase.context.AddExperimentInfo(output);
output.SetDescription($"Aggregates of [{foldName}] evaluation - only averages");
output.SetAdditionalInfoEntry("Report type", "Average per classifier");
//output.AddExtra("Most relevant rows are annoted with [(mean)] word");
return(output);
}
public DataTable GetFullValidationTable(experimentExecutionContext context)
{
objectTable <DocumentSetCaseCollectionReport> tp = new objectTable <DocumentSetCaseCollectionReport>(nameof(DocumentSetCaseCollectionReport.Name), parent.validationCase.name + "_avg");
foreach (var pair in this)
{
foreach (var r in pair.Value)
{
tp.Add(r);
}
}
DataTable output = tp.GetDataTable();
String foldName = parent.validationCase.name;
parent.validationCase.context.AddExperimentInfo(output);
output.SetDescription($"Results of fold [{foldName}] evaluation, with all entries");
output.SetAdditionalInfoEntry("Report type", "All entries");
output.AddExtra("Most relevant rows are annoted with [(mean)] word");
return(output);
}
/// <summary>
/// Prepares for parallel execution.
/// </summary>
/// <param name="tools">The tools.</param>
/// <param name="_context">The context.</param>
public webProjectKnowledgeSet PrepareForParallelExecution(classifierTools tools, experimentExecutionContext _context)
{
if (caseKnowledgeSet == null)
{
caseKnowledgeSet = new webProjectKnowledgeSet();
}
if (items.Any())
{
experimentContext.notes.log("Mining Context was ready already.");
return(caseKnowledgeSet);
}
DateTime startTime = DateTime.Now;
experimentContext = _context;
List <webCaseKnowledge> cases = new List <webCaseKnowledge>();
folderNode classReportFolder = experimentContext.folder.Add("General", "General and diagnostic reports", "The folder contains general (outside k-folds) reports on analysied industries (categories), web sites and other diagnostic data");
// <---------------------------------------------------------------------------------------------------------------- [ performing pipeline ]
experimentContext.notes.log("Executing the Mining Context decomposition with the pipeline model");
foreach (IDocumentSetClass classSet in experimentContext.classes.GetClasses())
{
var pipelineContext = GetContextForPipeline(tools, classSet);
sitesByCategory.Add(classSet, new List <pipelineTaskMCSiteSubject>());
if (!pipelineContext.exitByType.ContainsKey(typeof(pipelineTaskMCSiteSubject)))
{
throw new aceGeneralException("Pipeline context output contains no web site subjects! Check the pipeline Site Task constructor.", null, pipelineContext, "Pipeline broken");
}
var sitesForContext = pipelineContext.exitByType[typeof(pipelineTaskMCSiteSubject)]; // <----- preparing
foreach (var site in sitesForContext)
{
tokenBySite.Add(site as pipelineTaskMCSiteSubject, new ConcurrentBag <pipelineTaskSubjectContentToken>());
sitesByCategory[classSet].Add(site as pipelineTaskMCSiteSubject);
webCaseKnowledge webCase = new webCaseKnowledge(site as pipelineTaskMCSiteSubject, classSet);
caseKnowledgeSet.Add(webCase);
cases.Add(webCase);
}
semanticFVExtractorKnowledge kn = new semanticFVExtractorKnowledge();
kn.name = classSet.name + "_general";
kn.relatedItemPureName = classSet.name;
kn.type = WebFVExtractorKnowledgeType.aboutCompleteCategory;
kn.Deploy(classReportFolder, experimentContext.logger);
knowledgeByClass.TryAdd(classSet, kn);
}
experimentContext.notes.log("Sorting tokens for all sites [in parallel]");
Parallel.ForEach(tokenBySite.Keys, site =>
{
var leafs = site.getAllLeafs();
foreach (var leaf in leafs)
{
pipelineTaskSubjectContentToken token = leaf as pipelineTaskSubjectContentToken;
if (token != null)
{
tokenBySite[site].Add(token);
}
}
});
foreach (var c in cases)
{
c.tokens = tokenBySite[c.MCSiteSubject];
}
experimentContext.notes.log("Building diagnostic TF-IDF master tables for all classes [in parallel]");
Boolean useIntegratedApproach = false;
if (useIntegratedApproach)
{
var valCase = experimentContext.validationCollections[experimentContext.masterExtractor.name].GetDiagnosticCase(experimentContext.classes);
Parallel.ForEach(sitesByCategory, pair =>
{
knowledgeByClass.TryAdd(pair.Key, experimentContext.masterExtractor.DoFVExtractionForClassViaCases(valCase.trainingCases[pair.Key.classID], pair.Key, valCase, experimentContext.tools, experimentContext.logger));
});
}
else
{
Parallel.ForEach(sitesByCategory, pair =>
{
IDocumentSetClass category = pair.Key;
List <pipelineTaskMCSiteSubject> sites = pair.Value;
var lt = BuildLemmaTableForClass(tools, category, sites);
lt.Save();
// lt.SaveAs(classReportFolder.pathFor(lt.info.Name), imbSCI.Data.enums.getWritableFileMode.overwrite);
});
}
experimentContext.notes.log("Saving lexic resource cache subset - for later reuse in case of repeated experiment run");
tools.SaveCache();
if (!useIntegratedApproach)
{
experimentContext.notes.log("Performing chunk construction for all web sites in all categories [in serial]");
foreach (IDocumentSetClass classSet in experimentContext.classes.GetClasses())
{
BuildChunksForClass(tools, classSet);
}
foreach (IDocumentSetClass classSet in experimentContext.classes.GetClasses())
{
experimentContext.masterExtractor.chunkTableConstructor.process(chunksByCategory[classSet], cnt_level.mcPage, knowledgeByClass[classSet].WLChunkTableOfIndustryClass, null, experimentContext.logger, false);
}
}
if (tools.operation.doCreateDiagnosticMatrixAtStart)
{
experimentContext.notes.log("Performing diagnostic analysis on all categories...[doCreateDiagnosticMatrixAtStart=true]");
folderNode matrixReport = classReportFolder.Add("clouds", "More reports on semantic cloud", "Directory contains exported DirectedGraphs, varous matrix derivates, combined cloud and other diagnostic things");
List <lemmaSemanticCloud> clouds = new List <lemmaSemanticCloud>();
List <lemmaSemanticCloud> filteredClouds = new List <lemmaSemanticCloud>();
var converter = lemmaSemanticCloud.GetDGMLConverter();
foreach (IDocumentSetClass classSet in experimentContext.classes.GetClasses())
{
// experimentContext.masterExtractor.chunkTableConstructor.process(chunksByCategory[classSet], cnt_level.mcPage, knowledgeByClass[classSet].WLChunkTableOfIndustryClass, null, experimentContext.logger, false);
var cloud = experimentContext.masterExtractor.CloudConstructor.process(knowledgeByClass[classSet].WLChunkTableOfIndustryClass, knowledgeByClass[classSet].WLTableOfIndustryClass, knowledgeByClass[classSet].semanticCloud, experimentContext.logger, tokenBySite.Keys.ToList(), tools.GetLemmaResource());
knowledgeByClass[classSet].semanticCloud.className = classSet.name;
clouds.Add(cloud);
if (experimentContext.tools.operation.doUseSimpleGraphs)
{
cloud.GetSimpleGraph(true).Save(matrixReport.pathFor("cloud_initial_" + classSet.name, imbSCI.Data.enums.getWritableFileMode.none, "Initial version of full-sample set, diagnostic Semantic Cloud for category [" + classSet.name + "]"));
}
else
{
converter.ConvertToDMGL(cloud).Save(matrixReport.pathFor("cloud_initial_" + classSet.name, imbSCI.Data.enums.getWritableFileMode.none, "Initial version of full-sample set, diagnostic Semantic Cloud for category [" + classSet.name + "]"));
}
knowledgeByClass[classSet].semanticCloudFiltered = knowledgeByClass[classSet].semanticCloud.CloneIntoType <lemmaSemanticCloud>(true);
knowledgeByClass[classSet].semanticCloudFiltered.className = classSet.name;
filteredClouds.Add(knowledgeByClass[classSet].semanticCloudFiltered);
}
cloudMatrix matrix = new cloudMatrix("CloudMatrix", "Diagnostic cloud matrix created from the complete sample set of [" + clouds.Count() + "] classes");
matrix.build(filteredClouds, experimentContext.logger);
lemmaSemanticCloud mergedCloudInitial = matrix.GetUnifiedCloud();
mergedCloudInitial.Save(matrixReport.pathFor("unified_initial_cloud.xml", imbSCI.Data.enums.getWritableFileMode.overwrite, "Serialized object - Initial version of Semantic Cloud built as union of full-sample set Semantic Clouds of all categories"));
var reductions = matrix.TransformClouds(experimentContext.masterExtractor.settings.semanticCloudFilter, experimentContext.logger);
var p = matrixReport.pathFor("reductions_nodes.txt", imbSCI.Data.enums.getWritableFileMode.overwrite, "Report on Cloud Matrix transformation process");
File.WriteAllLines(p, reductions);
matrix.BuildTable(experimentContext.masterExtractor.settings.semanticCloudFilter, cloudMatrixDataTableType.initialState | cloudMatrixDataTableType.maxCloudFrequency | cloudMatrixDataTableType.absoluteValues).GetReportAndSave(matrixReport, appManager.AppInfo, "matrix_max_cf_initial", true, experimentContext.tools.operation.doReportsInParalell);
matrix.BuildTable(experimentContext.masterExtractor.settings.semanticCloudFilter, cloudMatrixDataTableType.initialState | cloudMatrixDataTableType.overlapSize | cloudMatrixDataTableType.absoluteValues).GetReportAndSave(matrixReport, appManager.AppInfo, "matrix_overlap_size_initial", true, experimentContext.tools.operation.doReportsInParalell);
matrix.BuildTable(experimentContext.masterExtractor.settings.semanticCloudFilter, cloudMatrixDataTableType.initialState | cloudMatrixDataTableType.overlapValue | cloudMatrixDataTableType.absoluteValues).GetReportAndSave(matrixReport, appManager.AppInfo, "matrix_overlap_value_initial", true, experimentContext.tools.operation.doReportsInParalell);
matrix.ExportTextReports(matrixReport, true, "matrix_cf");
matrix.ExportTextReports(matrixReport, false, "matrix_cf");
lemmaSemanticCloud mergedCloudAfterReduction = matrix.GetUnifiedCloud();
mergedCloudAfterReduction.Save(matrixReport.pathFor("unified_reduced_cloud.xml", imbSCI.Data.enums.getWritableFileMode.overwrite, "Serialized object -Version of all-categories diagnostic Semantic Cloud, after Cloud Matrix filter was applied"));
if (experimentContext.tools.operation.doUseSimpleGraphs)
{
mergedCloudInitial.GetSimpleGraph(true).Save(matrixReport.pathFor("unified_initial_cloud", imbSCI.Data.enums.getWritableFileMode.overwrite, "DirectedGraphML file - unified Semantic Cloud, before Cloud Matrix filter was applied - Open this in VisualStudo)"));
}
else
{
converter = lemmaSemanticCloud.GetDGMLConverter();
converter.ConvertToDMGL(mergedCloudInitial).Save(matrixReport.pathFor("unified_initial_cloud", imbSCI.Data.enums.getWritableFileMode.overwrite, "DirectedGraphML file - unified Semantic Cloud, before Cloud Matrix filter was applied - Open this in VisualStudo)"));
}
// <-------- analysis -----------------------------------------------------------------------------------
DataTableTypeExtended <freeGraphReport> cloudReports = new DataTableTypeExtended <freeGraphReport>();
foreach (var cl in filteredClouds)
{
freeGraphReport fgReport = new freeGraphReport(cl);
fgReport.Save(matrixReport);
cloudReports.AddRow(fgReport);
}
freeGraphReport unifiedReport = new freeGraphReport(mergedCloudAfterReduction);
unifiedReport.Save(matrixReport);
cloudReports.AddRow(unifiedReport);
cloudReports.GetReportAndSave(matrixReport, appManager.AppInfo, "analysis_SemanticClouds");
// <-------- analysis -----------------------------------------------------------------------------------
foreach (IDocumentSetClass classSet in experimentContext.classes.GetClasses())
{
var cloud = knowledgeByClass[classSet].semanticCloudFiltered; // .WLChunkTableOfIndustryClass, knowledgeByClass[classSet].WLTableOfIndustryClass, knowledgeByClass[classSet].semanticCloud, experimentContext.logger, tokenBySite.Keys.ToList());
if (experimentContext.tools.operation.doUseSimpleGraphs)
{
cloud.GetSimpleGraph(true).Save(matrixReport.pathFor("unified_initial_cloud", imbSCI.Data.enums.getWritableFileMode.overwrite, "DirectedGraphML file - unified Semantic Cloud, before Cloud Matrix filter was applied - Open this in VisualStudo)"));
}
else
{
converter = lemmaSemanticCloud.GetDGMLConverter();
converter.ConvertToDMGL(cloud).Save(matrixReport.pathFor("unified_initial_cloud", imbSCI.Data.enums.getWritableFileMode.overwrite, "DirectedGraphML file - unified Semantic Cloud, before Cloud Matrix filter was applied - Open this in VisualStudo)"));
}
//converter.ConvertToDMGL(cloud).Save(matrixReport.pathFor("cloud_reduced_" + classSet.name, imbSCI.Data.enums.getWritableFileMode.none, "DirectedGraphML file - Initial version of Semantic Cloud built as union of full-sample set Semantic Clouds of all categories (Open this with VS)"), imbSCI.Data.enums.getWritableFileMode.overwrite);
}
instanceCountCollection <String> tfcounter = new instanceCountCollection <string>();
foreach (IDocumentSetClass classSet in experimentContext.classes.GetClasses())
{
var wlt = knowledgeByClass[classSet].WLTableOfIndustryClass.GetDataTable();
wlt.DefaultView.Sort = "termFrequency desc";
var sorted = wlt.DefaultView.ToTable();
var tbl = wlt.GetClonedShema <DataTable>(true);
tbl.CopyRowsFrom(sorted, 0, 100);
tbl.GetReportAndSave(classReportFolder, appManager.AppInfo, classSet.name + "_WebLemma", true, experimentContext.tools.operation.doReportsInParalell);
var cht = knowledgeByClass[classSet].WLChunkTableOfIndustryClass.GetDataTable();
cht.DefaultView.Sort = "termFrequency desc";
var csorted = cht.DefaultView.ToTable();
tbl = cht.GetClonedShema <DataTable>(true);
tbl.CopyRowsFrom(csorted, 0, 100);
tbl.GetReportAndSave(classReportFolder, appManager.AppInfo, classSet.name + "_Chunks", true, experimentContext.tools.operation.doReportsInParalell);
tfcounter.AddInstanceRange(knowledgeByClass[classSet].WLTableOfIndustryClass.unresolved);
knowledgeByClass[classSet].OnBeforeSave();
}
List <String> countSorted = tfcounter.getSorted();
StringBuilder sb = new StringBuilder();
foreach (String s in countSorted)
{
sb.AppendLine(String.Format("{1} : {0}", s, tfcounter[s]));
}
String pt = classReportFolder.pathFor("unresolved_tokens.txt", imbSCI.Data.enums.getWritableFileMode.none, "Cloud Frequency list of all unresolved letter-only tokens");
File.WriteAllText(pt, sb.ToString());
}
if (tools.operation.doFullDiagnosticReport)
{
experimentContext.notes.log("Generating full diagnostic report on classes...");
DataTable rep = null;
foreach (IDocumentSetClass classSet in experimentContext.classes.GetClasses())
{
rep = this.GetClassKnowledgeReport(classSet, rep);
}
rep.SetAdditionalInfoEntry("Experiment", experimentContext.setup.name);
rep.AddExtra("Experiment: " + experimentContext.setup.name);
rep.AddExtra("Info: " + experimentContext.setup.description);
rep.SetDescription("Structural report for all classes in the experiment");
rep.GetReportAndSave(classReportFolder, appManager.AppInfo, "structural_class_report", true, experimentContext.tools.operation.doReportsInParalell);
}
classReportFolder.generateReadmeFiles(appManager.AppInfo);
experimentContext.notes.log("Mining Context preprocessing done in [" + DateTime.Now.Subtract(startTime).TotalMinutes.ToString("F2") + "] minutes");
return(caseKnowledgeSet);
}
public void MakeReports(experimentExecutionContext context, folderNode folder)
{
meanClassifierReport = new DocumentSetCaseCollectionReport(extractor.name);
aceDictionary2D <IWebPostClassifier, kFoldValidationCase, DocumentSetCaseCollectionReport> tempStructure = new aceDictionary2D <IWebPostClassifier, kFoldValidationCase, DocumentSetCaseCollectionReport>();
DSCCReports firstCase = null;
List <IWebPostClassifier> classifiers = new List <IWebPostClassifier>();
foreach (var kFoldCasePair in this)
{
if (firstCase == null)
{
firstCase = kFoldCasePair.Value;
}
foreach (var pair in kFoldCasePair.Value.avgReports)
{
tempStructure[pair.Key, kFoldCasePair.Key] = pair.Value;
if (!classifiers.Contains(pair.Key))
{
classifiers.Add(pair.Key);
}
}
}
// DataSet dataSet = new DataSet(context.setup.name);
// <---------- CREATING AVERAGE TABLE -----------------------------------------------------
var tpAvgMacro = new DataTableTypeExtended <DocumentSetCaseCollectionReport>(context.setup.name + " summary", "Cross k-fold averages measures, fold-level measures are computed by macro-average method");
var tpAvgMicro = new DataTableTypeExtended <DocumentSetCaseCollectionReport>(context.setup.name + " summary", "Cross k-fold averages measures, fold-level measures are computed by micro-average method");
List <DocumentSetCaseCollectionReport> macroaverages = new List <DocumentSetCaseCollectionReport>();
DataTableTypeExtended <DocumentSetCaseCollectionReport> EMperKFolds = new DataTableTypeExtended <DocumentSetCaseCollectionReport>(extractor.name + "_allReports");
foreach (IWebPostClassifier classifier in classifiers)
{
// < ---- report on each classifier
context.logger.log("-- producing report about [" + classifier.name + "]");
//objectTable<DocumentSetCaseCollectionReport> tp = new objectTable<DocumentSetCaseCollectionReport>(nameof(DocumentSetCaseCollectionReport.Name), classifier + "_sum");
DocumentSetCaseCollectionReport avg = new DocumentSetCaseCollectionReport(classifier.name + " macro-averaging, k-fold avg. ");
DocumentSetCaseCollectionReport rep_eval = new DocumentSetCaseCollectionReport(classifier.name + " micro-averaging, k-fold avg.");
rep_eval.Classifier = classifier.name;
classificationEvalMetricSet metrics = new classificationEvalMetricSet();
classificationEval eval = new classificationEval();
//eval = metrics[classifier.name];
Int32 c = 0;
foreach (KeyValuePair <kFoldValidationCase, DSCCReports> kFoldCasePair in this)
{
DocumentSetCaseCollectionReport rep = kFoldCasePair.Value.avgReports[classifier];
kFoldValidationCase vCase = kFoldCasePair.Key;
classificationEvalMetricSet met = rep.GetSetMetrics();
if (met != null)
{
foreach (IDocumentSetClass cl in context.classes.GetClasses())
{
eval = eval + met[cl.name];
}
}
rep.Name = classifier.name + "_" + vCase.name;
avg.AddValues(rep);
EMperKFolds.AddRow(rep);
c++;
}
rep_eval.AddValues(metrics, classificationMetricComputation.microAveraging);
avg.Classifier = classifier.name;
avg.DivideValues(c);
// <<< detecting the best performed classifier in all evaluation folds
if (avg.F1measure > highestF1Value)
{
highestF1Value = avg.F1measure;
topClassifierReport = avg;
}
meanClassifierReport.AddValues(avg);
// -----------------
EMperKFolds.AddRow(avg);
tpAvgMacro.AddRow(avg);
macroaverages.Add(avg);
if (DOMAKE_MICROaverage)
{
tpAvgMicro.AddRow(rep_eval);
}
// tp.Add(rep_eval);
if (context.tools.operation.DoMakeReportForEachClassifier)
{
DataTable cTable = EMperKFolds;
cTable.SetTitle($"{classifier.name} report");
cTable.SetDescription("Summary " + context.setup.validationSetup.k + "-fold validation report for [" + classifier.name + "]");
cTable.SetAdditionalInfoEntry("FV Extractor", extractor.name);
cTable.SetAdditionalInfoEntry("Classifier", classifier.name);
cTable.SetAdditionalInfoEntry("Class name", classifier.GetType().Name);
cTable.SetAdditionalInfoEntry("Correct", rep_eval.Correct);
cTable.SetAdditionalInfoEntry("Wrong", rep_eval.Wrong);
//cTable.SetAdditionalInfoEntry("Precision", rep_eval.Precision);
//cTable.SetAdditionalInfoEntry("Recall", rep_eval.Recall);
//cTable.SetAdditionalInfoEntry("F1", rep_eval.F1measure);
cTable.SetAdditionalInfoEntry("True Positives", metrics[classifier.name].truePositives);
cTable.SetAdditionalInfoEntry("False Negatives", metrics[classifier.name].falseNegatives);
cTable.SetAdditionalInfoEntry("False Positives", metrics[classifier.name].falsePositives);
cTable.AddExtra("Classifier: " + classifier.name + " [" + classifier.GetType().Name + "]");
var info = classifier.DescribeSelf();
info.ForEach(x => cTable.AddExtra(x));
cTable.AddExtra("-----------------------------------------------------------------------");
cTable.AddExtra("Precision, Recall and F1 measures expressed in this table are computed by macroaveraging shema");
// output.CopyRowsFrom(cTable);
cTable.GetReportAndSave(folder, appManager.AppInfo, extractor.name + "_classifier_" + classifier.name);
// dataSet.AddTable(cTable);
}
}
rangeFinderForDataTable rangerMacro = new rangeFinderForDataTable(tpAvgMacro, "Name");
meanClassifierReport.DivideValues(classifiers.Count);
if (macroaverages.Count > 0)
{
Double maxF1 = macroaverages.Max(x => x.F1measure);
Double minF1 = macroaverages.Min(x => x.F1measure);
List <String> minCaseNames = macroaverages.Where(x => x.F1measure == minF1).Select(x => x.Name).ToList();
List <String> maxCaseNames = macroaverages.Where(x => x.F1measure == maxF1).Select(x => x.Name).ToList();
var style = EMperKFolds.GetRowMetaSet().SetStyleForRowsWithValue <String>(DataRowInReportTypeEnum.dataHighlightA, nameof(DocumentSetCaseCollectionReport.Name), maxCaseNames);
EMperKFolds.GetRowMetaSet().AddUnit(style);
// style = tpAvgMacro.GetRowMetaSet().SetStyleForRowsWithValue<String>(DataRowInReportTypeEnum.dataHighlightC, nameof(DocumentSetCaseCollectionReport.Name), minCaseNames);
tpAvgMacro.SetAdditionalInfoEntry("FV Extractor", extractor.name);
if (DOMAKE_MICROaverage)
{
tpAvgMicro.SetAdditionalInfoEntry("FV Extractor", extractor.name);
}
List <String> averageNames = macroaverages.Select(x => x.Name).ToList();
var avg_style = EMperKFolds.GetRowMetaSet().SetStyleForRowsWithValue <String>(DataRowInReportTypeEnum.dataHighlightC, nameof(DocumentSetCaseCollectionReport.Name), averageNames);
foreach (var x in averageNames)
{
avg_style.AddMatch(x);
}
}
// ::: ------------------------------------------------------------------------------------------------- ::: --------------------------------------------------------------------- ::: //
tpAvgMacro.SetTitle($"{extractor.name} - macroaverage report");
if (DOMAKE_MICROaverage)
{
tpAvgMicro.SetTitle($"{extractor.name} - microaverage report");
}
tpAvgMacro.AddExtra("Complete report on " + context.setup.validationSetup.k + "-fold validation FVE [" + extractor.name + "]");
tpAvgMacro.AddExtra("Fold-level P, R and F1 measures are computed by macroaveraging method, values here are cross k-fold means.");
if (DOMAKE_MICROaverage)
{
tpAvgMicro.AddExtra("Complete " + context.setup.validationSetup.k + "-fold validation report for FVE [" + extractor.name + "]");
}
if (DOMAKE_MICROaverage)
{
tpAvgMicro.AddExtra("Fold-level P, R and F1 measures are computed by microaveraging method, values here are cross k-fold means.");
}
context.AddExperimentInfo(tpAvgMacro);
if (DOMAKE_MICROaverage)
{
context.AddExperimentInfo(tpAvgMicro);
}
tpAvgMacro.AddExtra(extractor.description);
if (extractor is semanticFVExtractor)
{
semanticFVExtractor semExtractor = (semanticFVExtractor)extractor;
semExtractor.termTableConstructor.DescribeSelf().ForEach(x => tpAvgMacro.AddExtra(x));
semExtractor.CloudConstructor.DescribeSelf().ForEach(x => tpAvgMacro.AddExtra(x));
semExtractor.termTableConstructor.DescribeSelf().ForEach(x => tpAvgMicro.AddExtra(x));
semExtractor.CloudConstructor.DescribeSelf().ForEach(x => tpAvgMicro.AddExtra(x));
}
context.logger.log("-- producing summary reports on [" + extractor.name + "]");
rangerMacro.AddRangeRows("Macroaverage ", tpAvgMacro, true,
imbSCI.Core.math.aggregation.dataPointAggregationType.min | imbSCI.Core.math.aggregation.dataPointAggregationType.max
| imbSCI.Core.math.aggregation.dataPointAggregationType.avg
| imbSCI.Core.math.aggregation.dataPointAggregationType.stdev);
tpAvgMacro.GetReportAndSave(folder, appManager.AppInfo, extractor.name + "_macroaverage_report", true, true);
EMperKFolds.AddExtra("The table shows average measures for each fold --- rows marked with colored background show averages for all folds, per classifier.");
EMperKFolds.GetReportAndSave(folder, appManager.AppInfo, extractor.name + "_allFolds", true, true);
if (DOMAKE_MICROaverage)
{
tpAvgMicro.GetReportAndSave(folder, appManager.AppInfo, extractor.name + "_microaverage_report", true, true);
}
//dataSet.GetReportVersion().serializeDataSet(extractor.name + "_classifiers_MultiSheetSummary", folder, imbSCI.Data.enums.reporting.dataTableExportEnum.excel, appManager.AppInfo);
}