/// <summary> /// Gets the report table on one collection /// </summary> /// <returns></returns> public DSCCReports GetReports() { //String repTitle = validationCase.name + " - " + validationCase.featureVectorExtractor.name; //String repName = repTitle.getCleanFileName(); DSCCReports repTable = new DSCCReports(); repTable.parent = this; foreach (IWebPostClassifier classifier in validationCase.context.setup.classifiers) { classificationEvalMetricSet metrics = GetMetrics(classifier); Int32 i = 0; DocumentSetCaseCollectionReport avgReport = new DocumentSetCaseCollectionReport(validationCase.name + "_" + classifier.name + "_" + "mean"); avgReport.Classifier = classifier.name; foreach (KeyValuePair <int, DocumentSetCaseCollection> pair in this) { var rep = new DocumentSetCaseCollectionReport(validationCase.name + " " + pair.Value.setClass.name + " " + classifier.name); //rep.kFoldCase = ; rep.Classifier = classifier.name; rep.Targets = metrics[pair.Value.setClass.name].correct + metrics[pair.Value.setClass.name].wrong; rep.Wrong = metrics[pair.Value.setClass.name].wrong; rep.Correct = metrics[pair.Value.setClass.name].correct; rep.Precision = metrics[pair.Value.setClass.name].GetPrecision(); rep.Recall = metrics[pair.Value.setClass.name].GetRecall(); rep.F1measure = metrics[pair.Value.setClass.name].GetF1(); repTable[classifier].Add(rep); avgReport.AddValues(rep); i++; } //DocumentSetCaseCollectionReport checkReport = new DocumentSetCaseCollectionReport(); //checkReport.kFoldCase = classifier.name + "(check)"; //checkReport.Classifier = classifier.name; //checkReport.Precision = metrics.GetPrecision(); //checkReport.Recall = metrics.GetRecall(); //checkReport.F1measure = metrics.GetF1(); avgReport.DivideValues(i); repTable[classifier].Add(avgReport); //repTable[classifier].Add(checkReport); avgReport.GetSetMetrics(metrics); repTable.avgReports.Add(classifier, avgReport); } // output.SetDescription("Report for " + validationCase.name + " sample evaluation"); // ds.Tables.Add(repTable.GetDataTable()); return(repTable); }
protected void runModel(experimentExecutionContext context, IWebFVExtractor model) { imbSCI.Core.screenOutputControl.logToConsoleControl.setAsOutput(context.logger, model.name); Int32 crashRetries = context.tools.operation.doRebootFVEOnCrashRetryLimit; aceDictionarySet <IDocumentSetClass, DocumentSetCaseCollection> casesByClasses = new aceDictionarySet <IDocumentSetClass, DocumentSetCaseCollection>(); DSCCReportSet kFoldReport = new DSCCReportSet(model); var valCol = context.validationCollections[model.name]; List <DocumentSetCaseCollectionSet> modelCaseResults = new List <DocumentSetCaseCollectionSet>(); crashRetries = context.tools.operation.doRebootFVEOnCrashRetryLimit; while (crashRetries > 0) { try { experimentNotes modelNotes = new experimentNotes(valCol.folder, "Fold-level experiment settings description notes"); modelNotes.AppendLine("# Notes on Feature Vector Extractor: " + model.name); var nts = model.DescribeSelf(); nts.ForEach(x => modelNotes.AppendLine(x)); context.logger.log("Executing k-fold cases with model [" + model.name + "]"); valCol.DescribeSampleDistribution(modelNotes); context.mainReport.valColVsModelVsSampleHash.Add("[" + model.name + "]".toWidthExact(20) + " [sample distribution hash: " + valCol.SampleDistributionHash + "]"); modelNotes.SaveNote(); ParallelOptions ops = new ParallelOptions(); ops.MaxDegreeOfParallelism = context.tools.operation.ParallelThreads; Parallel.ForEach <kFoldValidationCase>(valCol.GetCases(), ops, valCase => { model.DoFVEAndTraining(valCase, context.tools, context.logger); // <--------------------------------------------------------------------------------------- BUILDING FVE DocumentSetCaseCollectionSet results = model.DoClassification(valCase, context.tools, context.logger); if (!results.Any()) { throw new aceScienceException("DoClassification for [" + model.name + "] returned no results!", null, model, "DoClassification " + model.name + " failed!", context); } foreach (var pair in results) { DocumentSetCaseCollection cls = pair.Value; casesByClasses.Add(cls.setClass, cls); } valCase.evaluationResults = results; if (context.tools.DoResultReporting) { context.logger.log("producing reports on k-Fold case [" + valCase.name + "]"); DSCCReports r = results.GetReports(); var sumMeans = r.GetAverageTable(context); //.GetReportAndSave(valCase.folder, appManager.AppInfo, "CrossValidation_" + valCase.name); sumMeans.SetDescription("FVE report, aggregated for all categories - for fold [" + valCase.name + "]"); sumMeans.GetReportAndSave(valCase.folder, appManager.AppInfo, "CrossValidation_" + valCase.name, true, context.tools.operation.doReportsInParalell); var fveAndCase = r.GetFullValidationTable(context); fveAndCase.SetDescription("Per-category aggregate statistics, for each classifier, within fold [" + valCase.name + "], used for macro-averaging"); fveAndCase.GetReportAndSave(valCase.folder, appManager.AppInfo, "CrossValidation_extrainfo_" + valCase.name, true, context.tools.operation.doReportsInParalell); var fullCaseReport = results.GetReportOnAllCases(); fullCaseReport.GetReportAndSave(valCase.folder, appManager.AppInfo, "FullReport_" + valCase.name, true, context.tools.operation.doReportsInParalell); kFoldReport.Add(valCase, r); } context.logger.log("k-Fold case [" + valCase.name + "] completed"); context.notes.log("- - Experiment sequence for [" + valCase.name + "] fold completed"); if (context.tools.operation.doSaveKnowledgeForClasses) { valCase.knowledgeLibrary.SaveKnowledgeInstancesForClasses(valCase, context.logger); } }); foreach (var fold in valCol.GetCases()) // Parallel.ForEach<kFoldValidationCase>(valCol.GetCases(), ops, valCase => { modelCaseResults.Add(fold.evaluationResults); } crashRetries = 0; } catch (Exception ex) { crashRetries--; context.errorNotes.LogException("FVE Model crashed -- retries left [" + crashRetries + "] --- ", ex, model.name); context.logger.log(":::: REPEATING the model [" + model.name + "] ::: CRASHED [" + ex.Message + "] ::: RETRIES [" + crashRetries + "]"); imbACE.Services.terminal.aceTerminalInput.doBeepViaConsole(1200, 1000, 1); imbACE.Services.terminal.aceTerminalInput.doBeepViaConsole(2400, 1000, 1); imbSCI.Core.screenOutputControl.logToConsoleControl.setAsOutput(context.logger, "RETRIES[" + crashRetries + "]"); } } imbSCI.Core.screenOutputControl.logToConsoleControl.setAsOutput(context.logger, "Reporting"); valCol.knowledgeLibrary.SaveCaseKnowledgeInstances(context.logger); // DocumentSetCaseCollection second = null; if (modelCaseResults.Any()) { featureExtractionMetrics modelMetrics = new featureExtractionMetrics(model.name, "All"); DataTableTypeExtended <featureExtractionMetrics> modelVsCategoryMetrics = new DataTableTypeExtended <featureExtractionMetrics>(model.name, "Model metrics per category"); // <-------------------------------------- CATEGORIES REPORT ---------------------------------------------- DataTable allTable = modelCaseResults.First()[0].GetReportTable(false, false).GetClonedShema <DataTable>();; //valCol.GetCases().First().evaluationResults[0].GetReportTable(false, false); rangeFinderForDataTable ranger = new rangeFinderForDataTable(allTable, "name"); ranger.columnsToSignIn.Add("Case"); foreach (KeyValuePair <IDocumentSetClass, aceConcurrentBag <DocumentSetCaseCollection> > pair in casesByClasses) { DocumentSetCaseCollection first = null; DataTable repTable = null; ranger.prepareForNextAggregationBlock(allTable, "name"); foreach (DocumentSetCaseCollection cn in pair.Value) { foreach (var cni in cn) { if (cni != null) { cn.BuildRow(cni, allTable, false); } } } ranger.AddRangeRows(pair.Key.name, allTable, true, imbSCI.Core.math.aggregation.dataPointAggregationType.avg | imbSCI.Core.math.aggregation.dataPointAggregationType.stdev); var categoryMetrics = new featureExtractionMetrics(model.name, pair.Key.name); categoryMetrics.SetValues(ranger); modelVsCategoryMetrics.AddRow(categoryMetrics); modelMetrics.AddValues(categoryMetrics); categoryMetrics.saveObjectToXML(valCol.folder.pathFor(model.name + "_" + categoryMetrics.Name + ".xml", imbSCI.Data.enums.getWritableFileMode.overwrite, "FV and Category sample metrics, serialized object")); //context.notes.log("- - Creating report for category [" + pair.Key.name + "] completed"); //repTable.GetReportAndSave(valCol.folder, appManager.AppInfo, model.name + "_category_" + pair.Key.name); } modelMetrics.DivideValues(casesByClasses.Count); modelMetrics.saveObjectToXML(valCol.folder.pathFor(model.name + "_metrics.xml", imbSCI.Data.enums.getWritableFileMode.overwrite, "Cross-categories macroaveraged metrics of the FVE model [" + model.name + "]")); modelVsCategoryMetrics.AddRow(modelMetrics); modelVsCategoryMetrics.GetRowMetaSet().SetStyleForRowsWithValue <String>(DataRowInReportTypeEnum.dataHighlightA, "Name", modelMetrics.Name); modelVsCategoryMetrics.GetReportAndSave(valCol.folder, appManager.AppInfo, model.name + "_metrics", true, true); context.mainReport.AddModelMetrics(modelMetrics); context.notes.log("- Creating report for all categories [" + model.name + "] "); allTable.GetReportAndSave(valCol.folder, appManager.AppInfo, model.name + "_categories", true, context.tools.operation.doReportsInParalell); } kFoldReport.MakeReports(context, valCol.folder); context.mainReport.AddBestPerformer(kFoldReport.GetTopClassifierReport(), kFoldReport.meanClassifierReport, model); // <---------------- creation of complete report context.notes.log("- Experiment sequence with Feature Vector Extractor [" + model.name + "] completed"); context.notes.SaveNote(); // <------------- END OF THE MODEL ------------------------------------------------------------------------------------------------- }