public void RemoveAllModelsExcept(IWebFVExtractor model = null) { featureVectorExtractors_semantic.Clear(); models.Clear(); if (model != null) { models.Add(model); } }
public void connectContext(experimentExecutionContext _context, IWebFVExtractor _fve) { context = _context; extractor = _fve; caseFolder = folder.Add("cases", "Cases", "Directory with serialized DocumentSetCase knowledge, shared among k-fold validation folds for faster execution."); foreach (kFoldValidationCase setCase in items) { setCase.context = context; setCase.extractor = _fve; setCase.trainingCases.connectContext(this, setCase); setCase.evaluationCases.connectContext(this, setCase); } knowledgeLibrary = new WebFVExtractorKnowledgeLibrary(this); }
public void AddBestPerformer(DocumentSetCaseCollectionReport bestPerformingClassifier, DocumentSetCaseCollectionReport meanPerformance, IWebFVExtractor fveModel) { bestPerformingClassifiers.Add(bestPerformingClassifier); String line = "[" + fveModel.name + "] completed " + DateTime.Now.ToLongTimeString(); fveFinishedRecods.Add(line); meanPerformanceForExtractors.Add(meanPerformance); }
protected void runModel(experimentExecutionContext context, IWebFVExtractor model) { imbSCI.Core.screenOutputControl.logToConsoleControl.setAsOutput(context.logger, model.name); Int32 crashRetries = context.tools.operation.doRebootFVEOnCrashRetryLimit; aceDictionarySet <IDocumentSetClass, DocumentSetCaseCollection> casesByClasses = new aceDictionarySet <IDocumentSetClass, DocumentSetCaseCollection>(); DSCCReportSet kFoldReport = new DSCCReportSet(model); var valCol = context.validationCollections[model.name]; List <DocumentSetCaseCollectionSet> modelCaseResults = new List <DocumentSetCaseCollectionSet>(); crashRetries = context.tools.operation.doRebootFVEOnCrashRetryLimit; while (crashRetries > 0) { try { experimentNotes modelNotes = new experimentNotes(valCol.folder, "Fold-level experiment settings description notes"); modelNotes.AppendLine("# Notes on Feature Vector Extractor: " + model.name); var nts = model.DescribeSelf(); nts.ForEach(x => modelNotes.AppendLine(x)); context.logger.log("Executing k-fold cases with model [" + model.name + "]"); valCol.DescribeSampleDistribution(modelNotes); context.mainReport.valColVsModelVsSampleHash.Add("[" + model.name + "]".toWidthExact(20) + " [sample distribution hash: " + valCol.SampleDistributionHash + "]"); modelNotes.SaveNote(); ParallelOptions ops = new ParallelOptions(); ops.MaxDegreeOfParallelism = context.tools.operation.ParallelThreads; Parallel.ForEach <kFoldValidationCase>(valCol.GetCases(), ops, valCase => { model.DoFVEAndTraining(valCase, context.tools, context.logger); // <--------------------------------------------------------------------------------------- BUILDING FVE DocumentSetCaseCollectionSet results = model.DoClassification(valCase, context.tools, context.logger); if (!results.Any()) { throw new aceScienceException("DoClassification for [" + model.name + "] returned no results!", null, model, "DoClassification " + model.name + " failed!", context); } foreach (var pair in results) { DocumentSetCaseCollection cls = pair.Value; casesByClasses.Add(cls.setClass, cls); } valCase.evaluationResults = results; if (context.tools.DoResultReporting) { context.logger.log("producing reports on k-Fold case [" + valCase.name + "]"); DSCCReports r = results.GetReports(); var sumMeans = r.GetAverageTable(context); //.GetReportAndSave(valCase.folder, appManager.AppInfo, "CrossValidation_" + valCase.name); sumMeans.SetDescription("FVE report, aggregated for all categories - for fold [" + valCase.name + "]"); sumMeans.GetReportAndSave(valCase.folder, appManager.AppInfo, "CrossValidation_" + valCase.name, true, context.tools.operation.doReportsInParalell); var fveAndCase = r.GetFullValidationTable(context); fveAndCase.SetDescription("Per-category aggregate statistics, for each classifier, within fold [" + valCase.name + "], used for macro-averaging"); fveAndCase.GetReportAndSave(valCase.folder, appManager.AppInfo, "CrossValidation_extrainfo_" + valCase.name, true, context.tools.operation.doReportsInParalell); var fullCaseReport = results.GetReportOnAllCases(); fullCaseReport.GetReportAndSave(valCase.folder, appManager.AppInfo, "FullReport_" + valCase.name, true, context.tools.operation.doReportsInParalell); kFoldReport.Add(valCase, r); } context.logger.log("k-Fold case [" + valCase.name + "] completed"); context.notes.log("- - Experiment sequence for [" + valCase.name + "] fold completed"); if (context.tools.operation.doSaveKnowledgeForClasses) { valCase.knowledgeLibrary.SaveKnowledgeInstancesForClasses(valCase, context.logger); } }); foreach (var fold in valCol.GetCases()) // Parallel.ForEach<kFoldValidationCase>(valCol.GetCases(), ops, valCase => { modelCaseResults.Add(fold.evaluationResults); } crashRetries = 0; } catch (Exception ex) { crashRetries--; context.errorNotes.LogException("FVE Model crashed -- retries left [" + crashRetries + "] --- ", ex, model.name); context.logger.log(":::: REPEATING the model [" + model.name + "] ::: CRASHED [" + ex.Message + "] ::: RETRIES [" + crashRetries + "]"); imbACE.Services.terminal.aceTerminalInput.doBeepViaConsole(1200, 1000, 1); imbACE.Services.terminal.aceTerminalInput.doBeepViaConsole(2400, 1000, 1); imbSCI.Core.screenOutputControl.logToConsoleControl.setAsOutput(context.logger, "RETRIES[" + crashRetries + "]"); } } imbSCI.Core.screenOutputControl.logToConsoleControl.setAsOutput(context.logger, "Reporting"); valCol.knowledgeLibrary.SaveCaseKnowledgeInstances(context.logger); // DocumentSetCaseCollection second = null; if (modelCaseResults.Any()) { featureExtractionMetrics modelMetrics = new featureExtractionMetrics(model.name, "All"); DataTableTypeExtended <featureExtractionMetrics> modelVsCategoryMetrics = new DataTableTypeExtended <featureExtractionMetrics>(model.name, "Model metrics per category"); // <-------------------------------------- CATEGORIES REPORT ---------------------------------------------- DataTable allTable = modelCaseResults.First()[0].GetReportTable(false, false).GetClonedShema <DataTable>();; //valCol.GetCases().First().evaluationResults[0].GetReportTable(false, false); rangeFinderForDataTable ranger = new rangeFinderForDataTable(allTable, "name"); ranger.columnsToSignIn.Add("Case"); foreach (KeyValuePair <IDocumentSetClass, aceConcurrentBag <DocumentSetCaseCollection> > pair in casesByClasses) { DocumentSetCaseCollection first = null; DataTable repTable = null; ranger.prepareForNextAggregationBlock(allTable, "name"); foreach (DocumentSetCaseCollection cn in pair.Value) { foreach (var cni in cn) { if (cni != null) { cn.BuildRow(cni, allTable, false); } } } ranger.AddRangeRows(pair.Key.name, allTable, true, imbSCI.Core.math.aggregation.dataPointAggregationType.avg | imbSCI.Core.math.aggregation.dataPointAggregationType.stdev); var categoryMetrics = new featureExtractionMetrics(model.name, pair.Key.name); categoryMetrics.SetValues(ranger); modelVsCategoryMetrics.AddRow(categoryMetrics); modelMetrics.AddValues(categoryMetrics); categoryMetrics.saveObjectToXML(valCol.folder.pathFor(model.name + "_" + categoryMetrics.Name + ".xml", imbSCI.Data.enums.getWritableFileMode.overwrite, "FV and Category sample metrics, serialized object")); //context.notes.log("- - Creating report for category [" + pair.Key.name + "] completed"); //repTable.GetReportAndSave(valCol.folder, appManager.AppInfo, model.name + "_category_" + pair.Key.name); } modelMetrics.DivideValues(casesByClasses.Count); modelMetrics.saveObjectToXML(valCol.folder.pathFor(model.name + "_metrics.xml", imbSCI.Data.enums.getWritableFileMode.overwrite, "Cross-categories macroaveraged metrics of the FVE model [" + model.name + "]")); modelVsCategoryMetrics.AddRow(modelMetrics); modelVsCategoryMetrics.GetRowMetaSet().SetStyleForRowsWithValue <String>(DataRowInReportTypeEnum.dataHighlightA, "Name", modelMetrics.Name); modelVsCategoryMetrics.GetReportAndSave(valCol.folder, appManager.AppInfo, model.name + "_metrics", true, true); context.mainReport.AddModelMetrics(modelMetrics); context.notes.log("- Creating report for all categories [" + model.name + "] "); allTable.GetReportAndSave(valCol.folder, appManager.AppInfo, model.name + "_categories", true, context.tools.operation.doReportsInParalell); } kFoldReport.MakeReports(context, valCol.folder); context.mainReport.AddBestPerformer(kFoldReport.GetTopClassifierReport(), kFoldReport.meanClassifierReport, model); // <---------------- creation of complete report context.notes.log("- Experiment sequence with Feature Vector Extractor [" + model.name + "] completed"); context.notes.SaveNote(); // <------------- END OF THE MODEL ------------------------------------------------------------------------------------------------- }
public DSCCReportSet(IWebFVExtractor _extractor) { extractor = _extractor; }