/// <summary>
/// Sets the execution context.
/// </summary>
/// <param name="_manager">The manager.</param>
/// <param name="_setup">The setup.</param>
/// <param name="_tools">The tools.</param>
/// <param name="_classes">The classes.</param>
/// <param name="sufix">The sufix.</param>
/// <param name="chunker">The chunker.</param>
/// <param name="_masterExtractor">The master extractor.</param>
/// <param name="_logger">The logger.</param>
public void SetExecutionContext(experimentManager _manager, experimentSetup _setup, classifierTools _tools, DocumentSetClasses _classes, String sufix, chunkComposerBasic chunker, semanticFVExtractor _masterExtractor, ILogBuilder _logger = null)
{
if (_logger == null)
{
_logger = new builderForLog();
aceLog.consoleControl.setAsOutput(_logger, _setup.name);
}
logger = _logger;
chunkComposer = chunker;
setup = _setup;
tools = _tools;
tools.context = this;
classes = _classes;
// masterConstructor = _masterExtractor.termTableConstructor;
masterExtractor = _setup.featureVectorExtractors_semantic.First();
masterConstructor = masterExtractor.termTableConstructor;
manager = _manager;
String expContextName = "exp_" + setup.name.add(sufix, "_");
folder = manager.folder.Add(expContextName, "Experiment " + setup.name, "Directory with all information on the experiment [" + setup.name + "]");
errorNotesFolder = folder.Add("errors", "Error logs", "Directory with error reports produced if an exception occours. Normally, if everything was ok this folder should have only two files inside: directory_readme.txt and empty: note.txt).");
errorNotes = new experimentNotes(errorNotesFolder, "Notes (logs) about critical and non-critical errors that happen during experiment execution. If everything was ok - this file should remain empty");
notes = new experimentNotes(folder, "Notes on experiment setup and execution log");
aceLog.consoleControl.setAsOutput(notes, "Notes");
notes.log("Experiment [" + expContextName + "] initiated");
notes.AppendLine("About: " + setup.description);
notes.AppendHorizontalLine();
notes.SaveNote();
notes.AppendHeading("Feature extraction models");
var lnsc = chunkComposer.DescribeSelf();
lnsc.ForEach(x => notes.AppendLine(x));
notes.AppendLine(" - ");
List <String> mdn = new List <string>();
foreach (var md in setup.models)
{
if (mdn.Contains(md.name))
{
md.name += "_" + mdn.Count.ToString();
}
else
{
mdn.Add(md.name);
}
}
foreach (var md in setup.models)
{
String prefix = md.name;
md.classes = classes;
md.BuildFeatureVectorDefinition();
var lns = md.DescribeSelf();
lns.ForEach(x => notes.AppendLine(x));
kFoldValidationCollection validationCases = classes.BuildValidationCases(prefix, setup.validationSetup.k, tools.DoDebug, logger, folder, setup.validationSetup.randomize);
validationCases.pipelineCollection = pipelineCollection;
validationCases.connectContext(this, md);
validationCollections.Add(md.name, validationCases);
//md.postClassifiers = setup.classifiers;
}
}
protected void runModel(experimentExecutionContext context, IWebFVExtractor model)
{
imbSCI.Core.screenOutputControl.logToConsoleControl.setAsOutput(context.logger, model.name);
Int32 crashRetries = context.tools.operation.doRebootFVEOnCrashRetryLimit;
aceDictionarySet <IDocumentSetClass, DocumentSetCaseCollection> casesByClasses = new aceDictionarySet <IDocumentSetClass, DocumentSetCaseCollection>();
DSCCReportSet kFoldReport = new DSCCReportSet(model);
var valCol = context.validationCollections[model.name];
List <DocumentSetCaseCollectionSet> modelCaseResults = new List <DocumentSetCaseCollectionSet>();
crashRetries = context.tools.operation.doRebootFVEOnCrashRetryLimit;
while (crashRetries > 0)
{
try
{
experimentNotes modelNotes = new experimentNotes(valCol.folder, "Fold-level experiment settings description notes");
modelNotes.AppendLine("# Notes on Feature Vector Extractor: " + model.name);
var nts = model.DescribeSelf();
nts.ForEach(x => modelNotes.AppendLine(x));
context.logger.log("Executing k-fold cases with model [" + model.name + "]");
valCol.DescribeSampleDistribution(modelNotes);
context.mainReport.valColVsModelVsSampleHash.Add("[" + model.name + "]".toWidthExact(20) + " [sample distribution hash: " + valCol.SampleDistributionHash + "]");
modelNotes.SaveNote();
ParallelOptions ops = new ParallelOptions();
ops.MaxDegreeOfParallelism = context.tools.operation.ParallelThreads;
Parallel.ForEach <kFoldValidationCase>(valCol.GetCases(), ops, valCase =>
{
model.DoFVEAndTraining(valCase, context.tools, context.logger); // <--------------------------------------------------------------------------------------- BUILDING FVE
DocumentSetCaseCollectionSet results = model.DoClassification(valCase, context.tools, context.logger);
if (!results.Any())
{
throw new aceScienceException("DoClassification for [" + model.name + "] returned no results!", null, model, "DoClassification " + model.name + " failed!", context);
}
foreach (var pair in results)
{
DocumentSetCaseCollection cls = pair.Value;
casesByClasses.Add(cls.setClass, cls);
}
valCase.evaluationResults = results;
if (context.tools.DoResultReporting)
{
context.logger.log("producing reports on k-Fold case [" + valCase.name + "]");
DSCCReports r = results.GetReports();
var sumMeans = r.GetAverageTable(context); //.GetReportAndSave(valCase.folder, appManager.AppInfo, "CrossValidation_" + valCase.name);
sumMeans.SetDescription("FVE report, aggregated for all categories - for fold [" + valCase.name + "]");
sumMeans.GetReportAndSave(valCase.folder, appManager.AppInfo, "CrossValidation_" + valCase.name, true, context.tools.operation.doReportsInParalell);
var fveAndCase = r.GetFullValidationTable(context);
fveAndCase.SetDescription("Per-category aggregate statistics, for each classifier, within fold [" + valCase.name + "], used for macro-averaging");
fveAndCase.GetReportAndSave(valCase.folder, appManager.AppInfo, "CrossValidation_extrainfo_" + valCase.name, true, context.tools.operation.doReportsInParalell);
var fullCaseReport = results.GetReportOnAllCases();
fullCaseReport.GetReportAndSave(valCase.folder, appManager.AppInfo, "FullReport_" + valCase.name, true, context.tools.operation.doReportsInParalell);
kFoldReport.Add(valCase, r);
}
context.logger.log("k-Fold case [" + valCase.name + "] completed");
context.notes.log("- - Experiment sequence for [" + valCase.name + "] fold completed");
if (context.tools.operation.doSaveKnowledgeForClasses)
{
valCase.knowledgeLibrary.SaveKnowledgeInstancesForClasses(valCase, context.logger);
}
});
foreach (var fold in valCol.GetCases()) // Parallel.ForEach<kFoldValidationCase>(valCol.GetCases(), ops, valCase =>
{
modelCaseResults.Add(fold.evaluationResults);
}
crashRetries = 0;
}
catch (Exception ex)
{
crashRetries--;
context.errorNotes.LogException("FVE Model crashed -- retries left [" + crashRetries + "] --- ", ex, model.name);
context.logger.log(":::: REPEATING the model [" + model.name + "] ::: CRASHED [" + ex.Message + "] ::: RETRIES [" + crashRetries + "]");
imbACE.Services.terminal.aceTerminalInput.doBeepViaConsole(1200, 1000, 1);
imbACE.Services.terminal.aceTerminalInput.doBeepViaConsole(2400, 1000, 1);
imbSCI.Core.screenOutputControl.logToConsoleControl.setAsOutput(context.logger, "RETRIES[" + crashRetries + "]");
}
}
imbSCI.Core.screenOutputControl.logToConsoleControl.setAsOutput(context.logger, "Reporting");
valCol.knowledgeLibrary.SaveCaseKnowledgeInstances(context.logger);
// DocumentSetCaseCollection second = null;
if (modelCaseResults.Any())
{
featureExtractionMetrics modelMetrics = new featureExtractionMetrics(model.name, "All");
DataTableTypeExtended <featureExtractionMetrics> modelVsCategoryMetrics = new DataTableTypeExtended <featureExtractionMetrics>(model.name, "Model metrics per category");
// <-------------------------------------- CATEGORIES REPORT ----------------------------------------------
DataTable allTable = modelCaseResults.First()[0].GetReportTable(false, false).GetClonedShema <DataTable>();; //valCol.GetCases().First().evaluationResults[0].GetReportTable(false, false);
rangeFinderForDataTable ranger = new rangeFinderForDataTable(allTable, "name");
ranger.columnsToSignIn.Add("Case");
foreach (KeyValuePair <IDocumentSetClass, aceConcurrentBag <DocumentSetCaseCollection> > pair in casesByClasses)
{
DocumentSetCaseCollection first = null;
DataTable repTable = null;
ranger.prepareForNextAggregationBlock(allTable, "name");
foreach (DocumentSetCaseCollection cn in pair.Value)
{
foreach (var cni in cn)
{
if (cni != null)
{
cn.BuildRow(cni, allTable, false);
}
}
}
ranger.AddRangeRows(pair.Key.name, allTable, true, imbSCI.Core.math.aggregation.dataPointAggregationType.avg | imbSCI.Core.math.aggregation.dataPointAggregationType.stdev);
var categoryMetrics = new featureExtractionMetrics(model.name, pair.Key.name);
categoryMetrics.SetValues(ranger);
modelVsCategoryMetrics.AddRow(categoryMetrics);
modelMetrics.AddValues(categoryMetrics);
categoryMetrics.saveObjectToXML(valCol.folder.pathFor(model.name + "_" + categoryMetrics.Name + ".xml", imbSCI.Data.enums.getWritableFileMode.overwrite, "FV and Category sample metrics, serialized object"));
//context.notes.log("- - Creating report for category [" + pair.Key.name + "] completed");
//repTable.GetReportAndSave(valCol.folder, appManager.AppInfo, model.name + "_category_" + pair.Key.name);
}
modelMetrics.DivideValues(casesByClasses.Count);
modelMetrics.saveObjectToXML(valCol.folder.pathFor(model.name + "_metrics.xml", imbSCI.Data.enums.getWritableFileMode.overwrite, "Cross-categories macroaveraged metrics of the FVE model [" + model.name + "]"));
modelVsCategoryMetrics.AddRow(modelMetrics);
modelVsCategoryMetrics.GetRowMetaSet().SetStyleForRowsWithValue <String>(DataRowInReportTypeEnum.dataHighlightA, "Name", modelMetrics.Name);
modelVsCategoryMetrics.GetReportAndSave(valCol.folder, appManager.AppInfo, model.name + "_metrics", true, true);
context.mainReport.AddModelMetrics(modelMetrics);
context.notes.log("- Creating report for all categories [" + model.name + "] ");
allTable.GetReportAndSave(valCol.folder, appManager.AppInfo, model.name + "_categories", true, context.tools.operation.doReportsInParalell);
}
kFoldReport.MakeReports(context, valCol.folder);
context.mainReport.AddBestPerformer(kFoldReport.GetTopClassifierReport(), kFoldReport.meanClassifierReport, model);
// <---------------- creation of complete report
context.notes.log("- Experiment sequence with Feature Vector Extractor [" + model.name + "] completed");
context.notes.SaveNote();
// <------------- END OF THE MODEL -------------------------------------------------------------------------------------------------
}