public static secondaryReportOnFVE GetExperimentIntroductionLine(String experimentName, Int32 folds)
{
secondaryReportOnFVE output = new secondaryReportOnFVE();
output.Experiment = experimentName;
output.Folds = folds;
output.FVEHash = "------------";
output.FVEModel = "------";
output.FVPType = "------";
output.HTMLTagFactors = "------";
output.UID = experimentName;
output.Comment = "[separaptor row]";
return(output);
}
/// <summary>
/// Processes the specified logger.
/// </summary>
/// <param name="logger">The logger.</param>
public void Process(ILogBuilder logger)
{
Double F1 = Double.MinValue;
foreach (wlfClassifier.semanticFVExtractor fve in experiment.featureVectorExtractors_semantic)
{
secondaryReportOnFVE secReport = new secondaryReportOnFVE();
secReport.Experiment = experiment.name;
secReport.Folds = experiment.validationSetup.k;
String p = folder.path.Remove(folderRoot.path);
secReport.Path = p;
LoadSemanticClouds(fve, logger);
secReport.UpdateSecondaryRecord(fve);
secReport.Randomized = experiment.validationSetup.randomize;
secReport.Classifiers = experiment.classifiers.Count;
rangeFinderWithData ranger = new rangeFinderWithData();
imbSCI.Core.math.classificationMetrics.classificationReportRowFlags flags = imbSCI.Core.math.classificationMetrics.classificationReportRowFlags.classifier;
if (useDataTable)
{
var dataTable = topperformers.GetDataTable();
foreach (DataRow topReport in dataTable.Rows) //report.bestPerformingClassifiers)
{
String name = topReport["Name"].toStringSafe();
Double f1 = 0;
Double.TryParse(topReport["F1measure"].ToString(), out f1);
if (name.Contains(fve.name))
{
secReport.Classifier = topReport["Classifier"].ToString();
secReport.F1Score = f1;
}
ranger.Learn(f1);
}
}
else
{
foreach (DocumentSetCaseCollectionReport topReport in topperformers.GetList()) //report.bestPerformingClassifiers)
{
ranger.Learn(topReport.F1measure);
if (topReport.Name.Contains(fve.name))
{
secReport.Classifier = topReport.Classifier;
secReport.F1Score = topReport.F1measure;
}
}
}
secReport.F1ScoreDeviation = ranger.doubleEntries.GetStdDeviation(false);
secReport.F1ScoreMean = ranger.Average;
// <------------------------------------ COMPUTING THE CLOUD METRICS
Int32 cn = 0;
Double nodeCount = 0;
Double linkCount = 0;
Double pingLength = 0;
if (semanticClouds.ContainsKey(fve.name))
{
foreach (lemmaSemanticCloud sc in semanticClouds[fve.name])
{
nodeCount += sc.nodes.Count();
linkCount += sc.links.Count();
pingLength += freeGraphExtensions.PingGraphSize(sc, sc.primaryNodes, true, freeGraphPingType.maximumPingLength);
cn++;
}
nodeCount = nodeCount.GetRatio(cn);
linkCount = linkCount.GetRatio(cn);
pingLength = pingLength.GetRatio(semanticClouds[fve.name].Count());
}
secReport.NodeCount = nodeCount;
secReport.LinkRatio = linkCount.GetRatio(nodeCount);
secReport.GraphDepth = pingLength.GetRatio(nodeCount);
// <------------------------------------------------------ FINISHED WITH CLOUD METRICS
items.Add(secReport);
if (secReport.F1Score > F1)
{
F1 = secReport.F1Score;
topPerformer = secReport;
}
//F1 = Math.Max(secReport.F1Score, F1);
logger.log("Collected data on [" + fve.name + "] [" + fve.description + "]");
}
}
/// <summary>
/// Generates the report set directory readme file.
/// </summary>
/// <param name="filename">The filename.</param>
public void GenerateParentReadmeFile(String filename = "experiments_readme.txt")
{
folderNode node = folder;
//if (folder.parent != null)
//{
// node = folder.parent as folderNode;
//}
//else
//{
// DirectoryInfo di = folder;
// DirectoryInfo dip = di.Parent;
// node = dip;
// node.caption = dip.Name;
// node.description = "Report data set on group of experiments";
//}
StringBuilder sb = new StringBuilder();
// <--------- ---- ---- -- ------ DESCRIPTION GENERATION
String ln = "# Report package [" + node.caption + "]";
sb.AppendLine(ln);
String line = "-".Repeat(ln.Length);
sb.AppendLine(line);
sb.AppendLine("This directory contains integral reports on performed experiments on web site classification, done with imbWBI library - part of imbVeles Framework.");
sb.AppendLine("Each subdirectory contains reports on one particular configuration, tested on a range of semantic term expansion (Stx) values.");
sb.AppendLine(line);
sb.AppendLine();
sb.AppendLine("Main directory structure:");
sb.AppendLine("-- exp_[experient code name]");
sb.AppendLine("-- exp_[experient code name]");
sb.AppendLine("-- ... ");
sb.AppendLine("-- Summary");
sb.AppendLine();
sb.AppendLine(line);
sb.AppendLine();
sb.AppendLine("## Experiment subdirectories");
sb.AppendLine("In the subdirectories named exp_[experiment code name] you will find summary text, spreadsheet and XML data files - describing the experiment.");
sb.AppendLine("Wherever an Excel spreadsheet file is generated (in XLSX format), you'll find a subfolder named [data] where the same data is exported in Comma Separated Values (CSV) format for easier consumption by 3rd party software.");
sb.AppendLine("-- In Excel spreadsheet files you'll find second sheet called LEGEND, where each column in the report is explained. The same column descriptions are saved in plain text format within [data] subfolders.");
sb.AppendLine("In each subdirectory of this data set, across the complete directory tree, you will find [directory_readme.txt] where content of the subdirectory is described.");
sb.AppendLine("Exact content of the experiment report subdirectores varies depending on reporting options used and version of the software.");
sb.AppendLine();
sb.AppendLine("However, there is general directory tree structure:");
sb.AppendLine("-- exp_[experient code name]");
sb.AppendLine("-- -- [name of FVE]_[sample randomization tag]_E[Stx] <- directory with report on experiment version, performed with Stx number of semantic term expansion steps");
sb.AppendLine("-- -- -- [name of FVE]_[sample randomization tag]_E[Stx]00[fold id] <- directory with report on [fold id] fold of k-fold schema used");
sb.AppendLine("-- -- -- -- cases <- XML serialized data on Category Knowledge, constructed in this fold (from the training sample subset)");
sb.AppendLine("-- -- General <- general report on the complete data set, different in scope depending on reporting options used");
sb.AppendLine("-- -- SharedKnowledge <- XML serialized data (Lemma Tables) on all Cases in the data set");
sb.AppendLine("-- -- errors <- Here you might find logs on exceptions, if an error during execution happen. ");
sb.AppendLine(" | However, if the note.txt file is not empty - it should have earlier time/date creation stamp then the rest of the report.");
sb.AppendLine(" | Only reports with error notes that should exist in this data set are the ones where Semantic Cloud");
sb.AppendLine(" | failed to be created because of small amount of training data in the fold.");
sb.AppendLine();
sb.AppendLine("## Summary directory");
sb.AppendLine("In the [Summary] directory, you will find aggregated overview reports on all experiments in this group. The reports are created separately for each [k] number of k-fold schemas.");
sb.AppendLine("These reports are given in spreadsheet format (Excel file, and CSV in [data] subfolder) and in form of native XML serialized objects.");
sb.AppendLine("Additional remarks:");
sb.AppendLine("-- Reports may contain some additional metrics, subreports and other records, not mentioned nor explained in the research article. Like: ModelMetrics.xml. ");
sb.AppendLine("-- These are some unfinished ideas, models for FVE evaluation that, at the end, were not used for research conclusions nor system validation.");
sb.AppendLine();
Double f1 = Double.MinValue;
secondaryReportOnFVE topFVE = null;
sb.AppendLine(line);
sb.AppendLine("## List of the experiments contained in this data set");
Int32 c = 1;
foreach (var item in items)
{
ln = "[" + c.ToString("D2") + "] " + item.experiment.name + " (sub experiments: " + item.items.Count.ToString() + ")";
sb.AppendLine(ln);
if (f1 < item.topPerformer.F1Score)
{
topFVE = item.topPerformer;
f1 = item.topPerformer.F1Score;
}
c++;
}
sb.AppendLine(line);
sb.AppendLine();
sb.AppendLine("imbVeles Framework | imbWBI | GNU GPL v3.0 | http://blog.veles.rs | Goran Grubić | [email protected]");
sb.AppendLine(line);
sb.AppendLine("File generated: " + DateTime.Now.ToLongDateString() + ", " + DateTime.Now.ToLongTimeString());
// <---------
String p = node.pathFor(filename, imbSCI.Data.enums.getWritableFileMode.overwrite, "ReadMe file describing experiment reports contained in this report set.");
File.WriteAllText(p, sb.ToString());
node.AttachSubfolders();
if (DoGenerateGraph)
{
var dgml = node.GetDirectedGraph(false, false, false, 3);
dgml.Layout = imbSCI.Graph.DGML.enums.GraphLayoutEnum.DependencyMatrix;
dgml.GraphDirection = imbSCI.Graph.DGML.enums.GraphDirectionEnum.LeftToRight;
dgml.Save(node.pathFor("directoryGraph.dgml", imbSCI.Data.enums.getWritableFileMode.overwrite, "Directory structure Directed Graph in Microsoft DGML format - open in Visual Studio", true));
//var dot = dgml.ConvertToDOT();
//dot.Save(node.pathFor("directoryGraph.dot", imbSCI.Data.enums.getWritableFileMode.overwrite, "Directory structure Directed Graph in GraphVIZ DOT graph language format", true));
}
node.generateReadmeFiles(appManager.AppInfo);
}
