/// <summary>
/// Gets the sum data set.
/// </summary>
/// <param name="datasets">The datasets.</param>
/// <param name="dataSetName">Name of the data set.</param>
/// <returns></returns>
public static DataSet GetSumDataSet(this IEnumerable <DataSet> datasets, string dataSetName = "dataset")
{
DataSet dss = null;
aceDictionarySet <string, DataTable> tableCumul = new aceDictionarySet <string, DataTable>();
List <DataTable> fradt = new List <DataTable>();
foreach (DataSet ds in datasets)
{
if (dss == null)
{
dss = ds;
}
foreach (DataTable tb in ds.Tables)
{
tableCumul.Add(tb.TableName, tb);
}
}
DataSet fin = new DataSet(dataSetName);
foreach (DataTable tb in dss.Tables)
{
fin.AddTable(GetAggregatedTable(tableCumul[tb.TableName], "SumOf_" + tb.TableName, dataPointAggregationAspect.overlapMultiTable)); //GetSumTable(dataSetName + "_" + tb.TableName + "_sum_" + dss.Tables.Count));
}
return(fin);
}
/// <summary>
/// Gets all callable needles for all types registered,
/// </summary>
/// <returns></returns>
public aceDictionarySet <Type, String> GetAllRegistrations()
{
aceDictionarySet <Type, String> output = new aceDictionarySet <Type, string>();
foreach (var p in pluginTypesByName)
{
output.Add(p.Value, p.Key);
}
foreach (var p in pluginTypesByPathName)
{
output.Add(p.Value, p.Key);
}
if (supportDirtyNaming)
{
foreach (var p in dirtyDictionary)
{
output.Add(p.Value, p.Key);
}
}
return(output);
}
/// <summary>
/// Returns objects groupped by the same value of the specified column
/// </summary>
/// <typeparam name="TGroup">Value type of the column</typeparam>
/// <param name="column">The column to use for groups</param>
/// <param name="expression">Optional row filter expression</param>
/// <returns></returns>
public aceDictionarySet <TGroup, T> GetGroups <TGroup>(string column, string expression = "")
{
if (!table.Columns.Contains(column))
{
throw new ArgumentException("There is no property [" + column + "] in the table row type [" + type.Name + "]");
}
aceDictionarySet <TGroup, T> output = new aceDictionarySet <TGroup, T>();
List <T> list = new List <T>();
DataRow[] rows = null;
if (expression.isNullOrEmpty())
{
rows = table.Select();
}
else
{
rows = table.Select(expression);
}
foreach (DataRow dr in rows)
{
TGroup key = (TGroup)dr[column];
output.Add(key, (T)GetObjectFromRow(dr));
}
return(output);
}
public void RefineStyle(classificationReportStyleDefinition style, IEnumerable <classificationReportExpanded> reports)
{
local_style = style.CloneViaXML();
if (local_style.AutoNewDataColumns || local_style.AutoHideDataColumnsWithSameData)
{
aceDictionarySet <String, String> knownValues = new aceDictionarySet <string, string>();
var expData = reports.Select(x => x.data);
foreach (var cd in expData)
{
foreach (var cl in cd)
{
if (local_style.AutoNewDataColumns)
{
if (!local_style.dataColumns.Any(x => x.key == cl.key))
{
local_style.dataColumns.Add(cl);
}
}
if (!knownValues[cl.key].Contains(cl.value))
{
knownValues.Add(cl.key, cl.value);
}
}
}
if (local_style.AutoHideDataColumnsWithSameData)
{
List <String> columnsToHide = new List <string>();
foreach (var pair in knownValues)
{
if (pair.Value.Count < 2)
{
columnsToHide.Add(pair.Key);
}
else if (pair.Value.Count == 0)
{
columnsToHide.Add(pair.Key);
}
}
var toRemove = local_style.dataColumns.Where(x => columnsToHide.Contains(x.key)).ToList();
toRemove.ForEach(x => local_style.dataColumns.Remove(x));
}
}
}
public String DescribeSampleDistribution(ILogBuilder modelNotes)
{
if (modelNotes == null)
{
modelNotes = new builderForLog();
}
var l = modelNotes.Length;
foreach (var vc in GetCases())
{
modelNotes.AppendHeading("Fold: " + vc.name, 2);
var categoryCaseList = new aceDictionarySet <String, String>();
foreach (validationCaseCollection vcc in vc.trainingCases)
{
foreach (string vccs in vcc)
{
categoryCaseList.Add(vcc.className, "[T] " + vccs);
}
}
foreach (validationCaseCollection vcc in vc.evaluationCases)
{
foreach (string vccs in vcc)
{
categoryCaseList.Add(vcc.className, "[E] " + vccs);
}
}
foreach (var k in categoryCaseList.Keys)
{
modelNotes.AppendHeading("Category: " + k, 3);
foreach (var s in categoryCaseList[k])
{
modelNotes.AppendLine(s);
}
}
}
SampleDistributionNote = modelNotes.GetContent(l);
SampleDistributionHash = md5.GetMd5Hash(SampleDistributionNote);
return(SampleDistributionNote);
}
public static void SaveFeatures(folderNode _folder, aceDictionarySet <FeatureCWPTermClass, String> dict)
{
if (_folder != null)
{
foreach (FeatureCWPTermClass key in dict.Keys)
{
String p = _folder.pathFor("features_" + key.ToString() + ".txt", imbSCI.Data.enums.getWritableFileMode.overwrite, "List of features extracted as [" + key + "]");
StringBuilder sb = new StringBuilder();
if (dict[key].Count > 0)
{
foreach (String feature in dict[key])
{
sb.Append(feature + " ");
}
File.WriteAllText(p, sb.ToString());
}
}
}
}
public static DataTable GetParallelAggregates <T>(this List <IEnumerable <T> > sources, string column_snap, string column_prefix, dataPointAggregationType column_sideAggregates, params string[] column_toInclude)
{
settingsEntriesForObject sEO = new settingsEntriesForObject(typeof(T));
settingsPropertyEntry sPE_snap = sEO.spes[column_snap];
settingsPropertyEntry sPE_prefix = sEO.spes[column_prefix];
List <settingsPropertyEntry> sPE_toInclude = new List <settingsPropertyEntry>();
foreach (string toInclude in column_toInclude)
{
sPE_toInclude.Add(sEO.spes[toInclude]);
}
List <dataPointAggregationType> side_aggregates = column_sideAggregates.getEnumListFromFlags <dataPointAggregationType>();
Dictionary <dataPointAggregationType, settingsPropertyEntry> sPE_sideAggregates = new Dictionary <dataPointAggregationType, settingsPropertyEntry>();
Dictionary <settingsPropertyEntry, dataPointAggregationType> sPE_sideAggregatesContra = new Dictionary <settingsPropertyEntry, dataPointAggregationType>();
foreach (settingsPropertyEntry toInclude in sPE_toInclude)
{
foreach (dataPointAggregationType sideType in side_aggregates)
{
settingsPropertyEntry sPE = new settingsPropertyEntry(toInclude.pi);
sPE.type = typeof(double);
sPE.name = sPE.name + "_" + sideType.ToString();
sPE_sideAggregates.Add(sideType, sPE);
sPE_sideAggregatesContra.Add(sPE, sideType);
}
}
// <---------------------------- preparing data
Dictionary <string, IEnumerable <T> > dataByPrefix = new Dictionary <string, IEnumerable <T> >();
int c = 0;
foreach (IEnumerable <T> s in sources)
{
T firstItem = s.FirstOrDefault <T>();
if (firstItem != null)
{
string prefix = firstItem.imbGetPropertySafe(sPE_prefix.pi).toStringSafe(c.ToString("D3"));
dataByPrefix.Add(prefix, s);
}
c++;
}
// <----- DataColumn Index
aceDictionarySet <string, DataColumn> columnsByPrefix = new aceDictionarySet <string, DataColumn>();
aceDictionarySet <string, DataColumn> columnsSideAggregationByPrefix = new aceDictionarySet <string, DataColumn>();
// <------------------------- building Shema
DataTable output = new DataTable();
output.TableName = "ParallelAggregate_by_" + column_snap;
DataColumn col_recordID = output.Add("ID", "Row ordinal number", "ID", typeof(int), dataPointImportance.normal, "D3").SetUnit("#");
settingsPropertyEntry sPE_recID = col_recordID.GetSPE();
DataColumn col_snap = output.Add(sPE_snap);
aceDictionary2D <settingsPropertyEntry, dataPointAggregationType, DataColumn> columnsByAggregationType = new aceDictionary2D <settingsPropertyEntry, dataPointAggregationType, DataColumn>();
aceDictionarySet <settingsPropertyEntry, DataColumn> columnsBySource = new aceDictionarySet <settingsPropertyEntry, DataColumn>();
foreach (settingsPropertyEntry toInclude in sPE_toInclude)
{
foreach (var pair in dataByPrefix)
{
DataColumn nColumn = output.Add(toInclude);
nColumn.ColumnName = pair.Key + "_" + nColumn.ColumnName;
nColumn.SetGroup(pair.Key);
columnsByPrefix.Add(pair.Key, nColumn);
columnsBySource.Add(toInclude, nColumn);
}
foreach (var pair2 in sPE_sideAggregatesContra)
{
DataColumn nColumn2 = output.Add(toInclude);
nColumn2.SetGroup("Aggregate");
// columnsSideAggregationByPrefix.Add(pair.Key, nColumn);
}
}
// <----------------------------------------------------------- collecting rows
aceDictionary2D <string, settingsPropertyEntry, object> dataRowBySnapValue = new aceDictionary2D <string, settingsPropertyEntry, object>();
int riMax = 0;
foreach (string prefix in dataByPrefix.Keys)
{
IEnumerable <T> s = dataByPrefix[prefix];
int ri = 0;
foreach (T si in s)
{
ri++;
string snapValue = si.imbGetPropertySafe(sPE_snap.pi).toStringSafe();
dataRowBySnapValue[snapValue, sPE_snap] = snapValue;
dataRowBySnapValue[snapValue, sPE_recID] = ri;
foreach (settingsPropertyEntry toInclude in sPE_toInclude)
{
foreach (var pair in columnsByPrefix[prefix])
{
var spe = dataColumnRenderingSetup.GetSPE(pair);
dataRowBySnapValue[snapValue, spe] = si.imbGetPropertySafe(spe.pi);
}
}
riMax = Math.Max(ri, riMax);
}
}
foreach (string prefix in dataByPrefix.Keys)
{
}
//List<Double> data = new List<Double>();
//foreach (var pair2 in columnsSideAggregationByPrefix[prefix])
//{
// var spe2 = pair.GetSPE();
// dataRowBySnapValue[snapValue, spe2] = si.imbGetPropertySafe(spe2.pi);
//}
return(output);
}
/// <summary>
/// Populates the graph report
/// </summary>
/// <param name="graph">The graph.</param>
public void Deploy(freeGraph graph)
{
name = graph.name;
description = "Analysis of graph [" + graph.name + "] structure and other key metrics: " + graph.description;
List <Double> ws = new List <double>();
//instanceCountCollection<freeGraphNodeBase> linkPerNodeFrequency = new instanceCountCollection<freeGraphNodeBase>();
aceDictionarySet <Int32, freeGraphNodeBase> nodesByNumberOfLinks = new aceDictionarySet <int, freeGraphNodeBase>();
foreach (var node in graph.nodes)
{
ws.Add(node.weight);
Int32 lc = graph.CountLinks(node.name, true, true);
nodesByNumberOfLinks.Add(lc, node);
}
if (!ws.Any())
{
EmptyGraph = true;
return;
}
TotalWeight = ws.Sum();
AvgWeight = ws.Average();
StdWeight = ws.GetStdDeviation(false);
NodeCount = graph.nodes.Count;
LinkRatio = graph.links.GetRatio(graph.nodes);
List <String> processed = new List <String>();
List <Int32> linkFrequencies = nodesByNumberOfLinks.Keys.OrderByDescending(x => x).ToList();
Int32 maxLinkFreq = linkFrequencies.Max();
foreach (Int32 freq in linkFrequencies)
{
List <freeGraphNodeBase> nextSet = nodesByNumberOfLinks[freq].ToList();
foreach (freeGraphNodeBase n in nextSet)
{
if (!processed.Contains(n.name))
{
var result = graph.GetLinkedNodes(new String[] { n.name }, 100, true, true, false);
freeGraphIsland island = new freeGraphIsland();
island.Add(result);
if (island.type != freeGraphIslandType.none)
{
islands.Add(island);
processed.AddRange(island.nodes);
}
//result.ForEach(x => processed.Add(x.name));
}
}
}
List <freeGraphIsland> _binodal = new List <freeGraphIsland>();
List <freeGraphIsland> _trinodal = new List <freeGraphIsland>();
List <freeGraphIsland> _polinodal = new List <freeGraphIsland>();
freeGraphIsland continent = null;
if (islands.Any())
{
Int32 max = 0;
continent = islands.First();
foreach (var island in islands)
{
Int32 nc = island.nodes.Count;
if (nc == 2)
{
Binodal += nc;
BinodalN++;
}
if (nc == 3)
{
Trinodal += nc;
TrinodalN++;
}
if (nc > 3)
{
Polinodal += nc;
PolinodalN++;
}
if (max < nc)
{
max = nc;
continent = island;
}
}
}
AllIslands = islands.Count;
PolinodalRatio = Polinodal.GetRatio(graph.nodes.Count);
if (continent != null)
{
var contNodes = graph.nodes.Where(x => continent.nodes.Contains(x.name)).ToList();
//var contNodes = ;
ContinentMass = continent.nodes.Count;
Double cw = contNodes.Sum(x => x.weight);
ContinentWeightRate = cw.GetRatio(TotalWeight);
ContinentMassRate = continent.nodes.Count.GetRatio(graph.nodes.Count);
ContinentSize = graph.PingGraphSize(nodesByNumberOfLinks[maxLinkFreq], true, freeGraphPingType.maximumPingLength);
}
MainIsland = continent;
}
/// <summary>Performs post-processing of data collected by the workload plugin</summary>
/// <remarks><para>Loads all saved DataTables, groups rows in averages for each measure group and creates summary table with all experiments</para></remarks>
/// <param name="searchPattern">pattern used to select input files</param>
/// <param name="groupColumn">column name used for row grouping</param>
/// <param name="overviewColumns">columns to include in overview table</param>
/// <seealso cref="aceOperationSetExecutorBase"/>
public void aceOperation_runWorkloadData(
[Description("pattern used to select input files")] string searchPattern = "results*.xml",
[Description("column name used for row grouping")] string groupColumn = "measureGroup")
// [Description("columns to include in overview table")] String overviewColumns = "DataLoad,CrawlerIterations,ContentPages,dlcMaximum")
{
aceOperation_selectFiles(searchPattern, "index\\workload", true);
folder = folder["index\\workload"];
List <DataTable> tables = new List <DataTable>();
dataPointAggregationType aggType = dataPointAggregationType.avg;
int ci = 1;
int c = selectedFiles.Count();
output.log("[" + c + "] DataTable in the cue.");
List <DataTable> allTables = new List <DataTable>();
DataSet dSet = new DataSet();
aceDictionarySet <string, DataTable> byCrawler = new aceDictionarySet <string, DataTable>();
aceDictionarySet <string, DataTableForStatistics> byCrawlerRT = new aceDictionarySet <string, DataTableForStatistics>();
DataTableForStatistics rt = null;
foreach (FileInfo fi in selectedFiles)
{
try
{
objectTable <reportPlugIn_workloadEntry> workloadEntry = new objectTable <reportPlugIn_workloadEntry>(fi.FullName, true, "EntryID", "");
objectTable <reportPlugIn_workloadEntry> workloadGrouped = new objectTable <reportPlugIn_workloadEntry>("EntryID", "aggregated");
aceDictionarySet <int, reportPlugIn_workloadEntry> workloadGroups = workloadEntry.GetGroups <int>(groupColumn, "terminationWarning = 0");
collectionAggregationResultSet <reportPlugIn_workloadEntry> aggregateSet = new collectionAggregationResultSet <reportPlugIn_workloadEntry>();
foreach (var set in workloadGroups)
{
collectionAggregationResult <reportPlugIn_workloadEntry> aggregates = null;
aggregates = set.Value.GetAggregates(aggType);
var aggregate = aggregates[aggType];
aggregate.measureGroup = set.Key;
aggregate.EntryID = set.Key.ToString("D5") + "_" + aggType.ToString();
workloadGrouped.AddOrUpdate(aggregate);
aggregateSet.Add(aggregate.EntryID + "_" + fi.Name, aggregates);
}
string filename = (fi.Name + "_" + groupColumn + "_" + aggType.ToString()).getFilename();
string n = reportPlugIn_workload_state.ExtractEntryID(aggregateSet.lastItem.EntryID) + dSet.Tables.Count.ToString("D2");
DataTable dt = workloadGrouped.GetDataTable(dSet, n);
dt.SetDescription("Collection of [" + aggregateSet.recordType.Name + "] records grouped by [" + groupColumn + "]");
dt.SetAggregationAspect(dataPointAggregationAspect.subSetOfRows);
dt.SetAggregationOriginCount(aggregateSet.Count);
dt.SetAdditionalInfoEntry("Aggregation Type:", aggType);
dt.SetAdditionalInfoEntry("Data source file:", fi.Name);
dt.SetAdditionalInfoEntries("Last", aggregateSet.lastItem, typeof(string));
dt.SetTitle(n);
byCrawler.Add(aggregateSet.firstItem.Crawler, dt);
// dt.TableName = n;
// dSet.AddTable(dt);
rt = dt.GetReportAndSave(folder, imbWEMManager.authorNotation, n.getFilename(), true);
byCrawlerRT.Add(aggregateSet.firstItem.Crawler, rt);
response.AppendLine("[" + ci + " / " + c + "] DataTable [" + fi.Name + "] had [" + workloadGroups.Keys.Count() + "] groups. Result saved as: " + filename);
ci++;
} catch (Exception ex)
{
output.log("[" + ci + " / " + c + "] DataTable [" + fi.FullName + "] failed.");
output.log(ex.Message);
}
}
output.log("[" + c + "] DataTable processed.");
dSet.serializeDataSet("workload_all", folder, dataTableExportEnum.excel, imbWEMManager.authorNotation);
foreach (string key in byCrawler.Keys)
{
string filename = key.getFilename();
DataSet sd = new DataSet(key);
foreach (DataTable dti in byCrawler[key])
{
sd.AddTable(dti.Copy());
}
sd.AddTable(byCrawlerRT[key].First().RenderLegend());
sd.serializeDataSet(filename, folder, dataTableExportEnum.excel, imbWEMManager.authorNotation);
}
}
/// <summary>Exporting domain list according to criteria specified</summary>
/// <remarks><para>It will select domains using existing data. If index name not specified it will scan index repository and ask user to pick one</para></remarks>
/// <param name="indexName">name of the index to harvest sample from - IndexID</param>
/// <param name="minPages">required min. number of crawled/indexed pages in the doman--</param>
/// <param name="minRelevant">required min. number of relevant pages in the index for the domain</param>
/// <seealso cref="aceOperationSetExecutorBase"/>
public void aceOperation_runIndexData(
[Description("name of the index to harvest sample from - IndexID")] string indexName = "MainIndex",
[Description("required min. number of crawled/indexed pages in the doman--")] int minPages = 30,
[Description("required min. number of relevant pages in the index for the domain")] int minRelevant = 10)
{
if ((indexName == "*") || indexName.isNullOrEmpty())
{
List <string> indexList = imbWEMManager.index.GetIndexList();
indexList.Add("*");
aceTerminalInput.askForOption("Choose index to work with - or confirm * to load all indexes:", "*", indexList);
}
indexDatabaseStandalone indexDb = new indexDatabaseStandalone(indexName);
imbWEMManager.index.OpenIndex(indexName, "plugin_dataLoader");
imbWEMManager.index.pageIndexTable.ReadOnlyMode = true;
imbWEMManager.index.domainIndexTable.ReadOnlyMode = true;
List <indexDomain> d_list = new List <indexDomain>();
List <indexPage> pages = imbWEMManager.index.pageIndexTable.GetPagesAndDomains(indexPageEvaluationEntryState.inTheIndex, out d_list);
aceDictionarySet <indexDomain, indexPage> dict = new aceDictionarySet <indexDomain, indexPage>();
List <string> list = new List <string>();
List <indexDomain> domains = new List <indexDomain>();
foreach (indexDomain domain in d_list)
{
List <indexPage> pl = Enumerable.Where(pages, x => x.url.Contains(domain.domain)).ToList();
dict.Add(domain, pl);
int prc = 0;
if (pl.Count() > minPages)
{
foreach (indexPage ip in pl)
{
if (ip.relevancyText == "isRelevant")
{
prc++;
}
if (prc > minRelevant)
{
output.AppendLine($" {domain.domain} P[_{pl.Count()}_] Pr[_{prc}_] --> accepted, stop counting");
// domains.Add(domain);
list.Add(domain.domain);
break;
}
}
}
}
// var domains = imbWEMManager.index.domainIndexTable.GetWhere(nameof(indexDomain.relevantPages) + " > " + minRelevant);
// domains = domains.Where(x => ((x.relevantPages + x.notRelevantPages) > minPages)).ToList();
string sampleName = indexName.add("Pa" + minPages + "Pr" + minRelevant, "_").add("txt", ".");
domains.ForEach(x => list.Add(x.url));
objectTable <indexDomain> dTable = new objectTable <indexDomain>("url", sampleName);
domains.ForEach(x => dTable.AddOrUpdate(x));
dTable.GetDataTable(null, sampleName).GetReportAndSave(folder, imbWEMManager.authorNotation, sampleName, true);
folder = imbWEMManager.index.folder;
string p = folder.pathFor(sampleName);
list.saveContentOnFilePath(p);
output.log("Exported sample saved to: " + p);
}
public industryLemmaRankTable process(webLemmaTermTable chunkTable, webLemmaTermTable termTable, industryLemmaRankTable output)
{
List <webLemmaTerm> allChunks = chunkTable.GetList();
var docSetFreq = allChunks.Where(x => x.documentSetFrequency > 1);
instanceCountCollection <String> termCounter = new instanceCountCollection <string>();
aceDictionarySet <String, String> dict = new aceDictionarySet <string, string>();
foreach (webLemmaTerm chunk in docSetFreq)
{
var lemmas = chunk.nominalForm.SplitSmart(textMapBase.SEPARATOR, "", true, true);
lemmas = lemmas.Where(x => x.Length > 2).ToList();
termCounter.AddInstanceRange(lemmas);
foreach (String lm in lemmas)
{
foreach (String lmi in lemmas)
{
if (lmi != lm)
{
dict[lm].AddUnique(lmi);
}
}
}
}
List <String> primaries = new List <string>();
foreach (var pair in termCounter)
{
if (termCounter[pair] > 1)
{
primaries.Add(pair);
industryLemmaTerm lemma = output.GetOrCreate(pair);
lemma.termType = industryLemmaTermType.primary;
lemma.weight = settings.PrimaryTermFactor * termTable[lemma.name].weight;
lemma.nominalForm = pair;
output.AddOrUpdate(lemma);
if (dict.ContainsKey(lemma.nominalForm))
{
foreach (String secLemmas in dict[lemma.nominalForm])
{
industryLemmaTerm lemmaSec = output.GetOrCreate(secLemmas);
if (lemmaSec.termType == industryLemmaTermType.none)
{
lemmaSec.termType = industryLemmaTermType.secondary;
lemmaSec.weight = settings.SecondaryTermFactor * termTable[lemmaSec.name].weight;
lemmaSec.nominalForm = secLemmas;
output.AddOrUpdate(lemmaSec);
}
}
}
}
}
//var reserveChunks = allChunks.Where(x => x.nominalForm.ContainsAny(primaries));
//aceDictionarySet<String, String> dictReserve = new aceDictionarySet<string, string>();
//foreach (webLemmaTerm chunk in reserveChunks)
//{
// var lemmas = chunk.nominalForm.SplitSmart(textMapBase.SEPARATOR, "", true, true);
// lemmas = lemmas.Where(x => x.Length > 2).ToList();
// String prim = lemmas.FirstOrDefault(x => primaries.Contains(x));
// if (!prim.isNullOrEmpty())
// {
// foreach (String lm in lemmas)
// {
// if (prim != lm)
// {
// dictReserve[prim].AddUnique(lm);
// }
// }
// }
//}
//foreach (String prim in primaries)
//{
// if (dictReserve.ContainsKey(prim))
// {
// foreach (String res in dictReserve[prim])
// {
// industryLemmaTerm resLemma = output.GetOrCreate(res);
// if (resLemma.termType == industryLemmaTermType.none)
// {
// resLemma.nominalForm = res;
// resLemma.weight = settings.ReserveTermFactor *termTable[resLemma.name].weight;
// resLemma.termType = industryLemmaTermType.reserve;
// }
// output.AddOrUpdate(resLemma);
// }
// }
//}
return(output);
}
protected void runModel(experimentExecutionContext context, IWebFVExtractor model)
{
imbSCI.Core.screenOutputControl.logToConsoleControl.setAsOutput(context.logger, model.name);
Int32 crashRetries = context.tools.operation.doRebootFVEOnCrashRetryLimit;
aceDictionarySet <IDocumentSetClass, DocumentSetCaseCollection> casesByClasses = new aceDictionarySet <IDocumentSetClass, DocumentSetCaseCollection>();
DSCCReportSet kFoldReport = new DSCCReportSet(model);
var valCol = context.validationCollections[model.name];
List <DocumentSetCaseCollectionSet> modelCaseResults = new List <DocumentSetCaseCollectionSet>();
crashRetries = context.tools.operation.doRebootFVEOnCrashRetryLimit;
while (crashRetries > 0)
{
try
{
experimentNotes modelNotes = new experimentNotes(valCol.folder, "Fold-level experiment settings description notes");
modelNotes.AppendLine("# Notes on Feature Vector Extractor: " + model.name);
var nts = model.DescribeSelf();
nts.ForEach(x => modelNotes.AppendLine(x));
context.logger.log("Executing k-fold cases with model [" + model.name + "]");
valCol.DescribeSampleDistribution(modelNotes);
context.mainReport.valColVsModelVsSampleHash.Add("[" + model.name + "]".toWidthExact(20) + " [sample distribution hash: " + valCol.SampleDistributionHash + "]");
modelNotes.SaveNote();
ParallelOptions ops = new ParallelOptions();
ops.MaxDegreeOfParallelism = context.tools.operation.ParallelThreads;
Parallel.ForEach <kFoldValidationCase>(valCol.GetCases(), ops, valCase =>
{
model.DoFVEAndTraining(valCase, context.tools, context.logger); // <--------------------------------------------------------------------------------------- BUILDING FVE
DocumentSetCaseCollectionSet results = model.DoClassification(valCase, context.tools, context.logger);
if (!results.Any())
{
throw new aceScienceException("DoClassification for [" + model.name + "] returned no results!", null, model, "DoClassification " + model.name + " failed!", context);
}
foreach (var pair in results)
{
DocumentSetCaseCollection cls = pair.Value;
casesByClasses.Add(cls.setClass, cls);
}
valCase.evaluationResults = results;
if (context.tools.DoResultReporting)
{
context.logger.log("producing reports on k-Fold case [" + valCase.name + "]");
DSCCReports r = results.GetReports();
var sumMeans = r.GetAverageTable(context); //.GetReportAndSave(valCase.folder, appManager.AppInfo, "CrossValidation_" + valCase.name);
sumMeans.SetDescription("FVE report, aggregated for all categories - for fold [" + valCase.name + "]");
sumMeans.GetReportAndSave(valCase.folder, appManager.AppInfo, "CrossValidation_" + valCase.name, true, context.tools.operation.doReportsInParalell);
var fveAndCase = r.GetFullValidationTable(context);
fveAndCase.SetDescription("Per-category aggregate statistics, for each classifier, within fold [" + valCase.name + "], used for macro-averaging");
fveAndCase.GetReportAndSave(valCase.folder, appManager.AppInfo, "CrossValidation_extrainfo_" + valCase.name, true, context.tools.operation.doReportsInParalell);
var fullCaseReport = results.GetReportOnAllCases();
fullCaseReport.GetReportAndSave(valCase.folder, appManager.AppInfo, "FullReport_" + valCase.name, true, context.tools.operation.doReportsInParalell);
kFoldReport.Add(valCase, r);
}
context.logger.log("k-Fold case [" + valCase.name + "] completed");
context.notes.log("- - Experiment sequence for [" + valCase.name + "] fold completed");
if (context.tools.operation.doSaveKnowledgeForClasses)
{
valCase.knowledgeLibrary.SaveKnowledgeInstancesForClasses(valCase, context.logger);
}
});
foreach (var fold in valCol.GetCases()) // Parallel.ForEach<kFoldValidationCase>(valCol.GetCases(), ops, valCase =>
{
modelCaseResults.Add(fold.evaluationResults);
}
crashRetries = 0;
}
catch (Exception ex)
{
crashRetries--;
context.errorNotes.LogException("FVE Model crashed -- retries left [" + crashRetries + "] --- ", ex, model.name);
context.logger.log(":::: REPEATING the model [" + model.name + "] ::: CRASHED [" + ex.Message + "] ::: RETRIES [" + crashRetries + "]");
imbACE.Services.terminal.aceTerminalInput.doBeepViaConsole(1200, 1000, 1);
imbACE.Services.terminal.aceTerminalInput.doBeepViaConsole(2400, 1000, 1);
imbSCI.Core.screenOutputControl.logToConsoleControl.setAsOutput(context.logger, "RETRIES[" + crashRetries + "]");
}
}
imbSCI.Core.screenOutputControl.logToConsoleControl.setAsOutput(context.logger, "Reporting");
valCol.knowledgeLibrary.SaveCaseKnowledgeInstances(context.logger);
// DocumentSetCaseCollection second = null;
if (modelCaseResults.Any())
{
featureExtractionMetrics modelMetrics = new featureExtractionMetrics(model.name, "All");
DataTableTypeExtended <featureExtractionMetrics> modelVsCategoryMetrics = new DataTableTypeExtended <featureExtractionMetrics>(model.name, "Model metrics per category");
// <-------------------------------------- CATEGORIES REPORT ----------------------------------------------
DataTable allTable = modelCaseResults.First()[0].GetReportTable(false, false).GetClonedShema <DataTable>();; //valCol.GetCases().First().evaluationResults[0].GetReportTable(false, false);
rangeFinderForDataTable ranger = new rangeFinderForDataTable(allTable, "name");
ranger.columnsToSignIn.Add("Case");
foreach (KeyValuePair <IDocumentSetClass, aceConcurrentBag <DocumentSetCaseCollection> > pair in casesByClasses)
{
DocumentSetCaseCollection first = null;
DataTable repTable = null;
ranger.prepareForNextAggregationBlock(allTable, "name");
foreach (DocumentSetCaseCollection cn in pair.Value)
{
foreach (var cni in cn)
{
if (cni != null)
{
cn.BuildRow(cni, allTable, false);
}
}
}
ranger.AddRangeRows(pair.Key.name, allTable, true, imbSCI.Core.math.aggregation.dataPointAggregationType.avg | imbSCI.Core.math.aggregation.dataPointAggregationType.stdev);
var categoryMetrics = new featureExtractionMetrics(model.name, pair.Key.name);
categoryMetrics.SetValues(ranger);
modelVsCategoryMetrics.AddRow(categoryMetrics);
modelMetrics.AddValues(categoryMetrics);
categoryMetrics.saveObjectToXML(valCol.folder.pathFor(model.name + "_" + categoryMetrics.Name + ".xml", imbSCI.Data.enums.getWritableFileMode.overwrite, "FV and Category sample metrics, serialized object"));
//context.notes.log("- - Creating report for category [" + pair.Key.name + "] completed");
//repTable.GetReportAndSave(valCol.folder, appManager.AppInfo, model.name + "_category_" + pair.Key.name);
}
modelMetrics.DivideValues(casesByClasses.Count);
modelMetrics.saveObjectToXML(valCol.folder.pathFor(model.name + "_metrics.xml", imbSCI.Data.enums.getWritableFileMode.overwrite, "Cross-categories macroaveraged metrics of the FVE model [" + model.name + "]"));
modelVsCategoryMetrics.AddRow(modelMetrics);
modelVsCategoryMetrics.GetRowMetaSet().SetStyleForRowsWithValue <String>(DataRowInReportTypeEnum.dataHighlightA, "Name", modelMetrics.Name);
modelVsCategoryMetrics.GetReportAndSave(valCol.folder, appManager.AppInfo, model.name + "_metrics", true, true);
context.mainReport.AddModelMetrics(modelMetrics);
context.notes.log("- Creating report for all categories [" + model.name + "] ");
allTable.GetReportAndSave(valCol.folder, appManager.AppInfo, model.name + "_categories", true, context.tools.operation.doReportsInParalell);
}
kFoldReport.MakeReports(context, valCol.folder);
context.mainReport.AddBestPerformer(kFoldReport.GetTopClassifierReport(), kFoldReport.meanClassifierReport, model);
// <---------------- creation of complete report
context.notes.log("- Experiment sequence with Feature Vector Extractor [" + model.name + "] completed");
context.notes.SaveNote();
// <------------- END OF THE MODEL -------------------------------------------------------------------------------------------------
}
/// <summary>
/// Builds the type of the tree node for.
/// </summary>
/// <param name="type">The type.</param>
/// <param name="source">The source.</param>
/// <param name="host">The host.</param>
/// <returns></returns>
internal static commandTreeDescription BuildTreeNodeForType(Type type, IAceOperationSetExecutor source, commandTreeDescription parent, String nameOverride = "", commandTreeNodeLevel level = commandTreeNodeLevel.type)
{
settingsMemberInfoEntry typeInfo = new settingsMemberInfoEntry(type);
commandTreeDescription output = parent.Add(nameOverride.or(typeInfo.name, typeInfo.displayName, type.Name));
commandTree host = parent.root as commandTree;
output.description = typeInfo.description;
output.nodeLevel = level;
output.helpLines.Add(typeInfo.info_helpTitle);
output.helpLines.Add(typeInfo.info_helpTips);
output.helpLines.AddRange(typeInfo.additionalInfo);
var methods = type.GetMethods(System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.Public | System.Reflection.BindingFlags.DeclaredOnly);
aceDictionarySet <String, commandTreeDescription> groups = new aceDictionarySet <string, commandTreeDescription>();
aceDictionarySet <String, commandTreeDescription> group_nodes = new aceDictionarySet <string, commandTreeDescription>();
foreach (MemberInfo __m in methods)
{
if (__m.Name.StartsWith(aceMenuItemMeta.METHOD_PREFIX))
{
commandTreeDescription desc = new commandTreeDescription();
desc.setByMemberInfo(__m);
desc.nodeLevel = commandTreeNodeLevel.group;
groups.Add(desc.category, desc);
}
}
foreach (String group in groups.Keys.OrderBy(x => x))
{
var ordered = groups[group].OrderBy(x => x.name);
commandTreeDescription gdesc = parent.Add(group);
gdesc.nodeLevel = commandTreeNodeLevel.group;
foreach (var cdesc in ordered)
{
cdesc.nodeLevel = commandTreeNodeLevel.command;
gdesc.Add(cdesc, cdesc.name);
host.flatAccess.Add(cdesc.path, cdesc);
}
}
var properties = type.GetProperties(BindingFlags.Instance | BindingFlags.Public | BindingFlags.DeclaredOnly);
foreach (PropertyInfo pi in properties)
{
if (pi.DeclaringType.Name == "aceCommandConsole")
{
continue;
}
if (pi.DeclaringType.Name == "aceAdvancedConsole")
{
continue;
}
if (pi.PropertyType.GetInterfaces().Contains(typeof(IAceOperationSetExecutor)))
{
var plugin_instance = source.imbGetPropertySafe(pi) as IAceOperationSetExecutor;
var plugin_node = BuildTreeNodeForType(pi.PropertyType, source, parent, pi.Name, commandTreeNodeLevel.plugin);
plugin_node.setByMemberInfo(pi);
host.plugins.Add(plugin_node.path, plugin_instance);
}
else if (pi.PropertyType.IsValueType || pi.PropertyType.isTextOrNumber() || pi.GetCustomAttributes(false).Any())
{
if (pi.CanWrite)
{
var prop = parent.Add(pi.Name);
prop.setByMemberInfo(pi);
prop.nodeLevel = commandTreeNodeLevel.parameter;
host.properties.Add(prop.path, pi);
}
}
else if (pi.PropertyType.IsClass || pi.GetCustomAttributes(false).Any())
{
if (!pi.PropertyType.IsGenericType)
{
var prop = parent.Add(pi.Name);
prop.setByMemberInfo(pi);
prop.nodeLevel = commandTreeNodeLevel.module;
host.modules.Add(prop.path, pi);
}
}
}
return(output);
}
public override void eventCrawlJobFinished(crawlerDomainTaskMachine __machine, modelSpiderTestRecord tRecord)
{
crawlerReportFolder = __machine.reporter.folder;
var wRecords = tRecord.GetChildRecords();
string fileprefix = tRecord.instance.name.getFilename();
List <DataTable> iterationTimelines = new List <DataTable>();
foreach (var wRecord in wRecords)
{
iterationTimelines.Add(wRecord.iterationTableRecord.GetDataTable());
}
int dlc_c = 0;
if (imbWEMManager.settings.directReportEngine.DR_ReportModules)
{
tRecord.frontierDLCDataTables[moduleIterationRecordSummary.fra_overview].GetAggregatedTable("fra_overview").GetReportAndSave(crawlerReportFolder, imbWEMManager.authorNotation, "fra_overview".add(fileprefix, "_"), true);
tRecord.frontierDLCDataTables[moduleIterationRecordSummary.all].GetAggregatedTable("fra_modules_all").GetReportAndSave(crawlerReportFolder, imbWEMManager.authorNotation, "fra_modules_all".add(fileprefix, "_"), true);
if (tRecord.frontierDLCDataTables[moduleIterationRecordSummary.language].Any())
{
tRecord.frontierDLCDataTables[moduleIterationRecordSummary.language].GetAggregatedTable("fra_module_language").GetReportAndSave(crawlerReportFolder, imbWEMManager.authorNotation, "fra_module_language_".add(fileprefix, "_"), true);
}
if (tRecord.frontierDLCDataTables[moduleIterationRecordSummary.structure].Any())
{
tRecord.frontierDLCDataTables[moduleIterationRecordSummary.structure].GetAggregatedTable("fra_modules_structure").GetReportAndSave(crawlerReportFolder, imbWEMManager.authorNotation, "fra_module_structure_".add(fileprefix, "_"), true);
}
if (tRecord.frontierDLCDataTables[moduleIterationRecordSummary.template].Any())
{
tRecord.frontierDLCDataTables[moduleIterationRecordSummary.template].GetAggregatedTable("fra_modules_template").GetReportAndSave(crawlerReportFolder, imbWEMManager.authorNotation, "fra_module_template".add(fileprefix, "_"), true);
}
if (tRecord.frontierDLCDataTables[moduleIterationRecordSummary.diversity].Any())
{
tRecord.frontierDLCDataTables[moduleIterationRecordSummary.diversity].GetAggregatedTable("fra_module_diversity").GetReportAndSave(crawlerReportFolder, imbWEMManager.authorNotation, "fra_module_diversity_".add(fileprefix, "_"), true);
}
string finalOverviewPath = crawlerReportFolder.pathFor("fra_modules_impact".add(fileprefix, "_"), getWritableFileMode.newOrExisting);
objectTable <moduleFinalOverview> finalOverview = new objectTable <moduleFinalOverview>(finalOverviewPath, false, "ModuleName", "module_impact");
finalOverview.description = "Aggregate (DLC and iterations) metrics on modules' impact to the result.";
aceDictionarySet <moduleIterationRecordSummary, moduleIterationRecord> moduleIterationsByModule = new aceDictionarySet <moduleIterationRecordSummary, moduleIterationRecord>();
List <moduleIterationRecordSummary> moduleActive = new List <moduleIterationRecordSummary>();
foreach (var wRecord in wRecords)
{
dlc_c++;
foreach (var pair in wRecord.frontierDLC.modRecords)
{
moduleIterationsByModule.Add(pair.Value.moduleSummaryEnum, pair.Value.GetList());
if (!moduleActive.Contains(pair.Value.moduleSummaryEnum))
{
moduleActive.Add(pair.Value.moduleSummaryEnum);
}
}
}
int modC = 0;
List <moduleFinalOverview> modList = new List <moduleFinalOverview>();
foreach (var modType in moduleActive)
{
moduleFinalOverview mfo = new moduleFinalOverview();
mfo.deploy(tRecord.instance.name, modType, moduleIterationsByModule[modType], dlc_c);
modC += moduleIterationsByModule[modType].Count;
finalOverview.AddOrUpdate(mfo);
modList.Add(mfo);
}
moduleFinalOverview mfoSum = new moduleFinalOverview();
mfoSum.deploySum(tRecord.instance.name, modList);
finalOverview.AddOrUpdate(mfoSum);
foreach (var mfo in modList)
{
mfo.SetTestIDAndSignature(tRecord.instance, imbWEMManager.index.experimentEntry.state, tRecord);
finalOverview.AddOrUpdate(mfo);
record_moduleImpact.AddOrUpdate(mfo);
}
mfoSum.SetTestIDAndSignature(tRecord.instance, imbWEMManager.index.experimentEntry.state, tRecord);
record_moduleImpact.AddOrUpdate(mfoSum);
// finalOverview.SaveAs(finalOverviewPath.add(".xml"));
DataTable fover = finalOverview.GetDataTable(null, mfoSum.Crawler);
fover.SetAggregationOriginCount(modC);
fover.SetAggregationAspect(dataPointAggregationAspect.onTableMultiRow);
fover.GetReportAndSave(crawlerReportFolder, imbWEMManager.authorNotation, "fra_modules_impact_overview", true);
record_moduleImpact.Save();
var midt = record_moduleImpact.GetDataTable(null, "Module impacts");
midt.AddExtra("The last benchmark metrics entry [" + imbWEMManager.index.experimentEntry.CrawlID + "] inserted on " + DateTime.Now.ToLongDateString() + " / " + DateTime.Now.ToLongTimeString());
midt.GetReportAndSave(imbWEMManager.index.experimentEntry.sessionReportFolder, imbWEMManager.authorNotation, "fra_modules_impact_".add(fileprefix, "_"));
}
else
{
dlc_c = tRecord.children.Count();
}
if (iterationTimelines.Any())
{
DataTable crawlTimeline = iterationTimelines.GetAggregatedTable("Crawler_Timeline", dataPointAggregationAspect.overlapMultiTable);
crawlTimeline.SetDescription("Iteration-synced aggregated performance timeline using DLC records [" + wRecords.Count + "] domains.");
crawlTimeline.GetReportAndSave(imbWEMManager.index.experimentEntry.sessionCrawlerFolder, imbWEMManager.authorNotation, "timeline_performance_".add(imbWEMManager.index.experimentEntry.Crawler));
}
//String atl = "timeline_performance".add(tRecord.instance.name, "_").add("xml", ".");
var domainPerfList = tRecord.lastDomainIterationTable.GetList();
var benchmark = new reportPlugIn_benchmarkResults(); //records.GetOrCreate(imbWEMManager.index.experimentEntry.TestID);
tRecord.performance.SetTestIDAndSignature(tRecord.instance, imbWEMManager.index.experimentEntry.state, tRecord);
tRecord.performance.jobTimeInMinutes = tRecord.cpuTaker.GetTimeSpanInMinutes();
record_performances.AddOrUpdate(tRecord.performance);
benchmark.SetTestIDAndSignature(tRecord.instance, imbWEMManager.index.experimentEntry.state, tRecord);
benchmark.CrawlTime = tRecord.cpuTaker.GetTimeSpanInMinutes(); //tRecord.cpuTaker.GetTimeSpan().TotalMinutes; //.timeFinish.Subtract(tRecord.timeStart).TotalMinutes;
benchmark.IP = domainPerfList.Average(x => x.IP);
benchmark.IPnominal = domainPerfList.Average(x => x.IPnominal);
benchmark.IP_collected = domainPerfList.Average(x => x.IP_collected);
benchmark.Lm_collected = domainPerfList.Average(x => x.Lm_collected);
benchmark.Lm_recall = domainPerfList.Average(x => x.Lm_recall);
benchmark.E_PP = domainPerfList.Average(x => x.E_PP);
benchmark.E_TP = domainPerfList.Average(x => x.E_TP);
benchmark.IP_recall = domainPerfList.Average(x => x.IP_recall);
benchmark.Page_recall = domainPerfList.Average(x => x.Page_recall);
benchmark.Term_recall = domainPerfList.Average(x => x.Term_recall);
var resourcesamples = tRecord.measureTaker.GetLastSamples(1000);
var lastsample = tRecord.measureTaker.GetLastTake();
benchmark.DataLoad = lastsample.bytesLoadedTotal / benchmark.CrawlTime;
benchmark.CPU = resourcesamples.Average(x => x.cpuRateOfProcess);
benchmark.RAM = resourcesamples.Average(x => x.physicalMemory);
records.AddOrUpdate(benchmark);
records.Save();
var dt = records.GetDataTable(null, imbWEMManager.index.experimentEntry.CrawlID);
dt.AddExtra("The last benchmark metrics entry [" + benchmark.Crawler + "] inserted on " + DateTime.Now.ToLongDateString() + " / " + DateTime.Now.ToLongTimeString());
dt.SetAdditionalInfoEntry("DLC Threads - TC", __machine.maxThreads);
dt.SetAdditionalInfoEntry("LoadTake - LT", tRecord.instance.settings.limitIterationNewLinks);
dt.SetAdditionalInfoEntry("PageLoads - PL", tRecord.instance.settings.limitTotalPageLoad);
dt.SetAdditionalInfoEntry("Sample size - DC", dlc_c);
dt.SetAdditionalInfoEntry("Session ID", imbWEMManager.index.experimentEntry.SessionID);
dt.GetReportAndSave(crawlerReportFolder, imbWEMManager.authorNotation, "result", true);
benchmark.GetUserManualSaved(crawlerReportFolder.pathFor("crawler\\result.txt"));
// crawlTimeline.saveObjectToXML(homeFolder.pathFor(atl));
// crawlTimeline.saveObjectToXML(reportFolder.pathFor(atl));
// all three modules summary
imbWEMManager.settings.directReportEngine.GetUserManualSaved(crawlerReportFolder["crawler"].pathFor("settings_reportEngine.txt"));
imbWEMManager.settings.crawlerJobEngine.GetUserManualSaved(crawlerReportFolder["crawler"].pathFor("settings_crawlJobEngine.txt"));
imbWEMManager.settings.executionLog.GetUserManualSaved(crawlerReportFolder["crawler"].pathFor("settings_executionLogs.txt"));
tRecord.instance.settings.GetUserManualSaved(crawlerReportFolder["crawler"].pathFor("settings_crawler.txt"));
record_performances.Save();
var perfDT = record_performances.GetDataTable(null, imbWEMManager.index.experimentEntry.CrawlID);
perfDT.AddExtra("The last benchmark metrics entry [" + benchmark.Crawler + "] inserted on " + DateTime.Now.ToLongDateString() + " / " + DateTime.Now.ToLongTimeString());
perfDT.GetReportAndSave(imbWEMManager.index.experimentEntry.sessionReportFolder, imbWEMManager.authorNotation, "crawl_performances", true);
}
