/// <summary>
/// Describes the counts.
/// </summary>
/// <param name="dataset">The dataset.</param>
/// <param name="output">The output.</param>
/// <returns></returns>
public static String DescribeCounts(this IEnumerable <WebSiteDocumentsSet> dataset, ITextRender output = null)
{
long start = 0;
if (output == null)
{
output = new builderForText();
}
else
{
start = output.Length;
}
Int32 wsc = 0;
Int32 tdc = 0;
foreach (var set in dataset)
{
Int32 td = set.CountDocumentsTotal();
wsc += set.Count;
tdc += td;
output.AppendLine(set.name + "\t S[" + set.Count.ToString("D5") + "] D[" + td.ToString("D6") + "]");
}
output.AppendHorizontalLine();
output.AppendLine("Total \t S[" + wsc.ToString("D5") + "] D[" + tdc.ToString("D6") + "]");
output.AppendHorizontalLine();
output.AppendComment("S = document sets, D = documents");
return(output.GetContent(start));
}
public String GetJSON(MetaEntityNamespaceCollection namespaces)
{
builderForText output = new builderForText();
WriteJSON(output, namespaces);
return(output.GetContent().Replace(Environment.NewLine + Environment.NewLine, Environment.NewLine));
}
//public static ITextRender getRenderFor(reportAPI api)
//{
//}
public static ITextRender getRenderFor(reportAPI api)
{
ITextRender output = null;
switch (api)
{
case reportAPI.imbXmlHtml:
output = new builderForHtml();
break;
//case reportAPI.EEPlus:
// output = new builderForTableDocument();
// break;
case reportAPI.textBuilder:
output = new builderForText();
break;
default:
output = new builderForMarkdown();
break;
}
output.settings.api = api;
return(output);
}
public void Report(folderNode folder)
{
builderForText output = new builderForText();
output.AppendLine($"Signature: \t\t\t {Signature}");
output.AppendLine($"XPathRoot: \t\t\t {XPathRoot}");
output.AppendLine($"JunctionSize: \t\t {JunctionSize}");
output.AppendLine($"JunctionSizeFrequency: \t {JunctionSizeFrequency}");
output.AppendLine($"Level: \t {Level}");
output.AppendLine($"Junction type: \t {type}");
output.AppendLine($"Template:"); // \t {JunctionSizeFrequency}");
output.AppendLine($"XSubPath: \t\t {Template.SubXPath}");
output.AppendLine($"Signature: \t\t {Template.Signature}");
output.AppendLine($"Query: \t\t {Template.BuildXPathQuery()}");
output.AppendLine($"Items: \t\t ");
foreach (T item in items)
{
output.AppendLine(item.path);
}
String op = folder.pathFor("JunctionPoint_" + Signature + ".txt", imbSCI.Data.enums.getWritableFileMode.overwrite);
File.WriteAllText(op, output.GetContent());
}
public static String toStringReport(systemStatusData input)
{
builderForText sb = new builderForText();
sb.AppendLine("Starting time: " + input.startedDateTime.ToShortTimeString());
sb.AppendLine("Running time: " +
String.Format("{0}:{1}:{2}", input.timeRunningTimeSpan.Hours,
input.timeRunningTimeSpan.Minutes, input.timeRunningTimeSpan.Seconds));
sb.AppendLine("Memory usage: " + input.memoryUsage);
sb.AppendLine("Threads count: " + input.threads);
sb.AppendLine("Process list: ");
foreach (Process p in input.ps)
{
try
{
sb.AppendLine(p.ProcessName + " [" + p.Id + "] " + p.MainModule.FileName);
}
catch (Exception ex)
{
sb.AppendLine(p.ProcessName + " [" + p.Id + "] not accessible :: " + ex.Message);
}
}
sb.prevTabLevel();
return(sb.ToString());
}
public void Report(folderNode folder)
{
builderForText reporter = new builderForText();
Detectors.Report(reporter);
Candidates.Report(reporter, nameof(Candidates));
AcceptedCandidates.Report(reporter, nameof(AcceptedCandidates));
DeclinedCandidates.Report(reporter, nameof(DeclinedCandidates));
DeclinedByDetectorCandidates.Report(reporter, nameof(DeclinedByDetectorCandidates));
NodeDictionaryGraphStyleSettings style = new NodeDictionaryGraphStyleSettings();
var initialGraph = InitialGraph.BuildDirectedGraph(style);
initialGraph.Save(folder.pathFor("InitialGraph.dgml", imbSCI.Data.enums.getWritableFileMode.overwrite, "Initial content graph of chunk detection"));
var finalGraph = CurrentGraph.BuildDirectedGraph(style);
finalGraph.Save(folder.pathFor("FinalGraph.dgml", imbSCI.Data.enums.getWritableFileMode.overwrite, "Final content graph of chunk detection"));
CurrentGraphStates.Save(folder, "StateGraph");
foreach (var chunk in DetectedChunks)
{
chunk.PublishAnnotation(initialGraph, "#f7941d", style);
}
initialGraph.Save(folder.pathFor("InitialGraphWithAnnotation.dgml", imbSCI.Data.enums.getWritableFileMode.overwrite, "Initial content graph of chunk detection with chunks annoted"));
folder.SaveText(reporter.GetContent(), "report.txt");
}
public static String ReportSave(this reportExpandedData report, folderNode folder, String filename, String fileDescription = "")
{
builderForText reporter = new builderForText();
if (filename.isNullOrEmpty())
{
filename = "expandedData";
}
filename = Path.GetFileNameWithoutExtension(filename);
foreach (reportExpandedDataPair entry in report)
{
if (entry.value.StartsWith("<?xml"))
{
String filepath = folder.pathFor(filename + "_" + entry.key + ".xml", Data.enums.getWritableFileMode.overwrite, "Stored type value from [" + filename + "]");
File.WriteAllText(filepath, entry.value);
}
else
{
reporter.AppendLine($"{entry.key} = {entry.value} \t\t\t //{entry.description}");
}
}
return(reporter.ReportSave(folder, filename, fileDescription));
}
//public Dictionary<HtmlNode, DocumentCluster> GetClusterByDocumentDictionary()
//{
// Dictionary<HtmlNode, DocumentCluster> output = new Dictionary<HtmlNode, DocumentCluster>();
// foreach (DocumentCluster cluster in this.)
// {
// var nodes = cluster.items.Select(x => x);
// foreach (HtmlNode node in nodes)
// {
// context.DeclarationConstruction_ClusterAnalysisContext.ClusterByDocuments.Add(node, cluster);
// }
// if (cluster.ClusterSeed != null)
// {
// context.DeclarationConstruction_ClusterAnalysisContext.ClusterByDocuments.Add(cluster.ClusterSeed, cluster);
// }
// }
//}
public void Publish(Dictionary <HtmlNode, HtmlSourceAndUrl> documentNodeDictionary, folderNode folderWithResults, DocumentSimilarityResult result)
{
folderWithResults.generateReadmeFiles(null);
var items = GetClusters <DocumentCluster>(true);
Dictionary <HtmlNode, string> labelsByDocument = result.GetLabelsByDocument();
if (!name.isNullOrEmpty())
{
folderWithResults = folderWithResults.Add(name, name, "Reports for cluster collection " + name);
}
builderForText reporter = new builderForText();
foreach (DocumentCluster cluster in items)
{
cluster.Publish(labelsByDocument, documentNodeDictionary, folderWithResults, result);
reporter.AppendPair(cluster.name, cluster.items.Count);
reporter.AppendPair("- range", cluster.range.Range);
}
String reportPath = folderWithResults.pathFor("report.txt", imbSCI.Data.enums.getWritableFileMode.overwrite);
String reportContent = reporter.GetContent();
File.WriteAllText(reportPath, reportContent);
}
public static void Publish(this StructureGraphInformationSet info, folderNode folder, String name, aceAuthorNotation notation = null)
{
DataTable dt = info.items.ReportToDataTable <StructureGraphInformation>(true);
dt.SetTitle(name + " records");
dt.GetReportAndSave(folder, notation);
dt = info.changes.ReportToDataTable <StructureGraphInformation>(true);
dt.SetTitle(name + " changes");
dt.GetReportAndSave(folder, notation);
builderForText output = new builderForText();
foreach (StructureGraphInformation item in info.items)
{
item.Report(null, output);
}
output.ReportSave(folder, name + "_records", "Structure graph entries");
output = new builderForText();
foreach (var item in info.changes)
{
item.Report(null, output);
}
output.ReportSave(folder, name + "_changes", "Structure graph changes log");
}
public void Report(builderForText reporter)
{
reporter.AppendLine("Detector names:");
reporter.nextTabLevel();
foreach (String detectorname in DetectorNames)
{
reporter.AppendLine(detectorname);
}
reporter.prevTabLevel();
}
/// <summary>
/// Ubacuje u izvestaj tabelu sa svom dobijenom metrikom
/// </summary>
/// <param name="sb"></param>
/// <returns></returns>
public builderForText makeReport(builderForText sb = null)
{
if (sb == null)
{
sb = new builderForText();
}
string[] fields = new string[] { "groupName", "codename", "serialNumber", "Value" };
return(sb);
}
public void Report(folderNode folder, ITextRender output)
{
if (output == null)
{
output = new builderForText();
}
this.ReportBase(output, true, name);
if (folder != null)
{
output.ReportSave(folder, name, "Node graph structure information");
}
}
public TypeDiagnostics(folderNode outputfolder, Type type)
{
CollectTypes(type);
builderForText debug = new builderForText();
TestTypes(debug);
debug.ReportSave(outputfolder, "typediagnostic_" + type.Name + ".txt", "Results of type diagnostics");
debug = new builderForText();
TestTypesForBinarySerialization(debug);
debug.ReportSave(outputfolder, "typediagnostic_" + type.Name + "_binary.txt", "Results of type diagnostics");
}
public MetaTableDescription ConstructDescription(SourceTable sourceTable, TableExtractionTask task, ITextRender logger)
{
//if (sourceTable[0, 0].Value.isNullOrEmpty())
//{
// sourceTable[0, 0].Value = "ID";
//}
MetaTableDescription metaDescription = null;
builderForText reporter = task.score.CurrentEntry().reporter;
switch (task.score.executionMode)
{
case ExtractionTaskEngineMode.Training:
SourceTableDescription sourceDesc = sourceContentAnalysis.GetDescription(sourceTable);
metaDescription = new imbSCI.DataExtraction.MetaTables.Descriptors.MetaTableDescription(sourceDesc, imbSCI.DataExtraction.MetaTables.Descriptors.MetaTableFormatType.vertical);
// task.tableDescription = metaDescription;
//task.score.CurrentEntry().metaTableDescription = metaDescription;
break;
case ExtractionTaskEngineMode.Validation:
default:
case ExtractionTaskEngineMode.Application:
if (task.tableDescription == null)
{
throw new Exception("Task [" + task.name + "] has no table description set.");
}
break;
}
if (folder != null)
{
String sp = folder.pathFor("UMTC_Construct_" + task.name + "_" + task.score.executionMode.toString() + ".txt");
File.WriteAllText(sp, reporter.GetContent());
}
return(metaDescription);
}
public void Report(builderForText reporter, String title)
{
if (!this.Any())
{
return;
}
reporter.AppendLine(title);
reporter.nextTabLevel();
for (int i = 0; i < this.Count; i++)
{
this[i].Report(reporter, i);
}
reporter.prevTabLevel();
}
public void Publish(Dictionary <HtmlNode, HtmlSourceAndUrl> documentNodeDictionary, folderNode folderWithResults, List <HtmlNode> reportOn = null)
{
if (reportOn == null)
{
reportOn = GetDocuments();
}
folderWithResults.generateReadmeFiles(null);
builderForText reporter = new builderForText();
foreach (var item in items)
{
if (item.IsRelatedTo(reportOn))
{
reporter.AppendLine("Pair [" + items.IndexOf(item) + "]");
item.Publish(documentNodeDictionary, folderWithResults, reporter);
}
}
Report(StructureSimilarityRange, reporter, "Structure similarity", "SS");
Report(ContentSimilarityRange, reporter, "Content similarity", "CS");
String reporterPath = folderWithResults.pathFor("report.txt");
File.WriteAllText(reporterPath, reporter.GetContent());
DataSet output = PublishDataSet(documentNodeDictionary, reportOn);
HeatMapRender hmRender = new HeatMapRender();
foreach (DataTable table in output.Tables)
{
HeatMapModel heatMap = new HeatMapModel(table);
var heatMapSVG = hmRender.Render(heatMap, folderWithResults.pathFor(table.TableName + ".svg", imbSCI.Data.enums.getWritableFileMode.overwrite, "Headmap render for " + table.TableName));
heatMapSVG.SaveJPEG(folderWithResults.pathFor(table.TableName + ".jpg", imbSCI.Data.enums.getWritableFileMode.overwrite, "Headmap render for " + table.TableName));
}
output.GetReportAndSave(folderWithResults, null, "");
}
public void Publish(Dictionary <HtmlNode, String> labelsByDocument, Dictionary <HtmlNode, HtmlSourceAndUrl> documentNodeDictionary, folderNode folderWithResults, DocumentSimilarityResult result)
{
var cluster = this;
folderNode cFolder = folderWithResults.Add(cluster.name, cluster.name, "Directory for cluster " + cluster.name);
result.Publish(documentNodeDictionary, cFolder, cluster.items);
builderForText reporter = new builderForText();
reporter.AppendHeading("Name: " + cluster.name);
reporter.AppendPair("Items", cluster.items.Count);
if (cluster.ClusterSeed != null)
{
reporter.AppendPair("Seed", labelsByDocument[cluster.ClusterSeed]);
}
foreach (var pair in cluster.range.GetDictionary())
{
reporter.AppendPair(pair.Key, pair.Value.ToString("F3"));
}
foreach (var item in cluster.items)
{
if (item != cluster.ClusterSeed)
{
if (cluster.scoreDictionary.ContainsKey(item))
{
String label = labelsByDocument[item];
Double score = cluster.scoreDictionary[item];
HtmlSourceAndUrl source = documentNodeDictionary[item];
reporter.AppendLine("-----------------------------------");
reporter.AppendLine(label + " => " + score.ToString("F3"));
reporter.AppendLine("Filepath: " + source.filepath);
reporter.AppendLine("Url: " + source.url);
}
}
}
String reportPath = cFolder.pathFor("report.txt", imbSCI.Data.enums.getWritableFileMode.overwrite);
String reportContent = reporter.GetContent();
File.WriteAllText(reportPath, reportContent);
}
public void Publish(folderNode folder, Boolean exportXML = true, Boolean exportDescribe = true, Boolean exportDatatable = true)
{
if (exportXML)
{
String xml = objectSerialization.ObjectToXML(this);
String x_path = folder.pathFor(name + "_report.xml", imbSCI.Data.enums.getWritableFileMode.overwrite, "Serialized dtructural report of a dataset");
File.WriteAllText(x_path, xml);
}
if (exportDescribe)
{
builderForText builderForText = new builderForText();
String t_path = folder.pathFor(name + "_report.txt", imbSCI.Data.enums.getWritableFileMode.overwrite, "Summary of structural report of a dataset");
Describe(builderForText);
File.WriteAllText(t_path, builderForText.GetContent());
}
if (exportDatatable)
{
DataTableTypeExtended <DatasetStructureReport> dt_reports = new DataTableTypeExtended <DatasetStructureReport>(name, "Dataset structure stats");
dt_reports.AddRow(this);
foreach (var ch in Children)
{
dt_reports.AddRow(ch);
}
dt_reports.GetReportAndSave(folder, null, name + "_report");
}
}
/// <summary>
/// Prepares the context.
/// </summary>
/// <param name="context">The context.</param>
/// <param name="log">The log.</param>
/// <returns></returns>
public static DocumentSelectResult PrepareContext(this OperationContext context, DocumentRankingMethod ranking, folderNode folder, ILogBuilder log)
{
DocumentSelectResult selectContext = new DocumentSelectResult();
selectContext.stemmingContext = context.stemmContext;
selectContext.spaceModel = context.spaceModel;
selectContext.folder = folder;
if (ranking != null)
{
selectContext.name = ranking.model.GetSignature();
selectContext.query = ranking.query;
builderForText builder = new builderForText();
ranking.Describe(builder);
builder.AppendLine("Selected features [" + selectContext.selectedFeatures.description + "].");
selectContext.description = builder.GetContent().Replace(Environment.NewLine, "");
}
selectContext.selectedFeatures = context.SelectedFeatures;
foreach (KeyValuePair <string, WebSiteDocuments> pair in context.webSiteByDomain)
{
selectContext.domainNameToGraph.Add(pair.Key, pair.Value?.extensions?.graph);
foreach (WebSiteDocument doc in pair.Value.documents)
{
DocumentSelectResultEntry entry = new DocumentSelectResultEntry();
TextDocument text = null;
string err = "";
//if (context.textDocuments.ContainsKey(doc.AssignedID))
//{
// text = context.textDocuments[doc.AssignedID];
//}
//else
//{
// err += "Failed to find text document for [" + doc.AssignedID + "]";
//}
SpaceDocumentModel spaceDocument = context.spaceModel.documents.FirstOrDefault(x => x.name == doc.AssignedID);
if (spaceDocument == null)
{
err += "Failed to find space model document for [" + doc.AssignedID + "]";
}
string dn = pair.Value.domain;
entry.SetEntry(dn, doc, spaceDocument, text);
if (!entry.HasTextOrSpaceModel)
{
log.log(err);
}
selectContext.items.Add(entry);
//entry.SetEntry( context.context.webDocumentByAssignedID[pair.Key], webDocIDToDomain[aID], webDocumentRegistry[aID], spaceDocumentRegistry[aID], textDocumentRegistry[aID]);
}
}
// PREPARATION OF MODEL
if (ranking != null)
{
ranking.model.Prepare(selectContext, log);
}
return(selectContext);
}