/// <summary>
/// Appends its data points into new or existing property collection
/// </summary>
/// <param name="data">Property collection to add data into</param>
/// <returns>Updated or newly created property collection</returns>
public PropertyCollection AppendDataFields(PropertyCollection data = null)
{
PropertyCollectionExtended dataExtended = new PropertyCollectionExtended();
dataExtended.Add("seed", seedLink.link.url, "Start url", "URL the spider started from");
dataExtended.Add("links", webActiveLinks.Count(), "Links", "Number of links discovered during the spider operation");
dataExtended.Add("pages", webPages.items.Count(), "Pages", "Number of pages discovered during the spider operation");
dataExtended.Add("pages_result", webPages.items.Count(), "Page set", "Pages accepted for further analysis");
dataExtended.Add("flags", flags, "Flags", "Flags about the spider operation");
return(dataExtended);
}
/// <summary>
/// Gets time series of changes
/// </summary>
/// <param name="instanceID">The instance identifier.</param>
/// <param name="process">The process.</param>
/// <returns></returns>
public DataTable getHistory(string instanceID, string process)
{
PropertyCollectionExtended shema = new PropertyCollectionExtended();
shema.Add(templateResultColumn.res_iteration, 0, "Iteration", "Time coordinate");
shema.Add(templateResultColumn.res_score, 0, "Score", "Total score sum");
shema.Add(templateResultColumn.res_change, 0, "Change", "Diference between current value and last iteration");
shema.Add(templateResultColumn.res_stability, 0, "Stability", "On score unchanged adds +1 per each iteration. If change occoured -1 is applied. Value is alyways above 0");
//PropertyEntryDictionary columns = shema.buildColumnDictionary(PropertyEntryColumn.entry_name | PropertyEntryColumn.entry_value )
KeyValuePair <int, int> last = new KeyValuePair <int, int>(0, 0);
List <PropertyCollection> rows = new List <PropertyCollection>();
DataTable dt = shema.buildDataTableVertical("Score history", "Data for [" + instanceID + "] during execution of [" + process + "]");
int lastChange = 0;
int lastStability = 0;
foreach (KeyValuePair <int, int> it in history)
{
PropertyCollection row = new PropertyCollection();
row.Add(templateResultColumn.res_iteration, it.Key);
row.Add(templateResultColumn.res_score, it.Value);
int change = it.Value - last.Value;
int stability = lastStability;
row.Add(templateResultColumn.res_change, change);
if (it.Value > 0)
{
if (change == lastChange)
{
stability++;
}
else
{
stability--;
}
}
row.Add(templateResultColumn.res_stability, stability);
lastChange = change;
lastStability = stability;
last = it;
rows.Add(row);
}
dt.addTableExtendedRows(rows);
return(dt);
}
/// <summary>
/// Gets entries for checked items
/// </summary>
/// <param name="targetValue">if set to <c>true</c> [target value].</param>
/// <returns></returns>
public PropertyCollectionExtended getCheckedItems(Boolean targetValue = true)
{
PropertyCollectionExtended output = new PropertyCollectionExtended();
foreach (KeyValuePair <object, PropertyEntry> pe in items.entries)
{
if (pe.Value.getProperBoolean(false, PropertyEntryColumn.entry_value) == targetValue)
{
output.Add(pe.Value);
}
}
return(output);
}
public PropertyCollectionExtended GetPCE()
{
PropertyCollectionExtended output = new PropertyCollectionExtended();
output.Add(nameof(DocumentSetsUniqueForA), DocumentSetsUniqueForA.Count, "Unique for A", "Number of document sets that are unique for dataset A");
output.Add(nameof(DocumentSetsUniqueForB), DocumentSetsUniqueForB.Count, "Unique for B", "Number of document sets that are unique for dataset B");
output.Add(nameof(DocumentSetsInCommonByName), DocumentSetsInCommonByName.Count, "In common", "Number of document sets that in common for datasets");
output.Add(nameof(TermsUniqueForA), TermsUniqueForA.Count, "Terms unique for A", "Number of terms that are unique for dataset A");
output.Add(nameof(TermsUniqueForB), TermsUniqueForB.Count, "Terms unique for B", "Number of terms that are unique for dataset B");
output.Add(nameof(TermsInCommon), TermsInCommon.Count, "In common", "Number of terms that in common for datasets");
return(output);
}
/// <summary>
/// Builds the parent score collection.
/// </summary>
/// <param name="node">The node.</param>
/// <returns></returns>
public static PropertyCollectionExtended buildParentScoreCollection(this linknodeElement node)
{
PropertyCollectionExtended pce = new PropertyCollectionExtended();
linknodeElement head = node;
do
{
if (head == null)
{
break;
}
pce.Add(head.name, head.score, head.name, head.path + " [" + head.level.ToString() + "]");
if (head.parent != null)
{
head = head.parent;
}
} while (head.parent != null);
return(pce);
}
/// <summary>
/// Appends its data points into new or existing property collection
/// </summary>
/// <param name="data">Property collection to add data into</param>
/// <returns>Updated or newly created property collection</returns>
public override PropertyCollectionExtended AppendDataFields(PropertyCollection data = null)
{
PropertyCollectionExtended dataExtended = new PropertyCollectionExtended();
dataExtended.Add("name", name, "Name", "Spider algorithm name");
dataExtended.Add("description", description, "Description", "Short description");
dataExtended.Add("linkrules", linkActiveRules.Count(), "Link rules", "Number of evaluation rules for links");
dataExtended.Add("pagerules", pageScoreRules.Count(), "Page rules", "Number of evaluation rules for pages");
dataExtended.Add("controlrules", controlRules.Count(), "Control rules", "Execution flow control rules");
dataExtended.Add("controlpagerules", controlPageRules.Count(), "Control page rules", "Number of control page rules");
dataExtended.Add("controllinkrules", controlLinkRules.Count(), "Control link rules", "Number of control link rules");
dataExtended.Add("dotokenization", settings.flags.HasFlag(spiderEvaluatorExecutionFlags.doTokenization), "Tokenization", "If content tokenization is performed for each page loaded");
dataExtended.Add("iterationlimit", settings.limitIterations, "Iteration limit", "Maximum number of iterations allowed");
dataExtended.Add("iterationnewlinks", settings.limitIterationNewLinks, "New links limit", "Maximum number of new links allowed per iteration");
dataExtended.Add("iterationtotallinks", settings.limitTotalLinks, "Total links limit", "Maximum number of links allowed for consideration");
dataExtended.Add("pageloadlimit", settings.limitTotalPageLoad, "Total pages load limit", "Maximum number of links allowed for consideration");
dataExtended.Add("language_native", language.languageNativeName, "Language native name", "Native name of the language used by the spider");
dataExtended.Add("language_english", language.languageEnglishName, "Language english name", "English name of the language used by the spider");
dataExtended.Add("language_iso", language.iso2code, "Language code", "Language ISO 2-letter code");
return(dataExtended);
}