public bool CheckStage(modelSpiderSiteRecord wRecord, spiderObjectiveSolutionSet oSet, spiderTask task)
{
bool okToLeave = false;
if (task.Count() == 0)
{
wRecord.logBuilder.log("> Spider task [i:" + task.iteration + "] have no tasks defined. Aborting the stage loop.");
okToLeave = true;
return(okToLeave);
}
// <----------------------------- OBJECTIVE SOLUTION SET
okToLeave = operation_executeObjectiveSolutionSet(oSet, wRecord);
if (okToLeave)
{
return(okToLeave);
}
// <----------------------------- SPIDER LIMITS OVERRIDERS ---------------|
if (stageIteration > wRecord.tRecord.instance.settings.limitIterations)
{
wRecord.log("> Spider settings (limit iterations) trigered abort at [" + stageIteration + "] Aborting the stage loop.");
okToLeave = true;
return(okToLeave);
}
// <----------------------------------------------------------------------|
// <----------------------------- SPIDER LIMITS OVERRIDERS ---------------|
if (wRecord.web.webPages.Count() > wRecord.tRecord.instance.settings.limitTotalPageLoad)
{
wRecord.log("> Spider settings (limit pages load) trigered abort at [" + wRecord.web.webPages.Count() + "] Aborting the stage loop.");
okToLeave = true;
return(okToLeave);
}
// <----------------------------------------------------------------------|
if (stageIteration > stageIterationLimit)
{
wRecord.logBuilder.log("> Stage [" + name + "] iteration limit reached [ " + stageIterationLimit + " ] -- aborting [" + objectives.Count + "] objectives and move on");
okToLeave = true;
return(okToLeave);
}
if (stageIteration > GLOBAL_stageIterationLimit)
{
Exception ex = new aceGeneralException("spiderStage [" + name + "] reached the " + nameof(GLOBAL_stageIterationLimit) + "(" + GLOBAL_stageIterationLimit.ToString() + ")");
throw ex;
}
stageIteration++;
return(okToLeave);
}
/// <summary>
/// E2: Applies passive link rules to new Active links
/// </summary>
/// <param name="wRecord">The s record.</param>
public override spiderObjectiveSolutionSet operation_applyLinkRules(modelSpiderSiteRecord wRecord)
{
spiderObjectiveSolutionSet output = new spiderObjectiveSolutionSet();
int c = 0;
foreach (spiderLink sLink in Enumerable.Where(wRecord.web.webActiveLinks, x => !x.flags.HasFlag(spiderLinkFlags.passiveEvaluated)))
{
foreach (spiderEvalRuleForLinkBase rule in linkPassiveRules)
{
sLink.marks.deploy(rule.evaluate(sLink));
}
c++;
sLink.flags |= spiderLinkFlags.passiveEvaluated;
}
if (c > 0)
{
wRecord.log("Passive evaluation of [" + c + "] new links");
}
/// cleaning rule memory
foreach (ruleActiveBase aRule in linkActiveRules)
{
aRule.startIteration(wRecord.iteration, wRecord);
}
/// perceiving current situation
foreach (spiderLink sLink in wRecord.web.webActiveLinks)
{
sLink.linkAge++; // <---------------------------------------------------- adding link age points
foreach (ruleActiveBase aRule in linkActiveRules)
{
aRule.learn(sLink);
}
}
/// apply update on results
foreach (spiderLink sLink in wRecord.web.webActiveLinks)
{
foreach (ruleActiveBase aRule in linkActiveRules)
{
sLink.marks.deploy(aRule.evaluate(sLink));
}
sLink.marks.calculate(wRecord.iteration);
}
// <----------------------------sorts the links
wRecord.web.webActiveLinks.items.Sort((x, y) => y.marks.score.CompareTo(x.marks.score));
foreach (controlObjectiveRuleBase aRule in controlRules)
{
aRule.startIteration(wRecord.iteration, wRecord);
output.listen(aRule.evaluate(wRecord));
}
return(output);
}
protected virtual void operation_doControlAndStats(modelSpiderSiteRecord wRecord)
{
var stats = wRecord.web.webActiveLinks.calculateTotalAndAvgScore();
wRecord.timeseries[wRecord.iteration].avg_score_l = stats.Item2;
// wRecord.timeseries[wRecord.iteration].tc_detected_l = stats.Item1;
// <---------------- Control rules
spiderObjectiveSolutionSet output = new spiderObjectiveSolutionSet();
/// cleaning rule memory
foreach (controlLinkRuleBase aRule in controlLinkRules)
{
aRule.startIteration(wRecord.iteration, wRecord);
}
/// perceiving current situation
foreach (spiderLink sLink in wRecord.web.webActiveLinks)
{
foreach (controlLinkRuleBase aRule in controlLinkRules)
{
aRule.learn(sLink, wRecord);
}
}
/// --------- TRIM BELOW ZERO ------------ ///
if (settings.FRONTIER_PullDecayModes.HasFlag(spiderPullDecayModes.belowZeroScoreRemoval))
{
foreach (spiderLink sLink in wRecord.web.webActiveLinks.ToList())
{
if (sLink.marks.score < 0)
{
wRecord.web.webActiveLinks.Remove(sLink);
wRecord.log("Link [" + sLink.url + "] had score below zero");
}
}
}
/// apply update on results
foreach (spiderLink sLink in wRecord.web.webActiveLinks)
{
foreach (controlLinkRuleBase aRule in controlLinkRules)
{
output.listen(aRule.evaluate(sLink, wRecord));
}
}
int removed = 0;
foreach (var link in output.links)
{ // <--------------------------------------------------- removes any links found at control solution set
wRecord.web.webActiveLinks.Remove(link);
removed++;
}
if (removed > 0)
{
wRecord.log("Control rules removed: " + removed.ToString() + " links from active links collection");
}
//wRecord.logBuilder.log("Link drop-out:" + output.links.Count + ". Now have " + wRecord.web.webActiveLinks.Count() + " links waiting.");
stats = wRecord.web.webActiveLinks.items.calculateTotalAndAvgScore();
wRecord.timeseries[wRecord.iteration].avg_scoreADO_l = stats.Item2;
wRecord.timeseries[wRecord.iteration].tc_scoreADO_l = stats.Item1;
wRecord.timeseries[wRecord.iteration].nw_ruledout_l = output.links.Count;
}
public override spiderObjectiveSolutionSet operation_applyLinkRules(modelSpiderSiteRecord wRecord)
{
spiderModuleData <spiderLink> dataInput = new spiderModuleData <spiderLink>();
dataInput.iteration = wRecord.iteration;
dataInput.active.AddRange(wRecord.web.webActiveLinks);
frontierRankingAlgorithmIterationRecord frontierReportEntry = null;
if (imbWEMManager.settings.directReportEngine.DR_ReportModules)
{
frontierReportEntry = wRecord.frontierDLC.reportStartOfFRA(wRecord.iteration, wRecord, dataInput); // <----------------- reporting on module activity -- START
}
foreach (ISpiderModuleBase module in modules)
{
module.startIteration(wRecord.iteration, wRecord);
}
bool breakExecution = false;
foreach (ISpiderModuleBase module in modules)
{
if (imbWEMManager.settings.directReportEngine.DR_ReportModules)
{
dataInput.moduleDLC = wRecord.frontierDLC.modRecords[module.GetType().Name];
dataInput.moduleDLCRecordTableEntry = dataInput.moduleDLC.StartNewRecord(wRecord.iteration);
}
spiderModuleData <spiderLink> dataOutput = null;
if (!breakExecution)
{
dataOutput = module.evaluate(dataInput, wRecord) as spiderModuleData <spiderLink>;
}
//dataInput.moduleDLC.reportEvaluateAlterRanking(dataOutput.active, wRecord, dataInput.moduleDLCRecordTableEntry, module as spiderModuleBase);
if (imbWEMManager.settings.directReportEngine.DR_ReportModules)
{
dataInput.moduleDLC.AddOrUpdate(dataInput.moduleDLCRecordTableEntry);
dataInput.moduleDLCRecordTableEntry.disposeResources();
}
if (!breakExecution)
{
dataInput = dataOutput.CreateNext();
if (dataInput.active.Count == 1)
{
wRecord.log("Module " + module.name + " returned single link instance -- skipping other modules");
breakExecution = true;
}
}
}
if (imbWEMManager.settings.directReportEngine.DR_ReportModules)
{
frontierReportEntry = wRecord.frontierDLC.reportEndOfFRA(wRecord, frontierReportEntry, dataInput); // <--------------------------------------------- reporting on module activity -- END
}
wRecord.currentModuleData = dataInput;
// <------------------ Objective control rules
spiderObjectiveSolutionSet output = new spiderObjectiveSolutionSet();
foreach (controlObjectiveRuleBase aRule in controlRules)
{
aRule.startIteration(wRecord.iteration, wRecord);
output.listen(aRule.evaluate(wRecord));
}
return(output);
}