public static PropertyCollection GetData(crawlerDomainTaskMachineSettings crawlerJobEngineSettings, ISpiderEvaluatorBase crawler)
{
PropertyCollection data = new PropertyCollection();
data[nameComposerFields.crawlerClassName] = crawler.GetType().Name;
data[nameComposerFields.crawlerTitleName] = crawler.name;
data[nameComposerFields.crawlerFileFriendlyName] = crawler.name.getCleanFilepath().Replace("-", "");
data[nameComposerFields.variablePLmax] = crawler.settings.limitTotalPageLoad;
data[nameComposerFields.variableLT] = crawler.settings.limitIterationNewLinks;
data[nameComposerFields.variableTCmax] = crawlerJobEngineSettings.TC_max;
return(data);
}
public static PropertyCollection GetData(ICrawlJobContext state, ISpiderEvaluatorBase crawler)
{
PropertyCollection data = new PropertyCollection();
data[nameComposerFields.crawlerClassName] = crawler.GetType().Name;
data[nameComposerFields.crawlerTitleName] = crawler.name;
data[nameComposerFields.crawlerFileFriendlyName] = crawler.name.getCleanFilepath().Replace("-", "");
data[nameComposerFields.variablePLmax] = crawler.settings.limitTotalPageLoad;
data[nameComposerFields.variableLT] = crawler.settings.limitIterationNewLinks;
//data[nameComposerFields.variableTCmax] = state.crawlerJobEngineSettings.TC_max;
data[nameComposerFields.sampleSize] = state.sampleList.Count();
//data[nameComposerFields.sampleFileSource] = state.sampleFile;
//data[nameComposerFields.sampleName] = state.sampleTags;
return(data);
}
public static void Describe(this ISpiderEvaluatorBase evaluator, ITextRender output)
{
output.AppendHeading("Crawler [" + evaluator.name + "]");
output.AppendPair("Class name", evaluator.GetType().Name, true, ": ");
output.AppendPair("Description", evaluator.description, true, ": ");
if (evaluator is spiderModularEvaluatorBase)
{
spiderModularEvaluatorBase evaluator_spiderModularEvaluatorBase = (spiderModularEvaluatorBase)evaluator;
foreach (var md in evaluator_spiderModularEvaluatorBase.modules)
{
md.DescribeModule(output);
}
}
if (evaluator is spiderEvaluatorSimpleBase)
{
spiderEvaluatorSimpleBase evaluator_spiderEvaluatorSimpleBase = (spiderEvaluatorSimpleBase)evaluator;
evaluator_spiderEvaluatorSimpleBase.linkActiveRules.ToList().ForEach(x => x.DescribeRule(output));
}
output.AppendHorizontalLine();
output.open("div", "General configuration", "Crawler configuration properties declared in common settings class");
evaluator.settings.GetUserManual(output, "", true, true);
output.close();
}