/// <summary>
/// Sets the web loader controler.
/// </summary>
/// <param name="__folder">The folder.</param>
/// <returns></returns>
public spiderWebLoaderControler SetWebLoaderControler(folderNode __folder)
{
webLoaderControler = new spiderWebLoaderControler();
if (tRecord != null)
{
webLoaderControler.prepare(tRecord.logBuilder, folder);
}
else
{
webLoaderControler.prepare(null, folder);
}
return(webLoaderControler);
}
/// <summary>Runs the current crawl job</summary>
/// <remarks><para>Starts crawl execution</para></remarks>
/// <seealso cref="aceOperationSetExecutorBase"/>
public void aceOperation_runRun()
{
IAceAdvancedConsole console = parent as IAceAdvancedConsole;
// your code
DateTime start = DateTime.Now;
if (context.aRecord == null)
{
output.log("Error: define Job before calling this command.");
return;
}
int Tdl_max = context.crawlerJobEngineSettings.Tdl_max;
int Tll_max = context.crawlerJobEngineSettings.Tll_max;
int TC_max = context.crawlerJobEngineSettings.TC_max;
var spiderEvals = context.aRecord.GetChildRecords();
context.aRecord.initializeSoft(context.sampleList);
int c = 0;
DirectoryInfo di = imbWEMManager.index.experimentManager.CurrentSession.sessionReportFolder;
var notation = appManager.AppInfo;
// ------------------ note creation -------------------
analyticJobNote note = new analyticJobNote(imbWEMManager.index.experimentEntry.sessionCrawlerFolder);
note.WriteAboutJob(context, console.workspace, console);
note.AppendLine("--------------- Crawl Job configuration overview -------------------------- ");
note.AppendLine(" Script var | Article - Description ");
note.AppendLine("--------------------------------------------------------------------------- ");
note.AppendLine(" Tdl_max | Tdl - Time limit per domain - in minutes | : " + Tdl_max);
note.AppendLine(" Tll_max | Tac - Time limit for inactivity - in minutes | : " + Tll_max);
note.AppendLine(" TC_max | TC - Maximum number of JLC threads allowed | : " + TC_max);
note.AppendLine("--------------------------------------------------------------------------- ");
note.AppendHorizontalLine();
note.AppendLine("-- if the test was finished without problem at the last line it will be message [RunJob completed] ---");
note.AppendLine("-- if not: something went wrong - check the logs ---");
note.AppendHorizontalLine();
note.SaveNote();
foreach (modelSpiderTestRecord tRecord in spiderEvals)
{
c++;
spiderWebLoaderControler controler = null;
directAnalyticReporter reporter = new directAnalyticReporter(imbWEMManager.index.experimentEntry.CrawlID, imbWEMManager.index.experimentEntry.sessionCrawlerFolder, notation);
context.pluginStack.InstallTo(imbWEMManager.index.plugins, plugInGroupEnum.index, true);
tRecord.performance = imbWEMManager.index.experimentEntry;
output.log(tRecord.instance.name + " crawl start");
crawlerDomainTaskMachine cDTM = new crawlerDomainTaskMachine(tRecord, context.aRecord.sample, reporter, di)
{
maxThreads = TC_max,
_timeLimitForDLC = Tdl_max,
TimeLimitForTask = Tll_max
};
//state.pluginStack
context.pluginStack.InstallTo(cDTM.plugins, plugInGroupEnum.engine, false);
context.pluginStack.InstallTo(tRecord.instance.plugins, plugInGroupEnum.crawler, false);
context.pluginStack.InstallTo(cDTM.reportPlugins, plugInGroupEnum.report, false);
cDTM.startAutoParallel(true); // ----- execution
output.log(tRecord.instance.name + " crawl finished");
cDTM.webLoaderControler.Save();
controler = cDTM.webLoaderControler;
reporter.reportCrawler(tRecord);
note.WriteAboutCrawlerRun(tRecord, cDTM);
if (console != null)
{
console.scriptRunning.getContent().saveStringToFile(imbWEMManager.index.experimentEntry.sessionCrawlerFolder.pathFor("script.ace"));
}
if (imbWEMManager.settings.directReportEngine.doPublishExperimentSessionTable)
{
imbWEMManager.index.experimentManager.AddOrUpdate(tRecord.performance as experimentSessionEntry);
}
}
imbWEMManager.index.CloseSession(spiderEvals);
output.AppendLine("RunJob done in: " + DateTime.Now.Subtract(start).TotalMinutes.ToString("#0.0##") + " min");
note.AppendLine("[RunJob completed]");
note.SaveNote();
// imbWEMManager.settings.Save(imbWEMManager.index.experimentEntry.sessionCrawlerFolder.pathFor("imbAnalyticEngineSettings.xml"));
var sl = context.sampleList.ToList();
sl.saveContentOnFilePath(note.folder.pathFor("sample.txt"));
}
