public crawlerDomainTaskMachine(modelSpiderTestRecord __tRecord, List <webSiteProfile> sample, directAnalyticReporter __reporter, folderNode __folder)
{
reporter = __reporter;
folder = __folder;
tRecord = __tRecord;
logger = new builderForLog();
aceLog.consoleControl.setAsOutput(logger, tRecord.name);
SetWebLoaderControler(__folder);
items = new crawlerDomainTaskCollection(tRecord, sample, this);
cpuTaker = new performanceCpu(tRecord.name);
dataLoadTaker = new performanceDataLoad(tRecord.name);
measureTaker = new performanceResources(tRecord.name, this);
cpuTaker.take();
dataLoadTaker.take();
measureTaker.take();
tRecord.cpuTaker = cpuTaker;
tRecord.dataLoadTaker = dataLoadTaker;
tRecord.measureTaker = measureTaker;
plugins = new enginePlugInCollection(this);
reportPlugins = new reportingPlugInCollection(reporter, this);
}
/// <summary>
/// Deploy spider record into index datatables
/// </summary>
/// <param name="tRecord">The t record.</param>
public void deploy(modelSpiderTestRecord tRecord)
{
wRecordDeployedAutosaveDisable = true;
pageIndexTable.ReadOnlyMode = false;
domainIndexTable.ReadOnlyMode = false;
int i = 0;
int c = 0;
int ic = tRecord.children.Count;
int ib = tRecord.children.Count / 10;
aceLog.log("Deploying crawl data into index (" + ic.ToString() + ")");
foreach (KeyValuePair <spiderWeb, modelSpiderSiteRecord> pair in tRecord.children)
{
i++;
c++;
if (!pair.Value.indexDeployedMe)
{
deployWRecord(pair.Value);
}
if (i >= ib)
{
aceLog.log("DLC processed: " + c.GetRatio(ic).ToString("P2"));
i = 0;
}
}
wRecordDeployedAutosaveDisable = false;
Save();
pageIndexTable.ReadOnlyMode = imbWEMManager.settings.indexEngine.doRunIndexInReadOnlyMode;
domainIndexTable.ReadOnlyMode = imbWEMManager.settings.indexEngine.doRunIndexInReadOnlyMode;
}
private void doPerform(crawlerDomainTaskMachine _machine, modelSpiderTestRecord tRecord, reportPlugIn_workload plugin, experimentSessionEntry entry)
{
switch (pluginState)
{
case workloadPluginState.active:
break;
case workloadPluginState.cooldown:
if (cooldownIndex > 0)
{
comment = comment.add($"Coolingdown [{cooldownIndex}]");
cooldownIndex = cooldownIndex - (thisSampleID - lastSampleID);
}
else
{
comment = comment.add($"Cooldown finished");
pluginState = workloadPluginState.active;
}
break;
case workloadPluginState.disabled:
break;
case workloadPluginState.none:
break;
case workloadPluginState.preparing:
if (tCPU.SampleState.HasFlag(measureTrendSampleState.macroMean))
{
pluginState = workloadPluginState.active;
plugin.loger.log("Workload plugin ready");
}
break;
case workloadPluginState.sampleTail:
break;
case workloadPluginState.terminating:
terminate(_machine);
break;
case workloadPluginState.wormingUp:
if (wormingUpIndex > 0)
{
comment = comment.add($"WormingUp [{wormingUpIndex}]");
wormingUpIndex = wormingUpIndex - (thisSampleID - lastSampleID);
}
else
{
comment = comment.add($"WormingUp finished");
pluginState = workloadPluginState.active;
}
break;
}
}
public override void reportCrawlFinished(directAnalyticReporter reporter, modelSpiderTestRecord tRecord)
{
foreach (ISpiderModuleBase module in modules)
{
module.reportCrawlFinished(reporter, tRecord);
}
}
public string doCreateEntry(crawlerDomainTaskMachine _machine, modelSpiderTestRecord tRecord, reportPlugIn_workload plugin, experimentSessionEntry entry)
{
string recID = "";
if (_machine != null)
{
thisSampleID = _machine.dataLoadTaker.CountTakes();
}
if (lastSampleID == -1)
{
lastSampleID = 0;
}
if (thisSampleID != lastSampleID)
{
RecordID++;
// <------------------ RECORD CREATION
recID = GetEntryID(RecordID, measureGroup);
lastEntry = plugin.records.GetOrCreate(recID);
lastEntry.RecordID = RecordID;
lastEntry.pluginState = pluginState.ToString();
switch (pluginState)
{
case workloadPluginState.active:
lastEntry.measureGroup = measureGroup;
break;
default:
lastEntry.measureGroup = -1;
break;
}
lastEntry.SetTestIDAndSignature(tRecord.instance, entry.state, tRecord);
lastEntry.terminationWarning = terminationWarning;
lastEntry.availableMemory = mMemory;
lastEntry.ContentPages = tContentPages.MicroMean;
lastEntry.cpuRateOfMachine = tCPUm.MicroMean;
lastEntry.cpuRateOfProcess = tCPU.MicroMean;
lastEntry.physicalMemory = tMemory.MicroMean;
lastEntry.CrawlerIterations = tIterations.MicroMean;
lastEntry.DataLoad = tDataLoad.MicroMean;
lastEntry.dlcDone = _machine.taskDone;
lastEntry.dlcRunning = _machine.taskRunning;
lastEntry.dlcWaiting = _machine.taskWaiting;
lastEntry.dlcMaximum = _machine.maxThreads;
plugin.records.AddOrUpdate(lastEntry);
}
lastSampleID = thisSampleID;
return(recID);
}
public void deployReport(modelSpiderTestRecord tRecord)
{
className = tRecord.instance.GetType().Name;
name = tRecord.instance.name;
description = tRecord.instance.description;
reportFolder = tRecord.reporter.folder.name;
settings = tRecord.instance.settings;
slot = tRecord.aRecord.GetChildRecords().IndexOf(tRecord);
}
public void doCheckCriteria(crawlerDomainTaskMachine _machine, modelSpiderTestRecord tRecord, reportPlugIn_workload plugin, experimentSessionEntry entry)
{
if (!tCPU.SampleState.HasFlag(measureTrendSampleState.macroMean))
{
pluginState = workloadPluginState.preparing;
}
if (plugin.plugin_settings.term_DLCFinished > 0) // <----- da li je aktiviran ovaj uslov
{
if (_machine.taskDone >= plugin.plugin_settings.term_DLCFinished)
{
terminate(_machine);
}
}
if (pluginState == workloadPluginState.active)
{
if (_machine.taskDone >= DLCDoneForNext)
{
doStartNextGroup(_machine, tRecord, plugin, entry);
stateUpdate(_machine, tRecord, plugin, entry);
plugin.records.Save(getWritableFileMode.overwrite);
}
if (isSampleTail)
{
if (plugin.plugin_settings.term_JLCinTail)
{
terminate(_machine);
}
else
{
pluginState = workloadPluginState.sampleTail;
}
}
if (isMemoryLimit)
{
if (plugin.plugin_settings.term_availableMemory > 0)
{
terminate(_machine);
}
else
{
pluginState = workloadPluginState.cooldown;
cooldownIndex = plugin.plugin_settings.warmingUpTicks;
}
}
}
}
public void WriteAboutCrawlerRun(modelSpiderTestRecord tRecord, crawlerDomainTaskMachine cDTM)
{
AppendHorizontalLine();
AppendLine("Crawler name: " + cDTM.tRecord.instance.name);
AppendLine("Crawler description: " + cDTM.tRecord.instance.description);
AppendLine("Session report folder: " + cDTM.folder.path);
AppendLine("Crawler report folder: " + imbWEMManager.index.experimentEntry.sessionCrawlerFolder.path);
AppendLine("--------------------------------------------------------------------------- ");
string settings = objectSerialization.ObjectToXML(tRecord.instance.settings);
string hash = md5.GetMd5Hash(settings);
var fileinfo = settings.saveStringToFile(
imbWEMManager.index.experimentEntry.sessionCrawlerFolder.pathFor(imbWEMManager.index.experimentEntry.SessionID.getFilename() + "_settings.xml").getWritableFile().FullName);
//state.setupHash_crawler = hash;
AppendLine("Crawler settings hash: " + hash);
AppendLine("Crawler complete hash: " + tRecord.instance.crawlerHash);
// AppendLine("Crawler settings file: " + fileinfo.Name);
AppendLine("--------------- Crawler configuration overview ---------------------------- ");
AppendLine("PL_max | PL - Page Load max per domain | : "+ tRecord.instance.settings.limitTotalPageLoad);
AppendLine("LT_t | LT - Load Take per iteration | : "+ tRecord.instance.settings.limitIterationNewLinks);
AppendLine("I_max | I_max - Iteration limit per DLC | : "+ tRecord.instance.settings.limitIterations);
AppendLine("PS_c * | - Page Select count (not used) | : " + tRecord.instance.settings.primaryPageSetSize);
AppendLine("--------------------------------------------------------------------------- ");
AppendLine("-- * parameteers not used in by this version ------------------------------ ");
AppendLine();
AppendLine("--------------------------------------------------------------------------- ");
var duration = DateTime.Now.Subtract(cDTM.startTime);
AppendLine("Start time: " + cDTM.startTime.ToShortTimeString());
AppendLine("Finish time: " + DateTime.Now.ToShortTimeString());
AppendLine("Duration (minutes): " + duration.TotalMinutes);
AppendLine("^-- includes post-crawl reporting and index database update");
AppendLine("Failed domains: " + cDTM.webLoaderControler.GetFailedDomains().Count());
AppendLine("^-- includes domains that were accesable but no links discovered ");
AppendLine("Failed URLs: " + cDTM.webLoaderControler.GetFailedURLsCount());
AppendHorizontalLine();
// cDTM.tRecord.instance.Describe(this);
}
/// <summary>
/// Here the webSiteProfile is used to create crawlDomainTask and wRecords
/// </summary>
/// <param name="__tRecord">The t record.</param>
/// <param name="__sample">The sample.</param>
/// <param name="__parent">The parent.</param>
public crawlerDomainTaskCollection(modelSpiderTestRecord __tRecord, List <webSiteProfile> __sample, crawlerDomainTaskMachine __parent)
{
sampleSize = __sample.Count();
tRecord = __tRecord;
parent = __parent;
foreach (webSiteProfile profile in __sample)
{
//var crawlerContext = tRecord.aRecord.crawledContextGlobalRegister.GetContext(profile.domain, tRecord.aRecord.sciProject.mainWebCrawler.mainSettings, profile, tRecord.aRecord.testRunStamp);
var task = new crawlerDomainTask(profile, this);
items.Enqueue(task);
}
}
public void stateUpdate(crawlerDomainTaskMachine _machine, modelSpiderTestRecord tRecord, reportPlugIn_workload plugin, experimentSessionEntry entry)
{
comment = "";
doReadData(_machine);
// <------------------ DATA COLLECTION
// <------------------ STATE DECISION
doCheckFacts(_machine, tRecord, plugin, entry);
doPerform(_machine, tRecord, plugin, entry);
doCreateEntry(_machine, tRecord, plugin, entry);
doCheckCriteria(_machine, tRecord, plugin, entry);
// <------------------ PRINTING OUT ----------------------------
plugin.loger.AppendHorizontalLine();
if (pluginState != workloadPluginState.disabled)
{
string st_in = pluginState.ToString();
if (pluginState == workloadPluginState.active)
{
st_in = "_" + st_in + "_";
}
plugin.loger.AppendLine(string.Format(STATUSLINE_ONE, st_in, lastEntry.RecordID.ToString("D3"), lastEntry.measureGroup, lastEntry.dlcMaximum, lastEntry.dlcRunning, lastEntry.dlcWaiting).toWidthExact(Console.BufferWidth - 11, "="));
}
plugin.loger.AppendLine(tMemory.GetTrendInline() + " | " + tCPU.GetTrendInline() + " | " + tCPUm.GetTrendInline());
plugin.loger.AppendLine(tDataLoad.GetTrendInline() + " | " + tContentPages.GetTrendInline() + " | " + tIterations.GetTrendInline());
//plugin.loger.AppendLine("--- Info: " );
if (pluginState != workloadPluginState.disabled)
{
plugin.loger.AppendLine(string.Format(STATUSLINE_TWO, mMemory.ToString("P2"),
lastEntry.terminationWarning.ToString("D3"), lastEntry.dlcDone, DLCDoneForNext, thisSampleID, lastSampleID).toWidthExact(Console.BufferWidth - 11, "="));
// plugin.loger.AppendLine(String.Format(STATUSLINE_TWO, mMemory.ToString("P2"), g).toWidthExact(Console.BufferWidth-11, "="));
}
}
public void eventCrawlJobFinished(crawlerDomainTaskMachine __machine, modelSpiderTestRecord __tRecord) // <---- ovaj se nikad ne poziva
{
foreach (reportPlugIn_base plug in allPlugins)
{
try
{
plug.eventCrawlJobFinished(__machine, __tRecord);
}
catch (Exception ex)
{
aceLog.log("Reporting Plugin [" + plug.name + "]:" + plug.GetType().Name + " at status report execution crashed: " + ex.Message);
crawlerErrorLog cel = new crawlerErrorLog(ex, null, null, crawlerErrorEnum.indexPlugin);
cel.SaveXML();
}
}
}
private void doStartNextGroup(crawlerDomainTaskMachine _machine, modelSpiderTestRecord tRecord, reportPlugIn_workload plugin, experimentSessionEntry entry)
{
doReadData(_machine);
DLCDoneForNext = _machine.taskDone + plugin.plugin_settings.stepUp_DLCCount;
pluginState = workloadPluginState.wormingUp;
string msg = $"Measure group {measureGroup} completed -- DLCs done: {_machine.taskDone}";
plugin.loger.log(msg);
comment = comment.add($"Group {measureGroup} done");
wormingUpIndex = plugin.plugin_settings.warmingUpTicks;
if (wormingUpIndex == 0)
{
pluginState = workloadPluginState.active;
}
measureGroup = measureGroup + 1;
_machine.maxThreads += plugin.plugin_settings.stepUp_step;
}
private void doCheckFacts(crawlerDomainTaskMachine _machine, modelSpiderTestRecord tRecord, reportPlugIn_workload plugin, experimentSessionEntry entry)
{
if (mMemory < plugin.plugin_settings.term_availableMemory)
{
warningUpDate("Available RAM [" + mMemory.ToString("P2") + "] is below the termination limit [" + plugin.plugin_settings.term_availableMemory.ToString("P2") + "]", true, plugin);
if (terminationWarning >= plugin.plugin_settings.term_warningCount)
{
isMemoryLimit = true;
}
else
{
}
}
else if (_machine.taskWaiting == 0)
{
warningUpDate("There is no DLCs waiting [" + _machine.taskWaiting + "] - no way to run DLCs up to TC_max [" + _machine.maxThreads + "]", true, plugin);
if (terminationWarning >= plugin.plugin_settings.term_warningCount)
{
isSampleTail = true;
}
}
else
{
if (terminationWarning > 0)
{
warningUpDate("All termination criteria clean", false, plugin);
}
}
if (_machine.taskRunning > _machine.maxThreads)
{
plugin.loger.log($" Running {_machine.taskRunning} more then TC_max {_machine.maxThreads} - switching to cooldown");
cooldownIndex = plugin.plugin_settings.warmingUpTicks;
pluginState = workloadPluginState.cooldown;
}
}
public override void eventStatusReport(crawlerDomainTaskMachine _machine, modelSpiderTestRecord tRecord)
{
plugin_state.stateUpdate(_machine, tRecord, this, imbWEMManager.index.experimentEntry);
}
//public override void eventDLCFinished<TParent>(TParent __parent, crawlerDomainTask __task, modelSpiderSiteRecord __wRecord) => eventDLCFinished(__parent as directReporterBase, __task, __wRecord);
//public override void eventCrawlJobFinished(analyticJob aJob, crawlerDomainTaskMachine __machine, modelSpiderTestRecord __tRecord) => eventCrawlJobFinished(__machine, __tRecord);
//public override void eventDLCInitiated<TParent>(TParent __parent, crawlerDomainTask __task, modelSpiderSiteRecord __wRecord) => eventDLCInitiated(__parent as directReporterBase, __task, __wRecord);
public override void eventCrawlJobFinished(analyticJob aJob, crawlerDomainTaskMachine __machine, modelSpiderTestRecord __tRecord)
{
}
public override void eventStatusReport(crawlerDomainTaskMachine crawlerDomainTaskMachine, modelSpiderTestRecord tRecord)
{
}
public override void reportCrawlFinished(directAnalyticReporter reporter, modelSpiderTestRecord tRecord)
{
}
public abstract void eventStatusReport(crawlerDomainTaskMachine crawlerDomainTaskMachine, modelSpiderTestRecord tRecord);
/// <summary>
/// Initializes a new instance of the <see cref="crawlerDomainTaskMachine"/> class.
/// </summary>
/// <param name="tRecord">The t record.</param>
/// <param name="sample">The sample.</param>
/// <param name="macro">The macro.</param>
public crawlerDomainTaskMachine(modelSpiderTestRecord tRecord, List <webSiteProfile> sample, analyticMacroBase macro)
{
items = new crawlerDomainTaskCollection(tRecord, sample, macro);
// maxThreads = imbWEMManager.settings.crawlerDomainThreadCountAllowed;
}
public override void eventAtEndOfCrawlJob(crawlerDomainTaskMachine _machine, modelSpiderTestRecord tRecord)
{
}
public override void eventCrawlJobFinished(crawlerDomainTaskMachine _machine, modelSpiderTestRecord tRecord)
{
records.Save();
records.GetDataTable(null, plugin_state.TestID).GetReportAndSave(imbWEMManager.index.experimentEntry.sessionCrawlerFolder, imbWEMManager.authorNotation, "workload_", true);
}
public override void eventCrawlJobFinished(analyticJob aJob, crawlerDomainTaskMachine __machine, modelSpiderTestRecord __tRecord)
{
// imbWEMManager.index.Recheck(loger);
}
/// <summary> /// Just before /// </summary> /// <param name="crawlerDomainTaskMachine">The crawler domain task machine.</param> /// <param name="tRecord">The t record.</param> public abstract void eventAtEndOfCrawlJob(crawlerDomainTaskMachine crawlerDomainTaskMachine, modelSpiderTestRecord tRecord);
public abstract void eventCrawlJobFinished(crawlerDomainTaskMachine __machine, modelSpiderTestRecord __tRecord);
/// <summary>
/// Executes the key page search. /// PREVAZIDJENO
/// </summary>
/// <param name="__evaluator">The evaluator.</param>
/// <param name="rootPageUrl">The root page URL.</param>
/// <param name="crawlerContext">The crawler context.</param>
/// <param name="webSiteReport">The web site report.</param>
/// <param name="siteProfile">The site profile.</param>
/// <returns></returns>
public void executeKeyPageSearch(ISpiderEvaluatorBase __evaluator, string rootPageUrl, modelWebSiteGeneralRecord wGeneralRecord, webSiteProfile siteProfile, modelSpiderTestRecord tRecord, modelSpiderTestGeneralRecord tGeneralRecord)
{
//evaluator = __evaluator;
//// <--------------------------------------------------------------------------------------
//spiderWeb web = new spiderWeb();
//tRecord.startChildRecord(web, rootPageUrl); // .siteRecords.Add(siteProfile);
//wGeneralRecord.recordStart(tGeneralRecord.testRunStamp, rootPageUrl);
//tRecord.childRecord.wGeneralRecord = wGeneralRecord;
//modelSpiderSiteRecord wRecord = tRecord.childRecord;
//wRecord.siteProfile = siteProfile;
//aceLog.consoleControl.setAsOutput(wRecord, "" + siteProfile.domain);
//tRecord.stageControl.prepare();
//crawlerAgentContext crawlerContext = wGeneralRecord.instance;
//evaluator.setStartPage(rootPageUrl,web , crawlerContext);
//spiderTask sTask = evaluator.getSpiderSingleTask(web.seedLink, tRecord.childRecord, 1);
//spiderTaskResult sResult = loader.runSpiderTask(sTask, crawlerContext, tRecord.childRecord);
//spiderStageControl sControl = tRecord.stageControl;
//spiderObjectiveSolutionSet solSet = null;
//sControl.stage.EnterStage(tRecord.childRecord, evaluator);
//do
//{
// //aceLog.consoleControl.setAsOutput(tRecord.childRecord);
// dataUnitSpiderIteration iDataUnit = tRecord.childRecord.timeseries.CreateEntry(null, sTask.iteration);
// evaluator.operation_receiveResult(sResult, wRecord);
// solSet = evaluator.operation_applyLinkRules(wRecord);
// sTask = evaluator.operation_GetLoadTask(wRecord);
// sResult = loader.runSpiderTask(sTask, crawlerContext, wRecord);
// evaluator.operation_detectCrossLinks(wRecord);
// tRecord.log("Spider [" + __evaluator.name + "] for [" + rootPageUrl + "] collected [" + wRecord.children.Count() + "] pages");
// iDataUnit.checkData();
// // <--------------------- Crawler objective control override
//} while (!sControl.stage.CheckStage(wRecord, solSet, sTask));
//wRecord.children.FinishAllStarted();
//wGeneralRecord.children.FinishAllStarted();
//tRecord.log("Spider [" + __evaluator.name + "] for [" + rootPageUrl + "] collected [" + wRecord.children.Count() + "] pages");
//List<spiderPage> pages = evaluator.operation_evaluatePages(tRecord.childRecord);
//wRecord.resultPageSet = pages;
//wGeneralRecord.recordFinish();
//tRecord.finishChildRecord();
//long memBefore = GC.GetTotalMemory(false);
//GC.Collect();
//long memAfter = GC.GetTotalMemory(true);
//long released = memBefore - memAfter;
//aceLog.log("Memory released: " + released.getMByteCountFormated() + "");
//aceLog.consoleControl.removeFromOutput(wRecord); //, "sp:" + tRecord.instance.name);
// <--------------------------------------------------------------------------------------
}
public override void eventAtInitiationOfCrawlJob(crawlerDomainTaskMachine _machine, modelSpiderTestRecord tRecord)
{
plugin_settings = imbWEMManager.settings.supportEngine.plugIn_workload_settings;
plugin_state.statePrepare(plugin_settings);
string ad = plugin_settings.stepUp_start.ToString() + plugin_settings.stepUp_step.ToString();
__recordPath = homeFolder.pathFor("results_" + tRecord.instance.name + ad + ".xml");
records = new objectTable <reportPlugIn_workloadEntry>(recordPath, false, __recordKeyProperty, plugin_state.TestID);
if (plugin_settings.stepUp_enabled)
{
plugin_state.pluginState = workloadPluginState.preparing;
_machine.maxThreads = plugin_settings.stepUp_start;
}
else
{
plugin_state.pluginState = workloadPluginState.disabled;
}
}
public abstract spiderStageControl helpMethodBuildStageControl(analyticJobRecord aRecord, modelSpiderTestRecord spiderRecord);
public void reportCrawler(modelSpiderTestRecord tRecord)
{
folderNode fn = folder[DRFolderEnum.crawler];
string fileprefix = tRecord.instance.name.getCleanFilePath(); //tRecord.name.getCleanFilepath();
if (REPORT_TIMELINE)
{
DataTable timeline = timeSeries.GetAggregatedTable("frontier_stats", dataPointAggregationAspect.overlapMultiTable); //.GetSumTable("timeline_" + fileprefix.Replace(" ", ""));
timeline.GetReportAndSave(folder[DRFolderEnum.crawler], notation, "frontier_stats" + fileprefix);
}
if (REPORT_ITERATION_URLS)
{
tRecord.allUrls = urlsLoaded.GetAllUnique();
tRecord.allDetectedUrls = urlsDetected.GetAllUnique();
saveOutput(tRecord.allDetectedUrls, folder[DRFolderEnum.crawler].pathFor("urls_detected.txt"));
saveOutput(tRecord.allUrls, folder[DRFolderEnum.crawler].pathFor("urls_loaded.txt"));
saveOutput(tRecord.relevantPages, folder[DRFolderEnum.crawler].pathFor("urls_relevant_loaded.txt"));
}
// Int32 iterations = tRecord.instance.settings.limitIterations;
DataTable cpuTable = tRecord.cpuTaker.GetDataTableBase("cpuMetrics").GetReportAndSave(folder[DRFolderEnum.crawler], notation, "cpu_" + fileprefix);
DataTable dataTable = tRecord.dataLoadTaker.GetDataTableBase("dataLoadMetrics").GetReportAndSave(folder[DRFolderEnum.crawler], notation, "dataload_" + fileprefix);
DataTable resourcesTable = tRecord.measureTaker.GetDataTableBase("resourceMetrics").GetReportAndSave(folder[DRFolderEnum.crawler], notation, "resource_" + fileprefix);
if (imbWEMManager.settings.directReportEngine.doPublishPerformance)
{
tRecord.performance.folderName = folder.name;
tRecord.performance.deploy(tRecord);
tRecord.performance.saveObjectToXML(folder[DRFolderEnum.crawler].pathFor("performance.xml"));
DataTable pTable = tRecord.performance.GetDataTable(true).GetReportAndSave(folder, notation, "crawler_performance" + fileprefix);
}
tRecord.lastDomainIterationTable.GetDataTable(null, imbWEMManager.index.experimentEntry.CrawlID).GetReportAndSave(folder, notation, "DLCs_performance_" + fileprefix);
tRecord.reporter = this;
signature.deployReport(tRecord);
//signature.notation = notation;
signature.saveObjectToXML(folder.pathFor("signature.xml"));
folder.generateReadmeFiles(notation);
fileunit tLog = new fileunit(folder[DRFolderEnum.logs].pathFor(fileprefix + ".txt"), false);
tLog.setContent(tRecord.logBuilder.ContentToString(true));
tLog.Save();
tRecord.instance.reportCrawlFinished(this, tRecord);
aceLog.consoleControl.setLogFileWriter();
}
public static void SetTestIDAndSignature(this IReportBenchmark target, ISpiderEvaluatorBase evaluator, ICrawlJobContext state, modelSpiderTestRecord tRecord)
{
target.TestSignature = evaluator.name + "|DC:" + state.sampleList.Count + "|PL:" + evaluator.settings.limitTotalPageLoad + "|LT:" + evaluator.settings.limitIterationNewLinks + "|IID:" + imbWEMManager.index.current_indexID + "|SID:" + imbWEMManager.index.indexSessionEntry.SessionID.add(evaluator.settings.SignatureSufix);
target.Crawler = tRecord.instance.name;
if (evaluator != null)
{
target.TestID = md5.GetMd5Hash(objectSerialization.ObjectToXML(imbWEMManager.settings)) + "-" + evaluator.crawlerHash + "-" + md5.GetMd5Hash(target.TestSignature).toWidthMaximum(3, "");
}
else if (state != null)
{
target.TestID = state.setupHash_global + "-" + state.setupHash_crawler;
}
else
{
target.TestID = md5.GetMd5Hash(target.GetHashCode().ToString());
}
}
public abstract void reportCrawlFinished(directAnalyticReporter reporter, modelSpiderTestRecord tRecord);
