public static diagramModel buildModel(this spiderWeb source, List <spiderPage> selectedPages, diagramModel output = null) { if (output == null) { output = new diagramModel("Web structure", "Representation of crawled section of the web site structure.", diagramDirectionEnum.TB); } if (!imbWEMManager.settings.postReportEngine.reportBuildDoGraphs) { return(output); } foreach (spiderPage sp in selectedPages) { string desc = sp.url + " sc(" + sp.marks.score + ")"; output.AddNode(desc, diagramNodeShapeEnum.circle, "", sp.originHash); } foreach (spiderLink sl in source.webLinks.items.Values) { if (selectedPages.Contains(sl.targetedPage) && selectedPages.Contains(sl.originPage)) { var from = output.GetNodeByHash(sl.originHash); var to = output.GetNodeByHash(sl.targetedPage.originHash); string desc = " i(" + sl.iterationDiscovery + ") sc(" + sl.marks.score + ")"; output.AddLink(from, to, diagramLinkTypeEnum.normal, desc, sl.originHash); } } return(output); }
/// <summary> /// Cration of wRecords /// </summary> /// <param name="__wProfile">The w profile.</param> /// <param name="__parent">The parent.</param> protected void initialization(webSiteProfile __wProfile, crawlerDomainTaskCollection __parent) { try { parent = __parent; wProfile = __wProfile; aRecord = parent.tRecord.aRecord; // crawler = aRecord.sciProject.mainWebCrawler; tRecord = parent.tRecord; // crawlerContext = aRecord.crawledContextGlobalRegister.GetContext(wProfile.domain, tRecord.instance.settings.legacySettings, wProfile, aRecord.testRunStamp); // tGeneralRecord = aRecord.tGeneralRecord; // aRecord.tGeneralRecord.startChildRecord(wProfile, wProfile.url); // wGeneralRecord = aRecord.tGeneralRecord.childRecord; // aRecord.tGeneralRecord.children.GetRecord(crawlerContext, true); evaluator = tRecord.instance.Clone(tRecord.logBuilder); stageControl = tRecord.stageControl; if (stageControl == null) { stageControl = new macroStageControlDeclareKeyPages("Default", "CrawlerDomainTask"); stageControl.stage.stageIterationLimit = evaluator.settings.limitIterations; } //stageControl = parent.aMacro.helpMethodBuildStageControl(aRecord, tRecord); web = new spiderWeb(); evaluator.setStartPage(wProfile.url, web); tRecord.startChildRecord(web, wProfile.url); wRecord = tRecord.childRecord; wRecord.wGeneralRecord = wGeneralRecord; wRecord.frontierDLC = new frontierRankingAlgorithmDLCRecord(wRecord); wRecord.spiderReal = evaluator; wRecord.spiderDLCTask = this; //wRecord.siteProfile = __wProfile; evaluator.plugins.eventDLCInitiated(evaluator as spiderEvaluatorBase, this, wRecord); // TimeLimitForOneItemLoad = parent.TimeLimitForOneItemLoad; } catch (Exception ex) { status = crawlerDomainTaskStatusEnum.aborted; var clog = crawlerErrorLog.CreateAndSave(ex, wRecord, this, crawlerErrorEnum.jobTaskInitiationError); } }
/// <summary> /// Sets the start page and inicializes all rule sets /// </summary> /// <param name="rootUrl">The root URL.</param> /// <param name="web">The web.</param> /// <param name="context">The context.</param> /// <returns></returns> public virtual spiderWeb setStartPage(string rootUrl, spiderWeb web) { // language = context.language; prepareAll(); spiderLink splink = web.setSeedUrl(rootUrl); //context.domain = web.seedLink.domain; //context.url = rootUrl; //context.siteDomain = context.domain; //context.clickDepth = 0; return(web); }
public spiderTask(int __iteration, spiderWeb __web) { iteration = __iteration; web = __web; }