public void StartSession(string __CrawlID, indexPerformanceEntry __indexID, string __SessionID, ICrawlJobContext __state) { CrawlID = __CrawlID; SessionID = __SessionID; state = __state; var tmpFolder = new folderNode("reportOutput", "reporting module", ""); sessionReportFolder = tmpFolder.createDirectory(SessionID, "", imbWEMManager.settings.directReportEngine.doAutoRenameSessionFolder); // Directory.CreateDirectory(path); TestID = CrawlID + "-" + SessionID; ReportPath = sessionReportFolder.path; sessionCrawlerFolder = sessionReportFolder.createDirectory(__CrawlID, "Report folder for Crawl [" + __CrawlID + "] - part of session: " + SessionID); indexSubFolder = imbWEMManager.index.folder; //.createDirectory(__indexID.IndexRepository, "Index folder fo sub index", false); TFIDF_ConstructFolder = imbWEMManager.index.folder; //.createDirectory(SessionID, "TFIDF cache files for this session", false); FileInfo master_file = GetTFIDF_Master_File(); if (globalTFIDFCompiled == null) { globalTFIDFCompiled = new weightTableCompiled(master_file.FullName, true, SessionID); globalTFIDFCompiled.ReadOnlyMode = true; } SampleRandomOrder = imbWEMManager.settings.crawlerJobEngine.doRandomizeSampleOrder; //SampleSource = state.sampleTags.add(state.sampleFile, ";"); SampleListHash = randomizeSample(); }
public static String GetCrawlFolderName(ISpiderEvaluatorBase spider, ICrawlJobContext state, String templateString) { stringTemplate template = new stringTemplate(templateString); PropertyCollection data = GetData(state, spider); return(template.applyToContent(data)); }
public static PropertyCollection GetData(ICrawlJobContext state, ISpiderEvaluatorBase crawler) { PropertyCollection data = new PropertyCollection(); data[nameComposerFields.crawlerClassName] = crawler.GetType().Name; data[nameComposerFields.crawlerTitleName] = crawler.name; data[nameComposerFields.crawlerFileFriendlyName] = crawler.name.getCleanFilepath().Replace("-", ""); data[nameComposerFields.variablePLmax] = crawler.settings.limitTotalPageLoad; data[nameComposerFields.variableLT] = crawler.settings.limitIterationNewLinks; //data[nameComposerFields.variableTCmax] = state.crawlerJobEngineSettings.TC_max; data[nameComposerFields.sampleSize] = state.sampleList.Count(); //data[nameComposerFields.sampleFileSource] = state.sampleFile; //data[nameComposerFields.sampleName] = state.sampleTags; return(data); }
/// <summary> /// Gets the session. /// </summary> /// <param name="sessionId">The session identifier.</param> /// <param name="crawlId">The crawl identifier.</param> /// <returns></returns> public indexPerformanceEntry StartSession(string crawlId, ICrawlJobContext state = null) { indexSessionEntry = indexSessionRecords.GetOrCreate(DateTime.Now.ToShortDateString() + "-" + DateTime.Now.ToShortTimeString()); indexSessionEntry.SessionID = experimentManager.SessionID; indexSessionEntry.CrawlID = crawlId; indexSessionEntry.IndexRepository = current_indexID; indexSessionEntry.Start = DateTime.Now; experimentEntry = experimentManager.StartSession(crawlId, indexSessionEntry, state); plugins = new indexPlugInCollection(experimentEntry); plugins.IsEnabled = true; domainIndexTable.deploySession(); //imbWEMManager.index.experimentManager.globalTFIDFSet.GetAggregateDataTable().saveObjectToXML(folder.pathFor("TFIFD_aggregate")); if (imbWEMManager.settings.indexEngine.doIndexPublishAndBackupOnOpenSession) { Publish(imbWEMManager.authorNotation, experimentEntry.indexSubFolder); } return(indexSessionEntry); }
public static void SetTestIDAndSignature(this IReportBenchmark target, ISpiderEvaluatorBase evaluator, ICrawlJobContext state, modelSpiderTestRecord tRecord) { target.TestSignature = evaluator.name + "|DC:" + state.sampleList.Count + "|PL:" + evaluator.settings.limitTotalPageLoad + "|LT:" + evaluator.settings.limitIterationNewLinks + "|IID:" + imbWEMManager.index.current_indexID + "|SID:" + imbWEMManager.index.indexSessionEntry.SessionID.add(evaluator.settings.SignatureSufix); target.Crawler = tRecord.instance.name; if (evaluator != null) { target.TestID = md5.GetMd5Hash(objectSerialization.ObjectToXML(imbWEMManager.settings)) + "-" + evaluator.crawlerHash + "-" + md5.GetMd5Hash(target.TestSignature).toWidthMaximum(3, ""); } else if (state != null) { target.TestID = state.setupHash_global + "-" + state.setupHash_crawler; } else { target.TestID = md5.GetMd5Hash(target.GetHashCode().ToString()); } }
public experimentSessionEntry StartSession(string CrawlID, indexPerformanceEntry indexID, ICrawlJobContext state =null) { var experiment = GetOrCreate(GetRecordID(CrawlID)); experiment.StartSession(CrawlID, indexID, SessionID,state); CurrentSession = experiment; UpdateRecord(experiment); return experiment; }
public void WriteAboutJob(ICrawlJobContext state, aceAdvancedConsoleWorkspace workspace, IAceAdvancedConsole console) { log(":: " + state.job.name + " ::"); AppendLine("Test ID: " + imbWEMManager.index.experimentEntry.TestID); AppendLine("Test signature: " + imbWEMManager.index.experimentEntry.TestSignature); AppendLine("Test description: " + state.job.description); AppendLine("Test report folder: " + folder.path); AppendHorizontalLine(); AppendLine("Sample size: " + state.sampleList.Count()); // AppendLine("Sample subsets: " + state.sampleTags); AppendLine("Sample block: " + state.aRecord.sampleBlockOrdinalNumber); AppendLine("Sample limit: " + state.aRecord.sampleTakeLimit); //AppendLine("Sample file name: " + state.sampleFile); AppendLine("Sample order hash: " + imbWEMManager.index.experimentEntry.SampleListHash); AppendLine("Index auto-preloaded: " + imbWEMManager.index.doAutoLoad); //AppendLine("Lexicon preloaded: " +semanticLexiconManager.lexiconCache.isLexiconPreloaded); AppendHorizontalLine(); AppendLine("Run stamp: " + state.aRecord.testRunStamp); string hash = md5.GetMd5Hash(objectSerialization.ObjectToXML(imbWEMManager.settings)); state.setupHash_global = hash; AppendLine("Settings [imbAnalyticEngine] hash: " + hash); AppendLine(" ^-- to make sure multiple tests were running under the same settings compare the hash"); if (console.scriptRunning != null) { AppendHorizontalLine(); AppendLine("Job started by ACE script: " + console.scriptRunning.info.Name); AppendLine("=== Content of the script == start ======"); AppendLine(console.scriptRunning.getContent()); AppendLine("=== Content of the script == end ======"); } AppendHorizontalLine(); if (imbWEMManager.commandArgs.Any()) { AppendLine("Application was started with commandline arguments: "); int a = 1; foreach (string arg in imbWEMManager.commandArgs) { AppendLine(a.ToString("D3") + " [" + arg + "]"); } AppendHorizontalLine(); } else { AppendLine("Application was started without commandline arguments "); AppendHorizontalLine(); } var process = Process.GetCurrentProcess(); process.Refresh(); AppendLine("Process name: " + process.ProcessName); AppendLine("Process ID: " + process.Id); AppendHorizontalLine(); }