public void StartSession(string __CrawlID, indexPerformanceEntry __indexID, string __SessionID, ICrawlJobContext __state)
{
CrawlID = __CrawlID;
SessionID = __SessionID;
state = __state;
var tmpFolder = new folderNode("reportOutput", "reporting module", "");
sessionReportFolder = tmpFolder.createDirectory(SessionID, "", imbWEMManager.settings.directReportEngine.doAutoRenameSessionFolder); // Directory.CreateDirectory(path);
TestID = CrawlID + "-" + SessionID;
ReportPath = sessionReportFolder.path;
sessionCrawlerFolder = sessionReportFolder.createDirectory(__CrawlID, "Report folder for Crawl [" + __CrawlID + "] - part of session: " + SessionID);
indexSubFolder = imbWEMManager.index.folder; //.createDirectory(__indexID.IndexRepository, "Index folder fo sub index", false);
TFIDF_ConstructFolder = imbWEMManager.index.folder; //.createDirectory(SessionID, "TFIDF cache files for this session", false);
FileInfo master_file = GetTFIDF_Master_File();
if (globalTFIDFCompiled == null)
{
globalTFIDFCompiled = new weightTableCompiled(master_file.FullName, true, SessionID);
globalTFIDFCompiled.ReadOnlyMode = true;
}
SampleRandomOrder = imbWEMManager.settings.crawlerJobEngine.doRandomizeSampleOrder;
//SampleSource = state.sampleTags.add(state.sampleFile, ";");
SampleListHash = randomizeSample();
}
public static String GetCrawlFolderName(ISpiderEvaluatorBase spider, ICrawlJobContext state, String templateString)
{
stringTemplate template = new stringTemplate(templateString);
PropertyCollection data = GetData(state, spider);
return(template.applyToContent(data));
}
public static PropertyCollection GetData(ICrawlJobContext state, ISpiderEvaluatorBase crawler)
{
PropertyCollection data = new PropertyCollection();
data[nameComposerFields.crawlerClassName] = crawler.GetType().Name;
data[nameComposerFields.crawlerTitleName] = crawler.name;
data[nameComposerFields.crawlerFileFriendlyName] = crawler.name.getCleanFilepath().Replace("-", "");
data[nameComposerFields.variablePLmax] = crawler.settings.limitTotalPageLoad;
data[nameComposerFields.variableLT] = crawler.settings.limitIterationNewLinks;
//data[nameComposerFields.variableTCmax] = state.crawlerJobEngineSettings.TC_max;
data[nameComposerFields.sampleSize] = state.sampleList.Count();
//data[nameComposerFields.sampleFileSource] = state.sampleFile;
//data[nameComposerFields.sampleName] = state.sampleTags;
return(data);
}
/// <summary>
/// Gets the session.
/// </summary>
/// <param name="sessionId">The session identifier.</param>
/// <param name="crawlId">The crawl identifier.</param>
/// <returns></returns>
public indexPerformanceEntry StartSession(string crawlId, ICrawlJobContext state = null)
{
indexSessionEntry = indexSessionRecords.GetOrCreate(DateTime.Now.ToShortDateString() + "-" + DateTime.Now.ToShortTimeString());
indexSessionEntry.SessionID = experimentManager.SessionID;
indexSessionEntry.CrawlID = crawlId;
indexSessionEntry.IndexRepository = current_indexID;
indexSessionEntry.Start = DateTime.Now;
experimentEntry = experimentManager.StartSession(crawlId, indexSessionEntry, state);
plugins = new indexPlugInCollection(experimentEntry);
plugins.IsEnabled = true;
domainIndexTable.deploySession();
//imbWEMManager.index.experimentManager.globalTFIDFSet.GetAggregateDataTable().saveObjectToXML(folder.pathFor("TFIFD_aggregate"));
if (imbWEMManager.settings.indexEngine.doIndexPublishAndBackupOnOpenSession)
{
Publish(imbWEMManager.authorNotation, experimentEntry.indexSubFolder);
}
return(indexSessionEntry);
}
public static void SetTestIDAndSignature(this IReportBenchmark target, ISpiderEvaluatorBase evaluator, ICrawlJobContext state, modelSpiderTestRecord tRecord)
{
target.TestSignature = evaluator.name + "|DC:" + state.sampleList.Count + "|PL:" + evaluator.settings.limitTotalPageLoad + "|LT:" + evaluator.settings.limitIterationNewLinks + "|IID:" + imbWEMManager.index.current_indexID + "|SID:" + imbWEMManager.index.indexSessionEntry.SessionID.add(evaluator.settings.SignatureSufix);
target.Crawler = tRecord.instance.name;
if (evaluator != null)
{
target.TestID = md5.GetMd5Hash(objectSerialization.ObjectToXML(imbWEMManager.settings)) + "-" + evaluator.crawlerHash + "-" + md5.GetMd5Hash(target.TestSignature).toWidthMaximum(3, "");
}
else if (state != null)
{
target.TestID = state.setupHash_global + "-" + state.setupHash_crawler;
}
else
{
target.TestID = md5.GetMd5Hash(target.GetHashCode().ToString());
}
}
public experimentSessionEntry StartSession(string CrawlID, indexPerformanceEntry indexID, ICrawlJobContext state =null)
{
var experiment = GetOrCreate(GetRecordID(CrawlID));
experiment.StartSession(CrawlID, indexID, SessionID,state);
CurrentSession = experiment;
UpdateRecord(experiment);
return experiment;
}
public void WriteAboutJob(ICrawlJobContext state, aceAdvancedConsoleWorkspace workspace, IAceAdvancedConsole console)
{
log(":: " + state.job.name + " ::");
AppendLine("Test ID: " + imbWEMManager.index.experimentEntry.TestID);
AppendLine("Test signature: " + imbWEMManager.index.experimentEntry.TestSignature);
AppendLine("Test description: " + state.job.description);
AppendLine("Test report folder: " + folder.path);
AppendHorizontalLine();
AppendLine("Sample size: " + state.sampleList.Count());
// AppendLine("Sample subsets: " + state.sampleTags);
AppendLine("Sample block: " + state.aRecord.sampleBlockOrdinalNumber);
AppendLine("Sample limit: " + state.aRecord.sampleTakeLimit);
//AppendLine("Sample file name: " + state.sampleFile);
AppendLine("Sample order hash: " + imbWEMManager.index.experimentEntry.SampleListHash);
AppendLine("Index auto-preloaded: " + imbWEMManager.index.doAutoLoad);
//AppendLine("Lexicon preloaded: " +semanticLexiconManager.lexiconCache.isLexiconPreloaded);
AppendHorizontalLine();
AppendLine("Run stamp: " + state.aRecord.testRunStamp);
string hash = md5.GetMd5Hash(objectSerialization.ObjectToXML(imbWEMManager.settings));
state.setupHash_global = hash;
AppendLine("Settings [imbAnalyticEngine] hash: " + hash);
AppendLine(" ^-- to make sure multiple tests were running under the same settings compare the hash");
if (console.scriptRunning != null)
{
AppendHorizontalLine();
AppendLine("Job started by ACE script: " + console.scriptRunning.info.Name);
AppendLine("=== Content of the script == start ======");
AppendLine(console.scriptRunning.getContent());
AppendLine("=== Content of the script == end ======");
}
AppendHorizontalLine();
if (imbWEMManager.commandArgs.Any())
{
AppendLine("Application was started with commandline arguments: ");
int a = 1;
foreach (string arg in imbWEMManager.commandArgs)
{
AppendLine(a.ToString("D3") + " [" + arg + "]");
}
AppendHorizontalLine();
}
else
{
AppendLine("Application was started without commandline arguments ");
AppendHorizontalLine();
}
var process = Process.GetCurrentProcess();
process.Refresh();
AppendLine("Process name: " + process.ProcessName);
AppendLine("Process ID: " + process.Id);
AppendHorizontalLine();
}
