public spiderTargetCollection(modelSpiderSiteRecord __wRecord)
{
wRecord = __wRecord;
string __domain = wRecord.domain;
string __spiderName = wRecord.spider.name;
name = __spiderName + " targets on " + __domain;
description = "Registry of unique absolute URLs discovered on the web site: " + __domain + " by the " + __spiderName + " crawler";
dlTargetLinkTokens = new termDocumentSet(GetHash("links_" + __domain + " " + __spiderName), "URL and anchor text tokens from links discovered on the web site: " + __domain + " by the " + __spiderName + " crawler");
if (wRecord.tRecord.instance.settings.doEnableDLC_TFIDF)
{
dlTargetPageTokens = new termDocumentSet(GetHash("pages_" + __domain + " " + __spiderName), "Content text tokens from loaded pages the web site: " + __domain + " by the " + __spiderName + " crawler");
}
}
/// <summary>
/// Gets the or create tfidf DLC.
/// </summary>
/// <param name="__wRecord">The w record.</param>
/// <param name="loger">The loger.</param>
/// <param name="__useExisting">if set to <c>true</c> [use existing].</param>
/// <param name="__saveToCache">if set to <c>true</c> [save to cache].</param>
/// <param name="evaluator">The evaluator.</param>
/// <returns></returns>
public weightTableCompiled GetOrCreateTFIDF_DLC(modelSpiderSiteRecord __wRecord, builderForLog loger, bool __useExisting, bool __saveToCache, multiLanguageEvaluator evaluator = null)
{
indexDomain idomain = imbWEMManager.index.domainIndexTable.GetOrCreate(__wRecord.domain);
FileInfo TFIDF_DLC_File = GetTFIDF_DLC_File(idomain, getWritableFileMode.existing);
weightTableCompiled TFIDF_DLC = null;
if (TFIDF_DLC_File.Exists && __useExisting)
{
TFIDF_DLC = new weightTableCompiled(TFIDF_DLC_File.FullName, true, idomain.domain + "_DLC_TF_IDF");
loger.log("DLC TF-IDF[" + TFIDF_DLC.Count + "] cache found for: " + idomain.domain);
return(TFIDF_DLC);
}
if (evaluator == null)
{
evaluator = __wRecord.tRecord.evaluator;
}
// <--------------- evaluator selection
if (imbWEMManager.settings.TFIDF.doUseHeuristicDLCTFIDFConstruction)
{
TFIDF_DLC = GetOrCreateTFIDF_DLC_Heuristic(__wRecord, loger, __useExisting, __saveToCache, evaluator);
}
else
{
loger.log("DLC TF-IDF construction for: " + idomain.domain + " initiated.");
termDocumentSet domainSet = new termDocumentSet("DomainTFIDF_source");
var tLoaded = __wRecord.context.targets.GetLoaded();
int tc = tLoaded.Count;
int ti = 0;
int ts = 10;
int c = 0;
int input_c = 0;
int output_c = 0;
double io_r = 0;
foreach (spiderTarget target in tLoaded)
{
ti++;
c++;
double tp = ti.GetRatio(tc);
if (target.IsRelevant)
{
var wordlist = GetTermsForPage(target, idomain, null, evaluator, loger);
input_c += wordlist.Count;
termDocument pageTF = domainSet.AddTable(target.pageHash) as termDocument;
pageTF.expansion = 1;
pageTF.AddTokens(wordlist, loger);
output_c += pageTF.Count();
}
if (c > 10)
{
c = 0;
io_r = output_c.GetRatio(input_c);
aceLog.consoleControl.writeToConsole("Pages processed [" + tp.ToString("P2") + "] Semantic compression rate: " + io_r.ToString("P2"), loger, false, 0);
}
}
loger.log("[" + idomain.domain + "] preprocess finished. DLC TF-IDF terms [" + domainSet.CountAllDocuments() + "]");
TFIDF_DLC = domainSet.AggregateDocument.GetCompiledTable(loger);
TFIDF_DLC.name = "DLC-TFIDF " + idomain.domain;
}
idomain.Lemmas = TFIDF_DLC.Count;
if (__saveToCache)
{
if (TFIDF_DLC.SaveAs(TFIDF_DLC_File.FullName, getWritableFileMode.overwrite))
{
loger.log("[" + idomain.domain + "] DLC TF-IDF compiled table cache saved to: " + TFIDF_DLC_File.FullName);
}
else
{
loger.log("[" + idomain.domain + "] DLC TF-IDF compiled table save failed");
}
}
imbWEMManager.index.domainIndexTable.AddOrUpdate(idomain);
return(TFIDF_DLC);
}
/// <summary>
/// Gets the tfidf master: loads from file or returns any existing instance
/// </summary>
/// <returns></returns>
public weightTableCompiled GetTFIDF_Master(builderForLog loger, bool __useExisting = true, bool __saveToCache = true)
{
bool rebuild = !__useExisting;
FileInfo master_file = GetTFIDF_Master_File();
if (globalTFIDFCompiled == null)
{
globalTFIDFCompiled = new weightTableCompiled(master_file.FullName, __useExisting, SessionID);
globalTFIDFCompiled.ReadOnlyMode = true;
}
if (globalTFIDFCompiled.Count == 0)
{
rebuild = true;
}
else
{
if (loger != null)
{
loger.log("Master table loaded [" + globalTFIDFCompiled.Count + "]");
}
}
if (rebuild)
{
int input_c = 0;
int output_c = 0;
List <weightTableCompiled> allDLC_TFIDFs = GetTFIDF_DLC_AllCached(loger);
if (loger != null)
{
loger.log("Rebuilding Master Table ");
}
termDocumentSet construct = new termDocumentSet(SessionID, "Temporary TF-IDF construct table for session: " + SessionID);
int tc = allDLC_TFIDFs.Count;
double tr = 0;
int c = 0;
foreach (weightTableCompiled dlc in allDLC_TFIDFs)
{
c++;
termDocument td = construct.Add(dlc) as termDocument;
input_c += td.Count();
tr = c.GetRatio(tc);
if (loger != null)
{
aceLog.consoleControl.writeToConsole(tr.ToString("P2") + " ", loger, false, 0);
}
// output_c = construct.AggregateDocument.Count();
}
globalTFIDFCompiled = construct.AggregateDocument.GetCompiledTable(loger);
output_c = construct.AggregateDocument.Count();
tr = input_c.GetRatio(output_c);
if (loger != null)
{
loger.log("Master Table - final semantic compression rate: [" + tr.ToString("P2") + "]");
}
}
if (__saveToCache)
{
if (loger != null)
{
loger.log("Master Table saved to:[" + master_file.FullName + "]"); // Namesemantic compression rate: [" + tr.ToString("P2") + "]");
}
globalTFIDFCompiled.SaveAs(master_file.FullName, getWritableFileMode.overwrite);
}
return(globalTFIDFCompiled);
}
