public Searcher(bool stem)
{
stemmed = stem;
stemming = new StemmingSequence();
parser = new Parser(StartDialog.stopWordsPath, false);
ranker = new Ranker(StartDialog.indexPath, stemmed, @"C:\Users\welta\Desktop\New folder\glove.840B.300d\glove.840B.300d.vec" /*@"glove.6B.100dc.vec"*/);
}
/**
* unused function for testing puposes
*/
private void RunSingleThread()
{
var watch = System.Diagnostics.Stopwatch.StartNew();
FileReader fr = new FileReader(this.path_to_corpus_show.Text);
Parser pr = new Parser(this.path_to_stopfile_show.Text, true);
StemmingSequence stemmer = new StemmingSequence();
Indexer ind = new Indexer(this.path_to_index_show.Text);
while (fr.hasNext())
{
Document[] docAr = fr.GetNextDocuments();
for (int i = 0; i < docAr.Length; i++)
{
Token[] parseResult = pr.processDoc(docAr[i]);
if (this.doStemming.Checked)
{
parseResult = stemmer.StemTokens(parseResult);
}
ind.ProcessBatch(parseResult);
Console.WriteLine(docAr[i].id);
}
}
watch.Stop();
Console.WriteLine("Total time: " + watch.ElapsedMilliseconds);
}
/**
* function used to create calculation threads
* executes all IO intencive tasks such as parsing and stemming
* mid pipeline thread
*/
private void CalculationTask(String stopWordsPath, ConcurrentQueue <Document[]> docQ, ConcurrentQueue <Token[]> tokenQ, Boolean doStemming)
{
Parser parser = new Parser(stopWordsPath, true);
StemmingSequence stemmer = new StemmingSequence();
while (!docQ.IsEmpty || !StartDialog.readFinished)
{
if (docQ.Count > 1000)
{
Thread.Sleep(500);
}
Document[] docAr;
StartDialog.qLok.WaitOne();
bool ok = docQ.TryDequeue(out docAr);
StartDialog.qLok.ReleaseMutex();
if (!ok)
{
continue;
}
for (int i = 0; i < docAr.Length; i++)
{
Token[] parseResult = parser.processDoc(docAr[i]);
if (doStemming)
{
parseResult = stemmer.StemTokens(parseResult);
}
tokenQ.Enqueue(parseResult);
Console.WriteLine("Doc done :" + docAr[i].id);
}
}
Console.WriteLine("Processing Complete");
StartDialog.calculationsFinished = true;
}
public Searcher(bool stem)
{
stemmed = stem;
stemming = new StemmingSequence();
parser = new Parser(StartDialog.stopWordsPath, false);
ranker = new Ranker(StartDialog.indexPath, stemmed, @"glove.6B.100dc.vec");
}
