/**
* unused function for testing puposes
*/
private void RunSingleThread()
{
var watch = System.Diagnostics.Stopwatch.StartNew();
FileReader fr = new FileReader(this.path_to_corpus_show.Text);
Parser pr = new Parser(this.path_to_stopfile_show.Text, true);
StemmingSequence stemmer = new StemmingSequence();
Indexer ind = new Indexer(this.path_to_index_show.Text);
while (fr.hasNext())
{
Document[] docAr = fr.GetNextDocuments();
for (int i = 0; i < docAr.Length; i++)
{
Token[] parseResult = pr.processDoc(docAr[i]);
if (this.doStemming.Checked)
{
parseResult = stemmer.StemTokens(parseResult);
}
ind.ProcessBatch(parseResult);
Console.WriteLine(docAr[i].id);
}
}
watch.Stop();
Console.WriteLine("Total time: " + watch.ElapsedMilliseconds);
}
/**
* function to create a write thread which writes all posting files, in end of indexing process executes index optimization
* function is IO bound
* pipeline end
*/
private void WriteTask(String indexPath, ConcurrentQueue <Token[]> tokenQ)
{
Stopwatch sw = new Stopwatch();
sw.Start();
Indexer ind = new Indexer(indexPath);
while (!tokenQ.IsEmpty || !StartDialog.calculationsFinished || !StartDialog.readFinished)
{
Token[] tokAr;
bool ok = tokenQ.TryDequeue(out tokAr);
if (!ok)
{
continue;
}
ind.ProcessBatch(tokAr);
}
Console.WriteLine("Write complete!");
this.BeginInvoke(new InvokeDelegate(optimizing));
ind.endSession();
this.BeginInvoke(new InvokeDelegate(updateProgress));
StartDialog.writeFinished = true;
}