/** * unused function for testing puposes */ private void RunSingleThread() { var watch = System.Diagnostics.Stopwatch.StartNew(); FileReader fr = new FileReader(this.path_to_corpus_show.Text); Parser pr = new Parser(this.path_to_stopfile_show.Text, true); StemmingSequence stemmer = new StemmingSequence(); Indexer ind = new Indexer(this.path_to_index_show.Text); while (fr.hasNext()) { Document[] docAr = fr.GetNextDocuments(); for (int i = 0; i < docAr.Length; i++) { Token[] parseResult = pr.processDoc(docAr[i]); if (this.doStemming.Checked) { parseResult = stemmer.StemTokens(parseResult); } ind.ProcessBatch(parseResult); Console.WriteLine(docAr[i].id); } } watch.Stop(); Console.WriteLine("Total time: " + watch.ElapsedMilliseconds); }
/** * function to create a write thread which writes all posting files, in end of indexing process executes index optimization * function is IO bound * pipeline end */ private void WriteTask(String indexPath, ConcurrentQueue <Token[]> tokenQ) { Stopwatch sw = new Stopwatch(); sw.Start(); Indexer ind = new Indexer(indexPath); while (!tokenQ.IsEmpty || !StartDialog.calculationsFinished || !StartDialog.readFinished) { Token[] tokAr; bool ok = tokenQ.TryDequeue(out tokAr); if (!ok) { continue; } ind.ProcessBatch(tokAr); } Console.WriteLine("Write complete!"); this.BeginInvoke(new InvokeDelegate(optimizing)); ind.endSession(); this.BeginInvoke(new InvokeDelegate(updateProgress)); StartDialog.writeFinished = true; }