public void TestStopWord5()
{
string randomSentence = "I think The Flintstones are almost forty years.";
RemoveStopWords RSW_obj = new RemoveStopWords();
randomSentence = RSW_obj.RemoveStopwords(randomSentence);
Assert.AreEqual(randomSentence, "think Flintstones years");
}
public void TestStopWord3()
{
string randomSentence = "I am hungry, Samuel and Jide, you two can " +
"send some money to me :)";
RemoveStopWords RSW_obj = new RemoveStopWords();
randomSentence = RSW_obj.RemoveStopwords(randomSentence);
Assert.AreEqual(randomSentence, "hungry Samuel Jide send money :)");
}
public void TestStopWord2()
{
string randomSentence = "It was a dark and stormy night...along a rough terrain " +
"was a young boy in a cape. (sound effects)";
RemoveStopWords RSW_obj = new RemoveStopWords();
randomSentence = RSW_obj.RemoveStopwords(randomSentence);
Assert.AreEqual(randomSentence, "dark stormy night rough terrain young boy cape (sound effects)");
}
public void TestStopWord1()
{
string randomSentence = "C# is cool, but it's not very funny sometimes when you" +
" encounter an exception.";
RemoveStopWords RSW_obj = new RemoveStopWords();
randomSentence = RSW_obj.RemoveStopwords(randomSentence);
Assert.AreEqual(randomSentence, "C# cool it's funny encounter exception");
}
public void TestStopWord4()
{
string randomSentence = "And I see fire, for-mall-y mountain, I see fire" +
"...burning something something, but computer cry, describe, what" +
" you mean o, I am not down, I am found.";
RemoveStopWords RSW_obj = new RemoveStopWords();
randomSentence = RSW_obj.RemoveStopwords(randomSentence);
Assert.AreEqual(randomSentence, "for-mall-y mountain burning mean o");
}
// This does the setup of the pipeline (backgroundWorker is another thread)
// This setups the word list objects (new objects) and then builds a list
// of threads to be executed. Each thread is a step in the pipeline. The
// order started doesn't much matter. New steps can be added by
// inserting a step {anywhere} within the thread list. Once all threads
// are ready to go, all are started. Threads are smart enough to process
// when they can and wait when they shouldn't be processing.
private void backgroundWorker1_DoWork(object sender, DoWorkEventArgs e)
{
this._wordList_step1 = new BlockingCollection <string>();
this._wordList_step2 = new BlockingCollection <string>();
this._wordList_step3 = new BlockingCollection <string>();
this._wordList_step4 = new BlockingCollection <string>();
this._wordList_step5 = new BlockingCollection <string>();
this._stopList = new BlockingCollection <string>();
List <Thread> threadList2 = new List <Thread>();
// Load the stop words. Nothing blocks on start
IProcessingStep stepStopWord = new LoadStopWords(txtStopListFile.Text, ref _stopList);
#if DEBUG
stepStopWord.ProcessingStarted += startInstrumentationTimer;
stepStopWord.ProcessingCompleted += stopInstrumentionTimer;
#endif
Thread stopWordThread = new Thread(new ThreadStart(stepStopWord.executeStep));
stopWordThread.Name = "Load Stop Words";
threadList2.Add(stopWordThread);
// Load the input. Nothing blocks on start
IProcessingStep stepInputFile = new DataInput(txtInputFile.Text, ref _wordList_step1);
#if DEBUG
stepInputFile.ProcessingStarted += startInstrumentationTimer;
stepInputFile.ProcessingCompleted += stopInstrumentionTimer;
#endif
Thread inputThread = new Thread(new ThreadStart(stepInputFile.executeStep));
inputThread.Name = "Load Input";
threadList2.Add(inputThread);
//Remove punctuation.
IProcessingStep stepRemovePunctuation = new RemovePunctuation(ref _wordList_step1, ref _wordList_step2);
#if DEBUG
stepRemovePunctuation.ProcessingStarted += startInstrumentationTimer;
stepRemovePunctuation.ProcessingCompleted += stopInstrumentionTimer;
#endif
Thread removePunctuationThread = new Thread(new ThreadStart(stepRemovePunctuation.executeStep));
removePunctuationThread.Name = "Remove Punctation";
threadList2.Add(removePunctuationThread);
//We should de-dup the list...
//IProcessingStep stepDeDup = new DeDup(ref _wordList_step2, ref _wordList_step3);
//Thread deDupThread = new Thread(new ThreadStart(stepDeDup.executeStep));
//threadList2.Add(deDupThread);
//Remove stop words - At this point, the stop list must be ready to go (100% loaded). There is logic in the
// 'RemoveStopWords' Processing Step to assure this.
IProcessingStep stepRemoveStopWords = new RemoveStopWords(_stopList, ref _wordList_step2, ref _wordList_step4);
#if DEBUG
stepRemoveStopWords.ProcessingStarted += startInstrumentationTimer;
stepRemoveStopWords.ProcessingCompleted += stopInstrumentionTimer;
#endif
Thread removeStopWords = new Thread(new ThreadStart(stepRemoveStopWords.executeStep));
removeStopWords.Name = "Remove Stop Words";
stepRemoveStopWords.SuspendProcessing += delegate(object delSender, SuspendRequestEventArgs evtArg) {
System.Diagnostics.Debug.WriteLine("Hit Suspend Processing. Waiting...");
Thread.Sleep(evtArg.SuspendTimeMillaSeconds);
removeStopWords = new Thread(new ThreadStart(stepRemoveStopWords.executeStep));
removeStopWords.Name = "Remove Stop Words re-created thread";
removeStopWords.Start();
};
threadList2.Add(removeStopWords);
//Stemming algorthum.
IProcessingStep stepStemming = new ApplyStemming(ref _wordList_step4, ref _wordList_step5);
#if DEBUG
stepStemming.ProcessingStarted += startInstrumentationTimer;
stepStemming.ProcessingCompleted += stopInstrumentionTimer;
#endif
Thread StemmingThread = new Thread(new ThreadStart(stepStemming.executeStep));
StemmingThread.Name = "Applying Stemming";
threadList2.Add(StemmingThread);
//20 most frequently occurring terms - DeDupping may need to be removed.. huh.
// Return data via completed event handler? Seems reasonable.
IProcessingStep stepOutput = new DataOutput_TwentyCommon(ref _wordList_step5);
#if DEBUG
stepOutput.ProcessingStarted += startInstrumentationTimer;
stepOutput.ProcessingCompleted += stopInstrumentionTimer;
#endif
Thread outputThread = new Thread(new ThreadStart(stepOutput.executeStep));
outputThread.Name = "Preparing Final Output";
threadList2.Add(outputThread);
//Start all the threads.
foreach (var th in threadList2)
{
th.Start();
}
//Clean up processsing (as a delegate); show the results and reset the main form.
stepOutput.ProcessingCompleted += delegate(object sender3, EventArgs evt3)
{
//MessageBox.Show("stepOutput Done.");
TwentyCommonEventArgs evtArgs = (TwentyCommonEventArgs)evt3;
ArrayList _terms = evtArgs.Terms;
frmResults _results = new frmResults(_terms);
_results.ShowDialog();
this.changeState(false);
};
}
