Пример #1
0
        public void TestStopWord5()
        {
            string          randomSentence = "I think The Flintstones are almost forty years.";
            RemoveStopWords RSW_obj        = new RemoveStopWords();

            randomSentence = RSW_obj.RemoveStopwords(randomSentence);

            Assert.AreEqual(randomSentence, "think Flintstones years");
        }
Пример #2
0
        public void TestStopWord3()
        {
            string randomSentence = "I am hungry, Samuel and Jide, you two can " +
                                    "send some money to me :)";
            RemoveStopWords RSW_obj = new RemoveStopWords();

            randomSentence = RSW_obj.RemoveStopwords(randomSentence);

            Assert.AreEqual(randomSentence, "hungry Samuel Jide send money :)");
        }
Пример #3
0
        public void TestStopWord2()
        {
            string randomSentence = "It was a dark and stormy night...along a rough terrain " +
                                    "was a young boy in a cape. (sound effects)";
            RemoveStopWords RSW_obj = new RemoveStopWords();

            randomSentence = RSW_obj.RemoveStopwords(randomSentence);

            Assert.AreEqual(randomSentence, "dark stormy night rough terrain young boy cape (sound effects)");
        }
Пример #4
0
        public void TestStopWord1()
        {
            string randomSentence = "C# is cool, but it's not very funny sometimes when you" +
                                    " encounter an exception.";
            RemoveStopWords RSW_obj = new RemoveStopWords();

            randomSentence = RSW_obj.RemoveStopwords(randomSentence);

            Assert.AreEqual(randomSentence, "C# cool it's funny encounter exception");
        }
Пример #5
0
        public void TestStopWord4()
        {
            string randomSentence = "And I see fire, for-mall-y mountain, I see fire" +
                                    "...burning something something, but computer cry, describe, what" +
                                    " you mean o, I am not down, I am found.";
            RemoveStopWords RSW_obj = new RemoveStopWords();

            randomSentence = RSW_obj.RemoveStopwords(randomSentence);

            Assert.AreEqual(randomSentence, "for-mall-y mountain burning mean o");
        }
Пример #6
0
        // This does the setup of the pipeline (backgroundWorker is another thread)
        //  This setups the word list objects (new objects) and then builds a list
        //  of threads to be executed. Each thread is a step in the pipeline. The
        //  order started doesn't much matter. New steps can be added by
        //  inserting a step {anywhere} within the thread list. Once all threads
        //  are ready to go, all are started. Threads are smart enough to process
        //  when they can and wait when they shouldn't be processing.
        private void backgroundWorker1_DoWork(object sender, DoWorkEventArgs e)
        {
            this._wordList_step1 = new BlockingCollection <string>();
            this._wordList_step2 = new BlockingCollection <string>();
            this._wordList_step3 = new BlockingCollection <string>();
            this._wordList_step4 = new BlockingCollection <string>();
            this._wordList_step5 = new BlockingCollection <string>();

            this._stopList = new BlockingCollection <string>();

            List <Thread> threadList2 = new List <Thread>();

            // Load the stop words. Nothing blocks on start
            IProcessingStep stepStopWord = new LoadStopWords(txtStopListFile.Text, ref _stopList);

#if DEBUG
            stepStopWord.ProcessingStarted   += startInstrumentationTimer;
            stepStopWord.ProcessingCompleted += stopInstrumentionTimer;
#endif
            Thread stopWordThread = new Thread(new ThreadStart(stepStopWord.executeStep));
            stopWordThread.Name = "Load Stop Words";
            threadList2.Add(stopWordThread);

            // Load the input. Nothing blocks on start
            IProcessingStep stepInputFile = new DataInput(txtInputFile.Text, ref _wordList_step1);
#if DEBUG
            stepInputFile.ProcessingStarted   += startInstrumentationTimer;
            stepInputFile.ProcessingCompleted += stopInstrumentionTimer;
#endif
            Thread inputThread = new Thread(new ThreadStart(stepInputFile.executeStep));
            inputThread.Name = "Load Input";
            threadList2.Add(inputThread);

            //Remove punctuation.
            IProcessingStep stepRemovePunctuation = new RemovePunctuation(ref _wordList_step1, ref _wordList_step2);
#if DEBUG
            stepRemovePunctuation.ProcessingStarted   += startInstrumentationTimer;
            stepRemovePunctuation.ProcessingCompleted += stopInstrumentionTimer;
#endif
            Thread removePunctuationThread = new Thread(new ThreadStart(stepRemovePunctuation.executeStep));
            removePunctuationThread.Name = "Remove Punctation";
            threadList2.Add(removePunctuationThread);

            //We should de-dup the list...
            //IProcessingStep stepDeDup = new DeDup(ref _wordList_step2, ref _wordList_step3);
            //Thread deDupThread = new Thread(new ThreadStart(stepDeDup.executeStep));
            //threadList2.Add(deDupThread);

            //Remove stop words - At this point, the stop list must be ready to go (100% loaded). There is logic in the
            // 'RemoveStopWords' Processing Step to assure this.
            IProcessingStep stepRemoveStopWords = new RemoveStopWords(_stopList, ref _wordList_step2, ref _wordList_step4);
#if DEBUG
            stepRemoveStopWords.ProcessingStarted   += startInstrumentationTimer;
            stepRemoveStopWords.ProcessingCompleted += stopInstrumentionTimer;
#endif
            Thread removeStopWords = new Thread(new ThreadStart(stepRemoveStopWords.executeStep));
            removeStopWords.Name = "Remove Stop Words";
            stepRemoveStopWords.SuspendProcessing += delegate(object delSender, SuspendRequestEventArgs evtArg) {
                System.Diagnostics.Debug.WriteLine("Hit Suspend Processing. Waiting...");
                Thread.Sleep(evtArg.SuspendTimeMillaSeconds);
                removeStopWords      = new Thread(new ThreadStart(stepRemoveStopWords.executeStep));
                removeStopWords.Name = "Remove Stop Words re-created thread";
                removeStopWords.Start();
            };
            threadList2.Add(removeStopWords);

            //Stemming algorthum.
            IProcessingStep stepStemming = new ApplyStemming(ref _wordList_step4, ref _wordList_step5);
#if DEBUG
            stepStemming.ProcessingStarted   += startInstrumentationTimer;
            stepStemming.ProcessingCompleted += stopInstrumentionTimer;
#endif
            Thread StemmingThread = new Thread(new ThreadStart(stepStemming.executeStep));
            StemmingThread.Name = "Applying Stemming";
            threadList2.Add(StemmingThread);

            //20 most frequently occurring terms - DeDupping may need to be removed.. huh.
            // Return data via completed event handler? Seems reasonable.
            IProcessingStep stepOutput = new DataOutput_TwentyCommon(ref _wordList_step5);
#if DEBUG
            stepOutput.ProcessingStarted   += startInstrumentationTimer;
            stepOutput.ProcessingCompleted += stopInstrumentionTimer;
#endif
            Thread outputThread = new Thread(new ThreadStart(stepOutput.executeStep));
            outputThread.Name = "Preparing Final Output";
            threadList2.Add(outputThread);

            //Start all the threads.
            foreach (var th in threadList2)
            {
                th.Start();
            }

            //Clean up processsing (as a delegate); show the results and reset the main form.
            stepOutput.ProcessingCompleted += delegate(object sender3, EventArgs evt3)
            {
                //MessageBox.Show("stepOutput Done.");
                TwentyCommonEventArgs evtArgs  = (TwentyCommonEventArgs)evt3;
                ArrayList             _terms   = evtArgs.Terms;
                frmResults            _results = new frmResults(_terms);
                _results.ShowDialog();

                this.changeState(false);
            };
        }