Beispiel #1
0
        public void ParseFile()
        {
            Console.WriteLine("Reading Training File...");
            var text            = Preprocessing.ReadFile(File);
            var corpus          = Preprocessing.DumbSentenceSegmentor(text);
            var tokenizedCorpus = Preprocessing.DumbTokenizer(corpus.ToString());

            tokenizedCorpus = new List <string>(tokenizedCorpus.RemoveAll(string.IsNullOrEmpty));
        }