static Dictionary <string, int> loadAndProcessData(string file)
{
// load training files from directory
string learn = new FileReader(file).getAllTxt();
// tokenize words
List <string> dataTokenized = new Token().treebankWordTokenizer(learn);
//List<string> dataTokenized = new Token().whiteSpaceTokenizer(learn);
// removing numbers & removing single letter character & based on stoplist
StopWords sw = new StopWords();
dataTokenized = sw.removeNumbers(dataTokenized);
dataTokenized = sw.removeSingleLetterWords(dataTokenized);
dataTokenized = sw.removeWordsBasedOnStopWordsList(dataTokenized);
// get frequency per word
var data = new Frequnce().getWordFrequence(dataTokenized);
return(data);
}
