private void IncVocabulary(FrequencyDocument doc)
{
foreach (var token in doc.Tokens.Keys)
{
if (!Vocabulary.Contains(token))
{
Vocabulary.Add(token);
}
}
}
private List <FrequencyDocument> TransformClassificatedDocuments(string folder, string classification)
{
string[] files = Directory.GetFiles(folder);
var docs = new List <FrequencyDocument>();
string text;
foreach (var file in files)
{
text = File.ReadAllText(file);
FrequencyDocument doc = TransformText(text);
doc.Classification = classification;
IncVocabulary(doc);
docs.Add(doc);
}
return(docs);
}
private FrequencyDocument NormalizeDocument(FrequencyDocument doc)
{
double length = 0;
foreach (var token in doc.Tokens)
{
length += token.Value * token.Value;
}
length = Math.Sqrt(length);
string[] keys = doc.Tokens.Keys.ToArray();
for (var i = 0; i < keys.Length; i++)
{
doc.Tokens[keys[i]] /= length;
}
return(doc);
}
public FrequencyDocument TransformText(string text)
{
string[] tokens = Tokenize(text);
var document = new FrequencyDocument
{
Classification = "-",
Tokens = new Dictionary <string, double>()
};
foreach (var token in tokens)
{
if (token == String.Empty)
{
continue;
}
string stripped = Regex.Replace(token, "[^a-zA-Z0-9]", "");
if (!StopWords.stopWordsList.Contains(stripped.ToLower()))
{
var english = new EnglishWord(stripped);
string stemedToken = english.Stem;
if (document.Tokens.Keys.Contains(token))
{
document.Tokens[token]++;
}
else
{
document.Tokens.Add(token, 1);
}
}
}
return(document);
}
