public static void RemoveDiacriticsInFile(string path)
{
string originalText = File.OpenText(path).ReadToEnd();
string textWithoutDiacritics = StringRoutines.MyDiacriticsRemover(originalText);
File.WriteAllText($"{TextFile.FileName(path)}_WITHOUT-DIACRITICS{TextFile.FileExtension(path)}", textWithoutDiacritics);
}
public string RemoveBadWords(NgramFile file, int fromFrequency)
{
string name = file.FileName;
string extension = file.FileExtension;
using (var goodWords_sw = new StreamWriter($"{name}_GOOD-WORDS{extension}"))
using (var badWords_sw = new StreamWriter($"{name}_BAD-WORDS{extension}"))
{
Ngram ngram;
while ((ngram = file.Next()) != null)
{
if (ngram.Frequency > fromFrequency || IsGoodWord(StringRoutines.MyDiacriticsRemover(ngram.ToString())))
{
goodWords_sw.WriteLine(ngram.Line);
}
else
{
badWords_sw.WriteLine(ngram.Line);
}
}
}
return($"{name}_GOOD-WORDS{extension}");
}
