public async Task can_read_ignoreutf_dic()
{
var filePath = @"files/ignoreutf.dic";
var ignoreChars = new[] { 1618, 1617, 1616, 1615, 1614, 1613, 1612, 1611 }.Select(i => (char)i).ToArray();
IEnumerable <string> expectedWords = new[]
{
"طِير",
"فَتحة",
"ضُمة",
"كِسرة",
"فتحًتان",
"ضمتانٌ",
"كسرتاٍن",
"شدّة",
"سكوْن"
};
foreach (var ignoreChar in ignoreChars)
{
expectedWords = expectedWords.Select(w => w.Replace(ignoreChar.ToString(), ""));
}
var actual = await WordListReader.ReadFileAsync(filePath);
actual.RootWords.Should().HaveCount(9);
actual.RootWords.ShouldBeEquivalentTo(expectedWords);
}
public async Task can_read_forbiddenword_dic()
{
var filePath = @"files/forbiddenword.dic";
var actual = await WordListReader.ReadFileAsync(filePath);
actual.RootWords.Should().HaveCount(4);
actual.RootWords.ShouldBeEquivalentTo(new[] {
"foo",
"bar",
"bars",
"foos"
});
actual["foo"][0].Flags.Should().ContainInOrder(new[] { 'S' });
actual["foo"][0].Morphs.ShouldBeEquivalentTo(new[] { "[1]" });
actual["foo"][1].Flags.Should().ContainInOrder(new[] { 'X', 'Y' });
actual["foo"][1].Morphs.ShouldBeEquivalentTo(new[] { "[2]" });
actual["foo"][2].Flags.Should().ContainInOrder(new[] { 'Y' });
actual["foo"][2].Morphs.ShouldBeEquivalentTo(new[] { "[3]" });
actual["foo"][3].Flags.Should().ContainInOrder(new[] { 'S' });
actual["foo"][3].Morphs.ShouldBeEquivalentTo(new[] { "[4]" });
actual["bar"][0].Flags.Should().ContainInOrder(new[] { 'S', 'Y' });
actual["bar"][0].Morphs.ShouldBeEquivalentTo(new[] { "[5]" });
actual["bars"][0].Flags.Should().ContainInOrder(new[] { 'X' });
actual["foos"][0].Flags.Should().ContainInOrder(new[] { 'X' });
}
public async Task can_read_compoundrule5_dic()
{
var filePath = @"files/compoundrule5.dic";
var actual = await WordListReader.ReadFileAsync(filePath);
actual.RootWords.Should().HaveCount(13);
actual.RootWords.ShouldBeEquivalentTo(new[] {
"0",
"1",
"2",
"3",
"4",
"5",
"6",
"7",
"8",
"9",
".",
"%",
"‰"
});
actual["0"][0].Flags.Should().ContainInOrder(new[] { 'N' });
actual["0"][0].Morphs.ShouldBeEquivalentTo(new[] { "po:num" });
actual["."][0].Flags.Should().ContainInOrder(new[] { '.' });
actual["."][0].Morphs.ShouldBeEquivalentTo(new[] { "po:sign_dot" });
actual["%"][0].Flags.Should().ContainInOrder(new[] { '%' });
actual["%"][0].Morphs.ShouldBeEquivalentTo(new[] { "po:sign_percent" });
actual["‰"][0].Flags.Should().ContainInOrder(new[] { '%' });
actual["‰"][0].Morphs.ShouldBeEquivalentTo(new[] { "po:sign_per_mille" });
}
public async Task can_read_compoundrule4_dic()
{
var filePath = @"files/compoundrule4.dic";
var actual = await WordListReader.ReadFileAsync(filePath);
actual.RootWords.Should().HaveCount(23);
actual.RootWords.ShouldBeEquivalentTo(new[] {
"0",
"1",
"2",
"3",
"4",
"5",
"6",
"7",
"8",
"9",
"0th",
"1st",
"1th",
"2nd",
"2th",
"3rd",
"3th",
"4th",
"5th",
"6th",
"7th",
"8th",
"9th"
});
}
public async Task can_read_morph_dic()
{
var filePath = @"files/morph.dic";
var actual = await WordListReader.ReadFileAsync(filePath);
actual.RootWords.Should().HaveCount(8);
actual.RootWords.ShouldBeEquivalentTo(new[] {
"drink",
"drank",
"drunk",
"eat",
"ate",
"eaten",
"phenomenon",
"phenomena"
});
actual["drink"].Should().HaveCount(2);
actual["drink"][0].Flags.Should().ContainInOrder(new[] { 'S' });
actual["drink"][0].Morphs.ShouldBeEquivalentTo(new[] { "po:noun" });
actual["drink"][1].Flags.Should().ContainInOrder(new[] { 'Q', 'R' });
actual["drink"][1].Morphs.Should().BeEquivalentTo(new[] { "po:verb", "al:drank", "al:drunk", "ts:present" });
actual["eaten"][0].Flags.Should().BeEmpty();
actual["eaten"][0].Morphs.Should().BeEquivalentTo(new[] { "po:verb", "st:eat", "is:past_2" });
}
public void Benchmark(BenchmarkContext context)
{
Task.WhenAll(DictionaryLoadArguments.Select(async testItem =>
{
await WordListReader.ReadFileAsync(testItem.DictionaryFilePath, testItem.Affix).ConfigureAwait(false);
DictionaryFilesLoaded.Increment();
})).Wait();
}
public void Benchmark(BenchmarkContext context)
{
foreach (var testItem in DictionaryLoadArguments)
{
WordListReader.ReadFile(testItem.DictionaryFilePath, testItem.Affix);
DictionaryFilesLoaded.Increment();
}
}
public async Task can_read_utf8_bom2_dic()
{
var filePath = @"files/utf8_bom2.dic";
var actual = await WordListReader.ReadFileAsync(filePath);
actual.RootWords.Should().HaveCount(1);
actual.RootWords.Should().ContainSingle("apéritif");
}
public async Task can_read_ngram_utf_fix_dic()
{
var filePath = @"files/ngram_utf_fix.dic";
var actual = await WordListReader.ReadFileAsync(filePath);
actual.RootWords.Should().HaveCount(1);
actual["человек"][0].Flags.Should().ContainInOrder(new[] { 2022, 2000, 101 }.OrderBy(x => x));
}
public async Task can_read_checkcompoundcaseutf_dic()
{
var filePath = @"files/checkcompoundcaseutf.dic";
var actual = await WordListReader.ReadFileAsync(filePath);
actual.RootWords.Should().HaveCount(2);
actual.RootWords.Should().BeEquivalentTo(new[] { "áoó", "Óoá" });
}
public async Task can_read_flaglong_dic()
{
var filePath = @"files/flaglong.dic";
var actual = await WordListReader.ReadFileAsync(filePath);
actual.RootWords.Should().HaveCount(1);
actual["foo"][0].Flags.Should().ContainInOrder(new[] { 'z' << 8 | 'x', '0' << 8 | '9' }.OrderBy(x => x));
}
private async Task BenchmarkAsync_Impl(BenchmarkContext context)
{
foreach (var testItem in DictionaryLoadArguments)
{
await WordListReader.ReadFileAsync(testItem.DictionaryFilePath, testItem.Affix).ConfigureAwait(false);
DictionaryFilesLoaded.Increment();
}
}
public async Task can_read_flagnum_dic()
{
var filePath = @"files/flagnum.dic";
var actual = await WordListReader.ReadFileAsync(filePath);
actual.RootWords.Should().HaveCount(1);
actual["foo"][0].Flags.Should().ContainInOrder(new[] { 999, 54321 });
}
public async Task can_read_arabic_dic()
{
var filePath = @"files/arabic.dic";
var actual = await WordListReader.ReadFileAsync(filePath);
actual.RootWords.Should().HaveCount(1);
actual["ب"].Should().HaveCount(1);
}
public async Task can_read_alias_dic()
{
var filePath = @"files/alias.dic";
var actual = await WordListReader.ReadFileAsync(filePath);
actual.RootWords.Should().HaveCount(1);
actual["foo"][0].Flags.Should().ContainInOrder(new[] { 'A', 'B' });
}
public async Task can_read_condition_utf_dic()
{
var filePath = @"files/condition_utf.dic";
var actual = await WordListReader.ReadFileAsync(filePath);
actual.RootWords.Should().HaveCount(1);
actual["óőó"][0].Flags.Should().ContainInOrder(new[] { 'P', 'S' });
}
public async Task can_read_compoundaffix_dic()
{
var filePath = @"files/compoundaffix.dic";
var actual = await WordListReader.ReadFileAsync(filePath);
actual.RootWords.Should().HaveCount(2);
actual["foo"][0].Flags.Should().ContainInOrder(new[] { 'X', 'P', 'S' }.OrderBy(x => x));
actual["bar"][0].Flags.Should().ContainInOrder(new[] { 'X', 'P', 'S' }.OrderBy(x => x));
}
public async Task can_read_complexprefixes2_dic()
{
var filePath = @"files/complexprefixes2.dic";
var actual = await WordListReader.ReadFileAsync(filePath);
actual.RootWords.Should().HaveCount(1);
actual["oruo"][0].Flags.Should().ContainInOrder(new[] { 'B', 'C' });
actual["oruo"][0].Morphs.Should().ContainInOrder(new[] { "]1_mets[" });
}
public async Task can_read_circumfix_dic()
{
var filePath = @"files/circumfix.dic";
var actual = await WordListReader.ReadFileAsync(filePath);
actual.RootWords.Should().HaveCount(1);
actual["nagy"][0].Flags.Should().ContainInOrder(new[] { 'C' });
actual["nagy"][0].Morphs.Should().ContainInOrder(new[] { "po:adj" });
}
public async Task can_read_warn_dic()
{
var filePath = @"files/warn.dic";
var actual = await WordListReader.ReadFileAsync(filePath);
actual.RootWords.Should().HaveCount(2);
actual.RootWords.Should().BeEquivalentTo(new[] { "foo", "bar" });
actual["foo"][0].Flags.Should().ContainInOrder(new[] { 'A', 'W' });
}
public async Task can_read_1975530_dic()
{
var filePath = @"files/1975530.dic";
var actual = await WordListReader.ReadFileAsync(filePath);
actual.RootWords.Should().HaveCount(2);
actual["أرى"][0].Flags.Should().ContainInOrder(new[] { 'x' });
actual["أيار"][0].Flags.Should().ContainInOrder(new[] { 'x' });
}
public async Task can_read_1706659_dic()
{
var filePath = @"files/1706659.dic";
var actual = await WordListReader.ReadFileAsync(filePath);
actual.RootWords.Should().HaveCount(3);
actual["arbeits"][0].Flags.Should().ContainInOrder(new[] { 'v' });
actual["scheu"][0].Flags.Should().ContainInOrder(new[] { 'A', 'w' });
actual["farbig"][0].Flags.Should().ContainInOrder(new[] { 'A' });
}
public async Task can_read_1592880_dic()
{
var filePath = @"files/1592880.dic";
var actual = await WordListReader.ReadFileAsync(filePath);
actual.RootWords.Should().BeEquivalentTo(new[] { "weg", "wege" });
actual["weg"][0].Flags.Should().ContainInOrder(new[] { 'Q', 'o', 'z' });
actual["weg"][1].Flags.Should().ContainInOrder(new[] { 'P' });
actual["wege"][0].Flags.Should().BeEmpty();
}
public async Task can_read_1695964_dic()
{
var filePath = @"files/1695964.dic";
var actual = await WordListReader.ReadFileAsync(filePath);
actual.RootWords.Should().HaveCount(1);
actual["Mull"].Should().HaveCount(2);
actual["Mull"][0].Flags.Should().ContainInOrder(new[] { 'e', 'h' });
actual["Mull"][1].Flags.Should().ContainInOrder(new[] { 'S' });
}
public async Task can_read_alias3_dic()
{
var filePath = @"files/alias3.dic";
var reversedStem = new string("[stem_1]".ToCharArray().Reverse().ToArray());
var actual = await WordListReader.ReadFileAsync(filePath);
actual.RootWords.Should().HaveCount(1);
actual["oruo"][0].Flags.Should().ContainInOrder(new[] { 'B', 'C' });
actual["oruo"][0].Morphs.Should().OnlyContain(x => x == reversedStem);
}
public async Task can_read_korean_dic()
{
var filePath = @"files/korean.dic";
var actual = await WordListReader.ReadFileAsync(filePath);
actual.RootWords.Should().HaveCount(2);
actual.RootWords.Should().BeEquivalentTo(new[] {
"들어오세요",
"안녕하세요"
});
}
public async Task can_read_allcaps2_dic()
{
var filePath = @"files/allcaps2.dic";
var actual = await WordListReader.ReadFileAsync(filePath);
actual.RootWords.Should().HaveCount(4);
actual["iPod"][0].Flags.Should().ContainInOrder(new[] { 's' });
actual["Ipod"][0].Flags.Should().ContainInOrder(new[] { 's', (char)SpecialFlags.OnlyUpcaseFlag });
actual["iPodos"][0].Flags.Should().ContainInOrder(new[] { '*' });
actual["ipodos"][0].Flags.Should().BeEmpty();
}
public async Task can_read_ignore_dic()
{
var filePath = @"files/ignore.dic";
var actual = await WordListReader.ReadFileAsync(filePath);
actual.RootWords.Should().HaveCount(2);
actual.RootWords.Should().BeEquivalentTo(new[] {
"xmpl",
"xprssn"
});
}
public async Task can_read_maputf_dic()
{
var filePath = @"files/maputf.dic";
var actual = await WordListReader.ReadFileAsync(filePath);
actual.RootWords.Should().HaveCount(3);
actual.RootWords.Should().BeEquivalentTo(new[] {
"Frühstück",
"tükörfúró",
"groß"
});
}
public async Task can_read_conditionalprefix_dic()
{
var filePath = @"files/conditionalprefix.dic";
var actual = await WordListReader.ReadFileAsync(filePath);
actual.RootWords.Should().HaveCount(1);
actual["drink"].Should().HaveCount(2);
actual["drink"][0].Flags.Should().ContainInOrder(new[] { 'Q', 'R' });
actual["drink"][0].Morphs.Should().BeEquivalentTo(new[] { "po:verb" });
actual["drink"][1].Flags.Should().ContainInOrder(new[] { 'S' });
actual["drink"][1].Morphs.Should().BeEquivalentTo(new[] { "po:noun" });
}
