public virtual void TestNynorskVocabulary()
{
Analyzer analyzer = Analyzer.NewAnonymous(createComponents: (fieldName, reader) =>
{
Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
return(new TokenStreamComponents(source, new NorwegianLightStemFilter(source, NorwegianStandard.NYNORSK)));
});
VocabularyAssert.AssertVocabulary(analyzer, GetDataFile("nn_light.txt"));
}
/// <summary>
/// For the supplied language, run the stemmer against all strings in voc.txt
/// The output should be the same as the string in output.txt
/// </summary>
private void AssertCorrectOutput(string snowballLanguage, string dataDirectory)
{
if (VERBOSE)
{
Console.WriteLine("checking snowball language: " + snowballLanguage);
}
Analyzer a = new AnalyzerAnonymousInnerClassHelper(this, snowballLanguage);
VocabularyAssert.AssertVocabulary(a, GetDataFile("TestSnowballVocabData.zip"), dataDirectory + "/voc.txt", dataDirectory + "/output.txt");
}
/// <summary>
/// For the supplied language, run the stemmer against all strings in voc.txt
/// The output should be the same as the string in output.txt
/// </summary>
private void AssertCorrectOutput(string snowballLanguage, string dataDirectory)
{
if (Verbose)
{
Console.WriteLine("checking snowball language: " + snowballLanguage);
}
Analyzer a = Analyzer.NewAnonymous(createComponents: (fieldName, reader) =>
{
Tokenizer t = new KeywordTokenizer(reader);
return(new TokenStreamComponents(t, new SnowballFilter(t, snowballLanguage)));
});
VocabularyAssert.AssertVocabulary(a, GetDataFile("TestSnowballVocabData.zip"), dataDirectory + "/voc.txt", dataDirectory + "/output.txt");
}
public virtual void TestPorterStemFilter()
{
VocabularyAssert.AssertVocabulary(a, GetDataFile("porterTestData.zip"), "voc.txt", "output.txt");
}
public virtual void TestStemming()
{
System.IO.Stream vocOut = this.GetType().getResourceAsStream("data.txt");
VocabularyAssert.AssertVocabulary(analyzer, vocOut);
vocOut.Dispose();
}
public virtual void TestNynorskVocabulary()
{
Analyzer analyzer = new AnalyzerAnonymousInnerClassHelper2(this);
VocabularyAssert.AssertVocabulary(analyzer, GetDataFile("nn_minimal.txt"));
}
public virtual void TestVocabulary()
{
VocabularyAssert.AssertVocabulary(analyzer, GetDataFile("nb_minimal.txt"));
}
public virtual void TestVocabulary()
{
VocabularyAssert.AssertVocabulary(analyzer, GetDataFile("frminimaltestdata.zip"), "frminimal.txt");
}
public virtual void TestVocabulary()
{
VocabularyAssert.AssertVocabulary(analyzer, GetDataFile("delighttestdata.zip"), "delight.txt");
}
public virtual void TestVocabulary()
{
// top 8k words or so: freq > 1000
VocabularyAssert.AssertVocabulary(a, GetDataFile("ckbtestdata.zip"), "testdata.txt");
}
public virtual void TestVocabulary()
{
VocabularyAssert.AssertVocabulary(a, GetDataFile("kstemTestData.zip"), "kstem_examples.txt");
}
public virtual void TestVocabulary()
{
VocabularyAssert.AssertVocabulary(analyzer, GetDataFile("ptrslptestdata.zip"), "ptrslp.txt");
}