public virtual void TestNynorskVocabulary() { Analyzer analyzer = Analyzer.NewAnonymous(createComponents: (fieldName, reader) => { Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false); return(new TokenStreamComponents(source, new NorwegianLightStemFilter(source, NorwegianStandard.NYNORSK))); }); VocabularyAssert.AssertVocabulary(analyzer, GetDataFile("nn_light.txt")); }
/// <summary> /// For the supplied language, run the stemmer against all strings in voc.txt /// The output should be the same as the string in output.txt /// </summary> private void AssertCorrectOutput(string snowballLanguage, string dataDirectory) { if (VERBOSE) { Console.WriteLine("checking snowball language: " + snowballLanguage); } Analyzer a = new AnalyzerAnonymousInnerClassHelper(this, snowballLanguage); VocabularyAssert.AssertVocabulary(a, GetDataFile("TestSnowballVocabData.zip"), dataDirectory + "/voc.txt", dataDirectory + "/output.txt"); }
/// <summary> /// For the supplied language, run the stemmer against all strings in voc.txt /// The output should be the same as the string in output.txt /// </summary> private void AssertCorrectOutput(string snowballLanguage, string dataDirectory) { if (Verbose) { Console.WriteLine("checking snowball language: " + snowballLanguage); } Analyzer a = Analyzer.NewAnonymous(createComponents: (fieldName, reader) => { Tokenizer t = new KeywordTokenizer(reader); return(new TokenStreamComponents(t, new SnowballFilter(t, snowballLanguage))); }); VocabularyAssert.AssertVocabulary(a, GetDataFile("TestSnowballVocabData.zip"), dataDirectory + "/voc.txt", dataDirectory + "/output.txt"); }
public virtual void TestPorterStemFilter() { VocabularyAssert.AssertVocabulary(a, GetDataFile("porterTestData.zip"), "voc.txt", "output.txt"); }
public virtual void TestStemming() { System.IO.Stream vocOut = this.GetType().getResourceAsStream("data.txt"); VocabularyAssert.AssertVocabulary(analyzer, vocOut); vocOut.Dispose(); }
public virtual void TestNynorskVocabulary() { Analyzer analyzer = new AnalyzerAnonymousInnerClassHelper2(this); VocabularyAssert.AssertVocabulary(analyzer, GetDataFile("nn_minimal.txt")); }
public virtual void TestVocabulary() { VocabularyAssert.AssertVocabulary(analyzer, GetDataFile("nb_minimal.txt")); }
public virtual void TestVocabulary() { VocabularyAssert.AssertVocabulary(analyzer, GetDataFile("frminimaltestdata.zip"), "frminimal.txt"); }
public virtual void TestVocabulary() { VocabularyAssert.AssertVocabulary(analyzer, GetDataFile("delighttestdata.zip"), "delight.txt"); }
public virtual void TestVocabulary() { // top 8k words or so: freq > 1000 VocabularyAssert.AssertVocabulary(a, GetDataFile("ckbtestdata.zip"), "testdata.txt"); }
public virtual void TestVocabulary() { VocabularyAssert.AssertVocabulary(a, GetDataFile("kstemTestData.zip"), "kstem_examples.txt"); }
public virtual void TestVocabulary() { VocabularyAssert.AssertVocabulary(analyzer, GetDataFile("ptrslptestdata.zip"), "ptrslp.txt"); }