public virtual void TestBehaviorNouns()
{
Analyzer a = new PersianAnalyzer(TEST_VERSION_CURRENT);
AssertAnalyzesTo(a, "برگ ها", new string[] { "برگ" });
AssertAnalyzesTo(a, "برگ‌ها", new string[] { "برگ" });
}
public virtual void TestReusableTokenStream()
{
Analyzer a = new PersianAnalyzer(TEST_VERSION_CURRENT);
AssertAnalyzesTo(a, "خورده مي شده بوده باشد", new string[] { "خورده" });
AssertAnalyzesTo(a, "برگ‌ها", new string[] { "برگ" });
}
public virtual void TestBehaviorVerbs()
{
Analyzer a = new PersianAnalyzer(TEST_VERSION_CURRENT);
// active present indicative
AssertAnalyzesTo(a, "می‌خورد", new string[] { "خورد" });
// active preterite indicative
AssertAnalyzesTo(a, "خورد", new string[] { "خورد" });
// active imperfective preterite indicative
AssertAnalyzesTo(a, "می‌خورد", new string[] { "خورد" });
// active future indicative
AssertAnalyzesTo(a, "خواهد خورد", new string[] { "خورد" });
// active present progressive indicative
AssertAnalyzesTo(a, "دارد می‌خورد", new string[] { "خورد" });
// active preterite progressive indicative
AssertAnalyzesTo(a, "داشت می‌خورد", new string[] { "خورد" });
// active perfect indicative
AssertAnalyzesTo(a, "خورده‌است", new string[] { "خورده" });
// active imperfective perfect indicative
AssertAnalyzesTo(a, "می‌خورده‌است", new string[] { "خورده" });
// active pluperfect indicative
AssertAnalyzesTo(a, "خورده بود", new string[] { "خورده" });
// active imperfective pluperfect indicative
AssertAnalyzesTo(a, "می‌خورده بود", new string[] { "خورده" });
// active preterite subjunctive
AssertAnalyzesTo(a, "خورده باشد", new string[] { "خورده" });
// active imperfective preterite subjunctive
AssertAnalyzesTo(a, "می‌خورده باشد", new string[] { "خورده" });
// active pluperfect subjunctive
AssertAnalyzesTo(a, "خورده بوده باشد", new string[] { "خورده" });
// active imperfective pluperfect subjunctive
AssertAnalyzesTo(a, "می‌خورده بوده باشد", new string[] { "خورده" });
// passive present indicative
AssertAnalyzesTo(a, "خورده می‌شود", new string[] { "خورده" });
// passive preterite indicative
AssertAnalyzesTo(a, "خورده شد", new string[] { "خورده" });
// passive imperfective preterite indicative
AssertAnalyzesTo(a, "خورده می‌شد", new string[] { "خورده" });
// passive perfect indicative
AssertAnalyzesTo(a, "خورده شده‌است", new string[] { "خورده" });
// passive imperfective perfect indicative
AssertAnalyzesTo(a, "خورده می‌شده‌است", new string[] { "خورده" });
// passive pluperfect indicative
AssertAnalyzesTo(a, "خورده شده بود", new string[] { "خورده" });
// passive imperfective pluperfect indicative
AssertAnalyzesTo(a, "خورده می‌شده بود", new string[] { "خورده" });
// passive future indicative
AssertAnalyzesTo(a, "خورده خواهد شد", new string[] { "خورده" });
// passive present progressive indicative
AssertAnalyzesTo(a, "دارد خورده می‌شود", new string[] { "خورده" });
// passive preterite progressive indicative
AssertAnalyzesTo(a, "داشت خورده می‌شد", new string[] { "خورده" });
// passive present subjunctive
AssertAnalyzesTo(a, "خورده شود", new string[] { "خورده" });
// passive preterite subjunctive
AssertAnalyzesTo(a, "خورده شده باشد", new string[] { "خورده" });
// passive imperfective preterite subjunctive
AssertAnalyzesTo(a, "خورده می‌شده باشد", new string[] { "خورده" });
// passive pluperfect subjunctive
AssertAnalyzesTo(a, "خورده شده بوده باشد", new string[] { "خورده" });
// passive imperfective pluperfect subjunctive
AssertAnalyzesTo(a, "خورده می‌شده بوده باشد", new string[] { "خورده" });
// active present subjunctive
AssertAnalyzesTo(a, "بخورد", new string[] { "بخورد" });
}
public virtual void TestCustomStopwords()
{
PersianAnalyzer a = new PersianAnalyzer(TEST_VERSION_CURRENT, new CharArraySet(TEST_VERSION_CURRENT, AsSet("the", "and", "a"), false));
AssertAnalyzesTo(a, "The quick brown fox.", new string[] { "quick", "brown", "fox" });
}
public virtual void TestBehaviorNonPersian()
{
Analyzer a = new PersianAnalyzer(TEST_VERSION_CURRENT);
AssertAnalyzesTo(a, "English test.", new string[] { "english", "test" });
}
public virtual void TestBehaviorVerbs()
{
Analyzer a = new PersianAnalyzer(TEST_VERSION_CURRENT);
// active present indicative
AssertAnalyzesTo(a, "می‌خورد", new string[] { "خورد" });
// active preterite indicative
AssertAnalyzesTo(a, "خورد", new string[] { "خورد" });
// active imperfective preterite indicative
AssertAnalyzesTo(a, "می‌خورد", new string[] { "خورد" });
// active future indicative
AssertAnalyzesTo(a, "خواهد خورد", new string[] { "خورد" });
// active present progressive indicative
AssertAnalyzesTo(a, "دارد می‌خورد", new string[] { "خورد" });
// active preterite progressive indicative
AssertAnalyzesTo(a, "داشت می‌خورد", new string[] { "خورد" });
// active perfect indicative
AssertAnalyzesTo(a, "خورده‌است", new string[] { "خورده" });
// active imperfective perfect indicative
AssertAnalyzesTo(a, "می‌خورده‌است", new string[] { "خورده" });
// active pluperfect indicative
AssertAnalyzesTo(a, "خورده بود", new string[] { "خورده" });
// active imperfective pluperfect indicative
AssertAnalyzesTo(a, "می‌خورده بود", new string[] { "خورده" });
// active preterite subjunctive
AssertAnalyzesTo(a, "خورده باشد", new string[] { "خورده" });
// active imperfective preterite subjunctive
AssertAnalyzesTo(a, "می‌خورده باشد", new string[] { "خورده" });
// active pluperfect subjunctive
AssertAnalyzesTo(a, "خورده بوده باشد", new string[] { "خورده" });
// active imperfective pluperfect subjunctive
AssertAnalyzesTo(a, "می‌خورده بوده باشد", new string[] { "خورده" });
// passive present indicative
AssertAnalyzesTo(a, "خورده می‌شود", new string[] { "خورده" });
// passive preterite indicative
AssertAnalyzesTo(a, "خورده شد", new string[] { "خورده" });
// passive imperfective preterite indicative
AssertAnalyzesTo(a, "خورده می‌شد", new string[] { "خورده" });
// passive perfect indicative
AssertAnalyzesTo(a, "خورده شده‌است", new string[] { "خورده" });
// passive imperfective perfect indicative
AssertAnalyzesTo(a, "خورده می‌شده‌است", new string[] { "خورده" });
// passive pluperfect indicative
AssertAnalyzesTo(a, "خورده شده بود", new string[] { "خورده" });
// passive imperfective pluperfect indicative
AssertAnalyzesTo(a, "خورده می‌شده بود", new string[] { "خورده" });
// passive future indicative
AssertAnalyzesTo(a, "خورده خواهد شد", new string[] { "خورده" });
// passive present progressive indicative
AssertAnalyzesTo(a, "دارد خورده می‌شود", new string[] { "خورده" });
// passive preterite progressive indicative
AssertAnalyzesTo(a, "داشت خورده می‌شد", new string[] { "خورده" });
// passive present subjunctive
AssertAnalyzesTo(a, "خورده شود", new string[] { "خورده" });
// passive preterite subjunctive
AssertAnalyzesTo(a, "خورده شده باشد", new string[] { "خورده" });
// passive imperfective preterite subjunctive
AssertAnalyzesTo(a, "خورده می‌شده باشد", new string[] { "خورده" });
// passive pluperfect subjunctive
AssertAnalyzesTo(a, "خورده شده بوده باشد", new string[] { "خورده" });
// passive imperfective pluperfect subjunctive
AssertAnalyzesTo(a, "خورده می‌شده بوده باشد", new string[] { "خورده" });
// active present subjunctive
AssertAnalyzesTo(a, "بخورد", new string[] { "بخورد" });
}
public virtual void TestReusableTokenStream()
{
Analyzer a = new PersianAnalyzer(TEST_VERSION_CURRENT);
AssertAnalyzesTo(a, "خورده مي شده بوده باشد", new string[] { "خورده" });
AssertAnalyzesTo(a, "برگ‌ها", new string[] { "برگ" });
}
public virtual void TestCustomStopwords()
{
PersianAnalyzer a = new PersianAnalyzer(TEST_VERSION_CURRENT, new CharArraySet(TEST_VERSION_CURRENT, AsSet("the", "and", "a"), false));
AssertAnalyzesTo(a, "The quick brown fox.", new string[] { "quick", "brown", "fox" });
}
public virtual void TestBehaviorNouns()
{
Analyzer a = new PersianAnalyzer(TEST_VERSION_CURRENT);
AssertAnalyzesTo(a, "برگ ها", new string[] { "برگ" });
AssertAnalyzesTo(a, "برگ‌ها", new string[] { "برگ" });
}
public virtual void TestBehaviorNonPersian()
{
Analyzer a = new PersianAnalyzer(TEST_VERSION_CURRENT);
AssertAnalyzesTo(a, "English test.", new string[] { "english", "test" });
}
/*
* Test that custom stopwords work, and are not case-sensitive.
*/
public void testCustomStopwords(){
PersianAnalyzer a = new PersianAnalyzer(Version.LUCENE_CURRENT, new String[] { "the", "and", "a" });
AssertAnalyzesTo(a, "The quick brown fox.", new String[] { "quick",
"brown", "fox" });
}
/*
* Basic test ensuring that reusableTokenStream works correctly.
*/
public void testReusableTokenStream(){
Analyzer a = new PersianAnalyzer(Version.LUCENE_CURRENT);
AssertAnalyzesToReuse(a, "خورده مي شده بوده باشد", new String[] { "خورده" });
AssertAnalyzesToReuse(a, "برگ‌ها", new String[] { "برگ" });
}
/*
* Test showing that non-persian text is treated very much like SimpleAnalyzer
* (lowercased, etc)
*/
public void testBehaviorNonPersian(){
Analyzer a = new PersianAnalyzer(Version.LUCENE_CURRENT);
AssertAnalyzesTo(a, "English test.", new String[] { "english", "test" });
}
/*
* This test shows how the combination of tokenization (breaking on zero-width
* non-joiner or space) and stopwords creates a light-stemming effect for
* nouns, removing the plural -ha.
*/
public void testBehaviorNouns(){
Analyzer a = new PersianAnalyzer(Version.LUCENE_CURRENT);
AssertAnalyzesTo(a, "برگ ها", new String[] { "برگ" });
AssertAnalyzesTo(a, "برگ‌ها", new String[] { "برگ" });
}
/*
* This test shows how the combination of tokenization and stopwords creates a
* light-stemming effect for verbs.
*
* In this case, these forms are presented with alternative orthography, using
* arabic yeh and whitespace. This yeh phenomenon is common for legacy text
* due to some previous bugs in Microsoft Windows.
*
* These verb forms are from http://en.wikipedia.org/wiki/Persian_grammar
*/
public void testBehaviorVerbsDefective(){
Analyzer a = new PersianAnalyzer(Version.LUCENE_CURRENT);
// active present indicative
AssertAnalyzesTo(a, "مي خورد", new String[] { "خورد" });
// active preterite indicative
AssertAnalyzesTo(a, "خورد", new String[] { "خورد" });
// active imperfective preterite indicative
AssertAnalyzesTo(a, "مي خورد", new String[] { "خورد" });
// active future indicative
AssertAnalyzesTo(a, "خواهد خورد", new String[] { "خورد" });
// active present progressive indicative
AssertAnalyzesTo(a, "دارد مي خورد", new String[] { "خورد" });
// active preterite progressive indicative
AssertAnalyzesTo(a, "داشت مي خورد", new String[] { "خورد" });
// active perfect indicative
AssertAnalyzesTo(a, "خورده است", new String[] { "خورده" });
// active imperfective perfect indicative
AssertAnalyzesTo(a, "مي خورده است", new String[] { "خورده" });
// active pluperfect indicative
AssertAnalyzesTo(a, "خورده بود", new String[] { "خورده" });
// active imperfective pluperfect indicative
AssertAnalyzesTo(a, "مي خورده بود", new String[] { "خورده" });
// active preterite subjunctive
AssertAnalyzesTo(a, "خورده باشد", new String[] { "خورده" });
// active imperfective preterite subjunctive
AssertAnalyzesTo(a, "مي خورده باشد", new String[] { "خورده" });
// active pluperfect subjunctive
AssertAnalyzesTo(a, "خورده بوده باشد", new String[] { "خورده" });
// active imperfective pluperfect subjunctive
AssertAnalyzesTo(a, "مي خورده بوده باشد", new String[] { "خورده" });
// passive present indicative
AssertAnalyzesTo(a, "خورده مي شود", new String[] { "خورده" });
// passive preterite indicative
AssertAnalyzesTo(a, "خورده شد", new String[] { "خورده" });
// passive imperfective preterite indicative
AssertAnalyzesTo(a, "خورده مي شد", new String[] { "خورده" });
// passive perfect indicative
AssertAnalyzesTo(a, "خورده شده است", new String[] { "خورده" });
// passive imperfective perfect indicative
AssertAnalyzesTo(a, "خورده مي شده است", new String[] { "خورده" });
// passive pluperfect indicative
AssertAnalyzesTo(a, "خورده شده بود", new String[] { "خورده" });
// passive imperfective pluperfect indicative
AssertAnalyzesTo(a, "خورده مي شده بود", new String[] { "خورده" });
// passive future indicative
AssertAnalyzesTo(a, "خورده خواهد شد", new String[] { "خورده" });
// passive present progressive indicative
AssertAnalyzesTo(a, "دارد خورده مي شود", new String[] { "خورده" });
// passive preterite progressive indicative
AssertAnalyzesTo(a, "داشت خورده مي شد", new String[] { "خورده" });
// passive present subjunctive
AssertAnalyzesTo(a, "خورده شود", new String[] { "خورده" });
// passive preterite subjunctive
AssertAnalyzesTo(a, "خورده شده باشد", new String[] { "خورده" });
// passive imperfective preterite subjunctive
AssertAnalyzesTo(a, "خورده مي شده باشد", new String[] { "خورده" });
// passive pluperfect subjunctive
AssertAnalyzesTo(a, "خورده شده بوده باشد", new String[] { "خورده" });
// passive imperfective pluperfect subjunctive
AssertAnalyzesTo(a, "خورده مي شده بوده باشد", new String[] { "خورده" });
// active present subjunctive
AssertAnalyzesTo(a, "بخورد", new String[] { "بخورد" });
}
