Пример #1
0
        /*
         * Test that custom stopwords work, and are not case-sensitive.
         */
        public void testCustomStopwords()
        {
            PersianAnalyzer a = new PersianAnalyzer(Version.LUCENE_CURRENT, new String[] { "the", "and", "a" });

            AssertAnalyzesTo(a, "The quick brown fox.", new String[] { "quick",
                                                                       "brown", "fox" });
        }
Пример #2
0
        /*
         * Basic test ensuring that reusableTokenStream works correctly.
         */
        public void testReusableTokenStream()
        {
            Analyzer a = new PersianAnalyzer(Version.LUCENE_CURRENT);

            AssertAnalyzesToReuse(a, "خورده مي شده بوده باشد", new String[] { "خورده" });
            AssertAnalyzesToReuse(a, "برگ‌ها", new String[] { "برگ" });
        }
Пример #3
0
        /*
         * This test shows how the combination of tokenization (breaking on zero-width
         * non-joiner or space) and stopwords creates a light-stemming effect for
         * nouns, removing the plural -ha.
         */
        public void testBehaviorNouns()
        {
            Analyzer a = new PersianAnalyzer(Version.LUCENE_CURRENT);

            AssertAnalyzesTo(a, "برگ ها", new String[] { "برگ" });
            AssertAnalyzesTo(a, "برگ‌ها", new String[] { "برگ" });
        }
Пример #4
0
        /*
         * This test shows how the combination of tokenization (breaking on zero-width
         * non-joiner), normalization (such as treating arabic YEH and farsi YEH the
         * same), and stopwords creates a light-stemming effect for verbs.
         *
         * These verb forms are from http://en.wikipedia.org/wiki/Persian_grammar
         */
        public void testBehaviorVerbs()
        {
            Analyzer a = new PersianAnalyzer(Version.LUCENE_CURRENT);

            // active present indicative
            AssertAnalyzesTo(a, "می‌خورد", new String[] { "خورد" });
            // active preterite indicative
            AssertAnalyzesTo(a, "خورد", new String[] { "خورد" });
            // active imperfective preterite indicative
            AssertAnalyzesTo(a, "می‌خورد", new String[] { "خورد" });
            // active future indicative
            AssertAnalyzesTo(a, "خواهد خورد", new String[] { "خورد" });
            // active present progressive indicative
            AssertAnalyzesTo(a, "دارد می‌خورد", new String[] { "خورد" });
            // active preterite progressive indicative
            AssertAnalyzesTo(a, "داشت می‌خورد", new String[] { "خورد" });

            // active perfect indicative
            AssertAnalyzesTo(a, "خورده‌است", new String[] { "خورده" });
            // active imperfective perfect indicative
            AssertAnalyzesTo(a, "می‌خورده‌است", new String[] { "خورده" });
            // active pluperfect indicative
            AssertAnalyzesTo(a, "خورده بود", new String[] { "خورده" });
            // active imperfective pluperfect indicative
            AssertAnalyzesTo(a, "می‌خورده بود", new String[] { "خورده" });
            // active preterite subjunctive
            AssertAnalyzesTo(a, "خورده باشد", new String[] { "خورده" });
            // active imperfective preterite subjunctive
            AssertAnalyzesTo(a, "می‌خورده باشد", new String[] { "خورده" });
            // active pluperfect subjunctive
            AssertAnalyzesTo(a, "خورده بوده باشد", new String[] { "خورده" });
            // active imperfective pluperfect subjunctive
            AssertAnalyzesTo(a, "می‌خورده بوده باشد", new String[] { "خورده" });
            // passive present indicative
            AssertAnalyzesTo(a, "خورده می‌شود", new String[] { "خورده" });
            // passive preterite indicative
            AssertAnalyzesTo(a, "خورده شد", new String[] { "خورده" });
            // passive imperfective preterite indicative
            AssertAnalyzesTo(a, "خورده می‌شد", new String[] { "خورده" });
            // passive perfect indicative
            AssertAnalyzesTo(a, "خورده شده‌است", new String[] { "خورده" });
            // passive imperfective perfect indicative
            AssertAnalyzesTo(a, "خورده می‌شده‌است", new String[] { "خورده" });
            // passive pluperfect indicative
            AssertAnalyzesTo(a, "خورده شده بود", new String[] { "خورده" });
            // passive imperfective pluperfect indicative
            AssertAnalyzesTo(a, "خورده می‌شده بود", new String[] { "خورده" });
            // passive future indicative
            AssertAnalyzesTo(a, "خورده خواهد شد", new String[] { "خورده" });
            // passive present progressive indicative
            AssertAnalyzesTo(a, "دارد خورده می‌شود", new String[] { "خورده" });
            // passive preterite progressive indicative
            AssertAnalyzesTo(a, "داشت خورده می‌شد", new String[] { "خورده" });
            // passive present subjunctive
            AssertAnalyzesTo(a, "خورده شود", new String[] { "خورده" });
            // passive preterite subjunctive
            AssertAnalyzesTo(a, "خورده شده باشد", new String[] { "خورده" });
            // passive imperfective preterite subjunctive
            AssertAnalyzesTo(a, "خورده می‌شده باشد", new String[] { "خورده" });
            // passive pluperfect subjunctive
            AssertAnalyzesTo(a, "خورده شده بوده باشد", new String[] { "خورده" });
            // passive imperfective pluperfect subjunctive
            AssertAnalyzesTo(a, "خورده می‌شده بوده باشد", new String[] { "خورده" });

            // active present subjunctive
            AssertAnalyzesTo(a, "بخورد", new String[] { "بخورد" });
        }
Пример #5
0
        /*
         * Test showing that non-persian text is treated very much like SimpleAnalyzer
         * (lowercased, etc)
         */
        public void testBehaviorNonPersian()
        {
            Analyzer a = new PersianAnalyzer(Version.LUCENE_CURRENT);

            AssertAnalyzesTo(a, "English test.", new String[] { "english", "test" });
        }