コード例 #1
0
        public void TestMixedText()
        {
            TextReader          reader  = new StringReader("การที่ได้ต้องแสดงว่างานดี  This is a test ກວ່າດອກ");
            ICUTokenizerFactory factory = new ICUTokenizerFactory(new Dictionary <string, string>());

            factory.Inform(new ClasspathResourceLoader(GetType()));
            TokenStream stream = factory.Create(reader);

            AssertTokenStreamContents(stream,
                                      new string[] { "การ", "ที่", "ได้", "ต้อง", "แสดง", "ว่า", "งาน", "ดี",
                                                     "This", "is", "a", "test", "ກວ່າ", "ດອກ" });
        }
コード例 #2
0
        public void TestKeywordTokenizeCyrillicAndThai()
        {
            TextReader reader = new StringReader
                                    ("Some English.  Немного русский.  ข้อความภาษาไทยเล็ก ๆ น้อย ๆ  More English.");
            IDictionary <string, string> args = new Dictionary <string, string>();

            args.Put(ICUTokenizerFactory.RULEFILES, "Cyrl:KeywordTokenizer.rbbi,Thai:KeywordTokenizer.rbbi");
            ICUTokenizerFactory factory = new ICUTokenizerFactory(args);

            factory.Inform(new ClasspathResourceLoader(GetType()));
            TokenStream stream = factory.Create(reader);

            AssertTokenStreamContents(stream, new string[] { "Some", "English",
                                                             "Немного русский.  ",
                                                             "ข้อความภาษาไทยเล็ก ๆ น้อย ๆ  ",
                                                             "More", "English" });
        }
コード例 #3
0
        public void TestTokenizeLatinDontBreakOnHyphens()
        {
            TextReader reader = new StringReader
                                    ("One-two punch.  Brang-, not brung-it.  This one--not that one--is the right one, -ish.");
            IDictionary <string, string> args = new Dictionary <string, string>();

            args.Put(ICUTokenizerFactory.RULEFILES, "Latn:Latin-dont-break-on-hyphens.rbbi");
            ICUTokenizerFactory factory = new ICUTokenizerFactory(args);

            factory.Inform(new ClasspathResourceLoader(GetType()));
            TokenStream stream = factory.Create(reader);

            AssertTokenStreamContents(stream,
                                      new string[] { "One-two", "punch",
                                                     "Brang", "not", "brung-it",
                                                     "This", "one", "not", "that", "one", "is", "the", "right", "one", "ish" });
        }
コード例 #4
0
        public void TestTokenizeLatinOnWhitespaceOnly()
        {
            // “ U+201C LEFT DOUBLE QUOTATION MARK; ” U+201D RIGHT DOUBLE QUOTATION MARK
            TextReader reader = new StringReader
                                    ("  Don't,break.at?/(punct)!  \u201Cnice\u201D\r\n\r\n85_At:all; `really\" +2=3$5,&813 !@#%$^)(*@#$   ");
            IDictionary <string, string> args = new Dictionary <string, string>();

            args.Put(ICUTokenizerFactory.RULEFILES, "Latn:Latin-break-only-on-whitespace.rbbi");
            ICUTokenizerFactory factory = new ICUTokenizerFactory(args);

            factory.Inform(new ClasspathResourceLoader(this.GetType()));
            TokenStream stream = factory.Create(reader);

            AssertTokenStreamContents(stream,
                                      new string[] { "Don't,break.at?/(punct)!", "\u201Cnice\u201D", "85_At:all;", "`really\"", "+2=3$5,&813", "!@#%$^)(*@#$" },
                                      new string[] { "<ALPHANUM>", "<ALPHANUM>", "<ALPHANUM>", "<ALPHANUM>", "<NUM>", "<OTHER>" });
        }