public void TestMixedText()
{
TextReader reader = new StringReader("การที่ได้ต้องแสดงว่างานดี This is a test ກວ່າດອກ");
ICUTokenizerFactory factory = new ICUTokenizerFactory(new Dictionary <string, string>());
factory.Inform(new ClasspathResourceLoader(GetType()));
TokenStream stream = factory.Create(reader);
AssertTokenStreamContents(stream,
new string[] { "การ", "ที่", "ได้", "ต้อง", "แสดง", "ว่า", "งาน", "ดี",
"This", "is", "a", "test", "ກວ່າ", "ດອກ" });
}
public void TestKeywordTokenizeCyrillicAndThai()
{
TextReader reader = new StringReader
("Some English. Немного русский. ข้อความภาษาไทยเล็ก ๆ น้อย ๆ More English.");
IDictionary <string, string> args = new Dictionary <string, string>();
args.Put(ICUTokenizerFactory.RULEFILES, "Cyrl:KeywordTokenizer.rbbi,Thai:KeywordTokenizer.rbbi");
ICUTokenizerFactory factory = new ICUTokenizerFactory(args);
factory.Inform(new ClasspathResourceLoader(GetType()));
TokenStream stream = factory.Create(reader);
AssertTokenStreamContents(stream, new string[] { "Some", "English",
"Немного русский. ",
"ข้อความภาษาไทยเล็ก ๆ น้อย ๆ ",
"More", "English" });
}
public void TestTokenizeLatinDontBreakOnHyphens()
{
TextReader reader = new StringReader
("One-two punch. Brang-, not brung-it. This one--not that one--is the right one, -ish.");
IDictionary <string, string> args = new Dictionary <string, string>();
args.Put(ICUTokenizerFactory.RULEFILES, "Latn:Latin-dont-break-on-hyphens.rbbi");
ICUTokenizerFactory factory = new ICUTokenizerFactory(args);
factory.Inform(new ClasspathResourceLoader(GetType()));
TokenStream stream = factory.Create(reader);
AssertTokenStreamContents(stream,
new string[] { "One-two", "punch",
"Brang", "not", "brung-it",
"This", "one", "not", "that", "one", "is", "the", "right", "one", "ish" });
}
public void TestTokenizeLatinOnWhitespaceOnly()
{
// “ U+201C LEFT DOUBLE QUOTATION MARK; ” U+201D RIGHT DOUBLE QUOTATION MARK
TextReader reader = new StringReader
(" Don't,break.at?/(punct)! \u201Cnice\u201D\r\n\r\n85_At:all; `really\" +2=3$5,&813 !@#%$^)(*@#$ ");
IDictionary <string, string> args = new Dictionary <string, string>();
args.Put(ICUTokenizerFactory.RULEFILES, "Latn:Latin-break-only-on-whitespace.rbbi");
ICUTokenizerFactory factory = new ICUTokenizerFactory(args);
factory.Inform(new ClasspathResourceLoader(this.GetType()));
TokenStream stream = factory.Create(reader);
AssertTokenStreamContents(stream,
new string[] { "Don't,break.at?/(punct)!", "\u201Cnice\u201D", "85_At:all;", "`really\"", "+2=3$5,&813", "!@#%$^)(*@#$" },
new string[] { "<ALPHANUM>", "<ALPHANUM>", "<ALPHANUM>", "<ALPHANUM>", "<NUM>", "<OTHER>" });
}