/*
* ChineseTokenizer tokenizes numbers as one token, but they are filtered by ChineseFilter
*/
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
//ORIGINAL LINE: public void testNumerics() throws Exception
public virtual void testNumerics()
{
Analyzer justTokenizer = new JustChineseTokenizerAnalyzer(this);
assertAnalyzesTo(justTokenizer, "中1234", new string[] {"中", "1234"});
// in this case the ChineseAnalyzer (which applies ChineseFilter) will remove the numeric token.
Analyzer a = new ChineseAnalyzer();
assertAnalyzesTo(a, "中1234", new string[] {"中"});
}
public virtual void TestNumerics()
{
Analyzer justTokenizer = new JustChineseTokenizerAnalyzer();
AssertAnalyzesTo(justTokenizer, "中1234", new string[] { "中", "1234" });
// in this case the ChineseAnalyzer (which applies ChineseFilter) will remove the numeric token.
Analyzer a = new ChineseAnalyzer();
AssertAnalyzesTo(a, "中1234", new string[] { "中" });
}
/*
* ChineseTokenizer tokenizes english similar to SimpleAnalyzer.
* it will lowercase terms automatically.
*
* ChineseFilter has an english stopword list, it also removes any single character tokens.
* the stopword list is case-sensitive.
*/
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
//ORIGINAL LINE: public void testEnglish() throws Exception
public virtual void testEnglish()
{
Analyzer chinese = new ChineseAnalyzer();
assertAnalyzesTo(chinese, "This is a Test. b c d", new string[] {"test"});
Analyzer justTokenizer = new JustChineseTokenizerAnalyzer(this);
assertAnalyzesTo(justTokenizer, "This is a Test. b c d", new string[] {"this", "is", "a", "test", "b", "c", "d"});
Analyzer justFilter = new JustChineseFilterAnalyzer(this);
assertAnalyzesTo(justFilter, "This is a Test. b c d", new string[] {"This", "Test."});
}
public virtual void TestEnglish()
{
Analyzer chinese = new ChineseAnalyzer();
AssertAnalyzesTo(chinese, "This is a Test. b c d", new string[] { "test" });
Analyzer justTokenizer = new JustChineseTokenizerAnalyzer();
AssertAnalyzesTo(justTokenizer, "This is a Test. b c d", new string[] { "this", "is", "a", "test", "b", "c", "d" });
Analyzer justFilter = new JustChineseFilterAnalyzer();
AssertAnalyzesTo(justFilter, "This is a Test. b c d", new string[] { "This", "Test." });
}
