You must specify the required Version compatibility when creating CharTokenizer:
public virtual void TestSimple() { Analyzer a = new SimpleAnalyzer(TEST_VERSION_CURRENT); AssertAnalyzesTo(a, "foo bar FOO BAR", new string[] { "foo", "bar", "foo", "bar" }); AssertAnalyzesTo(a, "foo bar . FOO <> BAR", new string[] { "foo", "bar", "foo", "bar" }); AssertAnalyzesTo(a, "foo.bar.FOO.BAR", new string[] { "foo", "bar", "foo", "bar" }); AssertAnalyzesTo(a, "U.S.A.", new string[] { "u", "s", "a" }); AssertAnalyzesTo(a, "C++", new string[] { "c" }); AssertAnalyzesTo(a, "B2B", new string[] { "b", "b" }); AssertAnalyzesTo(a, "2B", new string[] { "b" }); AssertAnalyzesTo(a, "\"QUOTED\" word", new string[] { "quoted", "word" }); }
public virtual void TestPerField() { string text = "Qwerty"; IDictionary<string, Analyzer> analyzerPerField = new Dictionary<string, Analyzer>(); analyzerPerField["special"] = new SimpleAnalyzer(TEST_VERSION_CURRENT); PerFieldAnalyzerWrapper analyzer = new PerFieldAnalyzerWrapper(new WhitespaceAnalyzer(TEST_VERSION_CURRENT), analyzerPerField); TokenStream tokenStream = analyzer.TokenStream("field", text); try { ICharTermAttribute termAtt = tokenStream.GetAttribute<ICharTermAttribute>(); tokenStream.Reset(); assertTrue(tokenStream.IncrementToken()); assertEquals("WhitespaceAnalyzer does not lowercase", "Qwerty", termAtt.ToString()); assertFalse(tokenStream.IncrementToken()); tokenStream.End(); } finally { IOUtils.CloseWhileHandlingException(tokenStream); } tokenStream = analyzer.TokenStream("special", text); try { ICharTermAttribute termAtt = tokenStream.GetAttribute<ICharTermAttribute>(); tokenStream.Reset(); assertTrue(tokenStream.IncrementToken()); assertEquals("SimpleAnalyzer lowercases", "qwerty", termAtt.ToString()); assertFalse(tokenStream.IncrementToken()); tokenStream.End(); } finally { IOUtils.CloseWhileHandlingException(tokenStream); } }