public virtual void TestOtherLetterOffset() { string s = "a天b"; ChineseTokenizer tokenizer = new ChineseTokenizer(new StringReader(s)); int correctStartOffset = 0; int correctEndOffset = 1; IOffsetAttribute offsetAtt = tokenizer.GetAttribute<IOffsetAttribute>(); tokenizer.Reset(); while (tokenizer.IncrementToken()) { assertEquals(correctStartOffset, offsetAtt.StartOffset()); assertEquals(correctEndOffset, offsetAtt.EndOffset()); correctStartOffset++; correctEndOffset++; } tokenizer.End(); tokenizer.Dispose(); }
public virtual void TestOtherLetterOffset() { string s = "a天b"; ChineseTokenizer tokenizer = new ChineseTokenizer(new StringReader(s)); int correctStartOffset = 0; int correctEndOffset = 1; IOffsetAttribute offsetAtt = tokenizer.GetAttribute <IOffsetAttribute>(); tokenizer.Reset(); while (tokenizer.IncrementToken()) { assertEquals(correctStartOffset, offsetAtt.StartOffset); assertEquals(correctEndOffset, offsetAtt.EndOffset); correctStartOffset++; correctEndOffset++; } tokenizer.End(); tokenizer.Dispose(); }