public void TestOtherLetterOffset() { String s = "a天b"; ChineseTokenizer tokenizer = new ChineseTokenizer(new StringReader(s)); int correctStartOffset = 0; int correctEndOffset = 1; IOffsetAttribute offsetAtt = tokenizer.GetAttribute<IOffsetAttribute>(); while (tokenizer.IncrementToken()) { Assert.AreEqual(correctStartOffset, offsetAtt.StartOffset); Assert.AreEqual(correctEndOffset, offsetAtt.EndOffset); correctStartOffset++; correctEndOffset++; } }
public virtual void TestOtherLetterOffset() { string s = "a天b"; ChineseTokenizer tokenizer = new ChineseTokenizer(new StringReader(s)); int correctStartOffset = 0; int correctEndOffset = 1; IOffsetAttribute offsetAtt = tokenizer.GetAttribute<IOffsetAttribute>(); tokenizer.Reset(); while (tokenizer.IncrementToken()) { assertEquals(correctStartOffset, offsetAtt.StartOffset()); assertEquals(correctEndOffset, offsetAtt.EndOffset()); correctStartOffset++; correctEndOffset++; } tokenizer.End(); tokenizer.Dispose(); }
public virtual void TestOtherLetterOffset() { string s = "a天b"; ChineseTokenizer tokenizer = new ChineseTokenizer(new StringReader(s)); int correctStartOffset = 0; int correctEndOffset = 1; IOffsetAttribute offsetAtt = tokenizer.GetAttribute <IOffsetAttribute>(); tokenizer.Reset(); while (tokenizer.IncrementToken()) { assertEquals(correctStartOffset, offsetAtt.StartOffset); assertEquals(correctEndOffset, offsetAtt.EndOffset); correctStartOffset++; correctEndOffset++; } tokenizer.End(); tokenizer.Dispose(); }
/// <summary> /// Creates a TokenStream which tokenizes all the text in the provided Reader. /// </summary> /// <returns>A TokenStream build from a ChineseTokenizer filtered with ChineseFilter.</returns> public override sealed TokenStream TokenStream(String fieldName, TextReader reader) { TokenStream result = new ChineseTokenizer(reader); result = new ChineseFilter(result); return result; }
/// <summary> /// Creates /// <see cref="TokenStreamComponents"/> /// used to tokenize all the text in the provided <see cref="TextReader"/>. /// </summary> /// <returns> <see cref="TokenStreamComponents"/> /// built from a <see cref="ChineseTokenizer"/> filtered with /// <see cref="ChineseFilter"/> </returns> protected internal override TokenStreamComponents CreateComponents(string fieldName, TextReader reader) { Tokenizer source = new ChineseTokenizer(reader); return(new TokenStreamComponents(source, new ChineseFilter(source))); }
public override TokenStreamComponents CreateComponents(string fieldName, TextReader reader) { Tokenizer source = new ChineseTokenizer(reader); return new TokenStreamComponents(source, new ChineseFilter(source)); }