private bool hasIllegalOffsets; // only if the length changed before this filter /// <summary> /// Construct a new <see cref="WordTokenFilter"/>. /// </summary> /// <param name="input"><see cref="TokenStream"/> of sentences.</param> public WordTokenFilter(TokenStream input) : base(input) { this.wordSegmenter = new WordSegmenter(); this.termAtt = AddAttribute <ICharTermAttribute>(); this.offsetAtt = AddAttribute <IOffsetAttribute>(); this.typeAtt = AddAttribute <ITypeAttribute>(); }