private bool hasIllegalOffsets; // only if the length changed before this filter
/// <summary>
/// Construct a new <see cref="WordTokenFilter"/>.
/// </summary>
/// <param name="input"><see cref="TokenStream"/> of sentences.</param>
public WordTokenFilter(TokenStream input)
: base(input)
{
this.wordSegmenter = new WordSegmenter();
this.termAtt = AddAttribute <ICharTermAttribute>();
this.offsetAtt = AddAttribute <IOffsetAttribute>();
this.typeAtt = AddAttribute <ITypeAttribute>();
}
