public RemoveDuplicatesTokenFilter(TokenStream in_Renamed) : base(in_Renamed) { termAtt = AddAttribute <ITermAttribute>(); posIncrAtt = AddAttribute <IPositionIncrementAttribute>(); termOff = AddAttribute <IOffsetAttribute>(); tokenList = null; }
public PartKeywordTokenizer(System.IO.TextReader input) : base(input) { done = false; sb = new StringBuilder(); termAtt = AddAttribute <ITermAttribute>(); offsetAtt = AddAttribute <IOffsetAttribute>(); ClearAttributes(); }
/** Constructs a filter which tokenizes words from the input stream. * @param input The token stream from a tokenizer */ public WhitespaceFilter(TokenStream input) : base(input) { seen = new HashSet <string>(); termAtt = AddAttribute <ITermAttribute>(); offsetAtt = AddAttribute <IOffsetAttribute>(); posIncrAtt = AddAttribute <IPositionIncrementAttribute>(); ClearAttributes(); }
public SpecialNoneWhiteSpaceFilter(TokenStream in_Renamed) : base(in_Renamed) { noneWhiteSpaceChars = CDRWhitespaceTokenizer.NoneWhiteSpaceChars; savedTerms = new List <SavedTerm>(); termAtt = AddAttribute <ITermAttribute>(); posIncrAtt = AddAttribute <IPositionIncrementAttribute>(); termOff = AddAttribute <IOffsetAttribute>(); }