public InputWindowToken(ShingleFilter outerInstance, AttributeSource attSource) { this.outerInstance = outerInstance; this.attSource = attSource; this.termAtt = attSource.GetAttribute <ICharTermAttribute>(); this.offsetAtt = attSource.GetAttribute <IOffsetAttribute>(); }
public override TokenStream Create(TokenStream input) { ShingleFilter r = new ShingleFilter(input, minShingleSize, maxShingleSize); r.SetOutputUnigrams(outputUnigrams); r.SetOutputUnigramsIfNoShingles(outputUnigramsIfNoShingles); r.SetTokenSeparator(tokenSeparator); r.SetFillerToken(fillerToken); return(r); }
protected override TokenStreamComponents WrapComponents(string fieldName, TokenStreamComponents components) { ShingleFilter filter = new ShingleFilter(components.TokenStream, minShingleSize, maxShingleSize); filter.SetMinShingleSize(minShingleSize); filter.SetMaxShingleSize(maxShingleSize); filter.SetTokenSeparator(tokenSeparator); filter.SetOutputUnigrams(outputUnigrams); filter.SetOutputUnigramsIfNoShingles(outputUnigramsIfNoShingles); filter.SetFillerToken(fillerToken); return(new TokenStreamComponents(components.Tokenizer, filter)); }
public CircularSequence(ShingleFilter shingleFilter) { this.outerInstance = shingleFilter; minValue = shingleFilter.outputUnigrams ? 1 : shingleFilter.minShingleSize; Reset(); }
public CircularSequence(ShingleFilter outerInstance) { this.outerInstance = outerInstance; minValue = outerInstance.outputUnigrams ? 1 : outerInstance.minShingleSize; Reset(); }