public InputWindowToken(ShingleFilter outerInstance, AttributeSource attSource)
{
this.outerInstance = outerInstance;
this.attSource = attSource;
this.termAtt = attSource.GetAttribute <ICharTermAttribute>();
this.offsetAtt = attSource.GetAttribute <IOffsetAttribute>();
}
public override TokenStream Create(TokenStream input)
{
ShingleFilter r = new ShingleFilter(input, minShingleSize, maxShingleSize);
r.SetOutputUnigrams(outputUnigrams);
r.SetOutputUnigramsIfNoShingles(outputUnigramsIfNoShingles);
r.SetTokenSeparator(tokenSeparator);
r.SetFillerToken(fillerToken);
return(r);
}
protected override TokenStreamComponents WrapComponents(string fieldName, TokenStreamComponents components)
{
ShingleFilter filter = new ShingleFilter(components.TokenStream, minShingleSize, maxShingleSize);
filter.SetMinShingleSize(minShingleSize);
filter.SetMaxShingleSize(maxShingleSize);
filter.SetTokenSeparator(tokenSeparator);
filter.SetOutputUnigrams(outputUnigrams);
filter.SetOutputUnigramsIfNoShingles(outputUnigramsIfNoShingles);
filter.SetFillerToken(fillerToken);
return(new TokenStreamComponents(components.Tokenizer, filter));
}
public CircularSequence(ShingleFilter shingleFilter)
{
this.outerInstance = shingleFilter;
minValue = shingleFilter.outputUnigrams ? 1 : shingleFilter.minShingleSize;
Reset();
}
public CircularSequence(ShingleFilter outerInstance)
{
this.outerInstance = outerInstance;
minValue = outerInstance.outputUnigrams ? 1 : outerInstance.minShingleSize;
Reset();
}
