protected override TokenStreamComponents WrapComponents(string fieldName, TokenStreamComponents components)
{
ShingleFilter filter = new ShingleFilter(components.TokenStream, minShingleSize, maxShingleSize);
filter.SetMinShingleSize(minShingleSize);
filter.SetMaxShingleSize(maxShingleSize);
filter.SetTokenSeparator(tokenSeparator);
filter.SetOutputUnigrams(outputUnigrams);
filter.SetOutputUnigramsIfNoShingles(outputUnigramsIfNoShingles);
filter.SetFillerToken(fillerToken);
return(new TokenStreamComponents(components.Tokenizer, filter));
}
public override TokenStream TokenStream(String fieldName, TextReader reader)
{
TokenStream wrapped;
try
{
wrapped = defaultAnalyzer.ReusableTokenStream(fieldName, reader);
}
catch (IOException)
{
wrapped = defaultAnalyzer.TokenStream(fieldName, reader);
}
ShingleFilter filter = new ShingleFilter(wrapped);
filter.SetMaxShingleSize(maxShingleSize);
filter.SetOutputUnigrams(outputUnigrams);
return(filter);
}
public override TokenStream TokenStream(String fieldName, TextReader reader)
{
TokenStream wrapped;
try
{
wrapped = defaultAnalyzer.ReusableTokenStream(fieldName, reader);
}
catch (IOException)
{
wrapped = defaultAnalyzer.TokenStream(fieldName, reader);
}
ShingleFilter filter = new ShingleFilter(wrapped);
filter.SetMaxShingleSize(maxShingleSize);
filter.SetOutputUnigrams(outputUnigrams);
return filter;
}