protected override TokenStreamComponents WrapComponents(string fieldName, TokenStreamComponents components) { ShingleFilter filter = new ShingleFilter(components.TokenStream, minShingleSize, maxShingleSize); filter.SetMinShingleSize(minShingleSize); filter.SetMaxShingleSize(maxShingleSize); filter.SetTokenSeparator(tokenSeparator); filter.SetOutputUnigrams(outputUnigrams); filter.SetOutputUnigramsIfNoShingles(outputUnigramsIfNoShingles); filter.SetFillerToken(fillerToken); return(new TokenStreamComponents(components.Tokenizer, filter)); }
public override TokenStream TokenStream(String fieldName, TextReader reader) { TokenStream wrapped; try { wrapped = defaultAnalyzer.ReusableTokenStream(fieldName, reader); } catch (IOException) { wrapped = defaultAnalyzer.TokenStream(fieldName, reader); } ShingleFilter filter = new ShingleFilter(wrapped); filter.SetMaxShingleSize(maxShingleSize); filter.SetOutputUnigrams(outputUnigrams); return(filter); }
public override TokenStream TokenStream(String fieldName, TextReader reader) { TokenStream wrapped; try { wrapped = defaultAnalyzer.ReusableTokenStream(fieldName, reader); } catch (IOException) { wrapped = defaultAnalyzer.TokenStream(fieldName, reader); } ShingleFilter filter = new ShingleFilter(wrapped); filter.SetMaxShingleSize(maxShingleSize); filter.SetOutputUnigrams(outputUnigrams); return filter; }