public override TokenStream TokenStream(string fieldName, TextReader reader)
{
// Emits the entire input as a single token.
TokenStream source = new KeywordTokenizer(reader);
var map = new Dictionary<char, char> {{'-', ' '}};
// replaces specified characters from the token stream and lowercases the result
return new MapCharFilter(map, source);
}
public override TokenStream ReusableTokenStream(System.String fieldName, System.IO.TextReader reader)
{
Tokenizer tokenizer = (Tokenizer)GetPreviousTokenStream();
if (tokenizer == null)
{
tokenizer = new KeywordTokenizer(reader);
SetPreviousTokenStream(tokenizer);
}
else
tokenizer.Reset(reader);
return tokenizer;
}
public override TokenStream ReusableTokenStream(System.String fieldName, System.IO.TextReader reader)
{
Tokenizer tokenizer = (Tokenizer)GetPreviousTokenStream();
if (tokenizer == null)
{
tokenizer = new KeywordTokenizer(reader);
SetPreviousTokenStream(tokenizer);
}
else
{
tokenizer.Reset(reader);
}
return(tokenizer);
}
public override TokenStream ReusableTokenStream(System.String fieldName, System.IO.TextReader reader)
{
if (overridesTokenStreamMethod)
{
// LUCENE-1678: force fallback to tokenStream() if we
// have been subclassed and that subclass overrides
// tokenStream but not reusableTokenStream
return TokenStream(fieldName, reader);
}
Tokenizer tokenizer = (Tokenizer) GetPreviousTokenStream();
if (tokenizer == null)
{
tokenizer = new KeywordTokenizer(reader);
SetPreviousTokenStream(tokenizer);
}
else
tokenizer.Reset(reader);
return tokenizer;
}
public override TokenStream ReusableTokenStream(System.String fieldName, System.IO.TextReader reader)
{
if (overridesTokenStreamMethod)
{
// LUCENE-1678: force fallback to tokenStream() if we
// have been subclassed and that subclass overrides
// tokenStream but not reusableTokenStream
return(TokenStream(fieldName, reader));
}
Tokenizer tokenizer = (Tokenizer)GetPreviousTokenStream();
if (tokenizer == null)
{
tokenizer = new KeywordTokenizer(reader);
SetPreviousTokenStream(tokenizer);
}
else
{
tokenizer.Reset(reader);
}
return(tokenizer);
}
public override TokenStream TokenStream(string fieldName, TextReader reader)
{
TokenStream result = new KeywordTokenizer(reader);
return new CollationKeyFilter(result, cultureInfo);
}
public override TokenStream TokenStream(string fieldName, System.IO.TextReader reader)
{
TokenStream tokenStream = new KeywordTokenizer(reader);
tokenStream = new LowerCaseFilter(tokenStream);
return tokenStream;
}
public override TokenStream TokenStream(string fieldName, TextReader reader)
{
TokenStream source = new KeywordTokenizer(reader);
var map = new Dictionary<char, char> {{'-', ' '}};
return new MapCharFilter(map, source);
}
