public override TokenStreamComponents CreateComponents(string fieldName, TextReader reader)
{
Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
TrivialLookaheadFilter filter = new TrivialLookaheadFilter(source);
return(new TokenStreamComponents(source, filter));
}
public void TestMissedFirstToken()
{
Analyzer analyzer = Analyzer.NewAnonymous(createComponents: (fieldName, reader) =>
{
Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
TrivialLookaheadFilter filter = new TrivialLookaheadFilter(source);
return(new TokenStreamComponents(source, filter));
});
AssertAnalyzesTo(analyzer,
"Only he who is running knows .",
new String[] {
"Only",
"Only-huh?",
"he",
"he-huh?",
"who",
"who-huh?",
"is",
"is-huh?",
"running",
"running-huh?",
"knows",
"knows-huh?",
".",
".-huh?"
});
}