public void CustomClass()
{
MyTokenizer tokenizer = new MyTokenizer();
Token[] tokens = tokenizer.Tokenize("A.Bērziņš ").ToArray();
Assert.AreEqual("A", tokens[0].Text);
Assert.AreEqual(".", tokens[1].Text);
Assert.AreEqual("Bērziņš", tokens[2].Text);
Assert.AreEqual(" ", tokens[3].Text);
}
public void CustomClass()
{
MyTokenizer tokenizer = new MyTokenizer();
Token[] tokens = tokenizer.Tokenize("A.Bērziņš ").ToArray();
Assert.AreEqual("A", tokens[0].Text);
Assert.AreEqual(".", tokens[1].Text);
Assert.AreEqual("Bērziņš", tokens[2].Text);
Assert.AreEqual(" ", tokens[3].Text);
}
protected override TokenStreamComponents CreateComponents(string fieldName, TextReader reader)
{
const LuceneVersion version = LuceneVersion.LUCENE_48;
Tokenizer baseTokenizer = new MyTokenizer(version, reader);
StandardFilter standardFilter = new StandardFilter(version, baseTokenizer);
WordDelimiterFilter wordDelimiterFilter = new WordDelimiterFilter(version, standardFilter,
WordDelimiterFlags.CATENATE_WORDS | WordDelimiterFlags.GENERATE_WORD_PARTS |
WordDelimiterFlags.PRESERVE_ORIGINAL | WordDelimiterFlags.SPLIT_ON_CASE_CHANGE, CharArraySet.EMPTY_SET);
LowerCaseFilter lcFilter = new LowerCaseFilter(version, wordDelimiterFilter);
return(new TokenStreamComponents(baseTokenizer, lcFilter));
}
