public void TestJIEbaTokenizer()
{
var tokenizer = new JieBaTokenizer(TextReader.Null, TokenizerMode.Default);
Assert.NotEmpty(tokenizer.StopWords);
Assert.True(tokenizer.StopWords.ContainsKey("是"));
Assert.True(tokenizer.StopWords.ContainsKey("什么"));
}
/// <summary>
///
/// </summary>
/// <param name="fieldName"></param>
/// <param name="reader"></param>
/// <returns></returns>
protected override TokenStreamComponents CreateComponents(string fieldName, TextReader reader)
{
var token = new JieBaTokenizer(reader, Mode);
var tokenStream = new LowerCaseFilter(Lucene.Net.Util.LuceneVersion.LUCENE_48, token);
tokenStream.AddAttribute <ICharTermAttribute>();
tokenStream.AddAttribute <IOffsetAttribute>();
return(new TokenStreamComponents(token, tokenStream));
}
public void JieBaLuceneTokenStreamAdapterTest()
{
string text = "SQLiteFtsTokenizer 只有在 PrepareToStart 方法时才能取得要切分的字符串。";
string text2 = "有时候我们需要同时执行一些操作，然后把这些操作的结果进行汇总，以达到用异步处理降低操作耗时的效果，此时我们会考虑使用Task，而Task.WhenAll则排上了用场。这样当我们在主程序中用到Person类的SayHello方法的时候程序就会报错。当然，如果第二个参数设置为false时，在调用该成员的时候不会报错，但是会报出警告。微信sqlite本地全文索引搜索是怎么做的呢？安卓平台不支持中文的tokenizer？";
LuceneTokenStreamAdapter adapter = new LuceneTokenStreamAdapter();
adapter.TokenStart += delegate
{
Tokenizer tokenizer = new JieBaTokenizer(new StringReader(adapter.InputString), TokenizerMode.Search);
tokenizer.Reset();
adapter.SetTokenStream(tokenizer);
};
var result = adapter.TestMe(text);
var result2 = adapter.TestMe(text2);
Assert.Pass();
}