public override TokenStream TokenStream(string fieldName, System.IO.TextReader reader)
{
var seg = new JiebaSegmenter();
TokenStream result = new JiebaTokenizer(seg, reader);
result = new LowerCaseFilter(result);
result = new StopFilter(true, result, StopWords);
return result;
}
public override TokenStream TokenStream(string fieldName, System.IO.TextReader reader)
{
var seg = new JiebaSegmenter();
TokenStream result = new JiebaTokenizer(seg, reader);
result = new LowerCaseFilter(result);
result = new StopFilter(true, result, StopWords);
return(result);
}
private static string GetKeyWordsSplitBySpace(string keywords, JiebaTokenizer tokenizer)
{
var result = new StringBuilder();
var words = tokenizer.Tokenize(keywords);
foreach (var word in words)
{
if (string.IsNullOrWhiteSpace(word.Word))
{
continue;
}
result.AppendFormat("{0} ", word.Word);
}
return(result.ToString().Trim());
}
private static string GetKeyWordsSplitBySpace(string keywords, JiebaTokenizer tokenizer)
{
var result = new StringBuilder();
var words = tokenizer.Tokenize(keywords);
foreach (var word in words)
{
if (string.IsNullOrWhiteSpace(word.Word))
{
continue;
}
result.AppendFormat("{0} ", word.Word);
}
return result.ToString().Trim();
}
