/// <summary>
/// 索引分词
///
/// </summary>
/// <param name="str"></param>
public void Segement_Index(string str = "中国科学院计算技术研究所的宗成庆教授正在教授自然语言处理课程")
{
var termList = IndexTokenizer.segment(str);
var docList = termList.ToList();
Console.WriteLine(docList);
}
public void IndexTokenizer()
{
var tokenizer = new IndexTokenizer();
var t1 = tokenizer.GetToken("t", null);
Assert.Equal(t1, "@1");
var t2 = tokenizer.GetToken("t", null);
Assert.Equal(t2, "@2");
}
