public void CutDemo()
{
var segmenter = new JiebaSegmenter();
var segments = segmenter.Cut("我来到北京清华大学", cutAll: true);
Console.WriteLine("【全模式】：{0}", string.Join("/ ", segments));
segments = segmenter.Cut("我来到北京清华大学"); // 默认为精确模式
Console.WriteLine("【精确模式】：{0}", string.Join("/ ", segments));
segments = segmenter.Cut("他来到了网易杭研大厦"); // 默认为精确模式，同时也使用HMM模型
Console.WriteLine("【新词识别】：{0}", string.Join("/ ", segments));
segments = segmenter.CutForSearch("小明硕士毕业于中国科学院计算所，后在日本京都大学深造"); // 搜索引擎模式
Console.WriteLine("【搜索引擎模式】：{0}", string.Join("/ ", segments));
segments = segmenter.Cut("结过婚的和尚未结过婚的");
Console.WriteLine("【歧义消除】：{0}", string.Join("/ ", segments));
segments = segmenter.Cut("北京大学生喝进口红酒");
Console.WriteLine("【歧义消除】：{0}", string.Join("/ ", segments));
segments = segmenter.Cut("在北京大学生活区喝进口红酒");
Console.WriteLine("【歧义消除】：{0}", string.Join("/ ", segments));
segments = segmenter.Cut("腾讯视频致力于打造中国最大的在线视频媒体平台,以丰富的内容、极致的观看体验");
Console.WriteLine("【精确模式】：{0}", string.Join("/ ", segments));
segmenter.DeleteWord("湖南");
segmenter.AddWord("湖南");
//segmenter.AddWord("长沙市");
segments = segmenter.Cut("湖南长沙市天心区");
Console.WriteLine("【精确模式】：{0}", string.Join("/ ", segments));
}
public void CutDemo()
{
var segmenter = new JiebaSegmenter();
var segments = segmenter.Cut("我来到北京清华大学", cutAll: true);
Console.WriteLine("【全模式】：{0}", string.Join("/ ", segments));
segments = segmenter.Cut("我来到北京清华大学"); // 默认为精确模式
Console.WriteLine("【精确模式】：{0}", string.Join("/ ", segments));
segments = segmenter.Cut("他来到了网易杭研大厦"); // 默认为精确模式，同时也使用HMM模型
Console.WriteLine("【新词识别】：{0}", string.Join("/ ", segments));
segments = segmenter.CutForSearch("小明硕士毕业于中国科学院计算所，后在日本京都大学深造"); // 搜索引擎模式
Console.WriteLine("【搜索引擎模式】：{0}", string.Join("/ ", segments));
segments = segmenter.Cut("结过婚的和尚未结过婚的");
Console.WriteLine("【歧义消除】：{0}", string.Join("/ ", segments));
segments = segmenter.Cut("北京大学生喝进口红酒");
Console.WriteLine("【歧义消除】：{0}", string.Join("/ ", segments));
segments = segmenter.Cut("在北京大学生活区喝进口红酒");
Console.WriteLine("【歧义消除】：{0}", string.Join("/ ", segments));
segments = segmenter.Cut("腾讯视频致力于打造中国最大的在线视频媒体平台,以丰富的内容、极致的观看体验");
Console.WriteLine("【精确模式】：{0}", string.Join("/ ", segments));
segmenter.DeleteWord("湖南");
segmenter.AddWord("湖南");
//segmenter.AddWord("长沙市");
segments = segmenter.Cut("湖南长沙市天心区");
Console.WriteLine("【精确模式】：{0}", string.Join("/ ", segments));
}
public void TestIssue46()
{
var seg = new JiebaSegmenter();
seg.DeleteWord("天半");
var segments = seg.CutAll("2天半").ToList();
Assert.That(segments, Contains.Item("天"));
Assert.That(segments, Contains.Item("半"));
}
public void TestAddWord()
{
var seg = new JiebaSegmenter();
var s = "小明最近在学习机器学习和自然语言处理";
var segments = seg.Cut(s);
Assert.That(segments, Contains.Item("机器"));
Assert.That(segments, Contains.Item("学习"));
seg.AddWord("机器学习");
segments = seg.Cut(s);
Assert.That(segments, Contains.Item("机器学习"));
Assert.That(segments, Is.Not.Contains("机器"));
// reset dict otherwise other test cases would be affected.
seg.DeleteWord("机器学习");
}
public void TestDeleteWord()
{
var seg = new JiebaSegmenter();
var s = "小明最近在学习机器学习和自然语言处理";
var segments = seg.Cut(s);
Assert.That(segments, Contains.Item("机器"));
Assert.That(segments, Is.Not.Contains("机器学习"));
seg.AddWord("机器学习");
segments = seg.Cut(s);
Assert.That(segments, Contains.Item("机器学习"));
Assert.That(segments, Is.Not.Contains("机器"));
seg.DeleteWord("机器学习");
segments = seg.Cut(s);
Assert.That(segments, Contains.Item("机器"));
Assert.That(segments, Is.Not.Contains("机器学习"));
}
public void TestAddWord()
{
var seg = new JiebaSegmenter();
var posSeg = new PosSegmenter(seg);
var tokens = posSeg.Cut("小明最近在学习自然语言处理").ToList();
var result = string.Join(" ", tokens.Select(token => $"{token.Word}/{token.Flag}"));
Console.WriteLine(result);
var lastToken = tokens.Last();
Assert.That(lastToken.Word, Is.EqualTo("处理"));
seg.AddWord("自然语言处理", tag: "n");
tokens = posSeg.Cut("小明最近在学习自然语言处理").ToList();
result = string.Join(" ", tokens.Select(token => $"{token.Word}/{token.Flag}"));
Console.WriteLine(result);
lastToken = tokens.Last();
Assert.That(lastToken.Word, Is.EqualTo("自然语言处理"));
Assert.That(lastToken.Flag, Is.EqualTo("n"));
seg.DeleteWord("自然语言处理");
}
protected void Page_Load(object sender, EventArgs e)
{
var segmenter = new JiebaSegmenter();
segmenter.DeleteWord("将军");
segmenter.DeleteWord("却说");
segmenter.DeleteWord("二人");
segmenter.DeleteWord("司马");
segmenter.DeleteWord("不可");
segmenter.DeleteWord("不能");
segmenter.DeleteWord("夏侯");
segmenter.DeleteWord("如此");
segmenter.DeleteWord("诸葛");
segmenter.DeleteWord("商议");
segmenter.DeleteWord("如何");
segmenter.DeleteWord("大喜");
segmenter.DeleteWord("军士");
segmenter.DeleteWord("左右");
segmenter.DeleteWord("引兵");
segmenter.DeleteWord("夫人");
string aimFile = @"./Resources/三国演义.txt";
string content = ReadData(aimFile);
Stopwatch sw = new Stopwatch();
sw.Start();
//搜索引擎模式分词
var wordsforSearch = segmenter.CutForSearch(content);
//定义数据结构persons ，放置人名和词频
Dictionary <string, int> persons = new Dictionary <string, int>();
//将要以JSON格式输出的字符串，将其写到JSON文件中，就可以实现，词云图
string jsonstr = "[";
int i = 0;
foreach (string item in wordsforSearch.Distinct <string>())
{
//对长度大于等于2并且小于等于4的词进行统计
if (item.Length >= 2 && item.Length <= 4)
{
if (!persons.ContainsKey(item))
{
int f = GetFrequence(wordsforSearch, item); //统计词频
persons.Add(item.Trim(), f);
if (f >= 100 && f != 2406) //出于测试需要只对频率100以上的关键词，制作词云
{
//第一个前不用加逗号，目的是构造一个A,B,C,D,.....，注意除了A之外，每一个字母前都有逗号
if (i == 0)
{
jsonstr += "{\"name\":\"" + item.Trim() + "\",\"value\":" + f + "}";
}
else
{
jsonstr += ",{\"name\":\"" + item.Trim() + "\",\"value\":" + f + "}";
}
i++;
}
}
}
}
jsonstr += "]";
WriteData("test.json", jsonstr);
persons = (from entry in persons
orderby entry.Value descending
select entry).ToDictionary(pair => pair.Key, pair => pair.Value);
string result = "";
foreach (var person in persons)
{
if (person.Value >= 100)
{
result += ("<br>" + person.Key + "-" + person.Value.ToString());
}
}
Response.Write(result);
sw.Stop();
TimeSpan ts2 = sw.Elapsed;
Response.Write("</br>Stopwatch总共花费{0}ms." + ts2.TotalMilliseconds.ToString());
}
static void Main(string[] args)
{
//var analyzer = new JieBaAnalyzer(TokenizerMode.Default);
var IndexWriterConfig = new IndexWriterConfig(LuceneVersion.LUCENE_48, new JieBaAnalyzer(TokenizerMode.Default));
var directory = FSDirectory.Open(new DirectoryInfo(AppDomain.CurrentDomain.BaseDirectory + "Lucene"));
var indexWriter = new IndexWriter(directory, IndexWriterConfig);
var document = new Document();
var fieldList = new List <Field>();
//var test = new StringField("id", "22", Field.Store.YES);
//var test = new StringField("id", "22", Field.Store.YES);
var fieldType = new FieldType();
//var newFeild = new Field("id", "22", Field.Store.YES, Field.Index.ANALYZED);
//var newFeild2 = new Field("soc", "呵呵", Field.Store.YES, Field.Index.ANALYZED);
//var newFeild3 = new Field("shot", "内容分类标准以及为读者提供的任何信息", Field.Store.YES, Field.Index.ANALYZED);
//var newFeild4 = new Field("content", "《人民日报》（电子版）的一切内容(包括但不限于文字、图片、PDF、图表、标志、标识、商标、版面设计、专栏目录与名称、内容分类标准以及为读者提供的任何信息)仅供人民网读者阅读、学习研究使用，未经人民网股份有限公司及/或相关权利人书面授权，任何单位及个人不得将《人民日报》（电子版）所登载、发布的内容用于商业性目的，包括但不限于转载、复制、发行、制作光盘、数据库、触摸展示等行为方式，或将之在非本站所属的服务器上作镜像。否则，人民网股份有限公司将采取包括但不限于网上公示、向有关部门举报、诉讼等一切合法手段，追究侵权者的法律责任。《人民日报》（电子版）的一切内容(包括但不限于文字、图片、PDF、图表、标志、标识、商标、版面设计、专栏目录与名称、内容分类标准以及为读者提供的任何信息)仅供人民网读者阅读、学习研究使用，未经人民网股份有限公司及/或相关权利人书面授权，任何单位及个人不得将《人民日报》（电子版）所登载、发布的内容用于商业性目的，包括但不限于转载、复制、发行、制作光盘、数据库、触摸展示等行为方式，或将之在非本站所属的服务器上作镜像。否则，人民网股份有限公司将采取包括但不限于网上公示、向有关部门举报、诉讼等一切合法手段，追究侵权者的法律责任。", Field.Store.YES, Field.Index.ANALYZED);
//fieldList.Add(newFeild);
//fieldList.Add(newFeild2);
//fieldList.Add(newFeild3);
//fieldList.Add(newFeild4);
fieldList.Add(new TextField("id", "22", Field.Store.YES));
fieldList.Add(new TextField("soc", "呵呵", Field.Store.YES));
fieldList.Add(new TextField("shot", "内容分类标准以及为读者提供的任何信息", Field.Store.YES));
fieldList.Add(new TextField("content", "《人民日报》（电子版）的一切内容(包括但不限于文字、图片、PDF、图表、标志、标识、商标、版面设计、专栏目录与名称、内容分类标准以及为读者提供的任何信息)仅供人民网读者阅读、学习研究使用，未经人民网股份有限公司及/或相关权利人书面授权，任何单位及个人不得将《人民日报》（电子版）所登载、发布的内容用于商业性目的，包括但不限于转载、复制、发行、制作光盘、数据库、触摸展示等行为方式，或将之在非本站所属的服务器上作镜像。否则，人民网股份有限公司将采取包括但不限于网上公示、向有关部门举报、诉讼等一切合法手段，追究侵权者的法律责任。《人民日报》（电子版）的一切内容(包括但不限于文字、图片、PDF、图表、标志、标识、商标、版面设计、专栏目录与名称、内容分类标准以及为读者提供的任何信息)仅供人民网读者阅读、学习研究使用，未经人民网股份有限公司及/或相关权利人书面授权，任何单位及个人不得将《人民日报》（电子版）所登载、发布的内容用于商业性目的，包括但不限于转载、复制、发行、制作光盘、数据库、触摸展示等行为方式，或将之在非本站所属的服务器上作镜像。否则，人民网股份有限公司将采取包括但不限于网上公示、向有关部门举报、诉讼等一切合法手段，追究侵权者的法律责任。", Field.Store.YES));
indexWriter.AddDocument(fieldList);
indexWriter.Commit();
while (true)
{
// 1、创建Directory
//var directory = FSDirectory.Open(FileSystems.getDefault().getPath(INDEX_PATH));
// 2、创建IndexReader
var directoryReader = DirectoryReader.Open(directory);
// 3、根据IndexReader创建IndexSearch
IndexSearcher indexSearcher = new IndexSearcher(directoryReader);
var queryK = Console.ReadLine();
// MultiFieldQueryParser表示多个域解析， 同时可以解析含空格的字符串，如果我们搜索"上海 中国"
var analyzer = new JieBaAnalyzer(TokenizerMode.Search);
String[] fields = { "soc", "content" };
Occur[] clauses = { Occur.SHOULD, Occur.SHOULD };
Query multiFieldQuery = MultiFieldQueryParser.Parse(LuceneVersion.LUCENE_48, queryK, fields, clauses, analyzer);
var bb = new Lucene.Net.Search.TermQuery(new Term("shot", queryK));
var fuzzy = new FuzzyQuery(new Term("content", queryK));
// 5、根据searcher搜索并且返回TopDocs
TopDocs topDocs = indexSearcher.Search(fuzzy, 100); // 搜索前100条结果
Console.WriteLine("找到: " + topDocs.TotalHits);
QueryScorer scorer = new QueryScorer(fuzzy, "content");
// 自定义高亮代码
SimpleHTMLFormatter htmlFormatter = new SimpleHTMLFormatter("<span style=\"backgroud:red\">", "</span>");
Highlighter highlighter = new Highlighter(htmlFormatter, scorer);
//highlighter.set(new SimpleSpanFragmenter(scorer));
foreach (var doc in topDocs.ScoreDocs)
{
var returnDoc = indexSearcher.Doc(doc.Doc);
//Console.WriteLine("soc : " + returnDoc.Get("soc"));
var resultHiligh = highlighter.GetBestFragments(analyzer, "content", returnDoc.Get("content"), 3);
Console.WriteLine(string.Join("", resultHiligh));
}
//Console.WriteLine("go... press enter ");
//Console.ReadLine();
}
// valindexConfig: IndexWriterConfig = new IndexWriterConfig(new StandardAnalyzer());
// indexConfig.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND)
//// indexConfig.setInfoStream(System.out)
// val directory:Directory = FSDirectory.open(Paths.get(indexPath))
//val indexWriter:IndexWriter = new IndexWriter(directory, indexConfig)
var segmenter = new JiebaSegmenter();
var segments = segmenter.Cut("我来到北京清华大学", cutAll: true);
Console.WriteLine("【全模式】：{0}", string.Join("/ ", segments));
segments = segmenter.Cut("我来到北京清华大学"); // 默认为精确模式
Console.WriteLine("【精确模式】：{0}", string.Join("/ ", segments));
segments = segmenter.Cut("他来到了网易杭研大厦"); // 默认为精确模式，同时也使用HMM模型
Console.WriteLine("【新词识别】：{0}", string.Join("/ ", segments));
segments = segmenter.CutForSearch("小明硕士毕业于中国科学院计算所，后在日本京都大学深造"); // 搜索引擎模式
Console.WriteLine("【搜索引擎模式】：{0}", string.Join("/ ", segments));
segments = segmenter.Cut("结过婚的和尚未结过婚的");
Console.WriteLine("【歧义消除】：{0}", string.Join("/ ", segments));
segments = segmenter.Cut("北京大学生喝进口红酒");
Console.WriteLine("【歧义消除】：{0}", string.Join("/ ", segments));
segments = segmenter.Cut("在北京大学生活区喝进口红酒");
Console.WriteLine("【歧义消除】：{0}", string.Join("/ ", segments));
segments = segmenter.Cut("腾讯视频致力于打造中国最大的在线视频媒体平台,以丰富的内容、极致的观看体验");
Console.WriteLine("【精确模式】：{0}", string.Join("/ ", segments));
segmenter.DeleteWord("湖南");
segmenter.AddWord("湖南");
//segmenter.AddWord("长沙市");
segments = segmenter.Cut("湖南长沙市天心区");
Console.WriteLine("【精确模式】：{0}", string.Join("/ ", segments));
Console.Read();
}
public void DeleteWord(string word)
{
segmenter.DeleteWord(word);
}
public void AnlayzeEntitySurroundWords(AnnouceDocument doc, string KeyWord)
{
//Program.Training.WriteLine("关键字：[" + KeyWord + "]");
JiebaSegmenter segmenter = new JiebaSegmenter();
segmenter.AddWord(KeyWord);
PosSegmenter posSeg = new PosSegmenter(segmenter);
foreach (var paragrah in doc.root.Children)
{
foreach (var sentence in paragrah.Children)
{
var segments = posSeg.Cut(sentence.Content).ToList(); // 默认为精确模式,寻找关键字的位置
for (int i = 0; i < segments.Count; i++)
{
if (segments[i].Word.Equals(KeyWord))
{
//前5个词语和后五个词语
var startInx = Math.Max(0, i - 5);
var EndInx = Math.Min(i + 5, segments.Count);
for (int s = startInx; s < i; s++)
{
if (segments[s].Flag == LTPTrainingNER.词性标点 && segments[s].Word != "：")
{
continue;
}
if (LeadingWordDict.ContainsKey(segments[s].Word))
{
LeadingWordDict[segments[s].Word]++;
}
else
{
LeadingWordDict.Add(segments[s].Word, 1);
}
//Program.Training.WriteLine("前导关键字：[" + segments[s] + "]");
//特别关注动词和冒号的情况
if (segments[s].Flag == LTPTrainingNER.动词)
{
if (LeadingVerbWordDict.ContainsKey(segments[s].Word))
{
LeadingVerbWordDict[segments[s].Word]++;
}
else
{
LeadingVerbWordDict.Add(segments[s].Word, 1);
}
//Program.Training.WriteLine("前导动词:" + segments[s].Word);
}
}
//Program.Training.WriteLine("关键字：[" + KeyWord + "]");
for (int s = i + 1; s < EndInx; s++)
{
if (segments[s].Flag == LTPTrainingNER.词性标点)
{
continue;
}
if (TrailingWordDict.ContainsKey(segments[s].Word))
{
TrailingWordDict[segments[s].Word]++;
}
else
{
TrailingWordDict.Add(segments[s].Word, 1);
}
//Program.Training.WriteLine("后续关键字：[" + segments[s] + "]");
}
break; //仅统计第一次出现
}
}
}
}
segmenter.DeleteWord(KeyWord);
}
protected void Page_Load(object sender, EventArgs e)
{
var segmenter = new JiebaSegmenter();
segmenter.DeleteWord("将军");
segmenter.DeleteWord("却说");
segmenter.DeleteWord("二人");
segmenter.DeleteWord("玄德曰");
segmenter.DeleteWord("孔明曰");
segmenter.DeleteWord("司马");
segmenter.DeleteWord("不可");
segmenter.DeleteWord("不能");
segmenter.DeleteWord("夏侯");
segmenter.DeleteWord("如此");
segmenter.DeleteWord("诸葛");
segmenter.DeleteWord("商议");
segmenter.DeleteWord("如何");
segmenter.DeleteWord("大喜");
segmenter.DeleteWord("军士");
segmenter.DeleteWord("左右");
segmenter.DeleteWord("引兵");
segmenter.DeleteWord("夫人");
string aimFile = @"./Resources/三国演义.txt";
string content = ReadData(aimFile);
Stopwatch sw = new Stopwatch();
sw.Start();
//搜索引擎模式分词
//var wordsforSearch = segmenter.CutForSearch(content);
//精确模式
var wordsforSearch = segmenter.Cut(content);
List <KV> list = new List <KV>();
//定义数据结构persons ，放置人名和词频
Dictionary <string, int> persons = new Dictionary <string, int>();
foreach (string item in wordsforSearch.Distinct <string>())
{
//对长度大于等于2并且小于等于4的词进行统计
if (item.Length >= 2 && item.Length <= 4)
{
if (!persons.ContainsKey(item))
{
int f = GetFrequence(wordsforSearch, item); //统计词频
persons.Add(item.Trim(), f);
if (f >= 100 && f != 2406) //出于测试需要只对频率100以上的关键词，制作词云
{
KV kv = new KV(item.Trim(), f);
list.Add(kv);
}
}
}
}
string output = JsonConvert.SerializeObject(list);
//将要以JSON格式输出的字符串，将其写到JSON文件中，就可以实现，词云图
WriteData("test.json", output);
persons = (from entry in persons
orderby entry.Value descending
select entry).ToDictionary(pair => pair.Key, pair => pair.Value);
string result = "";
foreach (var person in persons)
{
if (person.Value >= 100)
{
result += ("<br>" + person.Key + "-" + person.Value.ToString());
}
}
Response.Write(result);
sw.Stop();
TimeSpan ts2 = sw.Elapsed;
Response.Write("</br>Stopwatch总共花费{0}ms." + ts2.TotalMilliseconds.ToString());
}