public void TestCutLargeFile()
{
var weiCheng = File.ReadAllText(@"Resources\围城.txt");
var seg = new JiebaSegmenter();
seg.Cut("热身");
Console.WriteLine("Start to cut");
var n = 20;
var stopWatch = new Stopwatch();
// Accurate mode
stopWatch.Start();
for (var i = 0; i < n; i++)
{
seg.Cut(weiCheng);
}
stopWatch.Stop();
Console.WriteLine("Accurate mode: {0} ms", stopWatch.ElapsedMilliseconds / n);
// Full mode
stopWatch.Reset();
stopWatch.Start();
for (var i = 0; i < n; i++)
{
seg.Cut(weiCheng, true);
}
stopWatch.Stop();
Console.WriteLine("Full mode: {0} ms", stopWatch.ElapsedMilliseconds / n);
}
public void CutDemo()
{
var segmenter = new JiebaSegmenter();
var segments = segmenter.Cut("我来到北京清华大学", cutAll: true);
Console.WriteLine("【全模式】：{0}", string.Join("/ ", segments));
segments = segmenter.Cut("我来到北京清华大学"); // 默认为精确模式
Console.WriteLine("【精确模式】：{0}", string.Join("/ ", segments));
segments = segmenter.Cut("他来到了网易杭研大厦"); // 默认为精确模式，同时也使用HMM模型
Console.WriteLine("【新词识别】：{0}", string.Join("/ ", segments));
segments = segmenter.CutForSearch("小明硕士毕业于中国科学院计算所，后在日本京都大学深造"); // 搜索引擎模式
Console.WriteLine("【搜索引擎模式】：{0}", string.Join("/ ", segments));
segments = segmenter.Cut("结过婚的和尚未结过婚的");
Console.WriteLine("【歧义消除】：{0}", string.Join("/ ", segments));
segments = segmenter.Cut("北京大学生喝进口红酒");
Console.WriteLine("【歧义消除】：{0}", string.Join("/ ", segments));
segments = segmenter.Cut("在北京大学生活区喝进口红酒");
Console.WriteLine("【歧义消除】：{0}", string.Join("/ ", segments));
segments = segmenter.Cut("腾讯视频致力于打造中国最大的在线视频媒体平台,以丰富的内容、极致的观看体验");
Console.WriteLine("【精确模式】：{0}", string.Join("/ ", segments));
segmenter.DeleteWord("湖南");
segmenter.AddWord("湖南");
//segmenter.AddWord("长沙市");
segments = segmenter.Cut("湖南长沙市天心区");
Console.WriteLine("【精确模式】：{0}", string.Join("/ ", segments));
}
public void CutDemo()
{
var segmenter = new JiebaSegmenter();
var segments = segmenter.Cut("我来到北京清华大学", cutAll: true);
Console.WriteLine("【全模式】：{0}", string.Join("/ ", segments));
segments = segmenter.Cut("我来到北京清华大学"); // 默认为精确模式
Console.WriteLine("【精确模式】：{0}", string.Join("/ ", segments));
segments = segmenter.Cut("他来到了网易杭研大厦"); // 默认为精确模式，同时也使用HMM模型
Console.WriteLine("【新词识别】：{0}", string.Join("/ ", segments));
segments = segmenter.CutForSearch("小明硕士毕业于中国科学院计算所，后在日本京都大学深造"); // 搜索引擎模式
Console.WriteLine("【搜索引擎模式】：{0}", string.Join("/ ", segments));
segments = segmenter.Cut("结过婚的和尚未结过婚的");
Console.WriteLine("【歧义消除】：{0}", string.Join("/ ", segments));
}
public void TestCutLargeFile()
{
var fileName = @"Resources\围城.txt";
var weiCheng = File.ReadAllText(fileName);
var fileSize = (new FileInfo(fileName)).Length;
var seg = new JiebaSegmenter();
seg.Cut("热身一下");
Console.WriteLine("Start to cut");
const int n = 20;
var stopWatch = new Stopwatch();
// Accurate mode
stopWatch.Start();
for (var i = 0; i < n; i++)
{
seg.Cut(weiCheng);
}
stopWatch.Stop();
var timeConsumed = (double)stopWatch.ElapsedMilliseconds / (1000 * n);
Console.WriteLine("Accurate mode: {0} ms, average: {1} / second",
timeConsumed, fileSize / timeConsumed);
// Full mode
stopWatch.Reset();
stopWatch.Start();
for (var i = 0; i < n; i++)
{
seg.Cut(weiCheng, true);
}
stopWatch.Stop();
timeConsumed = (double)stopWatch.ElapsedMilliseconds / (1000 * n);
Console.WriteLine("Full mode: {0} ms, average: {1} / second",
timeConsumed, fileSize / timeConsumed);
}
static void Main(string[] args)
{
var segmenter = new JiebaSegmenter();
segmenter.LoadUserDict("userdict.txt");
var segments = segmenter.Cut("我来到北京清华大学", cutAll: true);
Console.WriteLine("【全模式】：{0}", string.Join("/ ", segments));
segments = segmenter.Cut("我来到北京清华大学"); // 默认为精确模式
Console.WriteLine("【精确模式】：{0}", string.Join("/ ", segments));
segments = segmenter.Cut("他来到了网易杭研大厦"); // 默认为精确模式，同时也使用HMM模型
Console.WriteLine("【新词识别】：{0}", string.Join("/ ", segments));
segments = segmenter.CutForSearch("小明硕士毕业于中国科学院计算所，后在日本京都大学深造"); // 搜索引擎模式
Console.WriteLine("【搜索引擎模式】：{0}", string.Join("/ ", segments));
segments = segmenter.Cut("结过婚的和尚未结过婚的");
Console.WriteLine("【歧义消除】：{0}", string.Join("/ ", segments));
segments = segmenter.Cut("linezerodemo机器学习学习机器");
Console.WriteLine("【用户字典】：{0}", string.Join("/ ", segments));
//词频统计
var s = "此领域探讨如何处理及运用自然语言。自然语言生成系统把计算机数据转化为自然语言。自然语言理解系统把自然语言转化为计算机程序更易于处理的形式。";
var freqs = new Counter <string>(segmenter.Cut(s));
foreach (var pair in freqs.MostCommon(5))
{
Console.WriteLine($"{pair.Key}: {pair.Value}");
}
Console.ReadKey();
}
public void TestCutAllSpecialWords()
{
// TODO: Enable this test case after confirming with jieba py.
var seg = new JiebaSegmenter();
seg.AddWord(".NET");
seg.AddWord("U.S.A.");
seg.AddWord("Steve Jobs");
seg.AddWord("Mac OS X");
var s = ".NET平台是微软推出的, U.S.A.是美国的简写";
var segments = seg.Cut(s);
Console.WriteLine("Cut: {0}", string.Join("/ ", segments));
segments = seg.Cut(s, cutAll: true);
Console.WriteLine("Cut All: {0}", string.Join("/ ", segments));
s = "Steve Jobs重新定义了手机";
segments = seg.Cut(s);
Console.WriteLine("Cut: {0}", string.Join("/ ", segments));
segments = seg.Cut(s, cutAll: true);
Console.WriteLine("Cut All: {0}", string.Join("/ ", segments));
s = "我们所熟悉的一个版本是Mac OS X 10.11 EI Capitan，在2015年推出。";
segments = seg.Cut(s);
Console.WriteLine("Cut: {0}", string.Join("/ ", segments));
segments = seg.Cut(s, cutAll: true);
Console.WriteLine("Cut All: {0}", string.Join("/ ", segments));
}
public void TestChineseDot()
{
var seg = new JiebaSegmenter();
seg.AddWord("艾尔肯·吐尼亚孜");
seg.AddWord("短P-R间期");
var s = "艾尔肯·吐尼亚孜新疆阿克苏人。 在短P-R间期。";
var segments = seg.Cut(s).ToList();
Assert.That(segments, Contains.Item("艾尔肯·吐尼亚孜"));
Assert.That(segments, Contains.Item("短P-R间期"));
}
static string GetMostCommon(string html)
{
var result = "<h2>词频统计</h2>";
var seg = new JiebaSegmenter();
var freqs = new Counter <string>(seg.Cut(html));
foreach (var item in freqs.MostCommon(100))
{
result = result + $"{item.Key}:{item.Value} <br/>";
}
return(result);
}
static void Main(string[] args)
{
WebGetter wg = new WebGetter(@"https://tw.news.yahoo.com/most-popular");
wg.setMethod("GET");
string html = wg.webReader();
Console.WriteLine(html);
WebDecoder wd = new WebDecoder();
wd.setRule(@"//ul[@id='stream-container-scroll-template']/li/div/div/div/div/div/img");
List <string> list = wd.htmlDecode(html);
int i = 0;
var segmenter = new JiebaSegmenter();
//segmenter.LoadUserDict(@"myDic.txt");
//segmenter.AddWord("陳菊",3,"nr");
//segmenter.AddWord("後果", 3, "n");
//segmenter.AddWord("高雄", 3);
//segmenter.AddWord("陳致中",3,"nr");
//segmenter.AddWord("這件事", 3);
//segmenter.AddWord("身材照",3,"n");
//segmenter.AddWord("道盡",3);
segmenter.AddWord("韓國瑜", 3, "nr");
segmenter.AddWord("台灣人", 3, "n");
List <string> words = new List <string>();
//segmenter.AddWord("市長", 3, "n");
StreamWriter sw = new StreamWriter("test.txt");
//var segments;
foreach (var tmp in list)
{
Console.WriteLine(i + ": " + tmp);
sw.WriteLine(i + ": " + tmp);
var segments = segmenter.Cut(tmp);
//foreach(var tmp2 in segments)
//{
// Console.WriteLine("\t" + tmp2);
// sw.WriteLine("\t" + tmp2);
// words.Add(tmp2);
//}
i++;
}
sw.Close();
words.Sort();
//foreach(var tmp in words)
//{
// Console.WriteLine(tmp);
//}
Console.ReadKey(true);
}
public void TestSpecialWords()
{
var seg = new JiebaSegmenter();
seg.AddWord(".NET");
seg.AddWord("U.S.A.");
var s = ".NET平台是微软推出的, U.S.A.是美国的简写";
var segments = seg.Cut(s);
foreach (var segment in segments)
{
Console.WriteLine(segment);
}
s = "Steve Jobs重新定义了手机";
segments = seg.Cut(s);
foreach (var segment in segments)
{
Console.WriteLine(segment);
}
}
public List <Token> Tokenize(string sentence, TokenizationOptions options)
{
Init();
var tokens = segmenter.Cut(sentence)
.Select(x => new Token
{
Text = x
}).ToList();
CorrectTokenPosition(sentence, tokens);
return(tokens);
}
public void TestCut()
{
var sw = new Stopwatch();
sw.Start();
var sb = new StringBuilder();
for (int i = 0; i < 20000; i++)
{
sb.AppendLine("PS: 我觉得开源有一个好处，就是能够敦促自己不断改进，避免敞帚自珍");
}
var text = sb.ToString();
var lines = Regex.Split(text, "\r?\n");
var seg = new JiebaSegmenter();
seg.Cut("热身");
var raw = seg.Cut(text);
Console.WriteLine(raw.Count());
sw.Stop();
Console.WriteLine(sw.Elapsed);
sw.Restart();
var processed = (from line in lines.AsParallel().AsOrdered()
select seg.Cut(line)).SelectMany(s => s);
Console.WriteLine(processed.Count());
sw.Stop();
Console.WriteLine(sw.Elapsed);
}
private void button1_Click(object sender, EventArgs e)
{
// Directory.CreateDirectory("./in/");
// Directory.CreateDirectory("./out/");
var segmenter = new JiebaSegmenter();
var file = File.OpenRead("./in/src.txt");
var rbuf = new StreamReader(file);
var txt = rbuf.ReadToEnd();
rbuf.Close();
file.Close();
int cnt = 1;
var sb = new StringBuilder();
for (int i = 0; i < txt.Length; i++)
{
var chr = txt[i];
if ((chr >= 'A' && chr <= 'Z') || (chr >= 'a' && chr <= 'z') || (chr >= '0' && chr <= '9') || (chr >= 0x4e00 && chr <= 0x9fa5))
{
sb.Append(chr);
}
else
{
if (sb.Length >= 80)
{
file = File.OpenWrite($"./out/无题{cnt}.txt");
var wbuf = new StreamWriter(file);
wbuf.WriteLine(" 无题 ");
wbuf.WriteLine("作者：佚名");
wbuf.WriteLine();
var splitWords = segmenter.Cut(sb.ToString());
int sp = 0;
while (sp < splitWords.Count())
{
wbuf.WriteLine(string.Join("", splitWords.Skip(sp).Take(3)));
sp += 3;
}
wbuf.Close();
file.Close();
cnt++;
sb.Clear();
}
}
if (cnt >= 1000)
{
break;
}
}
}
static void Main(string[] args)
{
JiebaSegmenter segmenter = new JiebaSegmenter();
segmenter.AddWord("學系");
String[] sets = new String[] { "資訊工程學系", "資訊管理學系", "應用化學學系", "土木工程學系", "外國語言學系" };
foreach (string s in sets)
{
Console.WriteLine(string.Join(" ", segmenter.Cut(s)));
}
Console.ReadKey();
}
static void Main(string[] args)
{
Encoding.RegisterProvider(CodePagesEncodingProvider.Instance);
//TestDemo test = new TestDemo();
//test.CutDemo();
//test.TokenizeDemo();
var segmenter = new JiebaSegmenter();
var segments = segmenter.Cut("我来到北京清华大学", cutAll: true);
Console.WriteLine("【全模式】：{0}", string.Join("/ ", segments));
segments = segmenter.Cut("我来到北京清华大学"); // 默认为精确模式
Console.WriteLine("【精确模式】：{0}", string.Join("/ ", segments));
segments = segmenter.Cut("他来到了网易杭研大厦"); // 默认为精确模式，同时也使用HMM模型
Console.WriteLine("【新词识别】：{0}", string.Join("/ ", segments));
segments = segmenter.CutForSearch("小明硕士毕业于中国科学院计算所，后在日本京都大学深造"); // 搜索引擎模式
Console.WriteLine("【搜索引擎模式】：{0}", string.Join("/ ", segments));
segments = segmenter.Cut("结过婚的和尚未结过婚的");
Console.WriteLine("【歧义消除】：{0}", string.Join("/ ", segments));
Console.ReadKey();
}
// ====程序代码====
public Document(int id)
{
ID = id;
using (var file = new StreamReader($"Data/Input{id}.txt")) {
switch (file.ReadLine())
{
case "大众点评上能否查询到该企业？能查到正在营业":
能查到正在营业 = true; break;
case "大众点评上能否查询到该企业？能查到曾经营业":
能查到曾经营业 = true; break;
case "大众点评上能否查询到该企业？无信息":
无营业信息 = true; break;
default:
throw new FormatException($"请检查 Input{id}.txt 的格式");
}
switch (file.ReadLine())
{
case "队员是否有成功的GPS定位？有":
GPS定位 = true; break;
case "队员是否有成功的GPS定位？没有":
GPS定位 = false; break;
default:
throw new FormatException($"请检查 Input{id}.txt 的格式");
}
if (file.ReadLine().Length != 0)
{
throw new FormatException($"请检查 Input{id}.txt 的格式");
}
var comment = new StringBuilder();
while (true)
{
var line = file.ReadLine();
if (line == null)
{
break;
}
comment.Append(line);
}
备注文本 = comment.ToString();
备注词汇 = Segmenter.Cut(备注文本)
.Where(k => !char.IsPunctuation(k[0])).ToArray();
}
}
public WordAnalyResult <List <string> > WordAnalyJieba(string text, Encoding encoding = null)
{
if (ConfigBase.Default.IsTraceStack)
{
LogService.AnyLog("Stack", new StackTrace().GetFrame(0).GetMethod().ToString());
}
if (string.IsNullOrEmpty(text))
{
return(WordAnalyResult <List <string> > .Empty);
}
var segmenter = new JiebaSegmenter();
return(new WordAnalyResult <List <string> >().SetData(segmenter.Cut(text).ToList()));
//var posSeg = new PosSegmenter(segmenter);
//return posSeg.Cut(text, hmm).Select(token => string.Format("{0}/{1}", token.Word, token.Flag));
}
public void TestHyphen()
{
var seg = new JiebaSegmenter();
seg.AddWord("cet-4");
var s = "你一定也考过cet-4了。";
var segments = seg.Cut(s).ToList();
Assert.That(segments, Contains.Item("cet-4"));
Console.WriteLine(segments);
foreach (var sm in segments)
{
Console.WriteLine(sm);
}
}
public string[] Cut(string src)
{
IEnumerable <string> tokens = segmenter.Cut(src);
return(tokens.ToArray());
//if (src.Length == 2)
//{
//}
//else
//{
// IEnumerable<string> tokens = segmenter.Cut(src);
// return tokens.ToArray();
//}
}
/// <summary>
/// 切词
/// </summary>
/// <returns></returns>
public List <string> JiebaCut()
{
JiebaSegmenter jiebaseg = new JiebaSegmenter();
var segment = jiebaseg.Cut(doc);
List <string> cutresult = new List <string>();
foreach (var i in segment)
{
if (stopwords.Contains(i))
{//不参与计算的词排除
continue;
}
cutresult.Add(i);
}
return(cutresult);
}
public List <string> JiebaCut()
{
JiebaSegmenter jiebaseg = new JiebaSegmenter();
//Console.WriteLine(doc);
var segment = jiebaseg.Cut(doc);
List <string> cutresult = new List <string>();
foreach (var i in segment)
{
if (!stopwords.Contains(i))
{
cutresult.Add(i);
}
}
return(cutresult);
}
/// <summary>
/// 生成分词文件
/// </summary>
/// <param name="path"></param>
/// <returns></returns>
public string Build(string path)
{
var html = System.IO.File.ReadAllText(path, System.Text.Encoding.UTF8);
var segmenter = new JiebaSegmenter();
segmenter.LoadUserDict("Files/dict.txt");
var segments = segmenter.Cut(html, cutAll: true); //全匹配
var jiebaFIle = $"{path}.jieba";
System.IO.File.WriteAllText(jiebaFIle, string.Join(" ", segments), System.Text.Encoding.UTF8); //生成分词文件
return(jiebaFIle);
}
public static IEnumerable <Token> CutToToken(this JiebaSegmenter segmenter, string text, bool cutAll = true)
{
var words = segmenter.Cut(text, cutAll).Where(s => !string.IsNullOrWhiteSpace(s)).ToArray();
var indexDic = new Dictionary <string, int>();
var tokenArray = new Token[words.Length];
var checkIndex = 0;
for (var i = 0; i < words.Length; i++)
{
var word = words[i];
checkIndex = text.IndexOf(word, indexDic.ContainsKey(word) ? indexDic[word] + 1 : checkIndex, StringComparison.Ordinal);
tokenArray[i] = new Token(word, checkIndex, checkIndex + word.Length);
indexDic[word] = checkIndex;
}
return(tokenArray);
}
public IEnumerable <string> Analytical(string input)
{
var segment = new JiebaSegmenter();
var features = new List <string>();
var stopWordsList = GetStopWords();
foreach (var feature in segment.Cut(input))
{
if (stopWordsList.Any(s => s.Contains(feature)))
{
continue;
}
features.Add(feature);
}
return(features);
}
//中文分词
public static List <string> ChineseSegmenter(string content)
{
if (segmenter == null)
{
segmenter = new JiebaSegmenter();
List <string> temp = new List <string>().Concat(TechnologyStack)
.Concat(ProgramLanguage).Concat(Job).ToList();
foreach (var word in temp)
{
segmenter.AddWord(word);
}
}
IEnumerable <string> segments = segmenter.Cut(content);
return(segments.ToList());
}
/// <summary>
/// 全文检索 查询
/// </summary>
/// <param name="keyword"></param>
/// <returns></returns>
public NpgsqlTsQuery GetSerachNpgsqlTsQuery(string keyword)
{
if (keyword.Contains('&'))
{
string[] keys = keyword.Split('&');
return(GetSerachNpgsqlTsQuery_And(keys));
}
if (keyword.Contains("|"))
{
string[] keys = keyword.Split('|');
return(GetSerachNpgsqlTsQuery_Or(keys));
}
NpgsqlTsQuery vector;
try
{
var segmenter = new JiebaSegmenter();
HtmlToTextHelper htmlToTextHelper = new HtmlToTextHelper();
if (string.IsNullOrWhiteSpace(keyword))
{
return(null);;
}
string noHtmlConent = htmlToTextHelper.Convert(keyword);
var list = segmenter.Cut(noHtmlConent, hmm: true);
var cutList = new List <string>();
foreach (var item in list)
{
if (item.Length > 1)
{
cutList.Add(item.ToUpper());
}
}
string str = string.Join(" & ", cutList);
vector = NpgsqlTsQuery.Parse(str);
}
catch (Exception ex)
{
return(null);
}
return(vector);
}
/// <summary>
/// 获取num个核心句
/// </summary>
/// <param name="text">文本</param>
/// <param name="num">核心句数</param>
public void GetList(string text, int num, int type)
{
keywordList.Clear();
//获取核心关键词列表
switch (type)
{
case 1:
{
TfidfExtractor te = new TfidfExtractor();
keywordList = te.ExtractTags(text, num).ToList();
}
break;
case 2:
{
TextRankExtractor te = new TextRankExtractor();
keywordList = te.ExtractTags(text, num).ToList();
}
break;
}
AllsentenceList.Clear();
keySentenceList.Clear();
//将文章拆为句子列表，并分词
text = text.Replace(Environment.NewLine.ToString(), " 。");
//text = text.Replace(" ", "");
AllsentenceList = text.Split('。', '？').Where(x => !string.IsNullOrEmpty(x) && x != "undefined").Select(x => x.Trim()).ToList();
List <Sentence> temp = new List <Sentence>();
for (int i = 0; i < AllsentenceList.Count; i++)
{
AllsentenceList[i] = AllsentenceList[i] + "。";
var sentence = segmenter.Cut(AllsentenceList[i]);
Sentence v = new Sentence();
v.Sen = string.Join(" ", sentence);
v.Index = i;
temp.Add(v);
}
GetSentenceList(keywordList, temp);
}
public void CutDemo()
{
var segmenter = new JiebaSegmenter();
var segments = segmenter.Cut("我来到北京清华大学", cutAll: true);
Console.WriteLine("【全模式】：{0}", string.Join("/ ", segments));
segments = segmenter.Cut("我来到北京清华大学"); // 默认为精确模式
Console.WriteLine("【精确模式】：{0}", string.Join("/ ", segments));
segments = segmenter.Cut("他来到了网易杭研大厦"); // 默认为精确模式，同时也使用HMM模型
Console.WriteLine("【新词识别】：{0}", string.Join("/ ", segments));
segments = segmenter.CutForSearch("小明硕士毕业于中国科学院计算所，后在日本京都大学深造"); // 搜索引擎模式
Console.WriteLine("【搜索引擎模式】：{0}", string.Join("/ ", segments));
segments = segmenter.Cut("结过婚的和尚未结过婚的");
Console.WriteLine("【歧义消除】：{0}", string.Join("/ ", segments));
segments = segmenter.Cut("北京大学生喝进口红酒");
Console.WriteLine("【歧义消除】：{0}", string.Join("/ ", segments));
segments = segmenter.Cut("在北京大学生活区喝进口红酒");
Console.WriteLine("【歧义消除】：{0}", string.Join("/ ", segments));
segments = segmenter.Cut("腾讯视频致力于打造中国最大的在线视频媒体平台,以丰富的内容、极致的观看体验");
Console.WriteLine("【精确模式】：{0}", string.Join("/ ", segments));
segmenter.DeleteWord("湖南");
segmenter.AddWord("湖南");
//segmenter.AddWord("长沙市");
segments = segmenter.Cut("湖南长沙市天心区");
Console.WriteLine("【精确模式】：{0}", string.Join("/ ", segments));
TokenizeDemo();
TokenizeSearchDemo();
PosCutDemo();
ExtractTagsDemo();
ExtractTagsDemo2();
TestWordFreq();
}
public static void method2()
{
string path = @"D:\c#\stopwords-master\baidu_stopwords.txt"; //路径
string str = File.ReadAllText(path);
var stop_words = str.Split('\n');
path = @"D:\c#\图云词频计算\files";
DirectoryInfo root = new DirectoryInfo(path);
foreach (FileInfo f in root.GetFiles())
{
string fullName = f.FullName;
var text = File.ReadAllText(fullName);
string pattern = @"abstract:[\S\s]+";
Regex regex = new Regex(pattern);
Match match = regex.Match(text);
if (match.Groups.Count != 0)
{
text = match.Groups[0].ToString().Substring(9);
}
else
{
text = "abstract"; //防止出现“”
}
if (text == "")
{
text = "abstract";
}
var segmenter = new JiebaSegmenter();
var segments = segmenter.Cut(text, cutAll: true); // 默认为精确模式
System.IO.StreamWriter file = new System.IO.StreamWriter(@"D:\c#\图云词频计算\words.txt", true); //写入到文件末尾 不覆盖
foreach (var temp in segments)
{
if (!stop_words.Contains(temp))
{
file.WriteLine(temp);
}
}
file.Close();
}
}
static void Main(string[] args)
{
if (args.Length < 1)
{
Console.WriteLine("No file specified");
return;
}
var result = new List<string>();
var filename = Path.GetFullPath(args[0]);
var lines = File.ReadAllLines(filename);
var segmenter = new JiebaSegmenter();
foreach (var line in lines)
{
result.Add(string.Join("/ ", segmenter.Cut(line)));
}
Console.WriteLine(string.Join(Environment.NewLine, result));
}
protected static List <string> CutKeyWord(string key)
{
var rs = new List <string>();
var segmenter = new JiebaSegmenter();
var list = segmenter.Cut(key);
if (list != null && list.Count() > 0)
{
foreach (var item in list)
{
if (string.IsNullOrEmpty(item) || item.Length <= 1)
{
continue;
}
rs.Add(item);
}
}
return(rs);
}
static void Main(string[] args)
{
if (args.Length < 1)
{
Console.WriteLine("No file specified");
return;
}
var result = new List <string>();
var filename = Path.GetFullPath(args[0]);
var lines = File.ReadAllLines(filename);
var segmenter = new JiebaSegmenter();
foreach (var line in lines)
{
result.Add(string.Join("/ ", segmenter.Cut(line)));
}
Console.WriteLine(string.Join(Environment.NewLine, result));
}
private void GoButtonClick(object sender, RoutedEventArgs e)
{
tb1.IsEnabled = false;
string srcText = tb1.Text;
if (srcText == "" || srcText == null)
{
MessageBox.Show("内容为空", "Error", MessageBoxButton.OK, MessageBoxImage.Error);
}
else
{
JiebaSegmenter segmenter = new JiebaSegmenter();
IEnumerable <string> segments = segmenter.Cut(srcText);
string outText = "";
foreach (string s in segments)
{
if (NeedChaos() && s.Length > 1)
{
Random rd = new Random();
int n1 = rd.Next(0, s.Length);
int n2 = rd.Next(0, s.Length);
while (n2 == n1)
{
n2 = rd.Next(0, s.Length);
}
char[] arr = s.ToCharArray();
arr[n1] = s[n2];
arr[n2] = s[n1];
string s_t = string.Join("", arr);
outText += s_t;
}
else
{
outText += s;
}
}
tb2.Text = outText;
tb1.IsEnabled = true;
}
}
public static void AnlayzeEntitySurroundWords(HTMLEngine.MyRootHtmlNode root, string KeyWord)
{
Program.Training.WriteLine("关键字：[" + KeyWord + "]");
JiebaSegmenter segmenter = new JiebaSegmenter();
segmenter.AddWord(KeyWord);
foreach (var paragrah in root.Children)
{
var segments = segmenter.Cut(paragrah.FullText.NormalizeKey()).ToList(); // 默认为精确模式
//Console.WriteLine("【精确模式】：{0}", string.Join("/ ", segments));
//寻找关键字的位置
for (int i = 0; i < segments.Count; i++)
{
if (segments[i].Equals(KeyWord))
{
//前5个词语和后五个词语
var startInx = Math.Max(0, i - 5);
var EndInx = Math.Min(i + 5, segments.Count);
for (int s = startInx; s < i; s++)
{
Program.Training.WriteLine("前导关键字：[" + segments[s] + "]");
if (segments[s] == "：")
{
var leading = "";
for (int l = startInx; l < s; l++)
{
leading += segments[l];
}
Console.WriteLine("冒号前导词：" + leading);
}
}
Program.Training.WriteLine("关键字：[" + KeyWord + "]");
for (int s = i + 1; s < EndInx; s++)
{
Program.Training.WriteLine("后续关键字：[" + segments[s] + "]");
}
}
}
}
}
/// <summary>
/// 获取分词结果
/// </summary>
/// <param name="input"></param>
/// <returns></returns>
public static Dictionary <string, int> GetResult(string input, string mode = "", bool checkRepetitiveWord = false)
{
Dictionary <string, int> res = new Dictionary <string, int>();
var segmenter = new JiebaSegmenter();
var words = segmenter.Cut(input);
var wordDict = new Dictionary <string, int>();
foreach (var word in words)
{
if (2 <= word.Length && StringChecker.IsHanZi(word) || StringChecker.IsEnglish(word))
{
wordDict[word] = 0;
}
}
res = FenCi.GetRepetitiveWordCount(input, wordDict);
return(res);
}
public void TestCutManySentences()
{
var text = GetTestSentences().Join(string.Empty);
var fileSize = 1532 * 100;
var seg = new JiebaSegmenter();
seg.Cut("热身一下");
Console.WriteLine("Start to cut");
const int n = 20;
var stopWatch = new Stopwatch();
// Accurate mode
stopWatch.Start();
for (var i = 0; i < n; i++)
{
seg.Cut(text);
}
stopWatch.Stop();
var timeConsumed = (double)stopWatch.ElapsedMilliseconds / (1000 * n);
Console.WriteLine("Accurate mode: {0} ms, average: {1} / second",
timeConsumed, fileSize / timeConsumed);
// Full mode
stopWatch.Reset();
stopWatch.Start();
for (var i = 0; i < n; i++)
{
seg.Cut(text, true);
}
stopWatch.Stop();
timeConsumed = (double)stopWatch.ElapsedMilliseconds / (1000 * n);
Console.WriteLine("Full mode: {0} ms, average: {1} / second",
timeConsumed, fileSize / timeConsumed);
}
public void TestCut()
{
var sw = new Stopwatch();
sw.Start();
var sb = new StringBuilder();
for (int i = 0; i < 20000; i++)
{
sb.AppendLine("PS: 我觉得开源有一个好处，就是能够敦促自己不断改进，避免敞帚自珍");
}
var text = sb.ToString();
var lines = Regex.Split(text, "\r?\n");
var seg = new JiebaSegmenter();
seg.Cut("热身");
var raw = seg.Cut(text);
Console.WriteLine(raw.Count());
sw.Stop();
Console.WriteLine(sw.Elapsed);
sw.Restart();
var processed = (from line in lines.AsParallel().AsOrdered()
select seg.Cut(line)).SelectMany(s => s);
Console.WriteLine(processed.Count());
sw.Stop();
Console.WriteLine(sw.Elapsed);
}
public void TestCutSpecialWords()
{
var seg = new JiebaSegmenter();
seg.AddWord(".NET");
seg.AddWord("U.S.A.");
var s = ".NET平台是微软推出的, U.S.A.是美国的简写";
var segments = seg.Cut(s);
foreach (var segment in segments)
{
Console.WriteLine(segment);
}
seg.LoadUserDict(@"Resources\user_dict.txt");
s = "Steve Jobs重新定义了手机";
segments = seg.Cut(s);
foreach (var segment in segments)
{
Console.WriteLine(segment);
}
s = "我们所熟悉的一个版本是Mac OS X 10.11 EI Capitan，在2015年推出。";
segments = seg.Cut(s);
foreach (var segment in segments)
{
Console.WriteLine(segment);
}
}
public void TestSpecialWords()
{
var seg = new JiebaSegmenter();
seg.AddWord(".NET");
seg.AddWord("U.S.A.");
var s = ".NET平台是微软推出的, U.S.A.是美国的简写";
var segments = seg.Cut(s);
foreach (var segment in segments)
{
Console.WriteLine(segment);
}
s = "Steve Jobs重新定义了手机";
segments = seg.Cut(s);
foreach (var segment in segments)
{
Console.WriteLine(segment);
}
}
public void TestCutAllSpecialWords()
{
// TODO: Enable this test case after confirming with jieba py.
var seg = new JiebaSegmenter();
seg.AddWord(".NET");
seg.AddWord("U.S.A.");
seg.AddWord("Steve Jobs");
seg.AddWord("Mac OS X");
var s = ".NET平台是微软推出的, U.S.A.是美国的简写";
var segments = seg.Cut(s);
Console.WriteLine("Cut: {0}", string.Join("/ ", segments));
segments = seg.Cut(s, cutAll: true);
Console.WriteLine("Cut All: {0}", string.Join("/ ", segments));
s = "Steve Jobs重新定义了手机";
segments = seg.Cut(s);
Console.WriteLine("Cut: {0}", string.Join("/ ", segments));
segments = seg.Cut(s, cutAll: true);
Console.WriteLine("Cut All: {0}", string.Join("/ ", segments));
s = "我们所熟悉的一个版本是Mac OS X 10.11 EI Capitan，在2015年推出。";
segments = seg.Cut(s);
Console.WriteLine("Cut: {0}", string.Join("/ ", segments));
segments = seg.Cut(s, cutAll: true);
Console.WriteLine("Cut All: {0}", string.Join("/ ", segments));
}
public void TestEnglishWordsCut()
{
var seg = new JiebaSegmenter();
var text = "HighestDegree";
CollectionAssert.AreEqual(new[] { text }, seg.Cut(text));
text = "HelloWorld";
CollectionAssert.AreEqual(new[] { text }, seg.Cut(text));
text = "HelloWorldle";
CollectionAssert.AreEqual(new[] { text }, seg.Cut(text));
text = "HelloWorldlee";
CollectionAssert.AreEqual(new[] { text }, seg.Cut(text));
}
public void TestAddWord()
{
var seg = new JiebaSegmenter();
var s = "小明最近在学习机器学习和自然语言处理";
var segments = seg.Cut(s);
Assert.That(segments, Contains.Item("机器"));
Assert.That(segments, Contains.Item("学习"));
seg.AddWord("机器学习");
segments = seg.Cut(s);
Assert.That(segments, Contains.Item("机器学习"));
Assert.That(segments, Is.Not.Contains("机器"));
}
private static void TestCutThenPrint(JiebaSegmenter segmenter, string s)
{
Console.WriteLine(string.Join("/ ", segmenter.Cut(s)));
}