//public ActionResult LoadVocabulary(string idgc, string typew)
//{
// //var jarRoot = @"D:\HsnSky\NLPTokenzie\stanford-corenlp-full-2015-12-09\edu\stanford\nlp";
// string urlRoot = System.IO.Path.Combine(Server.MapPath("~/Uploads/english-left3words"), "english-left3words-distsim.tagger");
// //var jarRoot = @"\stanford-corenlp-full-2015-12-09\edu\stanford\nlp";
// //var modelsDirectory = urlRoot + jarRoot + @"\models\pos-tagger\english-left3words";
// // Loading POS Tagger
// //var tagger = new MaxentTagger(urlRoot + @"\english-left3words-distsim.tagger");
// var tagger = new MaxentTagger(urlRoot);
// // Text for tagging
// var data = db.Sentenses.Where(e=>e.Comment.GroupCommentId==idgc).ToList();
// //var data = db.Comments.Where(e => e.GroupCommentId == idgc).ToList();
// foreach (var item in data)
// {
// var text = item.ContentReview;
// var sen = db.Sentenses.Find(item.Id);
// string str = "";
// //var text = "Quality hotel at great price Very clean. Free breakfast with good selection. Staff friendly and most helpful. A grat stay!";
// var sentences = MaxentTagger.tokenizeText(new StringReader(text)).toArray();
// foreach (ArrayList sentence in sentences)
// {
// try
// {
// var taggedSentence = tagger.tagSentence(sentence);
// string[] str1 = taggedSentence.ToString().Split(',', '[', ']');
// if (typew == "2")
// {
// #region Tách có tạo từ ghép
// //Kiểm tra từ ghép
// for (int i = 0; i < str1.Length; i++)
// {
// try
// {
// string[] str2 = str1[i].ToString().Split('/');
// string[] str3 = str1[i + 1].ToString().Split('/');
// string str4 = "";
// string type = "JJ";
// if (str2[1].Trim() == "JJ" && (str3[1].Trim() == "NN" || str3[1].Trim() == "NNS")) //Luật 1
// {
// str4 = str2[0].Trim() + " " + str3[0].Trim();
// str += str2[0].Trim() + "|" + str2[1].Trim() + " ;" + str3[0].Trim() + "|" + str3[1].Trim() + ";";
// }
// else if ((str2[1].Trim() == "RB" || str2[1].Trim() == "RBR" || str2[1].Trim() == "RBS") && str3[1].Trim() == "JJ") //Luật 2
// {
// str4 = str2[0].Trim() + " " + str3[0].Trim();
// str += str2[0].Trim() + "|" + str2[1].Trim() + " ;" + str3[0].Trim() + "|" + str3[1].Trim() + ";";
// }
// else if (str2[1].Trim() == "JJ" && str3[1].Trim() == "JJ") //Luật 3
// {
// str4 = str2[0].Trim() + " " + str3[0].Trim();
// str += str2[0].Trim() + "|" + str2[1].Trim() + " ;" + str3[0].Trim() + "|" + str3[1].Trim() + ";";
// }
// else if ((str2[1].Trim() == "NN" || str2[1].Trim() == "NNS") && str3[1].Trim() == "JJ") //Luật 4
// {
// str4 = str2[0].Trim() + " " + str3[0].Trim();
// str += str2[0].Trim() + "|" + str2[1].Trim() + " ;" + str3[0].Trim() + "|" + str3[1].Trim() + ";";
// }
// else if ((str2[1].Trim() == "RB" || str2[1].Trim() == "RBR" || str2[1].Trim() == "RBS")
// && (str3[1].Trim() == "VB" || str3[1].Trim() == "VBD" || str3[1].Trim() == "VBN" || str3[1].Trim() == "VBG")) //Luật 5
// {
// str4 = str2[0].Trim() + " " + str3[0].Trim();
// str += str2[0].Trim() + "|" + str2[1].Trim() + " ;" + str3[0].Trim() + "|" + str3[1].Trim() + ";";
// }
// else if (str2[1].Trim() == "JJ" || str2[1].Trim() == "NN" || str2[1].Trim() == "VB" || str2[1].Trim() == "RB" //Luật 6
// || str2[1].Trim() == "VBZ" || str2[1].Trim() == "NNP" || str2[1].Trim() == "NNS")
// {
// str4 = str2[0].Trim();
// type = str2[1].Trim();
// str += str2[0].Trim() + "|" + str2[1].Trim() + " ;";
// }
// //cập nhật từ điển
// if (str4.Trim().Count() > 1)
// {
// var data2 = db.Vocabularies.Where(e => e.Word.Trim().ToLower().Equals(str4) && e.GroupCommentId == idgc).ToList();
// if (data2.Count <= 0)
// {
// Vocabulary data3 = new Vocabulary();
// string idv = Public.GetID();
// while (db.Vocabularies.Where(e => e.Id == idv).Count() > 0)
// {
// idv = Public.GetID();
// }
// data3.Id = idv;
// data3.Word = str4.Trim();
// data3.TypeWord = type.Trim();
// data3.GroupCommentId = idgc;
// db.Vocabularies.Add(data3);
// db.SaveChanges();
// }
// }
// }
// catch { }
// }
// #endregion
// }
// else
// {
// #region Tách không tạo từ ghép
// foreach (var item2 in str1)
// {
// try
// {
// if (item2.Trim() != "")
// {
// string[] str2 = item2.ToString().Split('/');
// if (str2[1].Trim() == "JJ" || str2[1].Trim() == "NN" || str2[1].Trim() == "VB" || str2[1].Trim() == "RB"
// || str2[1].Trim() == "VBZ" || str2[1].Trim() == "NNP" || str2[1].Trim() == "NNS")
// {
// string str3 = str2[0].Trim().ToLower();
// //logs tách từ
// str += str2[0].Trim() + "|" + str2[1].Trim() + " ;";
// //cập nhật từ điển
// if (str2[0].Trim().Count() > 1)
// {
// var data2 = db.Vocabularies.Where(e => e.Word.Trim().ToLower().Equals(str3) && e.GroupCommentId == idgc).ToList();
// if (data2.Count <= 0)
// {
// Vocabulary data4 = new Vocabulary();
// string idv = Public.GetID();
// while (db.Vocabularies.Where(e => e.Id == idv).Count() > 0)
// {
// idv = Public.GetID();
// }
// data4.Id = idv;
// data4.Word = str2[0].Trim();
// data4.TypeWord = str2[1].Trim();
// data4.GroupCommentId = idgc;
// db.Vocabularies.Add(data4);
// db.SaveChanges();
// }
// }
// }
// }
// }
// catch { }
// }
// #endregion
// }
// }
// catch { }
// }
// sen.Logs = str;
// db.Entry(sen).State = EntityState.Modified;
// db.SaveChanges();
// }
// CountVocabulary(idgc);
// //IEnumerable<GroupComent> data6 = (IEnumerable<GroupComent>)db.GroupComents.ToList();
// //ViewBag.GroupComent = new SelectList(data, "Id", "Name", idgc);
// var data5 = db.Vocabularies.Where(e => e.GroupCommentId == idgc).ToList();
// return PartialView("_ListVocabulary", data5);
//}
public ActionResult LoadVocabulary()
{
string idgc = "";
string typew = "1";
Stopwatch sw = Stopwatch.StartNew();
FileStream fs = new FileStream("D:\\hoctap\\DoAnTotNghiep\\soucecode\\stopWord_1.txt", FileMode.Open);
StreamReader rd = new StreamReader(fs, Encoding.UTF8);
string line = "";
Dictionary <string, string> stopword = new Dictionary <string, string>();
while ((line = rd.ReadLine()) != null)
{
stopword.Add(line, line);
}
string urlRoot = System.IO.Path.Combine(Server.MapPath("~/Uploads/english-left3words"), "english-left3words-distsim.tagger");
var tagger = new MaxentTagger(urlRoot);
// Text for tagging
var dataGC = db.GroupComents.Where(e => e.ProductId == "170319111826335").ToList();
foreach (var itemGC in dataGC)
{
idgc = itemGC.Id;
var data = db.Comments.Where(e => e.GroupCommentId == idgc).ToList();
foreach (var item in data)
{
var text = item.Comment1;
//string str = "";
var sentences = MaxentTagger.tokenizeText(new java.io.StringReader(text)).toArray();
foreach (ArrayList sentence in sentences)
{
try
{
var taggedSentence = tagger.tagSentence(sentence);
string[] str1 = taggedSentence.ToString().Split(',', '[', ']');
if (typew == "2")
{
#region Tách có tạo từ ghép
//Kiểm tra từ ghép
for (int i = 0; i < str1.Length; i++)
{
try
{
string[] str2 = str1[i].ToString().Split('/');
string[] str3 = str1[i + 1].ToString().Split('/');
string str4 = "";
string type = "JJ";
if (str2[1].Trim() == "JJ" && (str3[1].Trim() == "NN" || str3[1].Trim() == "NNS")) //Luật 1
{
str4 = str2[0].Trim() + " " + str3[0].Trim();
//str += str2[0].Trim() + "|" + str2[1].Trim() + " ;" + str3[0].Trim() + "|" + str3[1].Trim() + ";";
}
else if ((str2[1].Trim() == "RB" || str2[1].Trim() == "RBR" || str2[1].Trim() == "RBS") && str3[1].Trim() == "JJ") //Luật 2
{
str4 = str2[0].Trim() + " " + str3[0].Trim();
//str += str2[0].Trim() + "|" + str2[1].Trim() + " ;" + str3[0].Trim() + "|" + str3[1].Trim() + ";";
}
else if (str2[1].Trim() == "JJ" && str3[1].Trim() == "JJ") //Luật 3
{
str4 = str2[0].Trim() + " " + str3[0].Trim();
//str += str2[0].Trim() + "|" + str2[1].Trim() + " ;" + str3[0].Trim() + "|" + str3[1].Trim() + ";";
}
else if ((str2[1].Trim() == "NN" || str2[1].Trim() == "NNS") && str3[1].Trim() == "JJ") //Luật 4
{
str4 = str2[0].Trim() + " " + str3[0].Trim();
//str += str2[0].Trim() + "|" + str2[1].Trim() + " ;" + str3[0].Trim() + "|" + str3[1].Trim() + ";";
}
else if ((str2[1].Trim() == "RB" || str2[1].Trim() == "RBR" || str2[1].Trim() == "RBS") &&
(str3[1].Trim() == "VB" || str3[1].Trim() == "VBD" || str3[1].Trim() == "VBN" || str3[1].Trim() == "VBG")) //Luật 5
{
str4 = str2[0].Trim() + " " + str3[0].Trim();
//str += str2[0].Trim() + "|" + str2[1].Trim() + " ;" + str3[0].Trim() + "|" + str3[1].Trim() + ";";
}
else if (str2[1].Trim() == "JJ" || str2[1].Trim() == "NN" ||
str2[1].Trim() == "RB" || str2[1].Trim() == "RBR" || str2[1].Trim() == "RBS" ||
str2[1].Trim() == "VBZ" || str2[1].Trim() == "VBD" || str2[1].Trim() == "VBN" ||
str2[1].Trim() == "VBG" || str2[1].Trim() == "VB" ||
str2[1].Trim() == "NNP" || str2[1].Trim() == "NNS")
{
str4 = str2[0].Trim();
type = str2[1].Trim();
//str += str2[0].Trim() + "|" + str2[1].Trim() + " ;";
}
//cập nhật từ điển
if (str4.Trim().Count() > 1)
{
var data2 = db.Vocabulary_1.Where(e => e.Word.Trim().ToLower().Equals(str4) && e.GroupCommentId == idgc).ToList();
if (data2.Count <= 0)
{
Vocabulary_1 data3 = new Vocabulary_1();
string idv = Public.GetID();
while (db.Vocabulary_1.Where(e => e.Id == idv).Count() > 0)
{
idv = Public.GetID();
}
data3.Id = idv;
data3.Word = str4.Trim();
data3.TypeWord = type.Trim();
data3.GroupCommentId = idgc;
data3.Type = 2;
db.Vocabulary_1.Add(data3);
db.SaveChanges();
}
else
{
var data6 = data2.FirstOrDefault();
data6.Counts++;
db.Entry(data6).State = EntityState.Modified;
db.SaveChanges();
}
}
}
catch { }
}
#endregion
}
else
{
idgc = "170319111826335";
#region Tách không tạo từ ghép
string se = "";
foreach (var item2 in str1)
{
try
{
string linei = "";
if (item2.Trim() != "")
{
string[] str2 = item2.ToString().Split('/');
if (str2[1].Trim() == "JJ" || str2[1].Trim() == "NN" ||
str2[1].Trim() == "RB" || str2[1].Trim() == "RBS" ||
str2[1].Trim() == "RBR" || str2[1].Trim() == "VBN" ||
str2[1].Trim() == "VBZ" || str2[1].Trim() == "VBD" ||
str2[1].Trim() == "VBG" || str2[1].Trim() == "VB" ||
str2[1].Trim() == "NNP" || str2[1].Trim() == "NNS")
{
string str3 = str2[0].Trim().ToLower();
if (stopword.TryGetValue(str3, out linei))
{
continue;
}
if (se == "")
{
se = str3;
}
else
{
se = String.Concat(se, String.Concat(" ", str3));
}
//cập nhật từ điển
if (str2[0].Trim().Count() > 1)
{
string typeword = str2[1].Trim();
var data2 = db.Vocabulary_1.Where(e => e.Word.Trim().ToLower().Equals(str3) && e.TypeWord.Trim().ToLower().Equals(typeword) && e.GroupCommentId == idgc).ToList();
if (data2.Count <= 0)
{
Vocabulary_1 data4 = new Vocabulary_1();
string idv = Public.GetID();
while (db.Vocabulary_1.Where(e => e.Id == idv).Count() > 0)
{
idv = Public.GetID();
}
data4.Id = idv;
data4.Word = str3;
data4.TypeWord = str2[1].Trim();
data4.GroupCommentId = idgc;
data4.Counts = 1;
data4.Type = 1;
db.Vocabulary_1.Add(data4);
db.SaveChanges();
}
else
{
var data6 = data2.FirstOrDefault();
data6.Counts++;
db.Entry(data6).State = EntityState.Modified;
db.SaveChanges();
}
}
}
}
}
catch { }
}
//tao cau loai bo tu dung
Sentensesnotword senotopword = new Sentensesnotword();
senotopword.Id = Public.GetID();;
senotopword.ContentReview = se.Trim();
senotopword.CommentId = item.Id;
db.Sentensesnotwords.Add(senotopword);
db.SaveChanges();
#endregion
}
}
catch { }
}
}
//sen.Logs = str;
//db.Entry(sen).State = EntityState.Modified;
//db.SaveChanges();
}
//CountVocabulary(idgc);
//IEnumerable<GroupComent> data6 = (IEnumerable<GroupComent>)db.GroupComents.ToList();
//ViewBag.GroupComent = new SelectList(data, "Id", "Name", idgc);
ViewBag.Time = sw.ElapsedMilliseconds;
sw.Stop();
var data5 = db.Vocabulary_1.ToList();
return(PartialView("_ListVocabulary", data5));
}
private void standarsetenword()
{
var stopword = db.StopWords.ToList();
Dictionary <string, string> dicstopword = new Dictionary <string, string>();
foreach (var kvp in stopword)
{
dicstopword.Add(kvp.StopWord1, kvp.StopWord1);
}
var listcore = db.CoreWords.ToList();
Dictionary <string, string> diccore = new Dictionary <string, string>();
foreach (var kvp in listcore)
{
//diccore.Add(kvp.core_word, kvp.core_word);
string[] strcore = kvp.core_word.Split(',');
foreach (var s in strcore)
{
diccore.Add(s, s);
}
}
string urlRoot = System.IO.Path.Combine(Server.MapPath("~/Uploads/english-left3words"), "english-left3words-distsim.tagger");
var tagger = new MaxentTagger(urlRoot);
var sen2 = db.Sentenses.ToList();
if (sen2.Count >= 1)
{
var cm = sen2;
//try
//{
if (cm.Count > 1)
{
foreach (var item in cm)
{
var text = item.ContentReview;
//string str = "";
var sentences = MaxentTagger.tokenizeText(new java.io.StringReader(text)).toArray();
foreach (ArrayList sentence in sentences)
{
string strse = sentence.toString();
strse = strse.Replace(",", "");
strse = strse.Replace("[", "");
strse = strse.Replace("]", "");
string ids = item.Id;
var taggedSentence = tagger.tagSentence(sentence);
string[] str1 = taggedSentence.ToString().Split(',', '[', ']');
string strtmp = "";
foreach (var item2 in str1)
{
string linei = "";
if (item2.Trim() != "")
{
string[] str2 = item2.ToString().Split('/');
try
{
if (str2.Count() == 2 && str2[1].Trim() != "")
{
if ((str2[1].Trim() != "NN" &&
!diccore.TryGetValue(str2[0].Trim().ToLower(), out linei) &&
str2[1].Trim() != "NNP" && str2[1].Trim() != "NNS") || dicstopword.TryGetValue(str2[0].Trim().ToLower(), out linei))
{
continue;
}
else
{
strtmp += " " + str2[0].Trim().ToLower();
}
}
}
catch { }
}
}
Sentensesnotword senotopword = new Sentensesnotword();
/* while (db.Sentensesnotwords.Where(e => e.Id == ids).Count() > 0)
* {
* ids = Public.GetID();
* }*/
if (strtmp.Trim() != "" && db.Sentensesnotwords.Where(e => e.Id == ids).Count() == 0)
{
senotopword.Id = ids;
senotopword.ContentReview = strtmp.Trim();
senotopword.CommentId = item.CommentId;
db.Sentensesnotwords.Add(senotopword);
db.SaveChanges();
}
}
}
}
//}
//catch { }
}
}
