public void WriteLog(string error, string error2, string url) { using (var db = new XKWEntities2()) { var log = new Log(); log.Content = error; log.Content2 = error2; log.CrawlerUrl = url; db.Log.Add(log); db.SaveChanges(); } }
public void UpdateQuestionJiaocaiSourceResultStatus(int id) { using (var db = new XKWEntities2()) { if ( db.QuestionJiaoCaiSource.Any(t => t.Id == id)) { var entity = db.QuestionJiaoCaiSource.FirstOrDefault(t => t.Id == id); entity.Status = true; db.SaveChanges(); } } }
public void UpdateQuestionJiaocaiSourceTotalCount(int id, int total) { using (var db = new XKWEntities2()) { if ( db.QuestionJiaoCaiSource.Any(t => t.Id == id)) { var entity = db.QuestionJiaoCaiSource.FirstOrDefault(t => t.Id == id); entity.Total = total; db.SaveChanges(); } } }
public void UpdateJiaocaiDetailStatus(int jiaocaiDetailId) { using (var db = new XKWEntities2()) { if ( db.JiaocaiDetail.Any(t => t.JiaoCaiDetailId == jiaocaiDetailId)) { var entity = db.JiaocaiDetail.FirstOrDefault(t => t.JiaoCaiDetailId == jiaocaiDetailId); entity.Status = true; db.SaveChanges(); } } }
public void AddSubjectGrade(int gradeId, int subjectId) { using (var db = new XKWEntities2()) { if (!db.SubjectGrade.Any(t => t.SubjectId == subjectId && t.GradeId == gradeId)) { var entity = new SubjectGrade(); entity.GradeId = gradeId; entity.SubjectId = subjectId; db.SubjectGrade.Add(entity); db.SaveChanges(); } } }
public void AddArea(int areaId, string name, string shortName) { using (var db = new XKWEntities2()) { if (!db.Area.Any(t => t.AreaId == areaId)) { var entity = new Area(); entity.AreaId = areaId; entity.Name = name; entity.ShortName = shortName; db.Area.Add(entity); db.SaveChanges(); } } }
public void AddCategoryId(int categoryId, string categoryName, int subjectId) { using (var db = new XKWEntities2()) { if (!db.SubjectCategory.Any(t => t.CategoryId == categoryId)) { var entity = new SubjectCategory(); entity.CategoryId = categoryId; entity.CategoryName = categoryName; entity.SubjectId = subjectId; db.SubjectCategory.Add(entity); db.SaveChanges(); } } }
public void AddSubject(int subjectId, string subjectName, int degreeId) { using (var db = new XKWEntities2()) { if (!db.Subject.Any(t => t.SubjectId == subjectId)) { var entity = new Subject(); entity.SubjectId = subjectId; entity.SubjectName = subjectName; entity.DegreeId = degreeId; db.Subject.Add(entity); db.SaveChanges(); } } }
public void AddJiaocai(int categoryId, int jiaocaiId, string jiaocaiName, string jiaocaiUrl) { using (var db = new XKWEntities2()) { if (db.JiaoCai.Any(t => t.JiaoCaiId == jiaocaiId)) { return; } var entity = new JiaoCai(); entity.CategoryId = categoryId; entity.JiaoCaiId = jiaocaiId; entity.JCName = jiaocaiName; entity.JiaoCaiUrl = jiaocaiUrl; db.JiaoCai.Add(entity); db.SaveChanges(); } }
public void AddJiaocaiDetail(int jiaocaiId, int parentId, int id, string name) { using (var db = new XKWEntities2()) { if (db.JiaocaiDetail.Any(t => t.JiaoCaiDetailId == id)) { return; } var entity = new JiaocaiDetail(); entity.JiaoCaiId = jiaocaiId; entity.JiaoCaiDetailParentId = parentId; entity.JiaoCaiDetailId = id; entity.JiaoCaiDetailName = name; db.JiaocaiDetail.Add(entity); db.SaveChanges(); } }
public void AddGrabPageSource(int areaId, int jiaocaiId, int total) { using (var db = new XKWEntities2()) { if ( db.QuestionJiaoCaiSource.Any( t => t.AreaId == areaId && t.JiaocaiId == jiaocaiId)) { return; } var entity = new QuestionJiaoCaiSource(); entity.AreaId = areaId; entity.JiaocaiId = jiaocaiId; entity.Total = total; db.QuestionJiaoCaiSource.Add(entity); db.SaveChanges(); } }
public void AddGrabPageList(int areaId, int jiaocaiDetailId, int gradeId, int total, int pageNum) { using (var db = new XKWEntities2()) { if ( db.QuestionPageList.Any( t => t.AreaId == areaId && t.JiaocaiDetailId == jiaocaiDetailId && t.GradeId == gradeId && t.PageNum == pageNum)) { return; } var entity = new QuestionPageList(); entity.AreaId = areaId; entity.JiaocaiDetailId = jiaocaiDetailId; entity.GradeId = gradeId; entity.Total = total; entity.PageNum = pageNum; db.QuestionPageList.Add(entity); db.SaveChanges(); } }
public void AddJiaocaiDetaiSource(int areaId, int jiaocaiId, int jiaocaiDetaiId, int total, string url) { using (var db = new XKWEntities2()) { if (db.QuestionJiaoCaiDetailSource.Any(t => t.JiaocaiDetailId == jiaocaiDetaiId && t.AreaId == areaId)) { return; } var entity = new QuestionJiaoCaiDetailSource(); entity.AreaId = areaId; entity.JiaocaiId = jiaocaiId; entity.JiaocaiDetailId = jiaocaiDetaiId; entity.Total = total; entity.CrawlerUrl = url; db.QuestionJiaoCaiDetailSource.Add(entity); db.SaveChanges(); } }
public void AddQuestionJiaocaiSourceResult(int areaId, int jiaocaiId, int sourceId, string html, int total, int pageNum, string crawlerUrl, List <QuestionXkw> entities) { using (var db = new XKWEntities2()) { if ( db.QuestionJiaocaiSourceResult.Any( t => t.JiaocaiId == jiaocaiId && t.AreaId == areaId && t.PageNum == pageNum)) { return; } var entity = new QuestionJiaocaiSourceResult(); entity.AreaId = areaId; entity.JiaocaiId = jiaocaiId; entity.Html = html; entity.Total = total; entity.PageNum = pageNum; entity.QuestionJiaoCaiSourceId = sourceId; entity.CrawlerUrl = crawlerUrl; db.QuestionJiaocaiSourceResult.Add(entity); db.SaveChanges(); } }
public List <QuestionXkw> AddQuestion2(string html, string questionHtml, int jiaocaiId, string subjectId, string sourceUrl, int areaId, int sourceId, int total, int pageNum, int QuestionJiaoCaiDetailSourceId, Dictionary <string, bool> dicImageStaus) { List <QuestionXkw> result = new List <QuestionXkw>(); var elements = NSoupClient.Parse(html).GetElementsByClass("quesbox"); foreach (var element in elements) { try { QuestionXkw entity = new QuestionXkw(); entity.OriginHtml = element.Html(); var detail = element.Select("div.join-sj>a")[0]; entity.QuestionId = detail.Attr("quesid").NullToInt(); entity.@class = detail.Attr("class").NullToString(); entity.guid = detail.Attr("guid").NullToString(); entity.childnum = detail.Attr("childnum").NullToInt(); entity.questitle = detail.Attr("questitle").NullToString(); entity.categories = detail.Attr("categories").NullToString(); entity.qyid = detail.Attr("qyid").NullToInt(); entity.qdid = detail.Attr("qdid").NullToInt(); entity.qyname = detail.Attr("qyname").NullToString(); entity.qdname = detail.Attr("qdname").NullToString(); entity.JiaocaiId = jiaocaiId; var source = element.Select("div.quesource")[0]; entity.source = source.Html(); entity.SourceUrl = sourceUrl; var questiontitle = element.Select("div.question-inner")[0]; entity.key = questiontitle.Attr("key").NullToString(); entity.question_text = questiontitle.Html(); var href = element.Select("a.detail")[0]; entity.CrawlerUrl = href.Attr("href").NullToString(); var str = entity.CrawlerUrl.Replace("http://zujuan.xkw.com/", "") .Replace("https://zujuan.xkw.com/", ""); var bankId = str.Substring(0, str.IndexOf("q", StringComparison.OrdinalIgnoreCase)); ; var analysisUrl = $"http://im.zujuan.xkw.com/Parse/{entity.QuestionId}/{bankId}/700/14/28/{entity.key}"; var answerUrl = $"http://im.zujuan.xkw.com/Answer/{entity.QuestionId}/{bankId}/700/14/28/{entity.key}"; var paths = new ParseQuestionXkw().SaveAnswerImage(entity.QuestionId.ToString(), entity.key, subjectId); entity.AnalysisImg = paths[0]; entity.AnswerImg = paths[1]; entity.QuestionAnalysis = analysisUrl; entity.QuestionAnswer = answerUrl; entity.CreateTime = DateTime.Now; entity.QuestionJiaoCaiDetailSourceId = QuestionJiaoCaiDetailSourceId; if (dicImageStaus.Any(t => entity.OriginHtml.IndexOf(t.Key, StringComparison.OrdinalIgnoreCase) > 0)) { entity.ImageStatus = false; } else { entity.ImageStatus = true; } result.Add(entity); } //catch (DbUpdateException exception) //{ // var msg = string.Empty; // foreach (var validationError in ((DbUpdateException)exception).Data) // { // var o = validationError; // } // throw new Exception(); //} //catch (DbEntityValidationException ex) //{ // var msg = string.Empty; // foreach (var validationError in ((DbEntityValidationException)ex).EntityValidationErrors) // foreach (var error in validationError.ValidationErrors) // msg += string.Format("Property: {0} Error: {1}", error.PropertyName, error.ErrorMessage); // var fail = new Exception(msg); // throw fail; //} catch (Exception ex) { WriteLog(questionHtml, ex.ToString(), sourceUrl); throw ex; } } try { using (var db = new XKWEntities2()) { db.QuestionXkw.AddRange(result); if ( !db.QuestionJiaocaiSourceDetailResult.Any( t => t.JiaocaiDetailId == jiaocaiId && t.AreaId == areaId && t.PageNum == pageNum)) { var entity = new QuestionJiaocaiSourceDetailResult(); entity.AreaId = areaId; entity.JiaocaiId = jiaocaiId; entity.Html = html; entity.Total = total; entity.PageNum = pageNum; entity.QuestionJiaoCaiDetailSourceId = sourceId; entity.CrawlerUrl = sourceUrl; db.QuestionJiaocaiSourceDetailResult.Add(entity); } db.SaveChanges(); } } catch (Exception ex) { WriteLog(questionHtml, ex.ToString(), sourceUrl); throw ex; } return(result); }