public void WriteLog(string error, string error2, string url)
{
using (var db = new XKWEntities2())
{
var log = new Log();
log.Content = error;
log.Content2 = error2;
log.CrawlerUrl = url;
db.Log.Add(log);
db.SaveChanges();
}
}
public void UpdateQuestionJiaocaiSourceResultStatus(int id)
{
using (var db = new XKWEntities2())
{
if (
db.QuestionJiaoCaiSource.Any(t => t.Id == id))
{
var entity = db.QuestionJiaoCaiSource.FirstOrDefault(t => t.Id == id);
entity.Status = true;
db.SaveChanges();
}
}
}
public void UpdateQuestionJiaocaiSourceTotalCount(int id, int total)
{
using (var db = new XKWEntities2())
{
if (
db.QuestionJiaoCaiSource.Any(t => t.Id == id))
{
var entity = db.QuestionJiaoCaiSource.FirstOrDefault(t => t.Id == id);
entity.Total = total;
db.SaveChanges();
}
}
}
public void UpdateJiaocaiDetailStatus(int jiaocaiDetailId)
{
using (var db = new XKWEntities2())
{
if (
db.JiaocaiDetail.Any(t => t.JiaoCaiDetailId == jiaocaiDetailId))
{
var entity = db.JiaocaiDetail.FirstOrDefault(t => t.JiaoCaiDetailId == jiaocaiDetailId);
entity.Status = true;
db.SaveChanges();
}
}
}
public void AddSubjectGrade(int gradeId, int subjectId)
{
using (var db = new XKWEntities2())
{
if (!db.SubjectGrade.Any(t => t.SubjectId == subjectId && t.GradeId == gradeId))
{
var entity = new SubjectGrade();
entity.GradeId = gradeId;
entity.SubjectId = subjectId;
db.SubjectGrade.Add(entity);
db.SaveChanges();
}
}
}
public void AddArea(int areaId, string name, string shortName)
{
using (var db = new XKWEntities2())
{
if (!db.Area.Any(t => t.AreaId == areaId))
{
var entity = new Area();
entity.AreaId = areaId;
entity.Name = name;
entity.ShortName = shortName;
db.Area.Add(entity);
db.SaveChanges();
}
}
}
public void AddCategoryId(int categoryId, string categoryName, int subjectId)
{
using (var db = new XKWEntities2())
{
if (!db.SubjectCategory.Any(t => t.CategoryId == categoryId))
{
var entity = new SubjectCategory();
entity.CategoryId = categoryId;
entity.CategoryName = categoryName;
entity.SubjectId = subjectId;
db.SubjectCategory.Add(entity);
db.SaveChanges();
}
}
}
public void AddSubject(int subjectId, string subjectName, int degreeId)
{
using (var db = new XKWEntities2())
{
if (!db.Subject.Any(t => t.SubjectId == subjectId))
{
var entity = new Subject();
entity.SubjectId = subjectId;
entity.SubjectName = subjectName;
entity.DegreeId = degreeId;
db.Subject.Add(entity);
db.SaveChanges();
}
}
}
public void AddJiaocai(int categoryId, int jiaocaiId, string jiaocaiName, string jiaocaiUrl)
{
using (var db = new XKWEntities2())
{
if (db.JiaoCai.Any(t => t.JiaoCaiId == jiaocaiId))
{
return;
}
var entity = new JiaoCai();
entity.CategoryId = categoryId;
entity.JiaoCaiId = jiaocaiId;
entity.JCName = jiaocaiName;
entity.JiaoCaiUrl = jiaocaiUrl;
db.JiaoCai.Add(entity);
db.SaveChanges();
}
}
public void AddJiaocaiDetail(int jiaocaiId, int parentId, int id, string name)
{
using (var db = new XKWEntities2())
{
if (db.JiaocaiDetail.Any(t => t.JiaoCaiDetailId == id))
{
return;
}
var entity = new JiaocaiDetail();
entity.JiaoCaiId = jiaocaiId;
entity.JiaoCaiDetailParentId = parentId;
entity.JiaoCaiDetailId = id;
entity.JiaoCaiDetailName = name;
db.JiaocaiDetail.Add(entity);
db.SaveChanges();
}
}
public void AddGrabPageSource(int areaId, int jiaocaiId, int total)
{
using (var db = new XKWEntities2())
{
if (
db.QuestionJiaoCaiSource.Any(
t => t.AreaId == areaId && t.JiaocaiId == jiaocaiId))
{
return;
}
var entity = new QuestionJiaoCaiSource();
entity.AreaId = areaId;
entity.JiaocaiId = jiaocaiId;
entity.Total = total;
db.QuestionJiaoCaiSource.Add(entity);
db.SaveChanges();
}
}
public void AddGrabPageList(int areaId, int jiaocaiDetailId, int gradeId, int total, int pageNum)
{
using (var db = new XKWEntities2())
{
if (
db.QuestionPageList.Any(
t => t.AreaId == areaId && t.JiaocaiDetailId == jiaocaiDetailId && t.GradeId == gradeId && t.PageNum == pageNum))
{
return;
}
var entity = new QuestionPageList();
entity.AreaId = areaId;
entity.JiaocaiDetailId = jiaocaiDetailId;
entity.GradeId = gradeId;
entity.Total = total;
entity.PageNum = pageNum;
db.QuestionPageList.Add(entity);
db.SaveChanges();
}
}
public void AddJiaocaiDetaiSource(int areaId, int jiaocaiId, int jiaocaiDetaiId, int total, string url)
{
using (var db = new XKWEntities2())
{
if (db.QuestionJiaoCaiDetailSource.Any(t => t.JiaocaiDetailId == jiaocaiDetaiId && t.AreaId == areaId))
{
return;
}
var entity = new QuestionJiaoCaiDetailSource();
entity.AreaId = areaId;
entity.JiaocaiId = jiaocaiId;
entity.JiaocaiDetailId = jiaocaiDetaiId;
entity.Total = total;
entity.CrawlerUrl = url;
db.QuestionJiaoCaiDetailSource.Add(entity);
db.SaveChanges();
}
}
public void AddQuestionJiaocaiSourceResult(int areaId, int jiaocaiId, int sourceId, string html, int total, int pageNum, string crawlerUrl, List <QuestionXkw> entities)
{
using (var db = new XKWEntities2())
{
if (
db.QuestionJiaocaiSourceResult.Any(
t => t.JiaocaiId == jiaocaiId && t.AreaId == areaId && t.PageNum == pageNum))
{
return;
}
var entity = new QuestionJiaocaiSourceResult();
entity.AreaId = areaId;
entity.JiaocaiId = jiaocaiId;
entity.Html = html;
entity.Total = total;
entity.PageNum = pageNum;
entity.QuestionJiaoCaiSourceId = sourceId;
entity.CrawlerUrl = crawlerUrl;
db.QuestionJiaocaiSourceResult.Add(entity);
db.SaveChanges();
}
}
public List <QuestionXkw> AddQuestion2(string html, string questionHtml, int jiaocaiId, string subjectId, string sourceUrl, int areaId, int sourceId, int total, int pageNum, int QuestionJiaoCaiDetailSourceId, Dictionary <string, bool> dicImageStaus)
{
List <QuestionXkw> result = new List <QuestionXkw>();
var elements = NSoupClient.Parse(html).GetElementsByClass("quesbox");
foreach (var element in elements)
{
try
{
QuestionXkw entity = new QuestionXkw();
entity.OriginHtml = element.Html();
var detail = element.Select("div.join-sj>a")[0];
entity.QuestionId = detail.Attr("quesid").NullToInt();
entity.@class = detail.Attr("class").NullToString();
entity.guid = detail.Attr("guid").NullToString();
entity.childnum = detail.Attr("childnum").NullToInt();
entity.questitle = detail.Attr("questitle").NullToString();
entity.categories = detail.Attr("categories").NullToString();
entity.qyid = detail.Attr("qyid").NullToInt();
entity.qdid = detail.Attr("qdid").NullToInt();
entity.qyname = detail.Attr("qyname").NullToString();
entity.qdname = detail.Attr("qdname").NullToString();
entity.JiaocaiId = jiaocaiId;
var source = element.Select("div.quesource")[0];
entity.source = source.Html();
entity.SourceUrl = sourceUrl;
var questiontitle = element.Select("div.question-inner")[0];
entity.key = questiontitle.Attr("key").NullToString();
entity.question_text = questiontitle.Html();
var href = element.Select("a.detail")[0];
entity.CrawlerUrl = href.Attr("href").NullToString();
var str =
entity.CrawlerUrl.Replace("http://zujuan.xkw.com/", "")
.Replace("https://zujuan.xkw.com/", "");
var bankId = str.Substring(0, str.IndexOf("q", StringComparison.OrdinalIgnoreCase));
;
var analysisUrl =
$"http://im.zujuan.xkw.com/Parse/{entity.QuestionId}/{bankId}/700/14/28/{entity.key}";
var answerUrl =
$"http://im.zujuan.xkw.com/Answer/{entity.QuestionId}/{bankId}/700/14/28/{entity.key}";
var paths = new ParseQuestionXkw().SaveAnswerImage(entity.QuestionId.ToString(), entity.key, subjectId);
entity.AnalysisImg = paths[0];
entity.AnswerImg = paths[1];
entity.QuestionAnalysis = analysisUrl;
entity.QuestionAnswer = answerUrl;
entity.CreateTime = DateTime.Now;
entity.QuestionJiaoCaiDetailSourceId = QuestionJiaoCaiDetailSourceId;
if (dicImageStaus.Any(t => entity.OriginHtml.IndexOf(t.Key, StringComparison.OrdinalIgnoreCase) > 0))
{
entity.ImageStatus = false;
}
else
{
entity.ImageStatus = true;
}
result.Add(entity);
}
//catch (DbUpdateException exception)
//{
// var msg = string.Empty;
// foreach (var validationError in ((DbUpdateException)exception).Data)
// {
// var o = validationError;
// }
// throw new Exception();
//}
//catch (DbEntityValidationException ex)
//{
// var msg = string.Empty;
// foreach (var validationError in ((DbEntityValidationException)ex).EntityValidationErrors)
// foreach (var error in validationError.ValidationErrors)
// msg += string.Format("Property: {0} Error: {1}", error.PropertyName, error.ErrorMessage);
// var fail = new Exception(msg);
// throw fail;
//}
catch (Exception ex)
{
WriteLog(questionHtml, ex.ToString(), sourceUrl);
throw ex;
}
}
try
{
using (var db = new XKWEntities2())
{
db.QuestionXkw.AddRange(result);
if (
!db.QuestionJiaocaiSourceDetailResult.Any(
t => t.JiaocaiDetailId == jiaocaiId && t.AreaId == areaId && t.PageNum == pageNum))
{
var entity = new QuestionJiaocaiSourceDetailResult();
entity.AreaId = areaId;
entity.JiaocaiId = jiaocaiId;
entity.Html = html;
entity.Total = total;
entity.PageNum = pageNum;
entity.QuestionJiaoCaiDetailSourceId = sourceId;
entity.CrawlerUrl = sourceUrl;
db.QuestionJiaocaiSourceDetailResult.Add(entity);
}
db.SaveChanges();
}
}
catch (Exception ex)
{
WriteLog(questionHtml, ex.ToString(), sourceUrl);
throw ex;
}
return(result);
}