public void Execute(IJobExecutionContext context) { try { DateTime dt = DateTime.Now; string shortdate = dt.ToString("yyyy-MM-dd"); var bookdata = _helper.Get(null, "http://ac.qq.com/Comic/all/search/time/vip/3/page/1"); IQuery <VIPFreeComic> vfcq = dbcontext.Query <VIPFreeComic>(); string pattern_page = "共<em>(?<key1>.*?)</em>个结果"; MatchCollection matches_page = Regex.Matches(bookdata, pattern_page, RegexOptions.IgnoreCase | RegexOptions.Singleline); int pagecount = (int)Math.Ceiling(int.Parse(matches_page[0].Groups[1].Value) / 12.0); List <VIPFreeComic> vfclst = new List <VIPFreeComic>(); for (int i = 1; i <= pagecount; i++) { var bookdata2 = _helper.Get(null, "http://ac.qq.com/Comic/all/search/time/vip/3/page/" + i); bookdata2 = StringHelper.MergeSpace(bookdata2); string pattern2 = "<h3 class=\"ret-works-title clearfix\"> <a href=\"(?<key1>.*?)\" target=\"_blank\" title=\"(?<key2>.*?)\">(?<key3>.*?)</a>"; MatchCollection matches2 = Regex.Matches(bookdata2, pattern2, RegexOptions.IgnoreCase | RegexOptions.Singleline); for (int j = 0; j < matches2.Count; j++) { string bookurl = "http://ac.qq.com" + matches2[j].Groups["key1"].Value; string comicname = matches2[j].Groups["key2"].Value; string comicid = (int)Source.QQ + "_" + bookurl.Split('/').LastOrDefault(); vfclst.Add(new VIPFreeComic() { bookurl = bookurl, comicid = comicid, comicname = comicname, modify = dt, shortdate = shortdate, source = Source.QQ }); } } List <VIPFreeComic> vfcdblst = vfcq.Where(x => x.source == Source.QQ).ToList(); if (vfclst.Count > 0) { List <VIPFreeComic> delete = vfcdblst.Except(vfclst, new VIPFreeComic_Comparer()).ToList(); List <VIPFreeComic> add = vfclst.Except(vfcdblst, new VIPFreeComic_Comparer()).ToList(); if (delete.Count > 0) { List <string> deleteidlst = delete.Select(x => x.comicid).ToList(); dbcontext.Delete <VIPFreeComic>(x => deleteidlst.Contains(x.comicid)); } if (add.Count > 0) { dbcontext.BulkInsert(add); } } } catch (Exception ex) { throw; } }
public static void BulkCopyTest() { List <TestEntity> models = new List <TestEntity>(); models.Add(new TestEntity() { Id = 14, F_Byte = 1, F_Int16 = 1, F_Int32 = 1, F_Int64 = 1, F_Double = 1, F_Float = 1, F_Decimal = 1, F_Bool = true, F_DateTime = DateTime.Now, F_String = "lu" }); models.Add(new TestEntity() { Id = 15, F_Byte = 1, F_Int16 = 1, F_Int32 = 1, F_Int64 = 1, F_Double = 1, F_Float = 1, F_Decimal = 1, F_Bool = true, F_DateTime = DateTime.Now, F_Guid = Guid.NewGuid(), F_String = "so" }); object result = null; //SqlConnection conn = new SqlConnection(DbHelper.ConnectionString); //conn.BulkCopy1<TestEntity>(models, int.MaxValue, null, null, null); context.BeginTransaction(); result = context.Query <User>().Select(a => new UserLite() { Id = a.Id, Name = a.Name }).ToList(); context.BulkInsert(models, null, null, false); result = context.Query <User>().Select(a => new UserLite() { Id = a.Id, Name = a.Name }).ToList(); context.CommitTransaction(); Console.WriteLine(1); Console.ReadKey(); }
public static void BulkInsert() { List <User> models = new List <User>(); models.Add(new User() { Name = "lu", Age = 18, Gender = Gender.Woman, CityId = 1, OpTime = DateTime.Now }); models.Add(new User() { Name = "shuxin", Age = 18, Gender = Gender.Man, CityId = 1, OpTime = DateTime.Now }); /* 利用 SqlBulkCopy 批量插入数据 */ context.BulkInsert(models, batchSize: null, bulkCopyTimeout: null, keepIdentity: false); ConsoleHelper.WriteLineAndReadKey(1); }
public void Execute(IJobExecutionContext context) { DateTime dt = DateTime.Now; string shortdate = dt.ToString("yyyy-MM-dd"); IQuery <Chapter> cpq = dbcontext.Query <Chapter>(); List <Chapter> cplst = cpq.Where(x => x.source == Source.Zymk && x.downstatus == DownChapter.待处理链接).Take(200).ToList(); List <int> ids = cplst.Select(x => x.Id).ToList(); dbcontext.Update <Chapter>(a => ids.Contains(a.Id), a => new Chapter() { downstatus = DownChapter.处理完链接, modify = dt }); List <Chapter> chapterlst = new List <Chapter>(); foreach (var cp in cplst) { try { string chapterpage = cp.chapterurl.Replace("http://www.zymk.cn/", ""); var imgdata = _helper.Get(null, chapterpage); Regex imgpath = new Regex("chapter_addr:\"(?<key1>.*?)\","); Regex starindex = new Regex("start_var:(?<key1>.*?),"); Regex totalimg = new Regex("end_var:(?<key1>.*?),"); Regex pageid = new Regex("chapter_id:(?<key1>.*?),"); Regex domain = new Regex("domain:\"(?<key1>.*?)\","); Regex comic_size = new Regex("middle:\"(?<key1>.*?)\""); var _imgpath = imgpath.Match(imgdata).Groups["key1"].Value.Replace("\\\\", "#").Replace("\\", "").Replace("#", "\\"); var _totalimg = totalimg.Match(imgdata).Groups["key1"].Value; var _pageid = pageid.Match(imgdata).Groups["key1"].Value; var _domain = domain.Match(imgdata).Groups["key1"].Value; var _comic_size = comic_size.Match(imgdata).Groups["key1"].Value; int start = int.Parse(starindex.Match(imgdata).Groups["key1"].Value.Trim()); List <Page> pglst = new List <Page>(); int imgcount = int.Parse(_totalimg); imgcount = imgcount - start; string imgdecodepath = DecodeHelper.Decode(_imgpath, int.Parse(_pageid)); for (int i = 0; i <= imgcount; i++) { string pgsource = "http://mhpic." + _domain + "/comic/" + imgdecodepath + start + ".jpg" + _comic_size; pglst.Add(new Page() { pagesource = pgsource, chapterid = cp.chapterid, modify = dt, shortdate = shortdate, sort = i + 1, source = cp.source, pagelocal = "", }); start = start + 1; } dbcontext.BulkInsert(pglst); } catch (Exception ex) { logger.Error(ex.Message); cp.downstatus = DownChapter.待处理链接; cp.modify = dt; dbcontext.Update(cp); Err_ChapterJob err = new Err_ChapterJob(); err.bookurl = cp.chapterurl; err.source = cp.source; err.errtype = ErrChapter.解析出错; err.modify = dt; err.shortdate = shortdate; err.message = ex.Message; err = dbcontext.Insert(err); continue; } } }
public void Execute(IJobExecutionContext context) { DateTime dt = DateTime.Now; string shortdate = dt.ToString("yyyy-MM-dd"); string yesterday = dt.AddDays(-1).ToString("yyyy-MM-dd"); IQuery <Comic> q = dbcontext.Query <Comic>(); IQuery <Chapter> cpq = dbcontext.Query <Chapter>(); List <Comic> comiclst = q.Where(a => a.source == Source.dongmanmanhua && a.shortdate == shortdate).Take(200).ToList(); List <int> ids = comiclst.Select(x => x.Id).ToList(); dbcontext.Update <Comic>(a => ids.Contains(a.Id), a => new Comic() { shortdate = yesterday, modify = dt }); List <Chapter> chapterlst = new List <Chapter>(); foreach (var comic in comiclst) { List <Chapter> cplst = cpq.Where(a => a.comicid == comic.Id && a.source == Source.dongmanmanhua).ToList(); if (cplst.Count == 0) { try { string bookurl = comic.bookurl.Replace("https://www.dongmanmanhua.cn/", ""); var bookdata = _helper.Get(null, bookurl); string pattern = "<li id=\"episode_(?<key1>.*?)\" data-episode-no=\"(?<key2>.*?)\">(?<key3>.*?)</li>"; MatchCollection matches = Regex.Matches(bookdata, pattern, RegexOptions.IgnoreCase | RegexOptions.Singleline); int pagecount = (int)Math.Ceiling(int.Parse(matches[0].Groups[1].Value) / 10.0); for (int i = 0; i < matches.Count; i++) { var lihtml = matches[i].Groups["key3"].Value; Regex reg1 = new Regex("href=\"(?<key1>.*?)\""); Match match1 = reg1.Match(lihtml); string chapterurl = match1.Groups["key1"].Value; Regex reg2 = new Regex("<span class=\"tx\">#(?<key1>.*?)</span>"); Match match2 = reg2.Match(lihtml); int sort = int.Parse(match2.Groups["key1"].Value); Regex reg3 = new Regex("<span class=\"subj\"><span>(?<key1>.*?)</span></span>"); Match match3 = reg3.Match(lihtml); string chaptername = match3.Groups["key1"].Value; Regex reg4 = new Regex("src=\"(?<key1>.*?)\""); Match match4 = reg4.Match(lihtml); string chaptersource = match4.Groups["key1"].Value; chapterlst.Add(new Chapter() { chapterid = comic.comicid + "_" + sort, chaptername = chaptername, chapterurl = "https:" + chapterurl, sort = sort, comicid = comic.Id, retry = 0, source = comic.source, downstatus = DownChapter.待处理链接, isvip = "0", chaptersource = chaptersource, chapterlocal = "", modify = dt, shortdate = shortdate, }); } for (int i = 2; i <= pagecount; i++) { var bookdata2 = _helper.Get(null, bookurl + "&page=" + i); string pattern2 = "<li id=\"episode_(?<key1>.*?)\" data-episode-no=\"(?<key2>.*?)\">(?<key3>.*?)</li>"; MatchCollection matches2 = Regex.Matches(bookdata2, pattern2, RegexOptions.IgnoreCase | RegexOptions.Singleline); for (int j = 0; j < matches2.Count; j++) { var lihtml = matches2[j].Groups["key3"].Value; Regex reg1 = new Regex("href=\"(?<key1>.*?)\""); Match match1 = reg1.Match(lihtml); string chapterurl = match1.Groups["key1"].Value; Regex reg2 = new Regex("<span class=\"tx\">#(?<key1>.*?)</span>"); Match match2 = reg2.Match(lihtml); int sort = int.Parse(match2.Groups["key1"].Value); Regex reg3 = new Regex("<span class=\"subj\"><span>(?<key1>.*?)</span></span>"); Match match3 = reg3.Match(lihtml); string chaptername = match3.Groups["key1"].Value; Regex reg4 = new Regex("src=\"(?<key1>.*?)\""); Match match4 = reg4.Match(lihtml); string chaptersource = match4.Groups["key1"].Value; chapterlst.Add(new Chapter() { chapterid = comic.comicid + "_" + sort, chaptername = chaptername, chapterurl = "https:" + chapterurl, sort = sort, comicid = comic.Id, retry = 0, source = comic.source, downstatus = DownChapter.待处理链接, isvip = "0", chaptersource = chaptersource, chapterlocal = "", modify = dt, shortdate = shortdate, }); } } } catch (Exception ex) { logger.Error(ex.Message); comic.shortdate = shortdate; comic.modify = dt; dbcontext.Update(comic); Err_ChapterJob err = new Err_ChapterJob(); err.bookurl = comic.bookurl; err.source = comic.source; err.errtype = ErrChapter.解析出错; err.modify = dt; err.shortdate = shortdate; err.message = ex.Message; err = dbcontext.Insert(err); continue; } } } if (chapterlst.Count > 0) { dbcontext.BulkInsert(chapterlst); } }
public void Execute(IJobExecutionContext context) { DateTime dt = DateTime.Now; string shortdate = dt.ToString("yyyy-MM-dd"); string yesterday = dt.AddDays(-1).ToString("yyyy-MM-dd"); IQuery <Comic> q = dbcontext.Query <Comic>(); IQuery <Chapter> cpq = dbcontext.Query <Chapter>(); List <Comic> comiclst = q.Where(a => a.source == Source.QQ && a.shortdate == shortdate).Take(200).ToList(); List <int> ids = comiclst.Select(x => x.Id).ToList(); dbcontext.Update <Comic>(a => ids.Contains(a.Id), a => new Comic() { shortdate = yesterday, modify = dt }); List <Chapter> chapterlst = new List <Chapter>(); foreach (var comic in comiclst) { List <Chapter> cplst = cpq.Where(a => a.comicid == comic.Id && a.source == Source.QQ).ToList(); if (cplst.Count == 0) { try { string bookurl = comic.bookurl.Replace("http://ac.qq.com/", ""); var bookdata = _helper.Get(null, bookurl); //class=\"chapter-page-all works-(?<key1>.*?)-list\" Regex reg1 = new Regex("<ol class=\"chapter-page-all works-chapter-list\">(?<key1>.*?)</ol>", RegexOptions.IgnoreCase | RegexOptions.Singleline); Match match1 = reg1.Match(bookdata); string htmlSource = match1.Groups["key1"].Value; string pattern = "<a target=\"_blank\" title=\"(?<key1>.*?)\" href=\"(?<key2>.*?)\">(?<key3>.*?)</a>"; MatchCollection matches = Regex.Matches(htmlSource, pattern, RegexOptions.IgnoreCase | RegexOptions.Singleline); for (int i = 0; i < matches.Count; i++) { chapterlst.Add(new Chapter() { chapterid = comic.comicid + "_" + matches[i].Groups["key2"].Value.Split('/').LastOrDefault(), chaptername = matches[i].Groups["key3"].Value.Trim(), chapterurl = "http://ac.qq.com" + matches[i].Groups["key2"].Value, sort = i + 1, comicid = comic.Id, retry = 0, source = comic.source, downstatus = DownChapter.待处理链接, isvip = "0", chaptersource = "", chapterlocal = "", modify = dt, shortdate = shortdate, }); } } catch (Exception ex) { logger.Error(ex.Message); comic.shortdate = shortdate; comic.modify = dt; dbcontext.Update(comic); Err_ChapterJob err = new Err_ChapterJob(); err.bookurl = comic.bookurl; err.source = comic.source; err.errtype = ErrChapter.解析出错; err.modify = dt; err.shortdate = shortdate; err.message = ex.Message; err = dbcontext.Insert(err); continue; } } } if (chapterlst.Count > 0) { dbcontext.BulkInsert(chapterlst); } }
public void Execute(IJobExecutionContext context) { DateTime dt = DateTime.Now; string shortdate = dt.ToString("yyyy-MM-dd"); string yesterday = dt.AddDays(-1).ToString("yyyy-MM-dd"); IQuery <Page> pq = dbcontext.Query <Page>(); IQuery <Notice> nq = dbcontext.Query <Notice>(); IQuery <Chapter> cpq = dbcontext.Query <Chapter>();//x.comicid == "1_622585" List <Chapter> cplst = cpq.Where(x => x.source == Source.QQ && x.downstatus == DownChapter.完图片 && x.updatedate != shortdate).Take(200).ToList(); List <int> ids = cplst.Select(x => x.Id).ToList(); dbcontext.Update <Chapter>(a => ids.Contains(a.Id), a => new Chapter() { updatedate = shortdate, modify = dt }); List <Chapter> chapterlst = new List <Chapter>(); foreach (var cp in cplst) { try { string chapterpage = cp.chapterurl.Replace("http://ac.qq.com", ""); var imgdata = _helper.Get(null, chapterpage); Regex rex = new Regex("var DATA = '(?<key1>.*?)',"); Match match = rex.Match(imgdata); string key = match.Groups["key1"].Value; if (string.IsNullOrEmpty(key)) { cp.downstatus = DownChapter.待处理链接; cp.modify = dt; dbcontext.Update(cp); Err_ChapterJob err = new Err_ChapterJob(); err.bookurl = cp.chapterurl; err.source = cp.source; err.errtype = ErrChapter.解析出错; err.modify = dt; err.shortdate = shortdate; err.message = "DATA解析失败"; err = dbcontext.Insert(err); continue; } var s = DecodeHelper.QQPageDecode(key.Substring(1)); var t = JsonHelper.DeserializeJsonToObject <QQ_Page_Api>(s); if (t.chapter.vipStatus == 1) { List <Page> pglst = new List <Page>(); for (int i = 0; i < t.picture.Count; i++) { pglst.Add(new Page() { chapterid = cp.chapterid, modify = dt, shortdate = shortdate, sort = i + 1, source = cp.source, pagelocal = "", pagesource = t.picture[i].url }); } List <Page> pgdblst = pq.Where(x => x.chapterid == cp.chapterid).ToList(); if (pgdblst.Count != pglst.Count) { List <int> idlst = pgdblst.Select(x => x.Id).ToList(); dbcontext.Delete <Page>(x => idlst.Contains(x.Id)); dbcontext.BulkInsert(pglst); cp.downstatus = DownChapter.处理完链接; cp.modify = dt; dbcontext.Update(cp); Notice notice = new Notice(); notice.noticeid = cp.chapterid; notice.noticestatus = NoticeStatus.等待处理; notice.noticetype = NoticeType.章节更新; notice.source = cp.source; notice.shortdate = shortdate; notice.modify = dt; var nqwait = nq.Where(x => x.noticeid == cp.chapterid && x.noticestatus == NoticeStatus.等待处理 && x.noticetype == NoticeType.章节更新).FirstOrDefault(); if (nqwait == null) { dbcontext.Insert(notice); } continue; } } else { selenium.Navigate().GoToUrl(cp.chapterurl); IList <IWebElement> frames = selenium.FindElements(By.TagName("iframe")); IWebElement controlPanelFrame = null; //commentFrame ptlogin_iframe foreach (var frame in frames) { if (frame.GetAttribute("id") == "iframeAll") { controlPanelFrame = frame; break; } } if (controlPanelFrame != null) //QQ登录 { selenium.SwitchTo().Frame(controlPanelFrame); IReadOnlyCollection <IWebElement> switchtoElement = selenium.FindElements(By.Id("switcher_plogin")); if (switchtoElement != null && switchtoElement.Count > 0) { switchtoElement.First().Click(); selenium.FindElement(By.Id("u")).Clear(); selenium.FindElement(By.Id("u")).SendKeys("2806126975"); selenium.FindElement(By.Id("p")).Clear(); selenium.FindElement(By.Id("p")).SendKeys("rby123456"); selenium.FindElement(By.Id("login_button")).Click(); } selenium.SwitchTo().DefaultContent(); } frames = selenium.FindElements(By.TagName("iframe")); IWebElement checkVipFrame = null; foreach (var frame in frames) { if (frame.GetAttribute("id") == "checkVipFrame") { checkVipFrame = frame; break; } } if (checkVipFrame != null) { selenium.SwitchTo().Frame(checkVipFrame); //自动购买 IReadOnlyCollection <IWebElement> checkAutoElement = selenium.FindElements(By.Id("check_auto_next")); IReadOnlyCollection <IWebElement> singlBbuyElement = selenium.FindElements(By.ClassName("single_buy")); if (checkAutoElement != null && singlBbuyElement != null && checkAutoElement.Count > 0 && singlBbuyElement.Count > 0) { if (singlBbuyElement.First().Text.Trim().StartsWith("点券不足")) { MailMessage msg = new MailMessage(); continue; } else { checkAutoElement.First().Click(); singlBbuyElement.First().Click(); } } } Match match1 = rex.Match(selenium.PageSource); key = match1.Groups["key1"].Value; if (string.IsNullOrEmpty(key)) { cp.downstatus = DownChapter.待处理链接; cp.modify = dt; dbcontext.Update(cp); Err_ChapterJob err = new Err_ChapterJob(); err.bookurl = cp.chapterurl; err.source = cp.source; err.errtype = ErrChapter.解析出错; err.modify = dt; err.shortdate = shortdate; err.message = "DATA解析失败"; err = dbcontext.Insert(err); continue; } s = DecodeHelper.QQPageDecode(key.Substring(1)); t = JsonHelper.DeserializeJsonToObject <QQ_Page_Api>(s); List <Page> pglst = new List <Page>(); for (int i = 0; i < t.picture.Count; i++) { pglst.Add(new Page() { chapterid = cp.chapterid, modify = dt, shortdate = shortdate, sort = i + 1, source = cp.source, pagelocal = "", pagesource = t.picture[i].url }); } List <Page> pgdblst = pq.Where(x => x.chapterid == cp.chapterid).ToList(); if (pgdblst.Count != pglst.Count) { List <int> idlst = pgdblst.Select(x => x.Id).ToList(); dbcontext.Delete <Page>(x => idlst.Contains(x.Id)); dbcontext.BulkInsert(pglst); cp.downstatus = DownChapter.处理完链接; cp.modify = dt; dbcontext.Update(cp); Notice notice = new Notice(); notice.noticeid = cp.chapterid; notice.noticestatus = NoticeStatus.等待处理; notice.noticetype = NoticeType.章节更新; notice.source = cp.source; notice.shortdate = shortdate; notice.modify = dt; var nqwait = nq.Where(x => x.noticeid == cp.chapterid && x.noticestatus == NoticeStatus.等待处理 && x.noticetype == NoticeType.章节更新).FirstOrDefault(); if (nqwait == null) { dbcontext.Insert(notice); } continue; } } } catch (Exception ex) { logger.Error(ex.Message); cp.updatedate = yesterday; cp.modify = dt; dbcontext.Update(cp); Err_ChapterJob err = new Err_ChapterJob(); err.bookurl = cp.chapterurl; err.source = cp.source; err.errtype = ErrChapter.解析出错; err.modify = dt; err.shortdate = shortdate; err.message = ex.Message; err = dbcontext.Insert(err); continue; } } }
public void Execute(IJobExecutionContext context) { DateTime dt = DateTime.Now; string shortdate = dt.ToString("yyyy-MM-dd"); string yesterday = dt.AddDays(-1).ToString("yyyy-MM-dd"); IQuery <Comic> q = dbcontext.Query <Comic>(); IQuery <Chapter> cpq = dbcontext.Query <Chapter>(); List <Comic> comiclst = q.Where(a => a.source == Source.mh160 && a.shortdate == shortdate).Take(200).ToList(); List <int> ids = comiclst.Select(x => x.Id).ToList(); dbcontext.Update <Comic>(a => ids.Contains(a.Id), a => new Comic() { shortdate = yesterday, modify = dt }); List <Chapter> chapterlst = new List <Chapter>(); foreach (var comic in comiclst) { List <Chapter> cplst = cpq.Where(a => a.comicid == comic.comicid && a.source == Source.mh160).ToList(); if (cplst.Count == 0) { try { var bookdata = _helper.Get(comic.bookurl, Encoding.GetEncoding("gb2312")); bookdata = StringHelper.MergeSpace(bookdata); //string pattern = "<div class=\"plist pnormal\" id=\"play_0\"><li(?<key1>.*?)><a href=\"(?<key2>.*?)\" title=\"(?<key3>.*?)\" target=\"_blank\">(?<key4>.*?)</a></li>"; string pattern = "<div class=\"plist pnormal\" id=\"play_0\"> <ul>(?<key1>.*?)</ul> </div>"; MatchCollection matches = Regex.Matches(bookdata, pattern, RegexOptions.IgnoreCase | RegexOptions.Singleline); string pattern2 = "<a href=\"(?<key1>.*?)\" title=\"(?<key2>.*?)\" target=\"_blank\">(?<key3>.*?)</a>"; MatchCollection matches2 = Regex.Matches(matches[0].Groups["key1"].Value, pattern2, RegexOptions.IgnoreCase | RegexOptions.Singleline); for (int i = 0; i < matches2.Count; i++) { string chapterid = comic.comicid + "_" + matches2[i].Groups["key1"].Value.Split('/').LastOrDefault().Replace(".html", ""); string chaptername = matches2[i].Groups["key2"].Value; string chapterurl = "http://www.mh160.com" + matches2[i].Groups["key1"].Value; int sort = matches.Count - i; chapterlst.Add(new Chapter() { chapterid = chapterid, chaptername = chaptername, chapterurl = chapterurl, sort = sort, comicid = comic.comicid, retry = 0, source = comic.source, downstatus = DownChapter.待处理链接, isvip = "0", chaptersource = "", chapterlocal = "", modify = dt, shortdate = shortdate, }); } } catch (Exception ex) { logger.Error(ex.Message); comic.shortdate = shortdate; comic.modify = dt; dbcontext.Update(comic); Err_ChapterJob err = new Err_ChapterJob(); err.bookurl = comic.bookurl; err.source = comic.source; err.errtype = ErrChapter.解析出错; err.modify = dt; err.shortdate = shortdate; err.message = ex.Message; err = dbcontext.Insert(err); continue; } } } if (chapterlst.Count > 0) { dbcontext.BulkInsert(chapterlst); } }
public void Execute(IJobExecutionContext context) { DateTime dt = DateTime.Now; string ticks = dt.Ticks.ToString(); string shortdate = dt.ToString("yyyy-MM-dd"); string updatedatetime = shortdate + " " + ((dt.Hour / 0.5 + 1) * 0.5).ToString(); string yesterday = dt.AddDays(-1).ToString("yyyy-MM-dd"); IQuery <Comic> q = dbcontext.Query <Comic>(); IQuery <Chapter> cpq = dbcontext.Query <Chapter>(); IQuery <PageHis> phisq = dbcontext.Query <PageHis>(); IQuery <Page> pq = dbcontext.Query <Page>(); IQuery <Notice> nq = dbcontext.Query <Notice>(); List <Comic> comiclst = q.Where(a => a.source == Source.QQ && a.isoffline == false && (a.updatedatetime == null || a.updatedatetime != updatedatetime)).Take(200).ToList(); List <int> ids = comiclst.Select(x => x.Id).ToList(); dbcontext.Update <Comic>(a => ids.Contains(a.Id), a => new Comic() { updatedatetime = updatedatetime, modify = dt }); foreach (var comic in comiclst) { List <Chapter> cplst = cpq.Where(a => a.comicid == comic.comicid && a.source == Source.QQ).ToList(); List <Chapter> chapterlst = new List <Chapter>(); if (cplst.Count > 0) { try { string bookurl = comic.bookurl.Replace("http://ac.qq.com/", ""); var bookdata = _helper.Get(null, bookurl); //class=\"chapter-page-all works-(?<key1>.*?)-list\" Regex reg1 = new Regex("<ol class=\"chapter-page-all works-chapter-list\">(?<key1>.*?)</ol>", RegexOptions.IgnoreCase | RegexOptions.Singleline); Match match1 = reg1.Match(bookdata); string htmlSource = match1.Groups["key1"].Value; string pattern = "<a target=\"_blank\" title=\"(?<key1>.*?)\" href=\"(?<key2>.*?)\">(?<key3>.*?)</a>"; MatchCollection matches = Regex.Matches(htmlSource, pattern, RegexOptions.IgnoreCase | RegexOptions.Singleline); for (int i = 0; i < matches.Count; i++) { chapterlst.Add(new Chapter() { chapterid = comic.comicid + "_" + matches[i].Groups["key2"].Value.Split('/').LastOrDefault(), chaptername = matches[i].Groups["key3"].Value.Trim(), chapterurl = "http://ac.qq.com" + matches[i].Groups["key2"].Value, sort = i + 1, comicid = comic.comicid, retry = 0, source = comic.source, downstatus = DownChapter.待处理链接, isvip = "0", chaptersource = "", chapterlocal = "", modify = dt, shortdate = shortdate, ticks = ticks }); } int delete = cplst.Except(chapterlst, new Chapter_Comparer()).Count(); // 删章 List <Chapter> add = chapterlst.Except(cplst, new Chapter_Comparer()).ToList(); // 新增 //if (delete > 0) //{ // List<string> idlst = cplst.Select(x => x.chapterid).ToList(); // int phislstcount = phisq.Where(x => idlst.Contains(x.chapterid)).Count(); // if (phislstcount == 0) // { // List<Page> pglst = pq.Where(x => idlst.Contains(x.chapterid)).ToList(); // if (pglst.Count>0) // { // List<PageHis> phislst = new List<PageHis>(); // pglst.ForEach(x => // { // phislst.Add(new PageHis() // { // chapterid = x.chapterid, // modify = x.modify, // pagelocal = x.pagelocal, // pagesource = x.pagesource, // pagestate = x.pagestate, // shortdate = x.shortdate, // sort = x.sort, // source = x.source, // ticks = x.ticks // }); // }); // dbcontext.BulkInsert(phislst); // } // } // dbcontext.Delete<Page>(x => idlst.Contains(x.chapterid)); // dbcontext.Delete<Chapter>(x => idlst.Contains(x.chapterid)); // if (chapterlst.Count > 0) // { // dbcontext.BulkInsert(chapterlst); // } // Notice notice = new Notice(); // notice.noticeid = comic.comicid; // notice.noticestatus = NoticeStatus.等待处理; // notice.noticetype = NoticeType.目录变更; // notice.source = comic.source; // notice.shortdate = shortdate; // notice.modify = dt; // var nqwait = nq.Where(x => x.noticeid == comic.comicid && x.noticestatus == NoticeStatus.等待处理 && x.noticetype == NoticeType.目录变更).FirstOrDefault(); // if (nqwait == null) // { // dbcontext.Insert(notice); // } // continue; //} //else //{ // List<Chapter> mvadd = chapterlst.Except(add, new Chapter_Comparer()).ToList(); // string cplststr = string.Join(",", cplst.Select(x => x.chapterid).ToArray()); // string chapterlststr = string.Join(",", mvadd.Select(x => x.chapterid).ToArray()); // if (cplststr != chapterlststr) // 调序 // { // List<string> idlst = cplst.Select(x => x.chapterid).ToList(); // int phislstcount = phisq.Where(x => idlst.Contains(x.chapterid)).Count(); // if (phislstcount == 0) // { // List<Page> pglst = pq.Where(x => idlst.Contains(x.chapterid)).ToList(); // if (pglst.Count > 0) // { // List<PageHis> phislst = new List<PageHis>(); // pglst.ForEach(x => // { // phislst.Add(new PageHis() // { // chapterid = x.chapterid, // modify = x.modify, // pagelocal = x.pagelocal, // pagesource = x.pagesource, // pagestate = x.pagestate, // shortdate = x.shortdate, // sort = x.sort, // source = x.source, // ticks = x.ticks // }); // }); // dbcontext.BulkInsert(phislst); // } // } // dbcontext.Delete<Page>(x => idlst.Contains(x.chapterid)); // dbcontext.Delete<Chapter>(x => idlst.Contains(x.chapterid)); // if (chapterlst.Count > 0) // { // dbcontext.BulkInsert(chapterlst); // } // Notice notice = new Notice(); // notice.noticeid = comic.comicid; // notice.noticestatus = NoticeStatus.等待处理; // notice.noticetype = NoticeType.目录变更; // notice.source = comic.source; // notice.shortdate = shortdate; // notice.modify = dt; // var nqwait = nq.Where(x => x.noticeid == comic.comicid && x.noticestatus == NoticeStatus.等待处理 && x.noticetype == NoticeType.目录变更).FirstOrDefault(); // if (nqwait == null) // { // dbcontext.Insert(notice); // } // continue; // } //} if (add.Count > 0) { dbcontext.BulkInsert(add); } } catch (Exception ex) { logger.Error(ex.Message); comic.updatedatetime = ""; comic.modify = dt; dbcontext.Update(comic); Err_ChapterJob err = new Err_ChapterJob(); err.bookurl = comic.bookurl; err.source = comic.source; err.errtype = ErrChapter.解析出错; err.modify = dt; err.shortdate = shortdate; err.message = ex.Message; err = dbcontext.Insert(err); continue; } } } }
public void Execute(IJobExecutionContext context) { List <bkIm> bklst = ExcelHelper.Import(@"C:\ComicCrawer\GetComics\Excel\漫画抓取信息.xlsx").ToList(); List <Comic> comiclst = new List <Comic>(); IQuery <Comic> cq = dbcontext.Query <Comic>(); DateTime dt = DateTime.Now; string shortdate = dt.ToString("yyyy-MM-dd"); foreach (var bk in bklst) { var comic = cq.Where(x => x.bookurl == bk.bookurl).FirstOrDefault(); if (comic != null) { continue; } if (bk.bookurl.StartsWith("http://ac.qq.com")) { try { HttpHelper _helper = new HttpHelper("http://ac.qq.com"); string bookurl = bk.bookurl.Replace("http://ac.qq.com/", ""); var bookdata = _helper.Get(null, bookurl); Regex reg1 = new Regex("id=\"input_artistName\" value=\"(?<key1>.*?)\""); Match match1 = reg1.Match(bookdata); string authorname = match1.Groups["key1"].Value; Regex reg2 = new Regex("img src=\"(?<key1>.*?)\" alt=\"(?<key2>.*?)\" height=\"280\" width=\"210\""); Match match2 = reg2.Match(bookdata); string comiccover = match2.Groups["key1"].Value; string comicname = match2.Groups["key2"].Value; Regex reg3 = new Regex("<label class=\"works-intro-status\">(?<key1>.*?)</label>"); Match match3 = reg3.Match(bookdata); string isfinished = match3.Groups["key1"].Value; Regex reg4 = new Regex("<meta name=\"Description\" content=\"(?<key1>.*?),简介:(?<key2>.*?)《(?<key3>.*?)》的标签:(?<key4>.*?)\" />"); Match match4 = reg4.Match(bookdata); string bookdesc = match4.Groups["key2"].Value; string theme = match4.Groups["key4"].Value; comiclst.Add(new Comic() { authorname = authorname, bookurl = bk.bookurl, comiccoversource = comiccover, comiccoverlocal = "", comicdesc = bookdesc, comicid = bk.bookurl.Split('/').LastOrDefault(), comicname = comicname, isfinished = isfinished, theme = theme, isvip = "0", source = Source.QQ, stopcrawer = false, isoffline = false, recrawer = false, shortdate = shortdate, modify = dt, }); } catch (Exception ex) { Err_ComicJob err = new Err_ComicJob(); err.bookurl = bk.bookurl; err.errtype = ErrComic.解析出错; err.modify = dt; err.shortdate = shortdate; err.message = ex.Message; err = dbcontext.Insert(err); continue; } } else if (bk.bookurl.StartsWith("https://www.dongmanmanhua.cn")) { try { HttpHelper _helper = new HttpHelper("https://www.dongmanmanhua.cn"); string bookurl = bk.bookurl.Replace("https://www.dongmanmanhua.cn/", ""); var bookdata = _helper.Get(null, bookurl); Regex reg1 = new Regex("<meta property=\"og:image\" content=\"(?<key1>.*?)\" />"); Match match1 = reg1.Match(bookdata); string comiccover = match1.Groups["key1"].Value; Regex reg2 = new Regex("<meta property=\"com-dongman:webtoon:author\" content=\"(?<key1>.*?)\" />"); Match match2 = reg2.Match(bookdata); string authorname = match2.Groups["key1"].Value; Regex reg3 = new Regex("<p class=\"summary\">(?<key1>.*?)</p>", RegexOptions.IgnoreCase | RegexOptions.Singleline); Match match3 = reg3.Match(bookdata); string bookdesc = match3.Groups["key1"].Value; Regex reg4 = new Regex("<h1 class=\"subj\">(?<key1>.*?)</h1>"); Match match4 = reg4.Match(bookdata); string comicname = match4.Groups["key1"].Value; Regex reg5 = new Regex("txt_ico_up"); Match match5 = reg5.Match(bookdata); string isfinished = match5.Length > 0?"连载中": "已完结"; Regex reg6 = new Regex("<h2 class=\"genre(?<key1>.*?)\">(?<key2>.*?)</h2>"); Match match6 = reg6.Match(bookdata); string theme = match6.Groups["key2"].Value; comiclst.Add(new Comic() { authorname = authorname, bookurl = bk.bookurl, comiccoversource = comiccover, comiccoverlocal = "", comicdesc = bookdesc, comicid = bk.bookurl.Split('=').LastOrDefault(), comicname = comicname, isfinished = isfinished, theme = theme, isvip = "0", source = Source.dongmanmanhua, stopcrawer = false, isoffline = false, recrawer = false, shortdate = shortdate, modify = dt, }); } catch (Exception ex) { logger.Error(ex.Message); Err_ComicJob err = new Err_ComicJob(); err.bookurl = bk.bookurl; err.errtype = ErrComic.解析出错; err.modify = dt; err.shortdate = shortdate; err.message = ex.Message; err = dbcontext.Insert(err); continue; } } } if (comiclst.Count > 0) { dbcontext.BulkInsert(comiclst); } }
public void Execute(IJobExecutionContext context) { DateTime dt = DateTime.Now; string shortdate = dt.ToString("yyyy-MM-dd"); string yesterday = dt.AddDays(-1).ToString("yyyy-MM-dd"); IQuery <Chapter> cpq = dbcontext.Query <Chapter>(); List <Chapter> cplst = cpq.Where(x => x.source == Source.QQ && x.downstatus == DownChapter.待处理链接).Take(200).ToList(); List <int> ids = cplst.Select(x => x.Id).ToList(); dbcontext.Update <Chapter>(a => ids.Contains(a.Id), a => new Chapter() { downstatus = DownChapter.处理完链接, modify = dt }); List <Chapter> chapterlst = new List <Chapter>(); foreach (var cp in cplst) { try { string chapterpage = cp.chapterurl.Replace("http://ac.qq.com", ""); var imgdata = _helper.Get(null, chapterpage); Regex rex = new Regex("var DATA = '(?<key1>.*?)',"); Match match = rex.Match(imgdata); string key = match.Groups["key1"].Value; if (string.IsNullOrEmpty(key)) { cp.downstatus = DownChapter.待处理链接; cp.modify = dt; dbcontext.Update(dt); Err_ChapterJob err = new Err_ChapterJob(); err.bookurl = cp.chapterurl; err.source = cp.source; err.errtype = ErrChapter.解析出错; err.modify = dt; err.shortdate = shortdate; err.message = "DATA解析失败"; err = dbcontext.Insert(err); continue; } var s = DecodeHelper.QQPageDecode(key.Substring(1)); var t = JsonHelper.DeserializeJsonToObject <QQ_Page_Api>(s); if (t.chapter.vipStatus == 1) { List <Page> pglst = new List <Page>(); for (int i = 0; i < t.picture.Count; i++) { pglst.Add(new Page() { chapterid = cp.Id, modify = dt, shortdate = shortdate, sort = i + 1, source = cp.source, pagelocal = "", pagesource = t.picture[i].url }); } dbcontext.Update(cp); dbcontext.BulkInsert(pglst); } } catch (Exception ex) { logger.Error(ex.Message); cp.downstatus = DownChapter.待处理链接; cp.modify = dt; dbcontext.Update(cp); Err_ChapterJob err = new Err_ChapterJob(); err.bookurl = cp.chapterurl; err.source = cp.source; err.errtype = ErrChapter.解析出错; err.modify = dt; err.shortdate = shortdate; err.message = ex.Message; err = dbcontext.Insert(err); continue; } } }
public void Execute(IJobExecutionContext context) { DateTime dt = DateTime.Now; string shortdate = dt.ToString("yyyy-MM-dd"); string yesterday = dt.AddDays(-1).ToString("yyyy-MM-dd"); IQuery <Chapter> cpq = dbcontext.Query <Chapter>(); List <Chapter> cplst = cpq.Where(x => x.source == Source.dongmanmanhua && x.downstatus == DownChapter.待处理链接).Take(200).ToList(); List <int> ids = cplst.Select(x => x.Id).ToList(); dbcontext.Update <Chapter>(a => ids.Contains(a.Id), a => new Chapter() { downstatus = DownChapter.处理完链接, modify = dt }); List <Chapter> chapterlst = new List <Chapter>(); foreach (var cp in cplst) { List <Page> pglst = new List <Page>(); try { string chapterpage = cp.chapterurl.Replace("https://www.dongmanmanhua.cn", ""); var imgdata = _helper.Get(null, chapterpage); Regex reg1 = new Regex("documentURL: '(?<key1>.*?)'"); Match match1 = reg1.Match(imgdata); string docURl = match1.Groups["key1"].Value; if (string.IsNullOrEmpty(docURl)) { string pattern = "<div class=\"(?<key1>.*?)\" id=\"_imageList\">(?<key2>.*?)</div>"; MatchCollection matches = Regex.Matches(imgdata, pattern, RegexOptions.IgnoreCase | RegexOptions.Singleline); string imghtml = matches[0].Groups["key2"].Value; string pattern2 = "data-url=\"(?<key1>.*?)\""; MatchCollection matches2 = Regex.Matches(imghtml, pattern2); for (int i = 0; i < matches2.Count; i++) { pglst.Add(new Page() { chapterid = cp.chapterid, modify = dt, shortdate = shortdate, sort = i + 1, source = cp.source, pagelocal = "", pagesource = matches2[i].Groups["key1"].Value }); } } else { Regex reg2 = new Regex("stillcut: '(?<key1>.*?)'"); Match match2 = reg2.Match(imgdata); string stillcut = match2.Groups["key1"].Value; var bookdata = _helper.Get <dongmanmanhua_Page_Api>(null, "https://www.dongmanmanhua.cn" + docURl); string regstr = bookdata.assets.image.ToString(); string pattern3 = ": \"(?<key1>.*?)\""; MatchCollection matches3 = Regex.Matches(regstr, pattern3, RegexOptions.IgnoreCase | RegexOptions.Singleline); for (int i = 0; i < matches3.Count; i++) { pglst.Add(new Page() { chapterid = cp.chapterid, modify = dt, shortdate = shortdate, sort = i + 1, source = cp.source, pagelocal = "", pagesource = string.Format(stillcut.Replace("=filename", "0"), matches3[0].Groups["key1"].Value) }); } } dbcontext.BulkInsert(pglst); } catch (Exception ex) { logger.Error(ex.Message); cp.downstatus = DownChapter.待处理链接; cp.modify = dt; dbcontext.Update(cp); Err_ChapterJob err = new Err_ChapterJob(); err.bookurl = cp.chapterurl; err.source = cp.source; err.errtype = ErrChapter.解析出错; err.modify = dt; err.shortdate = shortdate; err.message = ex.Message; err = dbcontext.Insert(err); continue; } } }
public void Execute(IJobExecutionContext context) { //List<bkIm> bklst = ExcelHelper.Import(@"C:\Users\Administrator\Desktop\7家书单抓取.xlsx").ToList(); //List<bkIm> bklst = ExcelHelper.Import2(@"C:\Users\Administrator\Desktop\漫画抓取信息.xlsx").ToList(); List <bkIm> bklst = new List <bkIm>(); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/628520" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/627844" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/627229" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/626970" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/626968" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/626920" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/626916" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/626652" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/626455" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/624495" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/624347" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/624308" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/624012" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/623715" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/623595" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/623115" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/622968" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/622674" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/622498" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/621673" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/621253" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/620523" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/620493" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/553632" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/553579" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/553055" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/552920" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/552918" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/552839" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/552104" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/547900" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/547343" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/545168" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/544521" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/543172" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/542724" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/539105" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/536658" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/628956" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/628566" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/628481" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/628198" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/628180" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/628109" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/627912" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/627383" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/627300" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/627019" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/626907" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/626527" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/626362" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/624871" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/624601" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/623795" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/623201" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/623200" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/623199" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/623196" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/623195" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/622985" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/622973" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/622971" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/622754" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/622694" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/622561" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/622160" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/621727" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/621051" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/620725" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/553972" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/628458" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/553644" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/553026" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/552065" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/551693" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/551405" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/551386" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/549847" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/549277" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/548731" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/547243" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/545320" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/542861" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/538969" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/537982" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/534826" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/534796" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/534422" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/533555" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/533395" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/529810" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/518008" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/512742" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/626289" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/626269" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/622660" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/553518" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/553204" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/549673" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/549599" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/545131" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/537899" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/628565" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/628464" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/628428" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/628305" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/628269" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/627823" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/627456" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/626955" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/626819" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/626604" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/626465" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/626268" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/626267" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/624341" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/624152" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/624135" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/624009" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/623833" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/623720" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/623494" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/623492" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/623459" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/623283" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/623125" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/623009" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/622887" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/622856" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/622855" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/622854" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/622565" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/622365" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/622216" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/622108" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/621020" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/620928" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/553939" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/550484" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/549278" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/549274" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/549125" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/547358" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/545404" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/541345" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/540523" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/539443" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/536042" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/535934" }); //IQuery<VIPFreeComic> vfcq = dbcontext.Query<VIPFreeComic>(); //List<VIPFreeComic> vfclst = vfcq.Where(x => true).ToList(); //vfclst.ForEach(x => //{ // bklst.Add(new bkIm() // { // bookurl = x.bookurl // }); //}); List <Comic> comiclst = new List <Comic>(); IQuery <Comic> cq = dbcontext.Query <Comic>(); DateTime dt = DateTime.Now; string shortdate = dt.ToString("yyyy-MM-dd"); foreach (var bk in bklst) { if (!bk.bookurl.StartsWith("http://ac.qq.com")) { continue; } var comic = cq.Where(x => x.bookurl == bk.bookurl).FirstOrDefault(); if (comic != null) { continue; } if (bk.bookurl.StartsWith("http://ac.qq.com")) { try { HttpHelper _helper = new HttpHelper("http://ac.qq.com"); string bookurl = bk.bookurl.Replace("http://ac.qq.com/", ""); var bookdata = _helper.Get(null, bookurl); Regex reg1 = new Regex("id=\"input_artistName\" value=\"(?<key1>.*?)\""); Match match1 = reg1.Match(bookdata); string authorname = match1.Groups["key1"].Value; Regex reg2 = new Regex("img src=\"(?<key1>.*?)\" alt=\"(?<key2>.*?)\" height=\"280\" width=\"210\""); Match match2 = reg2.Match(bookdata); string comiccover = match2.Groups["key1"].Value; string comicname = match2.Groups["key2"].Value; Regex reg3 = new Regex("<label class=\"works-intro-status\">(?<key1>.*?)</label>"); Match match3 = reg3.Match(bookdata); string isfinished = match3.Groups["key1"].Value; Regex reg4 = new Regex("<meta name=\"Description\" content=\"(?<key1>.*?),简介:(?<key2>.*?)《(?<key3>.*?)》的标签:(?<key4>.*?)\" />"); Match match4 = reg4.Match(bookdata); string bookdesc = match4.Groups["key2"].Value; string theme = match4.Groups["key4"].Value; comiclst.Add(new Comic() { authorname = authorname, bookurl = bk.bookurl, comiccoversource = comiccover, comiccoverlocal = "", comicdesc = bookdesc, comicid = (int)Source.QQ + "_" + bk.bookurl.Split('/').LastOrDefault(), comicname = comicname, isfinished = isfinished, theme = theme, isvip = "0", source = Source.QQ, stopcrawer = false, isoffline = false, recrawer = false, shortdate = shortdate, modify = dt, }); } catch (Exception ex) { Err_ComicJob err = new Err_ComicJob(); err.bookurl = bk.bookurl; err.errtype = ErrComic.解析出错; err.modify = dt; err.shortdate = shortdate; err.message = ex.Message; err = dbcontext.Insert(err); continue; } } else if (bk.bookurl.StartsWith("https://www.dongmanmanhua.cn")) { try { HttpHelper _helper = new HttpHelper("https://www.dongmanmanhua.cn"); string bookurl = bk.bookurl.Replace("https://www.dongmanmanhua.cn/", ""); var bookdata = _helper.Get(null, bookurl); Regex reg1 = new Regex("<meta property=\"og:image\" content=\"(?<key1>.*?)\" />"); Match match1 = reg1.Match(bookdata); string comiccover = match1.Groups["key1"].Value; Regex reg2 = new Regex("<meta property=\"com-dongman:webtoon:author\" content=\"(?<key1>.*?)\" />"); Match match2 = reg2.Match(bookdata); string authorname = match2.Groups["key1"].Value; Regex reg3 = new Regex("<p class=\"summary\">(?<key1>.*?)</p>", RegexOptions.IgnoreCase | RegexOptions.Singleline); Match match3 = reg3.Match(bookdata); string bookdesc = match3.Groups["key1"].Value; Regex reg4 = new Regex("<h1 class=\"subj\">(?<key1>.*?)</h1>"); Match match4 = reg4.Match(bookdata); string comicname = match4.Groups["key1"].Value; Regex reg5 = new Regex("txt_ico_up"); Match match5 = reg5.Match(bookdata); string isfinished = match5.Length > 0 ? "连载中" : "已完结"; Regex reg6 = new Regex("<h2 class=\"genre(?<key1>.*?)\">(?<key2>.*?)</h2>"); Match match6 = reg6.Match(bookdata); string theme = match6.Groups["key2"].Value; comiclst.Add(new Comic() { authorname = authorname, bookurl = bk.bookurl, comiccoversource = comiccover, comiccoverlocal = "", comicdesc = bookdesc, comicid = (int)Source.dongmanmanhua + "_" + bk.bookurl.Split('=').LastOrDefault(), comicname = comicname, isfinished = isfinished, theme = theme, isvip = "0", source = Source.dongmanmanhua, stopcrawer = false, isoffline = false, recrawer = false, shortdate = shortdate, modify = dt, }); } catch (Exception ex) { logger.Error(ex.Message); Err_ComicJob err = new Err_ComicJob(); err.bookurl = bk.bookurl; err.errtype = ErrComic.解析出错; err.modify = dt; err.shortdate = shortdate; err.message = ex.Message; err = dbcontext.Insert(err); continue; } } else if (bk.bookurl.StartsWith("http://www.u17.com")) { try { HttpHelper _helper = new HttpHelper("http://www.u17.com"); string bookurl = bk.bookurl.Replace("http://www.u17.com/", ""); var bookdata = _helper.Get(null, bookurl); Regex reg1 = new Regex("<title>(?<key1>.*?)</title>"); Match match1 = reg1.Match(bookdata); string comicname = match1.Groups["key1"].Value.Split('_')[0]; string authorname = match1.Groups["key1"].Value.Split('_')[1]; Regex reg2 = new Regex("cover_url = \"(?<key1>.*?)\""); Match match2 = reg2.Match(bookdata); string comiccover = match2.Groups["key1"].Value; Regex reg3 = new Regex("id=\"words\">(?<key1>.*?)</p>", RegexOptions.IgnoreCase | RegexOptions.Singleline); Match match3 = reg3.Match(bookdata); string bookdesc = match3.Groups["key1"].Value.Trim(); Regex reg4 = new Regex("状态:(?<key1>.*?)<span(?<key2>.*?)>(?<key3>.*?)</span>", RegexOptions.IgnoreCase | RegexOptions.Singleline); Match match4 = reg4.Match(bookdata); string isfinished = match4.Groups["key3"].Value.Trim(); Regex reg5 = new Regex("txt_ico_up"); Match match5 = reg5.Match(bookdata); string theme = ""; string pattern = "class=\"label(?<key1>.*?)\" title=\"(?<key3>.*?)\">(?<key2>.*?)</a>"; MatchCollection match6 = Regex.Matches(bookdata, pattern); if (match6.Count == 0) { pattern = "class=\"(?<key1>.*?)_tag \" id=\"tag_(?<key3>.*?)\">(?<key2>.*?)</a>"; match6 = Regex.Matches(bookdata, pattern); } for (int j = 0; j < match6.Count; j++) { if (j == 0) { theme = match6[j].Groups["key2"].Value.Trim(); } else { theme = theme + "," + match6[j].Groups["key2"].Value.Trim(); } } comiclst.Add(new Comic() { comicname = comicname, authorname = authorname, bookurl = bk.bookurl, comiccoversource = comiccover, comiccoverlocal = "", comicdesc = bookdesc, comicid = (int)Source.U17 + "_" + bk.bookurl.Split('/').LastOrDefault().Replace(".html", ""), isfinished = isfinished, theme = theme, isvip = "0", source = Source.U17, stopcrawer = false, isoffline = false, recrawer = false, shortdate = shortdate, modify = dt, }); } catch (Exception ex) { logger.Error(ex.Message); Err_ComicJob err = new Err_ComicJob(); err.bookurl = bk.bookurl; err.errtype = ErrComic.解析出错; err.modify = dt; err.shortdate = shortdate; err.message = ex.Message; err = dbcontext.Insert(err); continue; } } else if (bk.bookurl.StartsWith("http://www.zymk.cn")) { try { HttpHelper _helper = new HttpHelper("http://www.zymk.cn"); string bookurl = bk.bookurl.Replace("http://www.zymk.cn/", ""); var bookdata = _helper.Get(null, bookurl); Regex reg1 = new Regex("<meta property=\"og:novel:book_name\" content=\"(?<key1>.*?)\">"); Match match1 = reg1.Match(bookdata); string comicname = match1.Groups["key1"].Value; Regex reg2 = new Regex("<meta property=\"og:novel:author\" content=\"(?<key1>.*?)\">"); Match match2 = reg2.Match(bookdata); string authorname = match2.Groups["key1"].Value; Regex reg3 = new Regex("<meta property=\"og:image\" content=\"(?<key1>.*?)\">"); Match match3 = reg3.Match(bookdata); string comiccover = match3.Groups["key1"].Value; Regex reg4 = new Regex("<div class=\"desc-con\">(?<key1>.*?)</div>", RegexOptions.IgnoreCase | RegexOptions.Singleline); Match match4 = reg4.Match(bookdata); string bookdesc = match4.Groups["key1"].Value.Trim(); Regex reg5 = new Regex("<meta property=\"og:novel:status\" content=\"(?<key1>.*?)\">"); Match match5 = reg5.Match(bookdata); string isfinished = match5.Groups["key1"].Value == "连载" ? "连载中" : "已完结"; string theme = ""; Regex reg6 = new Regex("<meta property=\"og:novel:category\" content=\"(?<key1>.*?)\">"); Match match6 = reg6.Match(bookdata); theme = string.Join(",", match6.Groups["key1"].Value.Trim().Split(' ')); comiclst.Add(new Comic() { comicname = comicname, authorname = authorname, bookurl = bk.bookurl, comiccoversource = comiccover, comiccoverlocal = "", comicdesc = bookdesc, comicid = (int)Source.Zymk + "_" + bk.bookurl.Split('/')[3], isfinished = isfinished, theme = theme, isvip = "0", source = Source.Zymk, stopcrawer = false, isoffline = false, recrawer = false, shortdate = shortdate, modify = dt, }); } catch (Exception ex) { logger.Error(ex.Message); Err_ComicJob err = new Err_ComicJob(); err.bookurl = bk.bookurl; err.errtype = ErrComic.解析出错; err.modify = dt; err.shortdate = shortdate; err.message = ex.Message; err = dbcontext.Insert(err); continue; } } else if (bk.bookurl.StartsWith("http://www.manhuatai.com")) { try { HttpHelper _helper = new HttpHelper("http://www.manhuatai.com"); string bookurl = bk.bookurl.Replace("http://www.manhuatai.com/", ""); var bookdata = _helper.Get(null, bookurl); Regex reg1 = new Regex("<meta property=\"og:novel:book_name\" content=\"(?<key1>.*?)\">"); Match match1 = reg1.Match(bookdata); string comicname = match1.Groups["key1"].Value; Regex reg2 = new Regex("<meta property=\"og:novel:author\" content=\"(?<key1>.*?)\">"); Match match2 = reg2.Match(bookdata); string authorname = match2.Groups["key1"].Value; Regex reg3 = new Regex("<meta property=\"og:image\" content=\"(?<key1>.*?)\">"); Match match3 = reg3.Match(bookdata); string comiccover = match3.Groups["key1"].Value; Regex reg4 = new Regex("<div class=\"wz clearfix t1\"><div>(?<key1>.*?)<a href=\"javascript:void(?<key2>.*?)\" target=\"_self\" class=\"wzrtitle\" onclick=\"openOrCloseSummary", RegexOptions.IgnoreCase | RegexOptions.Singleline); Match match4 = reg4.Match(bookdata); string bookdesc = match4.Groups["key1"].Value.Trim(); Regex reg5 = new Regex("<meta property=\"og:novel:status\" content=\"(?<key1>.*?)\">"); Match match5 = reg5.Match(bookdata); string isfinished = match5.Groups["key1"].Value == "连载中" ? "连载中" : "已完结"; string theme = ""; Regex reg6 = new Regex("<meta property=\"og:novel:category\" content=\"(?<key1>.*?)\">"); Match match6 = reg6.Match(bookdata); theme = string.Join(",", match6.Groups["key1"].Value.Trim().Split(' ')); comiclst.Add(new Comic() { comicname = comicname, authorname = authorname, bookurl = bk.bookurl, comiccoversource = comiccover, comiccoverlocal = "", comicdesc = bookdesc, comicid = (int)Source.Manhuatai + "_" + bk.bookurl.Split('/')[3], isfinished = isfinished, theme = theme, isvip = "0", source = Source.Manhuatai, stopcrawer = false, isoffline = false, recrawer = false, shortdate = shortdate, modify = dt, }); } catch (Exception ex) { logger.Error(ex.Message); Err_ComicJob err = new Err_ComicJob(); err.bookurl = bk.bookurl; err.errtype = ErrComic.解析出错; err.modify = dt; err.shortdate = shortdate; err.message = ex.Message; err = dbcontext.Insert(err); continue; } } else if (bk.bookurl.StartsWith("https://manhua.163.com")) { try { HttpHelper _helper = new HttpHelper("https://manhua.163.com"); string bookurl = bk.bookurl.Replace("https://manhua.163.com/", ""); var bookdata = _helper.Get(null, bookurl); Regex reg1 = new Regex("<meta name=\"keywords\" content=\"(?<key1>.*?)\">"); Match match1 = reg1.Match(bookdata); var arry = match1.Groups["key1"].Value.Split(',').ToList(); string comicname = arry[0]; string authorname = arry[arry.Count - 2]; arry.Remove(arry[0]); arry.Remove(arry[0]); arry.Remove(arry[arry.Count - 1]); arry.Remove(arry[arry.Count - 1]); string theme = string.Join(",", arry.Where(x => x.Length == 2).ToArray()); Regex reg3 = new Regex("<img class=\"sr-bcover\" src=\"(?<key1>.*?)\"/>"); Match match3 = reg3.Match(bookdata); string comiccover = match3.Groups["key1"].Value; Regex reg4 = new Regex("<dt>简介</dt>(?<key2>.*?)<dd>(?<key1>.*?)</dd>", RegexOptions.IgnoreCase | RegexOptions.Singleline); Match match4 = reg4.Match(bookdata); string bookdesc = match4.Groups["key1"].Value.Trim(); Regex reg5 = new Regex("<dt>状态</dt>(?<key2>.*?)<dd><a href=\"(?<key3>.*?)\" target=\"_blank\">(?<key1>.*?)</a></dd>", RegexOptions.IgnoreCase | RegexOptions.Singleline); Match match5 = reg5.Match(bookdata); string isfinished = match5.Groups["key1"].Value == "连载中" ? "连载中" : "已完结"; comiclst.Add(new Comic() { comicname = comicname, authorname = authorname, bookurl = bk.bookurl, comiccoversource = comiccover, comiccoverlocal = "", comicdesc = bookdesc, comicid = (int)Source._163 + "_" + bk.bookurl.Split('/')[4], isfinished = isfinished, theme = theme, isvip = "0", source = Source._163, stopcrawer = false, isoffline = false, recrawer = false, shortdate = shortdate, modify = dt, }); } catch (Exception ex) { logger.Error(ex.Message); Err_ComicJob err = new Err_ComicJob(); err.bookurl = bk.bookurl; err.errtype = ErrComic.解析出错; err.modify = dt; err.shortdate = shortdate; err.message = ex.Message; err = dbcontext.Insert(err); continue; } } else if (bk.bookurl.StartsWith("http://www.mh160.com")) { try { HttpWebHelper _helper = new HttpWebHelper(); var bookdata = _helper.Get(bk.bookurl, Encoding.GetEncoding("gb2312")); Regex reg1 = new Regex("<meta property=\"og:novel:book_name\" content=\"(?<key1>.*?)\">"); Match match1 = reg1.Match(bookdata); string comicname = match1.Groups["key1"].Value; Regex reg2 = new Regex("<meta property=\"og:novel:author\" content=\"(?<key1>.*?)\">"); Match match2 = reg2.Match(bookdata); string authorname = match2.Groups["key1"].Value; Regex reg3 = new Regex("<meta property=\"og:image\" content=\"(?<key1>.*?)\">"); Match match3 = reg3.Match(bookdata); string comiccover = match3.Groups["key1"].Value; Regex reg4 = new Regex("<div class=\"introduction\" id=\"intro1\"><p>(?<key1>.*?)</p>", RegexOptions.IgnoreCase | RegexOptions.Singleline); Match match4 = reg4.Match(bookdata); string bookdesc = match4.Groups["key1"].Value.Trim(); Regex reg5 = new Regex("<meta property=\"og:novel:status\" content=\"(?<key1>.*?)\">"); Match match5 = reg5.Match(bookdata); string isfinished = match5.Groups["key1"].Value == "连载中" ? "连载中" : "已完结"; string theme = ""; Regex reg6 = new Regex("<meta property=\"og:novel:category\" content=\"(?<key1>.*?)\">"); Match match6 = reg6.Match(bookdata); theme = string.Join(",", match6.Groups["key1"].Value.Trim().Split(' ')); comiclst.Add(new Comic() { comicname = comicname, authorname = authorname, bookurl = bk.bookurl, comiccoversource = comiccover, comiccoverlocal = "", comicdesc = bookdesc, comicid = (int)Source.mh160 + "_" + bk.bookurl.Split('/')[4], isfinished = isfinished, theme = theme, isvip = "0", source = Source.mh160, stopcrawer = false, isoffline = false, recrawer = false, shortdate = shortdate, modify = dt, }); } catch (Exception ex) { logger.Error(ex.Message); Err_ComicJob err = new Err_ComicJob(); err.bookurl = bk.bookurl; err.errtype = ErrComic.解析出错; err.modify = dt; err.shortdate = shortdate; err.message = ex.Message; err = dbcontext.Insert(err); continue; } } } if (comiclst.Count > 0) { //foreach (var item in comiclst) //{ // var comic = cq.Where(x => x.bookurl == item.bookurl).FirstOrDefault(); // comic.comicdesc = item.comicdesc; // dbcontext.Update(comic); //} dbcontext.BulkInsert(comiclst); } }
public void Execute(IJobExecutionContext context) { DateTime dt = DateTime.Now; string ticks = dt.Ticks.ToString(); string shortdate = dt.ToString("yyyy-MM-dd"); string updatedatetime = shortdate + " " + ((dt.Hour / 0.5 + 1) * 0.5).ToString(); string yesterday = dt.AddDays(-1).ToString("yyyy-MM-dd"); IQuery <Comic> q = dbcontext.Query <Comic>(); IQuery <Chapter> cpq = dbcontext.Query <Chapter>(); IQuery <PageHis> phisq = dbcontext.Query <PageHis>(); IQuery <Page> pq = dbcontext.Query <Page>(); IQuery <Notice> nq = dbcontext.Query <Notice>(); List <Comic> comiclst = q.Where(a => a.source == Source._163 && (a.updatedatetime == null || a.updatedatetime != updatedatetime)).Take(200).ToList(); List <int> ids = comiclst.Select(x => x.Id).ToList(); dbcontext.Update <Comic>(a => ids.Contains(a.Id), a => new Comic() { updatedatetime = updatedatetime, modify = dt }); foreach (var comic in comiclst) { List <Chapter> cplst = cpq.Where(a => a.comicid == comic.comicid && a.source == Source._163).ToList(); List <Chapter> chapterlst = new List <Chapter>(); if (cplst.Count > 0) { try { int sort = 0; var bookdata = _helper.Get <_163_Chapter_Api>(null, string.Format(chapterurl, comic.comicid.Replace("6_", ""))); if (bookdata.code == 200) { bookdata.catalog.sections.ForEach(x => { foreach (var item in x.sections) { sort = sort + 1; chapterlst.Add(new Chapter() { chapterid = comic.comicid + "_" + item.sectionId, chaptername = item.fullTitle, chapterurl = $"https://manhua.163.com/reader/{item.bookId}/{item.sectionId}#imgIndex=0", sort = sort, comicid = comic.comicid, retry = 0, source = comic.source, downstatus = DownChapter.待处理链接, isvip = item.needPay == 0 ? "0" : "1", chaptersource = "", chapterlocal = "", modify = dt, shortdate = shortdate, }); } }); } int delete = cplst.Except(chapterlst, new Chapter_Comparer()).Count(); // 删章 List <Chapter> add = chapterlst.Except(cplst, new Chapter_Comparer()).ToList(); // 新增 if (delete > 0) { List <string> idlst = cplst.Select(x => x.chapterid).ToList(); int phislstcount = phisq.Where(x => idlst.Contains(x.chapterid)).Count(); if (phislstcount == 0) { List <Page> pglst = pq.Where(x => idlst.Contains(x.chapterid)).ToList(); if (pglst.Count > 0) { List <PageHis> phislst = new List <PageHis>(); pglst.ForEach(x => { phislst.Add(new PageHis() { chapterid = x.chapterid, modify = x.modify, pagelocal = x.pagelocal, pagesource = x.pagesource, pagestate = x.pagestate, shortdate = x.shortdate, sort = x.sort, source = x.source, ticks = x.ticks }); }); dbcontext.BulkInsert(phislst); } } dbcontext.Delete <Page>(x => idlst.Contains(x.chapterid)); dbcontext.Delete <Chapter>(x => idlst.Contains(x.chapterid)); if (chapterlst.Count > 0) { dbcontext.BulkInsert(chapterlst); } Notice notice = new Notice(); notice.noticeid = comic.comicid; notice.noticestatus = NoticeStatus.等待处理; notice.noticetype = NoticeType.目录变更; notice.source = comic.source; notice.shortdate = shortdate; notice.modify = dt; var nqwait = nq.Where(x => x.noticeid == comic.comicid && x.noticestatus == NoticeStatus.等待处理 && x.noticetype == NoticeType.目录变更).FirstOrDefault(); if (nqwait == null) { dbcontext.Insert(notice); } continue; } else { List <Chapter> mvadd = chapterlst.Except(add, new Chapter_Comparer()).ToList(); string cplststr = string.Join(",", cplst.Select(x => x.chapterid).ToArray()); string chapterlststr = string.Join(",", mvadd.Select(x => x.chapterid).ToArray()); if (cplststr != chapterlststr) // 调序 { List <string> idlst = cplst.Select(x => x.chapterid).ToList(); int phislstcount = phisq.Where(x => idlst.Contains(x.chapterid)).Count(); if (phislstcount == 0) { List <Page> pglst = pq.Where(x => idlst.Contains(x.chapterid)).ToList(); if (pglst.Count > 0) { List <PageHis> phislst = new List <PageHis>(); pglst.ForEach(x => { phislst.Add(new PageHis() { chapterid = x.chapterid, modify = x.modify, pagelocal = x.pagelocal, pagesource = x.pagesource, pagestate = x.pagestate, shortdate = x.shortdate, sort = x.sort, source = x.source, ticks = x.ticks }); }); dbcontext.BulkInsert(phislst); } } dbcontext.Delete <Page>(x => idlst.Contains(x.chapterid)); dbcontext.Delete <Chapter>(x => idlst.Contains(x.chapterid)); if (chapterlst.Count > 0) { dbcontext.BulkInsert(chapterlst); } Notice notice = new Notice(); notice.noticeid = comic.comicid; notice.noticestatus = NoticeStatus.等待处理; notice.noticetype = NoticeType.目录变更; notice.source = comic.source; notice.shortdate = shortdate; notice.modify = dt; var nqwait = nq.Where(x => x.noticeid == comic.comicid && x.noticestatus == NoticeStatus.等待处理 && x.noticetype == NoticeType.目录变更).FirstOrDefault(); if (nqwait == null) { dbcontext.Insert(notice); } continue; } } if (add.Count > 0) { dbcontext.BulkInsert(add); } } catch (Exception ex) { logger.Error(ex.Message); comic.updatedatetime = ""; comic.modify = dt; dbcontext.Update(comic); Err_ChapterJob err = new Err_ChapterJob(); err.bookurl = comic.bookurl; err.source = comic.source; err.errtype = ErrChapter.解析出错; err.modify = dt; err.shortdate = shortdate; err.message = ex.Message; err = dbcontext.Insert(err); continue; } } } }
public void Execute(IJobExecutionContext context) { DateTime dt = DateTime.Now; string ticks = dt.Ticks.ToString(); string shortdate = dt.ToString("yyyy-MM-dd"); string updatedatetime = shortdate + " " + ((dt.Hour / 0.5 + 1) * 0.5).ToString(); string yesterday = dt.AddDays(-1).ToString("yyyy-MM-dd"); IQuery <Comic> q = dbcontext.Query <Comic>(); IQuery <Chapter> cpq = dbcontext.Query <Chapter>(); IQuery <PageHis> phisq = dbcontext.Query <PageHis>(); IQuery <Page> pq = dbcontext.Query <Page>(); IQuery <Notice> nq = dbcontext.Query <Notice>(); List <Comic> comiclst = q.Where(a => a.source == Source.Manhuatai && (a.updatedatetime == null || a.updatedatetime != updatedatetime)).Take(200).ToList(); List <int> ids = comiclst.Select(x => x.Id).ToList(); dbcontext.Update <Comic>(a => ids.Contains(a.Id), a => new Comic() { updatedatetime = updatedatetime, modify = dt }); foreach (var comic in comiclst) { List <Chapter> cplst = cpq.Where(a => a.comicid == comic.comicid && a.source == Source.Manhuatai).ToList(); List <Chapter> chapterlst = new List <Chapter>(); if (cplst.Count > 0) { try { string bookurl = comic.bookurl.Replace("http://www.manhuatai.com", ""); var bookdata = _helper.Get(null, bookurl); string pattern = "<li><a title=\"(?<key1>.*?)\" href=\"(?<key2>.*?)\"><span>(?<key3>.*?)</span></a></li>"; MatchCollection matches = Regex.Matches(bookdata, pattern, RegexOptions.IgnoreCase | RegexOptions.Singleline); for (int i = 0; i < matches.Count; i++) { string chapterid = comic.comicid + "_" + matches[i].Groups["key2"].Value.Replace(".html", ""); string chaptername = matches[i].Groups["key1"].Value; string chapterurl = comic.bookurl + matches[i].Groups["key2"].Value; int sort = matches.Count - i; chapterlst.Add(new Chapter() { chapterid = chapterid, chaptername = chaptername, chapterurl = chapterurl, sort = sort, comicid = comic.comicid, retry = 0, source = comic.source, downstatus = DownChapter.待处理链接, isvip = "0", chaptersource = "", chapterlocal = "", modify = dt, shortdate = shortdate, }); } int delete = cplst.Except(chapterlst, new Chapter_Comparer()).Count(); // 删章 List <Chapter> add = chapterlst.Except(cplst, new Chapter_Comparer()).ToList(); // 新增 if (delete > 0) { List <string> idlst = cplst.Select(x => x.chapterid).ToList(); dbcontext.Delete <Page>(x => idlst.Contains(x.chapterid)); dbcontext.Delete <Chapter>(x => idlst.Contains(x.chapterid)); if (chapterlst.Count > 0) { dbcontext.BulkInsert(chapterlst); } Notice notice = new Notice(); notice.noticeid = comic.comicid; notice.noticestatus = NoticeStatus.等待处理; notice.noticetype = NoticeType.目录变更; notice.source = comic.source; notice.shortdate = shortdate; notice.modify = dt; var nqwait = nq.Where(x => x.noticeid == comic.comicid && x.noticestatus == NoticeStatus.等待处理 && x.noticetype == NoticeType.目录变更).FirstOrDefault(); if (nqwait == null) { dbcontext.Insert(notice); } continue; } else { List <Chapter> mvadd = chapterlst.Except(add, new Chapter_Comparer()).ToList(); string cplststr = string.Join(",", cplst.Select(x => x.chapterid).ToArray()); string chapterlststr = string.Join(",", mvadd.Select(x => x.chapterid).ToArray()); if (cplststr != chapterlststr) // 调序 { List <string> idlst = cplst.Select(x => x.chapterid).ToList(); dbcontext.Delete <Page>(x => idlst.Contains(x.chapterid)); dbcontext.Delete <Chapter>(x => idlst.Contains(x.chapterid)); if (chapterlst.Count > 0) { dbcontext.BulkInsert(chapterlst); } Notice notice = new Notice(); notice.noticeid = comic.comicid; notice.noticestatus = NoticeStatus.等待处理; notice.noticetype = NoticeType.目录变更; notice.source = comic.source; notice.shortdate = shortdate; notice.modify = dt; var nqwait = nq.Where(x => x.noticeid == comic.comicid && x.noticestatus == NoticeStatus.等待处理 && x.noticetype == NoticeType.目录变更).FirstOrDefault(); if (nqwait == null) { dbcontext.Insert(notice); } continue; } } if (add.Count > 0) { dbcontext.BulkInsert(add); } } catch (Exception ex) { logger.Error(ex.Message); comic.updatedatetime = ""; comic.modify = dt; dbcontext.Update(comic); Err_ChapterJob err = new Err_ChapterJob(); err.bookurl = comic.bookurl; err.source = comic.source; err.errtype = ErrChapter.解析出错; err.modify = dt; err.shortdate = shortdate; err.message = ex.Message; err = dbcontext.Insert(err); continue; } } } }
public void Execute(IJobExecutionContext context) { DateTime dt = DateTime.Now; string shortdate = dt.ToString("yyyy-MM-dd"); string yesterday = dt.AddDays(-1).ToString("yyyy-MM-dd"); IQuery <Comic> q = dbcontext.Query <Comic>(); IQuery <Chapter> cpq = dbcontext.Query <Chapter>(); List <Comic> comiclst = q.Where(a => a.source == Source.Zymk && a.shortdate == shortdate).Take(200).ToList(); List <int> ids = comiclst.Select(x => x.Id).ToList(); dbcontext.Update <Comic>(a => ids.Contains(a.Id), a => new Comic() { shortdate = yesterday, modify = dt }); List <Chapter> chapterlst = new List <Chapter>(); foreach (var comic in comiclst) { List <Chapter> cplst = cpq.Where(a => a.comicid == comic.comicid && a.source == Source.Zymk).ToList(); if (cplst.Count == 0) { try { string bookurl = comic.bookurl.Replace("http://www.zymk.cn", ""); var bookdata = _helper.Get(null, bookurl); string pattern = "<li class=\"(?<key5>.*?)\" data-id=\"(?<key1>.*?)\"><a href=\"(?<key2>.*?)\" title=\"(?<key3>.*?)\">(?<key4>.*?)</a>"; MatchCollection matches = Regex.Matches(bookdata, pattern, RegexOptions.IgnoreCase | RegexOptions.Singleline); for (int i = 0; i < matches.Count; i++) { string chapterid = comic.comicid + "_" + matches[i].Groups["key1"].Value; string chaptername = matches[i].Groups["key4"].Value; string chapterurl = comic.bookurl + matches[i].Groups["key2"].Value; int sort = matches.Count - i; chapterlst.Add(new Chapter() { chapterid = chapterid, chaptername = chaptername, chapterurl = chapterurl, sort = sort, comicid = comic.comicid, retry = 0, source = comic.source, downstatus = DownChapter.待处理链接, isvip = "0", chaptersource = "", chapterlocal = "", modify = dt, shortdate = shortdate, }); } } catch (Exception ex) { logger.Error(ex.Message); comic.shortdate = shortdate; comic.modify = dt; dbcontext.Update(comic); Err_ChapterJob err = new Err_ChapterJob(); err.bookurl = comic.bookurl; err.source = comic.source; err.errtype = ErrChapter.解析出错; err.modify = dt; err.shortdate = shortdate; err.message = ex.Message; err = dbcontext.Insert(err); continue; } } } if (chapterlst.Count > 0) { dbcontext.BulkInsert(chapterlst); } }
public void Execute(IJobExecutionContext context) { DateTime dt = DateTime.Now; string shortdate = dt.ToString("yyyy-MM-dd"); IQuery <Chapter> cpq = dbcontext.Query <Chapter>(); List <Chapter> cplst = cpq.Where(x => x.source == Source._163 && x.downstatus == DownChapter.待处理链接 && x.isvip == "0").Take(200).ToList(); List <int> ids = cplst.Select(x => x.Id).ToList(); dbcontext.Update <Chapter>(a => ids.Contains(a.Id), a => new Chapter() { downstatus = DownChapter.处理完链接, modify = dt }); foreach (var cp in cplst) { try { string chapterpage = cp.chapterurl.Replace("https://manhua.163.com/", ""); var imgdata = _helper.Get(null, chapterpage); Regex rex1 = new Regex("imageId:"); var match1 = rex1.Match(imgdata); if (match1.Value.Trim() != "") { Regex rex2 = new Regex("url: window.IS_SUPPORT_WEBP (?<key1>.*?),"); var match2 = rex2.Matches(imgdata); List <Page> pglst = new List <Page>(); int sort = 0; foreach (var item in match2) { sort = sort + 1; var tt = ((Match)item).Groups["key1"].Value; var imgurl = tt.Split('"')[3].Substring(0, tt.Split('"')[3].IndexOf("%3D") + 3); pglst.Add(new Page() { pagesource = imgurl, chapterid = cp.chapterid, modify = dt, shortdate = shortdate, sort = sort, source = cp.source, pagelocal = "", }); } if (pglst.Count > 0) { dbcontext.BulkInsert(pglst); } } else { cp.downstatus = DownChapter.待处理链接; cp.modify = dt; dbcontext.Update(cp); } } catch (Exception ex) { logger.Error(ex.Message); cp.downstatus = DownChapter.待处理链接; cp.modify = dt; dbcontext.Update(cp); Err_ChapterJob err = new Err_ChapterJob(); err.bookurl = cp.chapterurl; err.source = cp.source; err.errtype = ErrChapter.解析出错; err.modify = dt; err.shortdate = shortdate; err.message = ex.Message; err = dbcontext.Insert(err); continue; } } }
public void Execute(IJobExecutionContext context) { DateTime dt = DateTime.Now; string shortdate = dt.ToString("yyyy-MM-dd"); string yesterday = dt.AddDays(-1).ToString("yyyy-MM-dd"); IQuery <Comic> q = dbcontext.Query <Comic>(); IQuery <Chapter> cpq = dbcontext.Query <Chapter>(); List <Comic> comiclst = q.Where(a => a.source == Source._163 && a.shortdate == shortdate).Take(200).ToList(); List <int> ids = comiclst.Select(x => x.Id).ToList(); dbcontext.Update <Comic>(a => ids.Contains(a.Id), a => new Comic() { shortdate = yesterday, modify = dt }); List <Chapter> chapterlst = new List <Chapter>(); foreach (var comic in comiclst) { List <Chapter> cplst = cpq.Where(a => a.comicid == comic.comicid && a.source == Source._163).ToList(); if (cplst.Count == 0) { try { int sort = 0; var bookdata = _helper.Get <_163_Chapter_Api>(null, string.Format(chapterurl, comic.comicid.Replace("6_", ""))); if (bookdata.code == 200) { bookdata.catalog.sections.ForEach(x => { foreach (var item in x.sections) { sort = sort + 1; chapterlst.Add(new Chapter() { chapterid = comic.comicid + "_" + item.sectionId, chaptername = item.fullTitle, chapterurl = $"https://manhua.163.com/reader/{item.bookId}/{item.sectionId}#imgIndex=0", sort = sort, comicid = comic.comicid, retry = 0, source = comic.source, downstatus = DownChapter.待处理链接, isvip = item.needPay == 0 ?"0":"1", chaptersource = "", chapterlocal = "", modify = dt, shortdate = shortdate, }); } }); } } catch (Exception ex) { logger.Error(ex.Message); comic.shortdate = shortdate; comic.modify = dt; dbcontext.Update(comic); Err_ChapterJob err = new Err_ChapterJob(); err.bookurl = comic.bookurl; err.source = comic.source; err.errtype = ErrChapter.解析出错; err.modify = dt; err.shortdate = shortdate; err.message = ex.Message; err = dbcontext.Insert(err); continue; } } } if (chapterlst.Count > 0) { dbcontext.BulkInsert(chapterlst); } }
public void Execute(IJobExecutionContext context) { DateTime dt = DateTime.Now; string shortdate = dt.ToString("yyyy-MM-dd"); string yesterday = dt.AddDays(-1).ToString("yyyy-MM-dd"); IQuery <Chapter> cpq = dbcontext.Query <Chapter>(); List <Chapter> cplst = cpq.Where(x => x.source == Source.dongmanmanhua && x.downstatus == DownChapter.待处理链接).Take(200).ToList(); List <int> ids = cplst.Select(x => x.Id).ToList(); dbcontext.Update <Chapter>(a => ids.Contains(a.Id), a => new Chapter() { downstatus = DownChapter.处理完链接, modify = dt }); List <Chapter> chapterlst = new List <Chapter>(); foreach (var cp in cplst) { try { string chapterpage = cp.chapterurl.Replace("https://www.dongmanmanhua.cn", ""); var imgdata = _helper.Get(null, chapterpage); string pattern = "<div class=\"(?<key1>.*?)\" id=\"_imageList\">(?<key2>.*?)</div>"; MatchCollection matches = Regex.Matches(imgdata, pattern, RegexOptions.IgnoreCase | RegexOptions.Singleline); string imghtml = matches[0].Groups["key2"].Value; string pattern2 = "data-url=\"(?<key1>.*?)\""; MatchCollection matches2 = Regex.Matches(imghtml, pattern2); List <Page> pglst = new List <Page>(); for (int i = 0; i < matches2.Count; i++) { pglst.Add(new Page() { chapterid = cp.Id, modify = dt, shortdate = shortdate, sort = i + 1, source = cp.source, pagelocal = "", pagesource = matches2[i].Groups["key1"].Value }); } dbcontext.BulkInsert(pglst); } catch (Exception ex) { logger.Error(ex.Message); cp.downstatus = DownChapter.待处理链接; cp.modify = dt; dbcontext.Update(cp); Err_ChapterJob err = new Err_ChapterJob(); err.bookurl = cp.chapterurl; err.source = cp.source; err.errtype = ErrChapter.解析出错; err.modify = dt; err.shortdate = shortdate; err.message = ex.Message; err = dbcontext.Insert(err); continue; } } }
public void Execute(IJobExecutionContext context) { DateTime dt = DateTime.Now; string shortdate = dt.ToString("yyyy-MM-dd"); IQuery <Chapter> cpq = dbcontext.Query <Chapter>(); List <Chapter> cplst = cpq.Where(x => x.source == Source.U17 && x.downstatus == DownChapter.待处理链接 && x.isvip == "0").Take(200).ToList(); List <int> ids = cplst.Select(x => x.Id).ToList(); dbcontext.Update <Chapter>(a => ids.Contains(a.Id), a => new Chapter() { downstatus = DownChapter.处理完链接, modify = dt }); foreach (var cp in cplst) { try { var imgdata = _helper.Get <U17_Page_Api>(null, string.Format(pageurl, cp.chapterid.Replace(cp.comicid + "_", ""))); List <Page> pglst = new List <Page>(); int sort = 0; imgdata.image_list.ForEach(x => { sort = sort + 1; pglst.Add(new Page() { pagesource = x.src, chapterid = cp.chapterid, modify = dt, shortdate = shortdate, sort = sort, source = cp.source, pagelocal = "", }); }); if (pglst.Count > 0) { dbcontext.BulkInsert(pglst); } } catch (Exception ex) { logger.Error(ex.Message); cp.downstatus = DownChapter.待处理链接; cp.modify = dt; dbcontext.Update(cp); Err_ChapterJob err = new Err_ChapterJob(); err.bookurl = cp.chapterurl; err.source = cp.source; err.errtype = ErrChapter.解析出错; err.modify = dt; err.shortdate = shortdate; err.message = ex.Message; err = dbcontext.Insert(err); continue; } } }
public void Execute(IJobExecutionContext context) { //logger.Info("QQ_page vip begin"); //string isStart = "IsStartBuyQQ".ValueOfAppSetting(); //if (isStart != null && isStart.Equals("1")) { DateTime dt = DateTime.Now; string shortdate = dt.ToString("yyyy-MM-dd"); string yesterday = dt.AddDays(-1).ToString("yyyy-MM-dd"); IQuery <Chapter> cpq = dbcontext.Query <Chapter>();//x.comicid == "1_524356" IQuery <PageHis> phisq = dbcontext.Query <PageHis>(); //List<Chapter> cplst = cpq.Where(x => x.source == Source.QQ && x.downstatus == DownChapter.待处理链接 && x.isvip.Equals("1")).Take(200).ToList(); List <Chapter> cplst = cpq.Where(x => x.source == Source.QQ && x.downstatus == DownChapter.完图片 && x.isvip.Equals("1")).Take(200).ToList(); List <int> ids = cplst.Select(x => x.Id).ToList(); dbcontext.Update <Chapter>(a => ids.Contains(a.Id), a => new Chapter() { downstatus = DownChapter.处理中, modify = dt }); List <Chapter> chapterlst = new List <Chapter>(); Regex rex = new Regex("var DATA = '(?<key1>.*?)',"); string errMsg; foreach (var cp in cplst) { try { IQuery <PageHis> cpHis = phisq.Where(x => x.chapterid == cp.chapterid); if (cpHis != null && cpHis.Count() > 0) { List <PageHis> cpHisList = cpHis.ToList(); List <Page> pglst = new List <Page>(); foreach (var page in cpHisList) { pglst.Add(new Page() { chapterid = page.chapterid, modify = dt, shortdate = shortdate, sort = page.sort, source = page.source, pagelocal = "", pagesource = page.pagesource }); } cp.downstatus = DownChapter.处理完链接; cp.modify = dt; dbcontext.Update(cp); dbcontext.BulkInsert(pglst); logger.Info("QQ_page vip syn history sucess:id=" + cp.Id); } else { errMsg = string.Empty; selenium.Navigate().GoToUrl(cp.chapterurl); IList <IWebElement> frames = selenium.FindElements(By.TagName("iframe")); IWebElement controlPanelFrame = null; foreach (var frame in frames) { if (frame.GetAttribute("id") == "iframeAll") { controlPanelFrame = frame; break; } } if (controlPanelFrame != null && controlPanelFrame.Displayed == true) //QQ登录 { try { selenium.SwitchTo().Frame(controlPanelFrame); IReadOnlyCollection <IWebElement> switchtoElement = selenium.FindElements(By.Id("switcher_plogin")); if (switchtoElement != null && switchtoElement.Count > 0 && switchtoElement.First().Displayed == true) { switchtoElement.First().Click(); selenium.FindElement(By.Id("u")).Clear(); selenium.FindElement(By.Id("u")).SendKeys("3283360259"); selenium.FindElement(By.Id("p")).Clear(); selenium.FindElement(By.Id("p")).SendKeys("xxxttt5544"); //selenium.FindElement(By.Id("u")).Clear(); //selenium.FindElement(By.Id("u")).SendKeys("1434299101"); //selenium.FindElement(By.Id("p")).Clear(); //selenium.FindElement(By.Id("p")).SendKeys("zhangyin123"); selenium.FindElement(By.Id("login_button")).Click(); } selenium.SwitchTo().DefaultContent(); } catch (Exception ex) { errMsg = "QQ 登录失败:" + ex.Message; logger.Error(errMsg); } } selenium.Navigate().GoToUrl("http://ac.qq.com/Home/buyList"); ICookieJar listCookie = selenium.Manage().Cookies; // IList<Cookie> listCookie = selenuim.Manage( ).Cookies.AllCookies;//只是显示 可以用Ilist对象 //显示初始Cookie的内容 Console.WriteLine("--------------------"); Console.WriteLine($"当前Cookie集合的数量:\t{listCookie.AllCookies.Count}"); for (int i = 0; i < listCookie.AllCookies.Count; i++) { Console.WriteLine($"Cookie的名称:{listCookie.AllCookies[i].Name}"); Console.WriteLine($"Cookie的值:{listCookie.AllCookies[i].Value}"); Console.WriteLine($"Cookie的所在域:{listCookie.AllCookies[i].Domain}"); Console.WriteLine($"Cookie的路径:{listCookie.AllCookies[i].Path}"); Console.WriteLine($"Cookie的过期时间:{listCookie.AllCookies[i].Expiry}"); Console.WriteLine("-----"); } frames = selenium.FindElements(By.TagName("iframe")); IWebElement checkVipFrame = null; foreach (var frame in frames) { if (frame.GetAttribute("id") == "checkVipFrame") { checkVipFrame = frame; break; } } if (checkVipFrame != null && checkVipFrame.Displayed == true) { try { //自动购买 selenium.SwitchTo().Frame(checkVipFrame); IReadOnlyCollection <IWebElement> checkAutoElement = selenium.FindElements(By.Id("check_auto_next")); IReadOnlyCollection <IWebElement> singlBbuyElement = selenium.FindElements(By.ClassName("single_buy")); if (checkAutoElement != null && singlBbuyElement != null && checkAutoElement.Count > 0 && singlBbuyElement.Count > 0 && checkAutoElement.First().Displayed == true) { if (singlBbuyElement.First().Text.IndexOf("点券不足") > -1) { //列表中未成功购买的数据还原成待处理 dbcontext.Update <Chapter>(a => ids.Contains(a.Id), a => new Chapter() { downstatus = DownChapter.待处理链接, modify = dt }); ////关闭购买,等待修改配置 //"IsStartBuyQQ".SetAppSettingValue("0"); if (isHasMoney) { Err_ChapterJob err = new Err_ChapterJob(); err.bookurl = cp.chapterurl; err.source = cp.source; err.errtype = ErrChapter.解析出错; err.modify = dt; err.shortdate = shortdate; err.message = "点券不足,请去充值!"; err = dbcontext.Insert(err); } isHasMoney = false; //Thread.Sleep(3600000); continue; } else { isHasMoney = true; } checkAutoElement.First().Click(); singlBbuyElement.First().Click(); } selenium.SwitchTo().DefaultContent(); } catch (Exception ex) { errMsg = "自动购买失败:" + ex.Message; logger.Error(errMsg); } } Match match1 = rex.Match(selenium.PageSource); string key = match1.Groups["key1"].Value; if (string.IsNullOrEmpty(key) || errMsg != string.Empty) { cp.downstatus = DownChapter.待处理链接; cp.modify = dt; dbcontext.Update(cp); Err_ChapterJob err = new Err_ChapterJob(); err.bookurl = cp.chapterurl; err.source = cp.source; err.errtype = ErrChapter.解析出错; err.modify = dt; err.shortdate = shortdate; err.message = errMsg != string.Empty ? errMsg : "DATA解析失败"; err = dbcontext.Insert(err); continue; } string s = DecodeHelper.QQPageDecode(key.Substring(1)); var t = JsonHelper.DeserializeJsonToObject <QQ_Page_Api>(s); if (t.picture.Count < 1) { cp.downstatus = DownChapter.待处理链接; cp.modify = dt; dbcontext.Update(cp); Err_ChapterJob err = new Err_ChapterJob(); err.bookurl = cp.chapterurl; err.source = cp.source; err.errtype = ErrChapter.解析出错; err.modify = dt; err.shortdate = shortdate; err.message = "访问付费章节内容时只存在一张图片"; err = dbcontext.Insert(err); continue; } List <Page> pglst = new List <Page>(); for (int i = 0; i < t.picture.Count; i++) { if (pglst.Exists(x => x.pagesource == t.picture[i].url) == false) { pglst.Add(new Page() { chapterid = cp.chapterid, modify = DateTime.Now, shortdate = shortdate, sort = i + 1, source = cp.source, pagelocal = "", pagesource = t.picture[i].url }); } else { Err_ChapterJob err = new Err_ChapterJob(); err.bookurl = cp.chapterurl; err.source = cp.source; err.errtype = ErrChapter.解析出错; err.modify = DateTime.Now; err.shortdate = shortdate; err.message = "存在重复图片"; err = dbcontext.Insert(err); } } cp.downstatus = DownChapter.处理完链接; cp.modify = dt; dbcontext.Update(cp); dbcontext.BulkInsert(pglst); ids.Remove(cp.Id); logger.Info("QQ_page vip buy sucess:id=" + cp.Id); } } catch (Exception ex) { if (ex.Message.IndexOf("Unexpected error. System.Net.WebException:") > -1) { InitChromeDriver(); } logger.Error(ex.Message); cp.downstatus = DownChapter.待处理链接; cp.modify = dt; dbcontext.Update(cp); Err_ChapterJob err = new Err_ChapterJob(); err.bookurl = cp.chapterurl; err.source = cp.source; err.errtype = ErrChapter.解析出错; err.modify = dt; err.shortdate = shortdate; err.message = ex.Message; err = dbcontext.Insert(err); continue; } } } }
public void Execute(IJobExecutionContext context) { //string keystr = "meyJjb21pYyI6eyJpZCI6NTM4NzQzLCJ0aXRsZSI6Ilx1NGUwN1x1OTFjY1x1NjY3NFx1NWRkZCIsImNvbGxlY3QiOiI0NzA0MSIsImlzSmFwYW5Db21pYyI6ZmFsc2UsImlzTGlnaHROb3ZlbCI6ZmFsc2UsImlzTGlnaHRDb21pYyI6ZmFsc2UsImlzRmluaXNoIjpmYWxzZSwiaXNSb2FzdGFibGUiOnRydWUsImVJZCI6IktsQlBTVU5BVmxWV0Jnc2ZBUVlPQUF3S0hFUldOQT09In0sImNoYXB0ZXIiOnsiY2lkIjo0OSwiY1RpdGxlIjoiXHU3YjJjMjVcdTU2ZGUgXHU2NDk1XHU4OGMyMlx1ZmYwOFx1NGUwYVx1ZmYwOSIsImNTZXEiOiI0OSIsInZpcFN0YXR1cyI6MiwicHJldkNpZCI6NDgsIm5leHRDaWQiOjUwLCJibGFua0ZpcnN0IjoxLCJjYW5SZWFkIjp0cnVlfSwicGljdHVyZSI6W3sicGlkIjoiMjY1MCIsIndpZHRoIjo4MDAsImhlaWdodCI6MTIwMCwidXJsIjoiaHR0cDpcL1wvYWMudGMucXEuY29tXC9zdG9yZV9maWxlX2Rvd25sb2FkP2J1aWQ9MTUwMTcmdWluPTE1MTA3MDk2NTgmZGlyX3BhdGg9XC8mbmFtZT0xNV8wOV8zNF8wY2E3NjllMDk1YTU0NTYwYzQ5Nzk3ZmEwNGY1NzQ5N18yNjUwLmpwZyJ9LHsicGlkIjoiMjY1MSIsIndpZHRoIjo4MDAsImhlaWdodCI6MTIwMCwidXJsIjoiaHR0cDpcL1wvYWMudGMucXEuY29tXC9zdG9yZV9maWxlX2Rvd25sb2FkP2J1aWQ9MTUwMTcmdWluPTE1MTA3MDk2NzImZGlyX3BhdGg9XC8mbmFtZT0xNV8wOV8zNF8yZmVmYmQ0Mzk3ODY5YmU5MGI4OTQxOTdmYTEwMGZmNF8yNjUxLmpwZyJ9LHsicGlkIjoiMjY1MiIsIndpZHRoIjo4MDAsImhlaWdodCI6MTIwMCwidXJsIjoiaHR0cDpcL1wvYWMudGMucXEuY29tXC9zdG9yZV9maWxlX2Rvd25sb2FkP2J1aWQ9MTUwMTcmdWluPTE1MTA3MDk2NzImZGlyX3BhdGg9XC8mbmFtZT0xNV8wOV8zNF9jODBiMTE0ZDIwOGIzZWMyZDkxNWM3MmRhODIwZDRhYl8yNjUyLmpwZyJ9LHsicGlkIjoiMjY1MyIsIndpZHRoIjo4MDAsImhlaWdodCI6MTIwMCwidXJsIjoiaHR0cDpcL1wvYWMudGMucXEuY29tXC9zdG9yZV9maWxlX2Rvd25sb2FkP2J1aWQ9MTUwMTcmdWluPTE1MTA3MDk2NzImZGlyX3BhdGg9XC8mbmFtZT0xNV8wOV8zNF9kYjI5MjI5MmM4NmVkN2U0MDIwN2Q2MDg4YjU2ODliYl8yNjUzLmpwZyJ9LHsicGlkIjoiMjY1NCIsIndpZHRoIjo4MDAsImhlaWdodCI6MTIwMCwidXJsIjoiaHR0cDpcL1wvYWMudGMucXEuY29tXC9zdG9yZV9maWxlX2Rvd25sb2FkP2J1aWQ9MTUwMTcmdWluPTE1MTA3MDk2NzImZGlyX3BhdGg9XC8mbmFtZT0xNV8wOV8zNF81YjEyOWVmMWJjY2NlZDQ3OWU1NDE3NTliMTIzZWJkNF8yNjU0LmpwZyJ9LHsicGlkIjoiMjY1NSIsIndpZHRoIjo4MDAsImhlaWdodCI6MTIwMCwidXJsIjoiaHR0cDpcL1wvYWMudGMucXEuY29tXC9zdG9yZV9maWxlX2Rvd25sb2FkP2J1aWQ9MTUwMTcmdWluPTE1MTA3MDk2NzImZGlyX3BhdGg9XC8mbmFtZT0xNV8wOV8zNF85MGY2MTg2MjMyNzcxYTE5NzVmOTVlM2RiYTQ3ZGIxZV8yNjU1LmpwZyJ9LHsicGlkIjoiMjY1NiIsIndpZHRoIjo4MDAsImhlaWdodCI6MTIwMCwidXJsIjoiaHR0cDpcL1wvYWMudGMucXEuY29tXC9zdG9yZV9maWxlX2Rvd25sb2FkP2J1aWQ9MTUwMTcmdWluPTE1MTA3MDk2ODYmZGlyX3BhdGg9XC8mbmFtZT0xNV8wOV8zNF9mZGVmMjExZmIxZjRlYWIxNGQzNDE1MjM1ZDNjOWIxNV8yNjU2LmpwZyJ9LHsicGlkIjoiMjY1NyIsIndpZHRoIjo4MDAsImhlaWdodCI6MTIwMCwidXJsIjoiaHR0cDpcL1wvYWMudGMucXEuY29tXC9zdG9yZV9maWxlX2Rvd25sb2FkP2J1aWQ9MTUwMTcmdWluPTE1MTA3MDk2ODYmZGlyX3BhdGg9XC8mbmFtZT0xNV8wOV8zNF9lNmQxZjAwNWQ1OTkxNzRkMjJlODI1MDE3MGVhMTM4N18yNjU3LmpwZyJ9LHsicGlkIjoiMjY1OCIsIndpZHRoIjo4MDAsImhlaWdodCI6MTIwMCwidXJsIjoiaHR0cDpcL1wvYWMudGMucXEuY29tXC9zdG9yZV9maWxlX2Rvd25sb2FkP2J1aWQ9MTUwMTcmdWluPTE1MTA3MDk2ODYmZGlyX3BhdGg9XC8mbmFtZT0xNV8wOV8zNF81NmI3YjI1YjVkZDM5YzdhNmU1OGNhYjY1Njc1MGY3MV8yNjU4LmpwZyJ9LHsicGlkIjoiMjY1OSIsIndpZHRoIjo4MDAsImhlaWdodCI6MTIwMCwidXJsIjoiaHR0cDpcL1wvYWMudGMucXEuY29tXC9zdG9yZV9maWxlX2Rvd25sb2FkP2J1aWQ9MTUwMTcmdWluPTE1MTA3MDk2ODYmZGlyX3BhdGg9XC8mbmFtZT0xNV8wOV8zNF9jODEyNGJiYzk2NTVlODVhMTk0ZTRiNDQwMzA2YTFjYV8yNjU5LmpwZyJ9LHsicGlkIjoiMjY2MCIsIndpZHRoIjo4MDAsImhlaWdodCI6MTIwMCwidXJsIjoiaHR0cDpcL1wvYWMudGMucXEuY29tXC9zdG9yZV9maWxlX2Rvd25sb2FkP2J1aWQ9MTUwMTcmdWluPTE1MTA3MDk2ODYmZGlyX3BhdGg9XC8mbmFtZT0xNV8wOV8zNF85ZDE0ZDYzOTY1Mjg0ZDUzY2MwMDkwN2QxZGIzY2YzNV8yNjYwLmpwZyJ9LHsicGlkIjoiMjY2MSIsIndpZHRoIjo4MDAsImhlaWdodCI6MTIwMCwidXJsIjoiaHR0cDpcL1wvYWMudGMucXEuY29tXC9zdG9yZV9maWxlX2Rvd25sb2FkP2J1aWQ9MTUwMTcmdWluPTE1MTA3MDk2ODYmZGlyX3BhdGg9XC8mbmFtZT0xNV8wOV8zNF9lZmU5OThmYjU1ZDJlZjYyMGFlMmIzYmM3ZjYyY2IzOV8yNjYxLmpwZyJ9LHsicGlkIjoiMjY2MiIsIndpZHRoIjo4NjAsImhlaWdodCI6MTQ1MywidXJsIjoiaHR0cDpcL1wvYWMudGMucXEuY29tXC9zdG9yZV9maWxlX2Rvd25sb2FkP2J1aWQ9MTUwMTcmdWluPTE1MTA3MDk2ODcmZGlyX3BhdGg9XC8mbmFtZT0xNV8wOV8zNF84NmYzY2QzMzgzYjA2N2JhOTRjYWQ4YjZlOThkMGY2NV8yNjYyLmpwZyJ9XSwiYWRzIjp7InRvcCI6eyJ0aXRsZSI6Ilx1NjVmNlx1N2E3YVx1NjA0Ylx1NGViYSIsInBpYyI6Imh0dHBzOlwvXC9tYW5odWEucXBpYy5jblwvb3BlcmF0aW9uXC8wXC8xM18wOV81NF9kOTU4YWYyOGU4NmYzNjhmYjhiN2EwMGMzMDZmODQ4OV8xNTEwNTM4MDQxNTMwLmpwZ1wvMCIsInVybCI6Imh0dHA6XC9cL2FjLnFxLmNvbVwvQ29taWNWaWV3XC9pbmRleFwvaWRcLzU1MDkyMVwvY2lkXC80Iiwid2lkdGgiOiI2NTAiLCJoZWlnaHQiOiIxMTAifSwibGVmdCI6W10sImJvdHRvbSI6eyJ0aXRsZSI6Ilx1NzNiMFx1NGUxNlx1NjBjNVx1NGViYVx1NjYyZlx1NWMzZVx1NzJkMCIsInBpYyI6Imh0dHBzOlwvXC9tYW5odWEucXBpYy5jblwvb3BlcmF0aW9uXC8wXC8xM18wOV81Ml84ZWJjZTJkZmJhMzZhZDhlYTA2YjEyODRkNzY1Mzc0Nl8xNTEwNTM3OTM3NDY2LmpwZ1wvMCIsInVybCI6Imh0dHA6XC9cL2FjLnFxLmNvbVwvQ29taWNWaWV3XC9pbmRleFwvaWRcLzYyNjg1MFwvY2lkXC8yIiwid2lkdGgiOiI2NTAiLCJoZWlnaHQiOiIxMTAifX0sImFydGlzdCI6eyJhdmF0YXIiOiJodHRwOlwvXC9xNC5xbG9nby5jblwvZz9iPXFxJms9d2g1c0ZOazAzUDV6RmlhOGljdGQ4ZW13JnM9NjQwJnQ9MTQ4MzMzOTY3NSIsIm5pY2siOiJcdTc3ZTVcdTk3ZjNcdTc5ZmJcdTUyYThcdTY1NzBcdTViNTciLCJ1aW5DcnlwdCI6IlVUSk1kVkpCZDAxTlFrSkVibGhzWmpOWlJqRjBaejA5In19"; //var ss = DecodeHelper.QQPageDecode(keystr.Substring(1)); //var tt = JsonHelper.DeserializeJsonToObject<QQ_Page_Api>(ss); //var redis = new RedisProxy(); //try //{ // redis.HashSet("QQ_Chapter", "185158", "185157"); //} //catch (Exception ex) //{ // throw; //} //var login_key = redis.Get<QQ_Login_Key>("QQ_1434299101"); DateTime dt = DateTime.Now; string ticks = dt.Ticks.ToString(); string shortdate = dt.ToString("yyyy-MM-dd"); string yesterday = dt.AddDays(-1).ToString("yyyy-MM-dd"); IQuery <Chapter> cpq = dbcontext.Query <Chapter>();//x.comicid == "1_622585" IQuery <VIPFreeComic> vfcq = dbcontext.Query <VIPFreeComic>(); List <Chapter> cplst; if (QQChapterTotal > 0) { cplst = cpq.Where(x => x.source == Source.QQ && x.downstatus == DownChapter.待处理链接 && x.isvip == "0" && x.Id % QQChapterTotal == QQChapterIndex).Take(200).ToList(); } else { cplst = cpq.Where(x => x.source == Source.QQ && x.downstatus == DownChapter.待处理链接 && x.isvip == "0").Take(200).ToList(); } List <int> ids = cplst.Select(x => x.Id).ToList(); dbcontext.Update <Chapter>(a => ids.Contains(a.Id), a => new Chapter() { ticks = ticks, downstatus = DownChapter.处理完链接, modify = dt }); List <Chapter> chapterlst = new List <Chapter>(); foreach (var cp in cplst) { try { //Dictionary<string, string> headers = new Dictionary<string, string>() //{ // {"Cookie","uin=o1434299101; skey="+login_key.skey+";" } //}; var imgdata = _helper.Get(cp.chapterurl, Encoding.GetEncoding("UTF-8"), null, "", null, "", null, "application/x-www-form-urlencoded; charset=UTF-8"); Regex rex = new Regex("var DATA = '(?<key1>.*?)',"); Match match = rex.Match(imgdata); string key = match.Groups["key1"].Value; if (string.IsNullOrEmpty(key)) { cp.downstatus = DownChapter.待处理链接; cp.modify = dt; cp.ticks = ticks; dbcontext.Update(cp); Err_ChapterJob err = new Err_ChapterJob(); err.bookurl = cp.chapterurl; err.source = cp.source; err.errtype = ErrChapter.解析出错; err.modify = dt; err.shortdate = shortdate; err.message = "DATA解析失败"; err = dbcontext.Insert(err); continue; } var s = DecodeHelper.QQPageDecode(key.Substring(1)); var t = JsonHelper.DeserializeJsonToObject <QQ_Page_Api>(s); if (t.chapter.canRead) { List <Page> pglst = new List <Page>(); for (int i = 0; i < t.picture.Count; i++) { pglst.Add(new Page() { chapterid = cp.chapterid, modify = dt, shortdate = shortdate, sort = i + 1, source = cp.source, pagelocal = "", ticks = ticks, pagesource = t.picture[i].url }); } cp.isvip = "0"; cp.ticks = ticks; cp.downstatus = DownChapter.处理完链接; cp.modify = dt; dbcontext.Update(cp); dbcontext.BulkInsert(pglst); } else { List <VIPFreeComic> vfclst = vfcq.Where(x => x.source == Source.QQ && x.comicid == cp.comicid).ToList(); cp.isvip = vfclst.Count == 0 ? "1" : "0"; cp.ticks = ticks; cp.downstatus = DownChapter.待处理链接; cp.modify = dt; dbcontext.Update(cp); } } catch (Exception ex) { cp.downstatus = DownChapter.待处理链接; cp.modify = dt; cp.ticks = ticks; dbcontext.Update(cp); Err_ChapterJob err = new Err_ChapterJob(); err.bookurl = cp.chapterurl; err.source = cp.source; err.errtype = ErrChapter.解析出错; err.modify = dt; err.shortdate = shortdate; err.message = ex.Message; err = dbcontext.Insert(err); continue; } } }
public void Execute(IJobExecutionContext context) { DateTime dt = DateTime.Now; string ticks = dt.Ticks.ToString(); string shortdate = dt.ToString("yyyy-MM-dd"); string updatedatetime = shortdate + " " + ((dt.Hour / 0.5 + 1) * 0.5).ToString(); string yesterday = dt.AddDays(-1).ToString("yyyy-MM-dd"); IQuery <Comic> q = dbcontext.Query <Comic>(); IQuery <Chapter> cpq = dbcontext.Query <Chapter>(); IQuery <PageHis> phisq = dbcontext.Query <PageHis>(); IQuery <Page> pq = dbcontext.Query <Page>(); IQuery <Notice> nq = dbcontext.Query <Notice>(); List <Comic> comiclst = q.Where(a => a.source == Source.dongmanmanhua && (a.updatedatetime == null || a.updatedatetime != updatedatetime)).Take(200).ToList(); List <int> ids = comiclst.Select(x => x.Id).ToList(); dbcontext.Update <Comic>(a => ids.Contains(a.Id), a => new Comic() { updatedatetime = updatedatetime, modify = dt }); foreach (var comic in comiclst) { List <Chapter> cplst = cpq.Where(a => a.comicid == comic.comicid && a.source == Source.dongmanmanhua).ToList(); List <Chapter> chapterlst = new List <Chapter>(); if (cplst.Count > 0) { try { string bookurl = comic.bookurl.Replace("https://www.dongmanmanhua.cn/", ""); var bookdata = _helper.Get(null, bookurl); string pattern = "<li id=\"episode_(?<key1>.*?)\" data-episode-no=\"(?<key2>.*?)\">(?<key3>.*?)</li>"; MatchCollection matches = Regex.Matches(bookdata, pattern, RegexOptions.IgnoreCase | RegexOptions.Singleline); int pagecount = (int)Math.Ceiling(int.Parse(matches[0].Groups[1].Value) / 10.0); for (int i = 0; i < matches.Count; i++) { var lihtml = matches[i].Groups["key3"].Value; Regex reg1 = new Regex("href=\"(?<key1>.*?)\""); Match match1 = reg1.Match(lihtml); string chapterurl = match1.Groups["key1"].Value; Regex reg2 = new Regex("<span class=\"tx\">#(?<key1>.*?)</span>"); Match match2 = reg2.Match(lihtml); int sort = int.Parse(match2.Groups["key1"].Value); //Regex reg3 = new Regex("<span class=\"subj\"><span>(?<key1>.*?)</span></span>"); //Match match3 = reg3.Match(lihtml); //string chaptername = match3.Groups["key1"].Value; Regex reg3 = new Regex("alt=\"(?<key1>.*?)\""); Match match3 = reg3.Match(lihtml); string chaptername = match3.Groups["key1"].Value; Regex reg4 = new Regex("src=\"(?<key1>.*?)\""); Match match4 = reg4.Match(lihtml); string chaptersource = match4.Groups["key1"].Value; chapterlst.Add(new Chapter() { chapterid = comic.comicid + "_" + sort, chaptername = chaptername, chapterurl = "https:" + chapterurl, sort = sort, comicid = comic.comicid, retry = 0, source = comic.source, downstatus = DownChapter.待处理链接, isvip = "0", chaptersource = chaptersource, chapterlocal = "", modify = dt, shortdate = shortdate, }); } for (int i = 2; i <= pagecount; i++) { var bookdata2 = _helper.Get(null, bookurl + "&page=" + i); string pattern2 = "<li id=\"episode_(?<key1>.*?)\" data-episode-no=\"(?<key2>.*?)\">(?<key3>.*?)</li>"; MatchCollection matches2 = Regex.Matches(bookdata2, pattern2, RegexOptions.IgnoreCase | RegexOptions.Singleline); for (int j = 0; j < matches2.Count; j++) { var lihtml = matches2[j].Groups["key3"].Value; Regex reg1 = new Regex("href=\"(?<key1>.*?)\""); Match match1 = reg1.Match(lihtml); string chapterurl = match1.Groups["key1"].Value; Regex reg2 = new Regex("<span class=\"tx\">#(?<key1>.*?)</span>"); Match match2 = reg2.Match(lihtml); int sort = int.Parse(match2.Groups["key1"].Value); //Regex reg3 = new Regex("<span class=\"subj\"><span>(?<key1>.*?)</span></span>"); //Match match3 = reg3.Match(lihtml); //string chaptername = match3.Groups["key1"].Value; Regex reg3 = new Regex("alt=\"(?<key1>.*?)\""); Match match3 = reg3.Match(lihtml); string chaptername = match3.Groups["key1"].Value; Regex reg4 = new Regex("src=\"(?<key1>.*?)\""); Match match4 = reg4.Match(lihtml); string chaptersource = match4.Groups["key1"].Value; chapterlst.Add(new Chapter() { chapterid = comic.comicid + "_" + sort, chaptername = chaptername, chapterurl = "https:" + chapterurl, sort = sort, comicid = comic.comicid, retry = 0, source = comic.source, downstatus = DownChapter.待处理链接, isvip = "0", chaptersource = chaptersource, chapterlocal = "", modify = dt, shortdate = shortdate, }); } } bool contextChange = false; if (chapterlst.Count > 0) { chapterlst = chapterlst.OrderBy(x => x.sort).ToList(); int cmpCount = cplst.Count >= chapterlst.Count ? chapterlst.Count : cplst.Count; for (int i = 0; i < cmpCount; i++) { if (cplst[i].chapterid != chapterlst[i].chapterid) { contextChange = true; } } } else { continue; } if (contextChange) { dbcontext.Delete <Chapter>(x => x.comicid == comic.comicid); if (chapterlst.Count > 0) { dbcontext.BulkInsert(chapterlst); Notice notice = new Notice(); notice.noticeid = comic.comicid; notice.noticestatus = NoticeStatus.等待处理; notice.noticetype = NoticeType.目录变更; notice.source = comic.source; notice.shortdate = shortdate; notice.modify = dt; //var nqwait = nq.Where(x => x.noticeid == comic.comicid && x.noticestatus == NoticeStatus.等待处理 && x.noticetype == NoticeType.目录变更).FirstOrDefault(); //if (nqwait == null) //{ dbcontext.Insert(notice); //} } } else { if (cplst.Count == chapterlst.Count) { continue; } if (cplst.Count > chapterlst.Count) { List <Chapter> needrmvlst = cplst.Except(chapterlst, new Chapter_Comparer()).ToList(); List <int> rmvidlst = needrmvlst.Select(x => x.Id).ToList(); logger.Error(string.Join(",", needrmvlst.Select(x => x.chapterid).ToArray())); dbcontext.Delete <Chapter>(x => rmvidlst.Contains(x.Id)); } if (cplst.Count < chapterlst.Count) { List <Chapter> needaddlst = chapterlst.Except(cplst, new Chapter_Comparer()).ToList(); dbcontext.BulkInsert(needaddlst); } } //int delete = cplst.Except(chapterlst, new Chapter_Comparer()).Count(); // 删章 //List<Chapter> add = chapterlst.Except(cplst, new Chapter_Comparer()).ToList(); // 新增 //Chapter maxChapter = cplst.OrderBy(x => x.sort).Max(); //if (delete > 0) //{ // List<string> idlst = cplst.Select(x => x.chapterid).ToList(); // dbcontext.Delete<Page>(x => idlst.Contains(x.chapterid)); // dbcontext.Delete<Chapter>(x => idlst.Contains(x.chapterid)); // if (chapterlst.Count > 0) // { // dbcontext.BulkInsert(chapterlst); // } // Notice notice = new Notice(); // notice.noticeid = comic.comicid; // notice.noticestatus = NoticeStatus.等待处理; // notice.noticetype = NoticeType.目录变更; // notice.source = comic.source; // notice.shortdate = shortdate; // notice.modify = dt; // var nqwait = nq.Where(x => x.noticeid == comic.comicid && x.noticestatus == NoticeStatus.等待处理 && x.noticetype == NoticeType.目录变更).FirstOrDefault(); // if (nqwait == null) // { // dbcontext.Insert(notice); // } // continue; //} //else //{ // List<Chapter> mvadd = chapterlst.Except(add, new Chapter_Comparer()).ToList(); // string cplststr = string.Join(",", cplst.Select(x => x.chapterid).ToArray()); // string chapterlststr = string.Join(",", mvadd.Select(x => x.chapterid).ToArray()); // if (cplststr != chapterlststr) // 调序 // { // List<string> idlst = cplst.Select(x => x.chapterid).ToList(); // dbcontext.Delete<Page>(x => idlst.Contains(x.chapterid)); // dbcontext.Delete<Chapter>(x => idlst.Contains(x.chapterid)); // if (chapterlst.Count > 0) // { // dbcontext.BulkInsert(chapterlst); // } // Notice notice = new Notice(); // notice.noticeid = comic.comicid; // notice.noticestatus = NoticeStatus.等待处理; // notice.noticetype = NoticeType.目录变更; // notice.source = comic.source; // notice.shortdate = shortdate; // notice.modify = dt; // var nqwait = nq.Where(x => x.noticeid == comic.comicid && x.noticestatus == NoticeStatus.等待处理 && x.noticetype == NoticeType.目录变更).FirstOrDefault(); // if (nqwait == null) // { // dbcontext.Insert(notice); // } // continue; // } //} //if (add.Count > 0) //{ // //int addsort = cplst.Count(); // //foreach (var item in add) // //{ // // item.sort = item.sort + addsort; // //} // dbcontext.BulkInsert(add); //} } catch (Exception ex) { logger.Error(ex.Message); comic.updatedatetime = ""; comic.modify = dt; dbcontext.Update(comic); Err_ChapterJob err = new Err_ChapterJob(); err.bookurl = comic.bookurl; err.source = comic.source; err.errtype = ErrChapter.解析出错; err.modify = dt; err.shortdate = shortdate; err.message = ex.Message; err = dbcontext.Insert(err); continue; } } } }
public void Execute(IJobExecutionContext context) { DateTime dt = DateTime.Now; string ticks = dt.Ticks.ToString(); string shortdate = dt.ToString("yyyy-MM-dd"); string yesterday = dt.AddDays(-1).ToString("yyyy-MM-dd"); IQuery <Comic> q = dbcontext.Query <Comic>(); IQuery <Chapter> cpq = dbcontext.Query <Chapter>(); //List<Comic> comiclst = q.Where(a => a.source == Source.U17 && a.shortdate == shortdate).Take(200).ToList(); List <Comic> comiclst = q.Where(a => a.source == Source.U17).Take(200).ToList(); List <int> ids = comiclst.Select(x => x.Id).ToList(); dbcontext.Update <Comic>(a => ids.Contains(a.Id), a => new Comic() { shortdate = yesterday, modify = dt }); List <Chapter> chapterlst = new List <Chapter>(); foreach (var comic in comiclst) { List <Chapter> cplst = cpq.Where(a => a.comicid == comic.comicid && a.source == Source.U17).ToList(); if (cplst.Count == 0) { try { string bookurl = comic.bookurl.Replace("http://www.u17.com/", ""); var bookdata = _helper.Get(null, bookurl); bookdata = StringHelper.MergeSpace(bookdata); string pattern = "<li id='cpt_read_(?<key1>.*?)'> <a id=\"cpt_(?<key2>.*?)\" href=\"(?<key3>.*?)\" title=(?<key5>.*?)>(?<key4>.*?)</a>"; MatchCollection matches = Regex.Matches(bookdata, pattern, RegexOptions.IgnoreCase | RegexOptions.Singleline); for (int i = 0; i < matches.Count; i++) { chapterlst.Add(new Chapter() { chapterid = comic.comicid + "_" + matches[i].Groups["key2"].Value, chaptername = StringHelper.ReplaceHtmlTag(matches[i].Groups["key4"].Value.Trim()), chapterurl = matches[i].Groups["key3"].Value, sort = i + 1, comicid = comic.comicid, retry = 0, source = comic.source, downstatus = DownChapter.待处理链接, isvip = matches[i].Groups["key5"].Value.IndexOf("_chapter") == -1? "0":"1", chaptersource = "", chapterlocal = "", modify = dt, shortdate = shortdate, ticks = ticks }); } } catch (Exception ex) { logger.Error(ex.Message); comic.shortdate = shortdate; comic.modify = dt; dbcontext.Update(comic); Err_ChapterJob err = new Err_ChapterJob(); err.bookurl = comic.bookurl; err.source = comic.source; err.errtype = ErrChapter.解析出错; err.modify = dt; err.shortdate = shortdate; err.message = ex.Message; err = dbcontext.Insert(err); continue; } } } if (chapterlst.Count > 0) { dbcontext.BulkInsert(chapterlst); } }