public void Execute(IJobExecutionContext context)
{
try
{
DateTime dt = DateTime.Now;
string shortdate = dt.ToString("yyyy-MM-dd");
var bookdata = _helper.Get(null, "http://ac.qq.com/Comic/all/search/time/vip/3/page/1");
IQuery <VIPFreeComic> vfcq = dbcontext.Query <VIPFreeComic>();
string pattern_page = "共<em>(?<key1>.*?)</em>个结果";
MatchCollection matches_page = Regex.Matches(bookdata, pattern_page, RegexOptions.IgnoreCase | RegexOptions.Singleline);
int pagecount = (int)Math.Ceiling(int.Parse(matches_page[0].Groups[1].Value) / 12.0);
List <VIPFreeComic> vfclst = new List <VIPFreeComic>();
for (int i = 1; i <= pagecount; i++)
{
var bookdata2 = _helper.Get(null, "http://ac.qq.com/Comic/all/search/time/vip/3/page/" + i);
bookdata2 = StringHelper.MergeSpace(bookdata2);
string pattern2 = "<h3 class=\"ret-works-title clearfix\"> <a href=\"(?<key1>.*?)\" target=\"_blank\" title=\"(?<key2>.*?)\">(?<key3>.*?)</a>";
MatchCollection matches2 = Regex.Matches(bookdata2, pattern2, RegexOptions.IgnoreCase | RegexOptions.Singleline);
for (int j = 0; j < matches2.Count; j++)
{
string bookurl = "http://ac.qq.com" + matches2[j].Groups["key1"].Value;
string comicname = matches2[j].Groups["key2"].Value;
string comicid = (int)Source.QQ + "_" + bookurl.Split('/').LastOrDefault();
vfclst.Add(new VIPFreeComic()
{
bookurl = bookurl,
comicid = comicid,
comicname = comicname,
modify = dt,
shortdate = shortdate,
source = Source.QQ
});
}
}
List <VIPFreeComic> vfcdblst = vfcq.Where(x => x.source == Source.QQ).ToList();
if (vfclst.Count > 0)
{
List <VIPFreeComic> delete = vfcdblst.Except(vfclst, new VIPFreeComic_Comparer()).ToList();
List <VIPFreeComic> add = vfclst.Except(vfcdblst, new VIPFreeComic_Comparer()).ToList();
if (delete.Count > 0)
{
List <string> deleteidlst = delete.Select(x => x.comicid).ToList();
dbcontext.Delete <VIPFreeComic>(x => deleteidlst.Contains(x.comicid));
}
if (add.Count > 0)
{
dbcontext.BulkInsert(add);
}
}
}
catch (Exception ex)
{
throw;
}
}
public static void BulkCopyTest()
{
List <TestEntity> models = new List <TestEntity>();
models.Add(new TestEntity()
{
Id = 14,
F_Byte = 1,
F_Int16 = 1,
F_Int32 = 1,
F_Int64 = 1,
F_Double = 1,
F_Float = 1,
F_Decimal = 1,
F_Bool = true,
F_DateTime = DateTime.Now,
F_String = "lu"
});
models.Add(new TestEntity()
{
Id = 15,
F_Byte = 1,
F_Int16 = 1,
F_Int32 = 1,
F_Int64 = 1,
F_Double = 1,
F_Float = 1,
F_Decimal = 1,
F_Bool = true,
F_DateTime = DateTime.Now,
F_Guid = Guid.NewGuid(),
F_String = "so"
});
object result = null;
//SqlConnection conn = new SqlConnection(DbHelper.ConnectionString);
//conn.BulkCopy1<TestEntity>(models, int.MaxValue, null, null, null);
context.BeginTransaction();
result = context.Query <User>().Select(a => new UserLite()
{
Id = a.Id, Name = a.Name
}).ToList();
context.BulkInsert(models, null, null, false);
result = context.Query <User>().Select(a => new UserLite()
{
Id = a.Id, Name = a.Name
}).ToList();
context.CommitTransaction();
Console.WriteLine(1);
Console.ReadKey();
}
public static void BulkInsert()
{
List <User> models = new List <User>();
models.Add(new User()
{
Name = "lu", Age = 18, Gender = Gender.Woman, CityId = 1, OpTime = DateTime.Now
});
models.Add(new User()
{
Name = "shuxin", Age = 18, Gender = Gender.Man, CityId = 1, OpTime = DateTime.Now
});
/* 利用 SqlBulkCopy 批量插入数据 */
context.BulkInsert(models, batchSize: null, bulkCopyTimeout: null, keepIdentity: false);
ConsoleHelper.WriteLineAndReadKey(1);
}
public void Execute(IJobExecutionContext context)
{
DateTime dt = DateTime.Now;
string shortdate = dt.ToString("yyyy-MM-dd");
IQuery <Chapter> cpq = dbcontext.Query <Chapter>();
List <Chapter> cplst = cpq.Where(x => x.source == Source.Zymk && x.downstatus == DownChapter.待处理链接).Take(200).ToList();
List <int> ids = cplst.Select(x => x.Id).ToList();
dbcontext.Update <Chapter>(a => ids.Contains(a.Id), a => new Chapter()
{
downstatus = DownChapter.处理完链接,
modify = dt
});
List <Chapter> chapterlst = new List <Chapter>();
foreach (var cp in cplst)
{
try
{
string chapterpage = cp.chapterurl.Replace("http://www.zymk.cn/", "");
var imgdata = _helper.Get(null, chapterpage);
Regex imgpath = new Regex("chapter_addr:\"(?<key1>.*?)\",");
Regex starindex = new Regex("start_var:(?<key1>.*?),");
Regex totalimg = new Regex("end_var:(?<key1>.*?),");
Regex pageid = new Regex("chapter_id:(?<key1>.*?),");
Regex domain = new Regex("domain:\"(?<key1>.*?)\",");
Regex comic_size = new Regex("middle:\"(?<key1>.*?)\"");
var _imgpath = imgpath.Match(imgdata).Groups["key1"].Value.Replace("\\\\", "#").Replace("\\", "").Replace("#", "\\");
var _totalimg = totalimg.Match(imgdata).Groups["key1"].Value;
var _pageid = pageid.Match(imgdata).Groups["key1"].Value;
var _domain = domain.Match(imgdata).Groups["key1"].Value;
var _comic_size = comic_size.Match(imgdata).Groups["key1"].Value;
int start = int.Parse(starindex.Match(imgdata).Groups["key1"].Value.Trim());
List <Page> pglst = new List <Page>();
int imgcount = int.Parse(_totalimg);
imgcount = imgcount - start;
string imgdecodepath = DecodeHelper.Decode(_imgpath, int.Parse(_pageid));
for (int i = 0; i <= imgcount; i++)
{
string pgsource = "http://mhpic." + _domain + "/comic/" + imgdecodepath + start + ".jpg" + _comic_size;
pglst.Add(new Page()
{
pagesource = pgsource,
chapterid = cp.chapterid,
modify = dt,
shortdate = shortdate,
sort = i + 1,
source = cp.source,
pagelocal = "",
});
start = start + 1;
}
dbcontext.BulkInsert(pglst);
}
catch (Exception ex)
{
logger.Error(ex.Message);
cp.downstatus = DownChapter.待处理链接;
cp.modify = dt;
dbcontext.Update(cp);
Err_ChapterJob err = new Err_ChapterJob();
err.bookurl = cp.chapterurl;
err.source = cp.source;
err.errtype = ErrChapter.解析出错;
err.modify = dt;
err.shortdate = shortdate;
err.message = ex.Message;
err = dbcontext.Insert(err);
continue;
}
}
}
public void Execute(IJobExecutionContext context)
{
DateTime dt = DateTime.Now;
string shortdate = dt.ToString("yyyy-MM-dd");
string yesterday = dt.AddDays(-1).ToString("yyyy-MM-dd");
IQuery <Comic> q = dbcontext.Query <Comic>();
IQuery <Chapter> cpq = dbcontext.Query <Chapter>();
List <Comic> comiclst = q.Where(a => a.source == Source.dongmanmanhua && a.shortdate == shortdate).Take(200).ToList();
List <int> ids = comiclst.Select(x => x.Id).ToList();
dbcontext.Update <Comic>(a => ids.Contains(a.Id), a => new Comic()
{
shortdate = yesterday,
modify = dt
});
List <Chapter> chapterlst = new List <Chapter>();
foreach (var comic in comiclst)
{
List <Chapter> cplst = cpq.Where(a => a.comicid == comic.Id && a.source == Source.dongmanmanhua).ToList();
if (cplst.Count == 0)
{
try
{
string bookurl = comic.bookurl.Replace("https://www.dongmanmanhua.cn/", "");
var bookdata = _helper.Get(null, bookurl);
string pattern = "<li id=\"episode_(?<key1>.*?)\" data-episode-no=\"(?<key2>.*?)\">(?<key3>.*?)</li>";
MatchCollection matches = Regex.Matches(bookdata, pattern, RegexOptions.IgnoreCase | RegexOptions.Singleline);
int pagecount = (int)Math.Ceiling(int.Parse(matches[0].Groups[1].Value) / 10.0);
for (int i = 0; i < matches.Count; i++)
{
var lihtml = matches[i].Groups["key3"].Value;
Regex reg1 = new Regex("href=\"(?<key1>.*?)\"");
Match match1 = reg1.Match(lihtml);
string chapterurl = match1.Groups["key1"].Value;
Regex reg2 = new Regex("<span class=\"tx\">#(?<key1>.*?)</span>");
Match match2 = reg2.Match(lihtml);
int sort = int.Parse(match2.Groups["key1"].Value);
Regex reg3 = new Regex("<span class=\"subj\"><span>(?<key1>.*?)</span></span>");
Match match3 = reg3.Match(lihtml);
string chaptername = match3.Groups["key1"].Value;
Regex reg4 = new Regex("src=\"(?<key1>.*?)\"");
Match match4 = reg4.Match(lihtml);
string chaptersource = match4.Groups["key1"].Value;
chapterlst.Add(new Chapter()
{
chapterid = comic.comicid + "_" + sort,
chaptername = chaptername,
chapterurl = "https:" + chapterurl,
sort = sort,
comicid = comic.Id,
retry = 0,
source = comic.source,
downstatus = DownChapter.待处理链接,
isvip = "0",
chaptersource = chaptersource,
chapterlocal = "",
modify = dt,
shortdate = shortdate,
});
}
for (int i = 2; i <= pagecount; i++)
{
var bookdata2 = _helper.Get(null, bookurl + "&page=" + i);
string pattern2 = "<li id=\"episode_(?<key1>.*?)\" data-episode-no=\"(?<key2>.*?)\">(?<key3>.*?)</li>";
MatchCollection matches2 = Regex.Matches(bookdata2, pattern2, RegexOptions.IgnoreCase | RegexOptions.Singleline);
for (int j = 0; j < matches2.Count; j++)
{
var lihtml = matches2[j].Groups["key3"].Value;
Regex reg1 = new Regex("href=\"(?<key1>.*?)\"");
Match match1 = reg1.Match(lihtml);
string chapterurl = match1.Groups["key1"].Value;
Regex reg2 = new Regex("<span class=\"tx\">#(?<key1>.*?)</span>");
Match match2 = reg2.Match(lihtml);
int sort = int.Parse(match2.Groups["key1"].Value);
Regex reg3 = new Regex("<span class=\"subj\"><span>(?<key1>.*?)</span></span>");
Match match3 = reg3.Match(lihtml);
string chaptername = match3.Groups["key1"].Value;
Regex reg4 = new Regex("src=\"(?<key1>.*?)\"");
Match match4 = reg4.Match(lihtml);
string chaptersource = match4.Groups["key1"].Value;
chapterlst.Add(new Chapter()
{
chapterid = comic.comicid + "_" + sort,
chaptername = chaptername,
chapterurl = "https:" + chapterurl,
sort = sort,
comicid = comic.Id,
retry = 0,
source = comic.source,
downstatus = DownChapter.待处理链接,
isvip = "0",
chaptersource = chaptersource,
chapterlocal = "",
modify = dt,
shortdate = shortdate,
});
}
}
}
catch (Exception ex)
{
logger.Error(ex.Message);
comic.shortdate = shortdate;
comic.modify = dt;
dbcontext.Update(comic);
Err_ChapterJob err = new Err_ChapterJob();
err.bookurl = comic.bookurl;
err.source = comic.source;
err.errtype = ErrChapter.解析出错;
err.modify = dt;
err.shortdate = shortdate;
err.message = ex.Message;
err = dbcontext.Insert(err);
continue;
}
}
}
if (chapterlst.Count > 0)
{
dbcontext.BulkInsert(chapterlst);
}
}
public void Execute(IJobExecutionContext context)
{
DateTime dt = DateTime.Now;
string shortdate = dt.ToString("yyyy-MM-dd");
string yesterday = dt.AddDays(-1).ToString("yyyy-MM-dd");
IQuery <Comic> q = dbcontext.Query <Comic>();
IQuery <Chapter> cpq = dbcontext.Query <Chapter>();
List <Comic> comiclst = q.Where(a => a.source == Source.QQ && a.shortdate == shortdate).Take(200).ToList();
List <int> ids = comiclst.Select(x => x.Id).ToList();
dbcontext.Update <Comic>(a => ids.Contains(a.Id), a => new Comic()
{
shortdate = yesterday,
modify = dt
});
List <Chapter> chapterlst = new List <Chapter>();
foreach (var comic in comiclst)
{
List <Chapter> cplst = cpq.Where(a => a.comicid == comic.Id && a.source == Source.QQ).ToList();
if (cplst.Count == 0)
{
try
{
string bookurl = comic.bookurl.Replace("http://ac.qq.com/", "");
var bookdata = _helper.Get(null, bookurl);
//class=\"chapter-page-all works-(?<key1>.*?)-list\"
Regex reg1 = new Regex("<ol class=\"chapter-page-all works-chapter-list\">(?<key1>.*?)</ol>", RegexOptions.IgnoreCase | RegexOptions.Singleline);
Match match1 = reg1.Match(bookdata);
string htmlSource = match1.Groups["key1"].Value;
string pattern = "<a target=\"_blank\" title=\"(?<key1>.*?)\" href=\"(?<key2>.*?)\">(?<key3>.*?)</a>";
MatchCollection matches = Regex.Matches(htmlSource, pattern, RegexOptions.IgnoreCase | RegexOptions.Singleline);
for (int i = 0; i < matches.Count; i++)
{
chapterlst.Add(new Chapter()
{
chapterid = comic.comicid + "_" + matches[i].Groups["key2"].Value.Split('/').LastOrDefault(),
chaptername = matches[i].Groups["key3"].Value.Trim(),
chapterurl = "http://ac.qq.com" + matches[i].Groups["key2"].Value,
sort = i + 1,
comicid = comic.Id,
retry = 0,
source = comic.source,
downstatus = DownChapter.待处理链接,
isvip = "0",
chaptersource = "",
chapterlocal = "",
modify = dt,
shortdate = shortdate,
});
}
}
catch (Exception ex)
{
logger.Error(ex.Message);
comic.shortdate = shortdate;
comic.modify = dt;
dbcontext.Update(comic);
Err_ChapterJob err = new Err_ChapterJob();
err.bookurl = comic.bookurl;
err.source = comic.source;
err.errtype = ErrChapter.解析出错;
err.modify = dt;
err.shortdate = shortdate;
err.message = ex.Message;
err = dbcontext.Insert(err);
continue;
}
}
}
if (chapterlst.Count > 0)
{
dbcontext.BulkInsert(chapterlst);
}
}
public void Execute(IJobExecutionContext context)
{
DateTime dt = DateTime.Now;
string shortdate = dt.ToString("yyyy-MM-dd");
string yesterday = dt.AddDays(-1).ToString("yyyy-MM-dd");
IQuery <Page> pq = dbcontext.Query <Page>();
IQuery <Notice> nq = dbcontext.Query <Notice>();
IQuery <Chapter> cpq = dbcontext.Query <Chapter>();//x.comicid == "1_622585"
List <Chapter> cplst = cpq.Where(x => x.source == Source.QQ && x.downstatus == DownChapter.完图片 && x.updatedate != shortdate).Take(200).ToList();
List <int> ids = cplst.Select(x => x.Id).ToList();
dbcontext.Update <Chapter>(a => ids.Contains(a.Id), a => new Chapter()
{
updatedate = shortdate,
modify = dt
});
List <Chapter> chapterlst = new List <Chapter>();
foreach (var cp in cplst)
{
try
{
string chapterpage = cp.chapterurl.Replace("http://ac.qq.com", "");
var imgdata = _helper.Get(null, chapterpage);
Regex rex = new Regex("var DATA = '(?<key1>.*?)',");
Match match = rex.Match(imgdata);
string key = match.Groups["key1"].Value;
if (string.IsNullOrEmpty(key))
{
cp.downstatus = DownChapter.待处理链接;
cp.modify = dt;
dbcontext.Update(cp);
Err_ChapterJob err = new Err_ChapterJob();
err.bookurl = cp.chapterurl;
err.source = cp.source;
err.errtype = ErrChapter.解析出错;
err.modify = dt;
err.shortdate = shortdate;
err.message = "DATA解析失败";
err = dbcontext.Insert(err);
continue;
}
var s = DecodeHelper.QQPageDecode(key.Substring(1));
var t = JsonHelper.DeserializeJsonToObject <QQ_Page_Api>(s);
if (t.chapter.vipStatus == 1)
{
List <Page> pglst = new List <Page>();
for (int i = 0; i < t.picture.Count; i++)
{
pglst.Add(new Page()
{
chapterid = cp.chapterid,
modify = dt,
shortdate = shortdate,
sort = i + 1,
source = cp.source,
pagelocal = "",
pagesource = t.picture[i].url
});
}
List <Page> pgdblst = pq.Where(x => x.chapterid == cp.chapterid).ToList();
if (pgdblst.Count != pglst.Count)
{
List <int> idlst = pgdblst.Select(x => x.Id).ToList();
dbcontext.Delete <Page>(x => idlst.Contains(x.Id));
dbcontext.BulkInsert(pglst);
cp.downstatus = DownChapter.处理完链接;
cp.modify = dt;
dbcontext.Update(cp);
Notice notice = new Notice();
notice.noticeid = cp.chapterid;
notice.noticestatus = NoticeStatus.等待处理;
notice.noticetype = NoticeType.章节更新;
notice.source = cp.source;
notice.shortdate = shortdate;
notice.modify = dt;
var nqwait = nq.Where(x => x.noticeid == cp.chapterid && x.noticestatus == NoticeStatus.等待处理 && x.noticetype == NoticeType.章节更新).FirstOrDefault();
if (nqwait == null)
{
dbcontext.Insert(notice);
}
continue;
}
}
else
{
selenium.Navigate().GoToUrl(cp.chapterurl);
IList <IWebElement> frames = selenium.FindElements(By.TagName("iframe"));
IWebElement controlPanelFrame = null; //commentFrame ptlogin_iframe
foreach (var frame in frames)
{
if (frame.GetAttribute("id") == "iframeAll")
{
controlPanelFrame = frame;
break;
}
}
if (controlPanelFrame != null) //QQ登录
{
selenium.SwitchTo().Frame(controlPanelFrame);
IReadOnlyCollection <IWebElement> switchtoElement = selenium.FindElements(By.Id("switcher_plogin"));
if (switchtoElement != null && switchtoElement.Count > 0)
{
switchtoElement.First().Click();
selenium.FindElement(By.Id("u")).Clear();
selenium.FindElement(By.Id("u")).SendKeys("2806126975");
selenium.FindElement(By.Id("p")).Clear();
selenium.FindElement(By.Id("p")).SendKeys("rby123456");
selenium.FindElement(By.Id("login_button")).Click();
}
selenium.SwitchTo().DefaultContent();
}
frames = selenium.FindElements(By.TagName("iframe"));
IWebElement checkVipFrame = null;
foreach (var frame in frames)
{
if (frame.GetAttribute("id") == "checkVipFrame")
{
checkVipFrame = frame;
break;
}
}
if (checkVipFrame != null)
{
selenium.SwitchTo().Frame(checkVipFrame);
//自动购买
IReadOnlyCollection <IWebElement> checkAutoElement = selenium.FindElements(By.Id("check_auto_next"));
IReadOnlyCollection <IWebElement> singlBbuyElement = selenium.FindElements(By.ClassName("single_buy"));
if (checkAutoElement != null && singlBbuyElement != null && checkAutoElement.Count > 0 && singlBbuyElement.Count > 0)
{
if (singlBbuyElement.First().Text.Trim().StartsWith("点券不足"))
{
MailMessage msg = new MailMessage();
continue;
}
else
{
checkAutoElement.First().Click();
singlBbuyElement.First().Click();
}
}
}
Match match1 = rex.Match(selenium.PageSource);
key = match1.Groups["key1"].Value;
if (string.IsNullOrEmpty(key))
{
cp.downstatus = DownChapter.待处理链接;
cp.modify = dt;
dbcontext.Update(cp);
Err_ChapterJob err = new Err_ChapterJob();
err.bookurl = cp.chapterurl;
err.source = cp.source;
err.errtype = ErrChapter.解析出错;
err.modify = dt;
err.shortdate = shortdate;
err.message = "DATA解析失败";
err = dbcontext.Insert(err);
continue;
}
s = DecodeHelper.QQPageDecode(key.Substring(1));
t = JsonHelper.DeserializeJsonToObject <QQ_Page_Api>(s);
List <Page> pglst = new List <Page>();
for (int i = 0; i < t.picture.Count; i++)
{
pglst.Add(new Page()
{
chapterid = cp.chapterid,
modify = dt,
shortdate = shortdate,
sort = i + 1,
source = cp.source,
pagelocal = "",
pagesource = t.picture[i].url
});
}
List <Page> pgdblst = pq.Where(x => x.chapterid == cp.chapterid).ToList();
if (pgdblst.Count != pglst.Count)
{
List <int> idlst = pgdblst.Select(x => x.Id).ToList();
dbcontext.Delete <Page>(x => idlst.Contains(x.Id));
dbcontext.BulkInsert(pglst);
cp.downstatus = DownChapter.处理完链接;
cp.modify = dt;
dbcontext.Update(cp);
Notice notice = new Notice();
notice.noticeid = cp.chapterid;
notice.noticestatus = NoticeStatus.等待处理;
notice.noticetype = NoticeType.章节更新;
notice.source = cp.source;
notice.shortdate = shortdate;
notice.modify = dt;
var nqwait = nq.Where(x => x.noticeid == cp.chapterid && x.noticestatus == NoticeStatus.等待处理 && x.noticetype == NoticeType.章节更新).FirstOrDefault();
if (nqwait == null)
{
dbcontext.Insert(notice);
}
continue;
}
}
}
catch (Exception ex)
{
logger.Error(ex.Message);
cp.updatedate = yesterday;
cp.modify = dt;
dbcontext.Update(cp);
Err_ChapterJob err = new Err_ChapterJob();
err.bookurl = cp.chapterurl;
err.source = cp.source;
err.errtype = ErrChapter.解析出错;
err.modify = dt;
err.shortdate = shortdate;
err.message = ex.Message;
err = dbcontext.Insert(err);
continue;
}
}
}
public void Execute(IJobExecutionContext context)
{
DateTime dt = DateTime.Now;
string shortdate = dt.ToString("yyyy-MM-dd");
string yesterday = dt.AddDays(-1).ToString("yyyy-MM-dd");
IQuery <Comic> q = dbcontext.Query <Comic>();
IQuery <Chapter> cpq = dbcontext.Query <Chapter>();
List <Comic> comiclst = q.Where(a => a.source == Source.mh160 && a.shortdate == shortdate).Take(200).ToList();
List <int> ids = comiclst.Select(x => x.Id).ToList();
dbcontext.Update <Comic>(a => ids.Contains(a.Id), a => new Comic()
{
shortdate = yesterday,
modify = dt
});
List <Chapter> chapterlst = new List <Chapter>();
foreach (var comic in comiclst)
{
List <Chapter> cplst = cpq.Where(a => a.comicid == comic.comicid && a.source == Source.mh160).ToList();
if (cplst.Count == 0)
{
try
{
var bookdata = _helper.Get(comic.bookurl, Encoding.GetEncoding("gb2312"));
bookdata = StringHelper.MergeSpace(bookdata);
//string pattern = "<div class=\"plist pnormal\" id=\"play_0\"><li(?<key1>.*?)><a href=\"(?<key2>.*?)\" title=\"(?<key3>.*?)\" target=\"_blank\">(?<key4>.*?)</a></li>";
string pattern = "<div class=\"plist pnormal\" id=\"play_0\"> <ul>(?<key1>.*?)</ul> </div>";
MatchCollection matches = Regex.Matches(bookdata, pattern, RegexOptions.IgnoreCase | RegexOptions.Singleline);
string pattern2 = "<a href=\"(?<key1>.*?)\" title=\"(?<key2>.*?)\" target=\"_blank\">(?<key3>.*?)</a>";
MatchCollection matches2 = Regex.Matches(matches[0].Groups["key1"].Value, pattern2, RegexOptions.IgnoreCase | RegexOptions.Singleline);
for (int i = 0; i < matches2.Count; i++)
{
string chapterid = comic.comicid + "_" + matches2[i].Groups["key1"].Value.Split('/').LastOrDefault().Replace(".html", "");
string chaptername = matches2[i].Groups["key2"].Value;
string chapterurl = "http://www.mh160.com" + matches2[i].Groups["key1"].Value;
int sort = matches.Count - i;
chapterlst.Add(new Chapter()
{
chapterid = chapterid,
chaptername = chaptername,
chapterurl = chapterurl,
sort = sort,
comicid = comic.comicid,
retry = 0,
source = comic.source,
downstatus = DownChapter.待处理链接,
isvip = "0",
chaptersource = "",
chapterlocal = "",
modify = dt,
shortdate = shortdate,
});
}
}
catch (Exception ex)
{
logger.Error(ex.Message);
comic.shortdate = shortdate;
comic.modify = dt;
dbcontext.Update(comic);
Err_ChapterJob err = new Err_ChapterJob();
err.bookurl = comic.bookurl;
err.source = comic.source;
err.errtype = ErrChapter.解析出错;
err.modify = dt;
err.shortdate = shortdate;
err.message = ex.Message;
err = dbcontext.Insert(err);
continue;
}
}
}
if (chapterlst.Count > 0)
{
dbcontext.BulkInsert(chapterlst);
}
}
public void Execute(IJobExecutionContext context)
{
DateTime dt = DateTime.Now;
string ticks = dt.Ticks.ToString();
string shortdate = dt.ToString("yyyy-MM-dd");
string updatedatetime = shortdate + " " + ((dt.Hour / 0.5 + 1) * 0.5).ToString();
string yesterday = dt.AddDays(-1).ToString("yyyy-MM-dd");
IQuery <Comic> q = dbcontext.Query <Comic>();
IQuery <Chapter> cpq = dbcontext.Query <Chapter>();
IQuery <PageHis> phisq = dbcontext.Query <PageHis>();
IQuery <Page> pq = dbcontext.Query <Page>();
IQuery <Notice> nq = dbcontext.Query <Notice>();
List <Comic> comiclst = q.Where(a => a.source == Source.QQ && a.isoffline == false && (a.updatedatetime == null || a.updatedatetime != updatedatetime)).Take(200).ToList();
List <int> ids = comiclst.Select(x => x.Id).ToList();
dbcontext.Update <Comic>(a => ids.Contains(a.Id), a => new Comic()
{
updatedatetime = updatedatetime,
modify = dt
});
foreach (var comic in comiclst)
{
List <Chapter> cplst = cpq.Where(a => a.comicid == comic.comicid && a.source == Source.QQ).ToList();
List <Chapter> chapterlst = new List <Chapter>();
if (cplst.Count > 0)
{
try
{
string bookurl = comic.bookurl.Replace("http://ac.qq.com/", "");
var bookdata = _helper.Get(null, bookurl);
//class=\"chapter-page-all works-(?<key1>.*?)-list\"
Regex reg1 = new Regex("<ol class=\"chapter-page-all works-chapter-list\">(?<key1>.*?)</ol>", RegexOptions.IgnoreCase | RegexOptions.Singleline);
Match match1 = reg1.Match(bookdata);
string htmlSource = match1.Groups["key1"].Value;
string pattern = "<a target=\"_blank\" title=\"(?<key1>.*?)\" href=\"(?<key2>.*?)\">(?<key3>.*?)</a>";
MatchCollection matches = Regex.Matches(htmlSource, pattern, RegexOptions.IgnoreCase | RegexOptions.Singleline);
for (int i = 0; i < matches.Count; i++)
{
chapterlst.Add(new Chapter()
{
chapterid = comic.comicid + "_" + matches[i].Groups["key2"].Value.Split('/').LastOrDefault(),
chaptername = matches[i].Groups["key3"].Value.Trim(),
chapterurl = "http://ac.qq.com" + matches[i].Groups["key2"].Value,
sort = i + 1,
comicid = comic.comicid,
retry = 0,
source = comic.source,
downstatus = DownChapter.待处理链接,
isvip = "0",
chaptersource = "",
chapterlocal = "",
modify = dt,
shortdate = shortdate,
ticks = ticks
});
}
int delete = cplst.Except(chapterlst, new Chapter_Comparer()).Count(); // 删章
List <Chapter> add = chapterlst.Except(cplst, new Chapter_Comparer()).ToList(); // 新增
//if (delete > 0)
//{
// List<string> idlst = cplst.Select(x => x.chapterid).ToList();
// int phislstcount = phisq.Where(x => idlst.Contains(x.chapterid)).Count();
// if (phislstcount == 0)
// {
// List<Page> pglst = pq.Where(x => idlst.Contains(x.chapterid)).ToList();
// if (pglst.Count>0)
// {
// List<PageHis> phislst = new List<PageHis>();
// pglst.ForEach(x =>
// {
// phislst.Add(new PageHis()
// {
// chapterid = x.chapterid,
// modify = x.modify,
// pagelocal = x.pagelocal,
// pagesource = x.pagesource,
// pagestate = x.pagestate,
// shortdate = x.shortdate,
// sort = x.sort,
// source = x.source,
// ticks = x.ticks
// });
// });
// dbcontext.BulkInsert(phislst);
// }
// }
// dbcontext.Delete<Page>(x => idlst.Contains(x.chapterid));
// dbcontext.Delete<Chapter>(x => idlst.Contains(x.chapterid));
// if (chapterlst.Count > 0)
// {
// dbcontext.BulkInsert(chapterlst);
// }
// Notice notice = new Notice();
// notice.noticeid = comic.comicid;
// notice.noticestatus = NoticeStatus.等待处理;
// notice.noticetype = NoticeType.目录变更;
// notice.source = comic.source;
// notice.shortdate = shortdate;
// notice.modify = dt;
// var nqwait = nq.Where(x => x.noticeid == comic.comicid && x.noticestatus == NoticeStatus.等待处理 && x.noticetype == NoticeType.目录变更).FirstOrDefault();
// if (nqwait == null)
// {
// dbcontext.Insert(notice);
// }
// continue;
//}
//else
//{
// List<Chapter> mvadd = chapterlst.Except(add, new Chapter_Comparer()).ToList();
// string cplststr = string.Join(",", cplst.Select(x => x.chapterid).ToArray());
// string chapterlststr = string.Join(",", mvadd.Select(x => x.chapterid).ToArray());
// if (cplststr != chapterlststr) // 调序
// {
// List<string> idlst = cplst.Select(x => x.chapterid).ToList();
// int phislstcount = phisq.Where(x => idlst.Contains(x.chapterid)).Count();
// if (phislstcount == 0)
// {
// List<Page> pglst = pq.Where(x => idlst.Contains(x.chapterid)).ToList();
// if (pglst.Count > 0)
// {
// List<PageHis> phislst = new List<PageHis>();
// pglst.ForEach(x =>
// {
// phislst.Add(new PageHis()
// {
// chapterid = x.chapterid,
// modify = x.modify,
// pagelocal = x.pagelocal,
// pagesource = x.pagesource,
// pagestate = x.pagestate,
// shortdate = x.shortdate,
// sort = x.sort,
// source = x.source,
// ticks = x.ticks
// });
// });
// dbcontext.BulkInsert(phislst);
// }
// }
// dbcontext.Delete<Page>(x => idlst.Contains(x.chapterid));
// dbcontext.Delete<Chapter>(x => idlst.Contains(x.chapterid));
// if (chapterlst.Count > 0)
// {
// dbcontext.BulkInsert(chapterlst);
// }
// Notice notice = new Notice();
// notice.noticeid = comic.comicid;
// notice.noticestatus = NoticeStatus.等待处理;
// notice.noticetype = NoticeType.目录变更;
// notice.source = comic.source;
// notice.shortdate = shortdate;
// notice.modify = dt;
// var nqwait = nq.Where(x => x.noticeid == comic.comicid && x.noticestatus == NoticeStatus.等待处理 && x.noticetype == NoticeType.目录变更).FirstOrDefault();
// if (nqwait == null)
// {
// dbcontext.Insert(notice);
// }
// continue;
// }
//}
if (add.Count > 0)
{
dbcontext.BulkInsert(add);
}
}
catch (Exception ex)
{
logger.Error(ex.Message);
comic.updatedatetime = "";
comic.modify = dt;
dbcontext.Update(comic);
Err_ChapterJob err = new Err_ChapterJob();
err.bookurl = comic.bookurl;
err.source = comic.source;
err.errtype = ErrChapter.解析出错;
err.modify = dt;
err.shortdate = shortdate;
err.message = ex.Message;
err = dbcontext.Insert(err);
continue;
}
}
}
}
public void Execute(IJobExecutionContext context)
{
List <bkIm> bklst = ExcelHelper.Import(@"C:\ComicCrawer\GetComics\Excel\漫画抓取信息.xlsx").ToList();
List <Comic> comiclst = new List <Comic>();
IQuery <Comic> cq = dbcontext.Query <Comic>();
DateTime dt = DateTime.Now;
string shortdate = dt.ToString("yyyy-MM-dd");
foreach (var bk in bklst)
{
var comic = cq.Where(x => x.bookurl == bk.bookurl).FirstOrDefault();
if (comic != null)
{
continue;
}
if (bk.bookurl.StartsWith("http://ac.qq.com"))
{
try
{
HttpHelper _helper = new HttpHelper("http://ac.qq.com");
string bookurl = bk.bookurl.Replace("http://ac.qq.com/", "");
var bookdata = _helper.Get(null, bookurl);
Regex reg1 = new Regex("id=\"input_artistName\" value=\"(?<key1>.*?)\"");
Match match1 = reg1.Match(bookdata);
string authorname = match1.Groups["key1"].Value;
Regex reg2 = new Regex("img src=\"(?<key1>.*?)\" alt=\"(?<key2>.*?)\" height=\"280\" width=\"210\"");
Match match2 = reg2.Match(bookdata);
string comiccover = match2.Groups["key1"].Value;
string comicname = match2.Groups["key2"].Value;
Regex reg3 = new Regex("<label class=\"works-intro-status\">(?<key1>.*?)</label>");
Match match3 = reg3.Match(bookdata);
string isfinished = match3.Groups["key1"].Value;
Regex reg4 = new Regex("<meta name=\"Description\" content=\"(?<key1>.*?)，简介：(?<key2>.*?)《(?<key3>.*?)》的标签：(?<key4>.*?)\" />");
Match match4 = reg4.Match(bookdata);
string bookdesc = match4.Groups["key2"].Value;
string theme = match4.Groups["key4"].Value;
comiclst.Add(new Comic()
{
authorname = authorname,
bookurl = bk.bookurl,
comiccoversource = comiccover,
comiccoverlocal = "",
comicdesc = bookdesc,
comicid = bk.bookurl.Split('/').LastOrDefault(),
comicname = comicname,
isfinished = isfinished,
theme = theme,
isvip = "0",
source = Source.QQ,
stopcrawer = false,
isoffline = false,
recrawer = false,
shortdate = shortdate,
modify = dt,
});
}
catch (Exception ex)
{
Err_ComicJob err = new Err_ComicJob();
err.bookurl = bk.bookurl;
err.errtype = ErrComic.解析出错;
err.modify = dt;
err.shortdate = shortdate;
err.message = ex.Message;
err = dbcontext.Insert(err);
continue;
}
}
else if (bk.bookurl.StartsWith("https://www.dongmanmanhua.cn"))
{
try
{
HttpHelper _helper = new HttpHelper("https://www.dongmanmanhua.cn");
string bookurl = bk.bookurl.Replace("https://www.dongmanmanhua.cn/", "");
var bookdata = _helper.Get(null, bookurl);
Regex reg1 = new Regex("<meta property=\"og:image\" content=\"(?<key1>.*?)\" />");
Match match1 = reg1.Match(bookdata);
string comiccover = match1.Groups["key1"].Value;
Regex reg2 = new Regex("<meta property=\"com-dongman:webtoon:author\" content=\"(?<key1>.*?)\" />");
Match match2 = reg2.Match(bookdata);
string authorname = match2.Groups["key1"].Value;
Regex reg3 = new Regex("<p class=\"summary\">(?<key1>.*?)</p>", RegexOptions.IgnoreCase | RegexOptions.Singleline);
Match match3 = reg3.Match(bookdata);
string bookdesc = match3.Groups["key1"].Value;
Regex reg4 = new Regex("<h1 class=\"subj\">(?<key1>.*?)</h1>");
Match match4 = reg4.Match(bookdata);
string comicname = match4.Groups["key1"].Value;
Regex reg5 = new Regex("txt_ico_up");
Match match5 = reg5.Match(bookdata);
string isfinished = match5.Length > 0?"连载中": "已完结";
Regex reg6 = new Regex("<h2 class=\"genre(?<key1>.*?)\">(?<key2>.*?)</h2>");
Match match6 = reg6.Match(bookdata);
string theme = match6.Groups["key2"].Value;
comiclst.Add(new Comic()
{
authorname = authorname,
bookurl = bk.bookurl,
comiccoversource = comiccover,
comiccoverlocal = "",
comicdesc = bookdesc,
comicid = bk.bookurl.Split('=').LastOrDefault(),
comicname = comicname,
isfinished = isfinished,
theme = theme,
isvip = "0",
source = Source.dongmanmanhua,
stopcrawer = false,
isoffline = false,
recrawer = false,
shortdate = shortdate,
modify = dt,
});
}
catch (Exception ex)
{
logger.Error(ex.Message);
Err_ComicJob err = new Err_ComicJob();
err.bookurl = bk.bookurl;
err.errtype = ErrComic.解析出错;
err.modify = dt;
err.shortdate = shortdate;
err.message = ex.Message;
err = dbcontext.Insert(err);
continue;
}
}
}
if (comiclst.Count > 0)
{
dbcontext.BulkInsert(comiclst);
}
}
public void Execute(IJobExecutionContext context)
{
DateTime dt = DateTime.Now;
string shortdate = dt.ToString("yyyy-MM-dd");
string yesterday = dt.AddDays(-1).ToString("yyyy-MM-dd");
IQuery <Chapter> cpq = dbcontext.Query <Chapter>();
List <Chapter> cplst = cpq.Where(x => x.source == Source.QQ && x.downstatus == DownChapter.待处理链接).Take(200).ToList();
List <int> ids = cplst.Select(x => x.Id).ToList();
dbcontext.Update <Chapter>(a => ids.Contains(a.Id), a => new Chapter()
{
downstatus = DownChapter.处理完链接,
modify = dt
});
List <Chapter> chapterlst = new List <Chapter>();
foreach (var cp in cplst)
{
try
{
string chapterpage = cp.chapterurl.Replace("http://ac.qq.com", "");
var imgdata = _helper.Get(null, chapterpage);
Regex rex = new Regex("var DATA = '(?<key1>.*?)',");
Match match = rex.Match(imgdata);
string key = match.Groups["key1"].Value;
if (string.IsNullOrEmpty(key))
{
cp.downstatus = DownChapter.待处理链接;
cp.modify = dt;
dbcontext.Update(dt);
Err_ChapterJob err = new Err_ChapterJob();
err.bookurl = cp.chapterurl;
err.source = cp.source;
err.errtype = ErrChapter.解析出错;
err.modify = dt;
err.shortdate = shortdate;
err.message = "DATA解析失败";
err = dbcontext.Insert(err);
continue;
}
var s = DecodeHelper.QQPageDecode(key.Substring(1));
var t = JsonHelper.DeserializeJsonToObject <QQ_Page_Api>(s);
if (t.chapter.vipStatus == 1)
{
List <Page> pglst = new List <Page>();
for (int i = 0; i < t.picture.Count; i++)
{
pglst.Add(new Page()
{
chapterid = cp.Id,
modify = dt,
shortdate = shortdate,
sort = i + 1,
source = cp.source,
pagelocal = "",
pagesource = t.picture[i].url
});
}
dbcontext.Update(cp);
dbcontext.BulkInsert(pglst);
}
}
catch (Exception ex)
{
logger.Error(ex.Message);
cp.downstatus = DownChapter.待处理链接;
cp.modify = dt;
dbcontext.Update(cp);
Err_ChapterJob err = new Err_ChapterJob();
err.bookurl = cp.chapterurl;
err.source = cp.source;
err.errtype = ErrChapter.解析出错;
err.modify = dt;
err.shortdate = shortdate;
err.message = ex.Message;
err = dbcontext.Insert(err);
continue;
}
}
}
public void Execute(IJobExecutionContext context)
{
DateTime dt = DateTime.Now;
string shortdate = dt.ToString("yyyy-MM-dd");
string yesterday = dt.AddDays(-1).ToString("yyyy-MM-dd");
IQuery <Chapter> cpq = dbcontext.Query <Chapter>();
List <Chapter> cplst = cpq.Where(x => x.source == Source.dongmanmanhua && x.downstatus == DownChapter.待处理链接).Take(200).ToList();
List <int> ids = cplst.Select(x => x.Id).ToList();
dbcontext.Update <Chapter>(a => ids.Contains(a.Id), a => new Chapter()
{
downstatus = DownChapter.处理完链接,
modify = dt
});
List <Chapter> chapterlst = new List <Chapter>();
foreach (var cp in cplst)
{
List <Page> pglst = new List <Page>();
try
{
string chapterpage = cp.chapterurl.Replace("https://www.dongmanmanhua.cn", "");
var imgdata = _helper.Get(null, chapterpage);
Regex reg1 = new Regex("documentURL: '(?<key1>.*?)'");
Match match1 = reg1.Match(imgdata);
string docURl = match1.Groups["key1"].Value;
if (string.IsNullOrEmpty(docURl))
{
string pattern = "<div class=\"(?<key1>.*?)\" id=\"_imageList\">(?<key2>.*?)</div>";
MatchCollection matches = Regex.Matches(imgdata, pattern, RegexOptions.IgnoreCase | RegexOptions.Singleline);
string imghtml = matches[0].Groups["key2"].Value;
string pattern2 = "data-url=\"(?<key1>.*?)\"";
MatchCollection matches2 = Regex.Matches(imghtml, pattern2);
for (int i = 0; i < matches2.Count; i++)
{
pglst.Add(new Page()
{
chapterid = cp.chapterid,
modify = dt,
shortdate = shortdate,
sort = i + 1,
source = cp.source,
pagelocal = "",
pagesource = matches2[i].Groups["key1"].Value
});
}
}
else
{
Regex reg2 = new Regex("stillcut: '(?<key1>.*?)'");
Match match2 = reg2.Match(imgdata);
string stillcut = match2.Groups["key1"].Value;
var bookdata = _helper.Get <dongmanmanhua_Page_Api>(null, "https://www.dongmanmanhua.cn" + docURl);
string regstr = bookdata.assets.image.ToString();
string pattern3 = ": \"(?<key1>.*?)\"";
MatchCollection matches3 = Regex.Matches(regstr, pattern3, RegexOptions.IgnoreCase | RegexOptions.Singleline);
for (int i = 0; i < matches3.Count; i++)
{
pglst.Add(new Page()
{
chapterid = cp.chapterid,
modify = dt,
shortdate = shortdate,
sort = i + 1,
source = cp.source,
pagelocal = "",
pagesource = string.Format(stillcut.Replace("=filename", "0"), matches3[0].Groups["key1"].Value)
});
}
}
dbcontext.BulkInsert(pglst);
}
catch (Exception ex)
{
logger.Error(ex.Message);
cp.downstatus = DownChapter.待处理链接;
cp.modify = dt;
dbcontext.Update(cp);
Err_ChapterJob err = new Err_ChapterJob();
err.bookurl = cp.chapterurl;
err.source = cp.source;
err.errtype = ErrChapter.解析出错;
err.modify = dt;
err.shortdate = shortdate;
err.message = ex.Message;
err = dbcontext.Insert(err);
continue;
}
}
}
public void Execute(IJobExecutionContext context)
{
//List<bkIm> bklst = ExcelHelper.Import(@"C:\Users\Administrator\Desktop\7家书单抓取.xlsx").ToList();
//List<bkIm> bklst = ExcelHelper.Import2(@"C:\Users\Administrator\Desktop\漫画抓取信息.xlsx").ToList();
List <bkIm> bklst = new List <bkIm>();
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/628520"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/627844"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/627229"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/626970"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/626968"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/626920"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/626916"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/626652"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/626455"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/624495"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/624347"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/624308"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/624012"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/623715"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/623595"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/623115"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/622968"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/622674"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/622498"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/621673"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/621253"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/620523"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/620493"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/553632"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/553579"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/553055"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/552920"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/552918"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/552839"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/552104"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/547900"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/547343"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/545168"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/544521"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/543172"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/542724"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/539105"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/536658"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/628956"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/628566"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/628481"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/628198"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/628180"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/628109"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/627912"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/627383"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/627300"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/627019"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/626907"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/626527"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/626362"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/624871"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/624601"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/623795"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/623201"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/623200"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/623199"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/623196"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/623195"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/622985"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/622973"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/622971"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/622754"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/622694"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/622561"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/622160"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/621727"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/621051"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/620725"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/553972"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/628458"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/553644"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/553026"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/552065"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/551693"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/551405"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/551386"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/549847"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/549277"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/548731"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/547243"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/545320"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/542861"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/538969"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/537982"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/534826"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/534796"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/534422"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/533555"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/533395"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/529810"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/518008"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/512742"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/626289"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/626269"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/622660"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/553518"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/553204"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/549673"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/549599"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/545131"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/537899"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/628565"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/628464"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/628428"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/628305"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/628269"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/627823"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/627456"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/626955"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/626819"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/626604"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/626465"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/626268"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/626267"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/624341"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/624152"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/624135"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/624009"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/623833"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/623720"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/623494"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/623492"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/623459"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/623283"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/623125"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/623009"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/622887"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/622856"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/622855"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/622854"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/622565"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/622365"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/622216"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/622108"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/621020"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/620928"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/553939"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/550484"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/549278"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/549274"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/549125"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/547358"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/545404"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/541345"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/540523"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/539443"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/536042"
});
bklst.Add(new bkIm()
{
bookurl = "http://ac.qq.com/Comic/comicInfo/id/535934"
});
//IQuery<VIPFreeComic> vfcq = dbcontext.Query<VIPFreeComic>();
//List<VIPFreeComic> vfclst = vfcq.Where(x => true).ToList();
//vfclst.ForEach(x =>
//{
// bklst.Add(new bkIm()
// {
// bookurl = x.bookurl
// });
//});
List <Comic> comiclst = new List <Comic>();
IQuery <Comic> cq = dbcontext.Query <Comic>();
DateTime dt = DateTime.Now;
string shortdate = dt.ToString("yyyy-MM-dd");
foreach (var bk in bklst)
{
if (!bk.bookurl.StartsWith("http://ac.qq.com"))
{
continue;
}
var comic = cq.Where(x => x.bookurl == bk.bookurl).FirstOrDefault();
if (comic != null)
{
continue;
}
if (bk.bookurl.StartsWith("http://ac.qq.com"))
{
try
{
HttpHelper _helper = new HttpHelper("http://ac.qq.com");
string bookurl = bk.bookurl.Replace("http://ac.qq.com/", "");
var bookdata = _helper.Get(null, bookurl);
Regex reg1 = new Regex("id=\"input_artistName\" value=\"(?<key1>.*?)\"");
Match match1 = reg1.Match(bookdata);
string authorname = match1.Groups["key1"].Value;
Regex reg2 = new Regex("img src=\"(?<key1>.*?)\" alt=\"(?<key2>.*?)\" height=\"280\" width=\"210\"");
Match match2 = reg2.Match(bookdata);
string comiccover = match2.Groups["key1"].Value;
string comicname = match2.Groups["key2"].Value;
Regex reg3 = new Regex("<label class=\"works-intro-status\">(?<key1>.*?)</label>");
Match match3 = reg3.Match(bookdata);
string isfinished = match3.Groups["key1"].Value;
Regex reg4 = new Regex("<meta name=\"Description\" content=\"(?<key1>.*?)，简介：(?<key2>.*?)《(?<key3>.*?)》的标签：(?<key4>.*?)\" />");
Match match4 = reg4.Match(bookdata);
string bookdesc = match4.Groups["key2"].Value;
string theme = match4.Groups["key4"].Value;
comiclst.Add(new Comic()
{
authorname = authorname,
bookurl = bk.bookurl,
comiccoversource = comiccover,
comiccoverlocal = "",
comicdesc = bookdesc,
comicid = (int)Source.QQ + "_" + bk.bookurl.Split('/').LastOrDefault(),
comicname = comicname,
isfinished = isfinished,
theme = theme,
isvip = "0",
source = Source.QQ,
stopcrawer = false,
isoffline = false,
recrawer = false,
shortdate = shortdate,
modify = dt,
});
}
catch (Exception ex)
{
Err_ComicJob err = new Err_ComicJob();
err.bookurl = bk.bookurl;
err.errtype = ErrComic.解析出错;
err.modify = dt;
err.shortdate = shortdate;
err.message = ex.Message;
err = dbcontext.Insert(err);
continue;
}
}
else if (bk.bookurl.StartsWith("https://www.dongmanmanhua.cn"))
{
try
{
HttpHelper _helper = new HttpHelper("https://www.dongmanmanhua.cn");
string bookurl = bk.bookurl.Replace("https://www.dongmanmanhua.cn/", "");
var bookdata = _helper.Get(null, bookurl);
Regex reg1 = new Regex("<meta property=\"og:image\" content=\"(?<key1>.*?)\" />");
Match match1 = reg1.Match(bookdata);
string comiccover = match1.Groups["key1"].Value;
Regex reg2 = new Regex("<meta property=\"com-dongman:webtoon:author\" content=\"(?<key1>.*?)\" />");
Match match2 = reg2.Match(bookdata);
string authorname = match2.Groups["key1"].Value;
Regex reg3 = new Regex("<p class=\"summary\">(?<key1>.*?)</p>", RegexOptions.IgnoreCase | RegexOptions.Singleline);
Match match3 = reg3.Match(bookdata);
string bookdesc = match3.Groups["key1"].Value;
Regex reg4 = new Regex("<h1 class=\"subj\">(?<key1>.*?)</h1>");
Match match4 = reg4.Match(bookdata);
string comicname = match4.Groups["key1"].Value;
Regex reg5 = new Regex("txt_ico_up");
Match match5 = reg5.Match(bookdata);
string isfinished = match5.Length > 0 ? "连载中" : "已完结";
Regex reg6 = new Regex("<h2 class=\"genre(?<key1>.*?)\">(?<key2>.*?)</h2>");
Match match6 = reg6.Match(bookdata);
string theme = match6.Groups["key2"].Value;
comiclst.Add(new Comic()
{
authorname = authorname,
bookurl = bk.bookurl,
comiccoversource = comiccover,
comiccoverlocal = "",
comicdesc = bookdesc,
comicid = (int)Source.dongmanmanhua + "_" + bk.bookurl.Split('=').LastOrDefault(),
comicname = comicname,
isfinished = isfinished,
theme = theme,
isvip = "0",
source = Source.dongmanmanhua,
stopcrawer = false,
isoffline = false,
recrawer = false,
shortdate = shortdate,
modify = dt,
});
}
catch (Exception ex)
{
logger.Error(ex.Message);
Err_ComicJob err = new Err_ComicJob();
err.bookurl = bk.bookurl;
err.errtype = ErrComic.解析出错;
err.modify = dt;
err.shortdate = shortdate;
err.message = ex.Message;
err = dbcontext.Insert(err);
continue;
}
}
else if (bk.bookurl.StartsWith("http://www.u17.com"))
{
try
{
HttpHelper _helper = new HttpHelper("http://www.u17.com");
string bookurl = bk.bookurl.Replace("http://www.u17.com/", "");
var bookdata = _helper.Get(null, bookurl);
Regex reg1 = new Regex("<title>(?<key1>.*?)</title>");
Match match1 = reg1.Match(bookdata);
string comicname = match1.Groups["key1"].Value.Split('_')[0];
string authorname = match1.Groups["key1"].Value.Split('_')[1];
Regex reg2 = new Regex("cover_url = \"(?<key1>.*?)\"");
Match match2 = reg2.Match(bookdata);
string comiccover = match2.Groups["key1"].Value;
Regex reg3 = new Regex("id=\"words\">(?<key1>.*?)</p>", RegexOptions.IgnoreCase | RegexOptions.Singleline);
Match match3 = reg3.Match(bookdata);
string bookdesc = match3.Groups["key1"].Value.Trim();
Regex reg4 = new Regex("状态：(?<key1>.*?)<span(?<key2>.*?)>(?<key3>.*?)</span>", RegexOptions.IgnoreCase | RegexOptions.Singleline);
Match match4 = reg4.Match(bookdata);
string isfinished = match4.Groups["key3"].Value.Trim();
Regex reg5 = new Regex("txt_ico_up");
Match match5 = reg5.Match(bookdata);
string theme = "";
string pattern = "class=\"label(?<key1>.*?)\" title=\"(?<key3>.*?)\">(?<key2>.*?)</a>";
MatchCollection match6 = Regex.Matches(bookdata, pattern);
if (match6.Count == 0)
{
pattern = "class=\"(?<key1>.*?)_tag \" id=\"tag_(?<key3>.*?)\">(?<key2>.*?)</a>";
match6 = Regex.Matches(bookdata, pattern);
}
for (int j = 0; j < match6.Count; j++)
{
if (j == 0)
{
theme = match6[j].Groups["key2"].Value.Trim();
}
else
{
theme = theme + "," + match6[j].Groups["key2"].Value.Trim();
}
}
comiclst.Add(new Comic()
{
comicname = comicname,
authorname = authorname,
bookurl = bk.bookurl,
comiccoversource = comiccover,
comiccoverlocal = "",
comicdesc = bookdesc,
comicid = (int)Source.U17 + "_" + bk.bookurl.Split('/').LastOrDefault().Replace(".html", ""),
isfinished = isfinished,
theme = theme,
isvip = "0",
source = Source.U17,
stopcrawer = false,
isoffline = false,
recrawer = false,
shortdate = shortdate,
modify = dt,
});
}
catch (Exception ex)
{
logger.Error(ex.Message);
Err_ComicJob err = new Err_ComicJob();
err.bookurl = bk.bookurl;
err.errtype = ErrComic.解析出错;
err.modify = dt;
err.shortdate = shortdate;
err.message = ex.Message;
err = dbcontext.Insert(err);
continue;
}
}
else if (bk.bookurl.StartsWith("http://www.zymk.cn"))
{
try
{
HttpHelper _helper = new HttpHelper("http://www.zymk.cn");
string bookurl = bk.bookurl.Replace("http://www.zymk.cn/", "");
var bookdata = _helper.Get(null, bookurl);
Regex reg1 = new Regex("<meta property=\"og:novel:book_name\" content=\"(?<key1>.*?)\">");
Match match1 = reg1.Match(bookdata);
string comicname = match1.Groups["key1"].Value;
Regex reg2 = new Regex("<meta property=\"og:novel:author\" content=\"(?<key1>.*?)\">");
Match match2 = reg2.Match(bookdata);
string authorname = match2.Groups["key1"].Value;
Regex reg3 = new Regex("<meta property=\"og:image\" content=\"(?<key1>.*?)\">");
Match match3 = reg3.Match(bookdata);
string comiccover = match3.Groups["key1"].Value;
Regex reg4 = new Regex("<div class=\"desc-con\">(?<key1>.*?)</div>", RegexOptions.IgnoreCase | RegexOptions.Singleline);
Match match4 = reg4.Match(bookdata);
string bookdesc = match4.Groups["key1"].Value.Trim();
Regex reg5 = new Regex("<meta property=\"og:novel:status\" content=\"(?<key1>.*?)\">");
Match match5 = reg5.Match(bookdata);
string isfinished = match5.Groups["key1"].Value == "连载" ? "连载中" : "已完结";
string theme = "";
Regex reg6 = new Regex("<meta property=\"og:novel:category\" content=\"(?<key1>.*?)\">");
Match match6 = reg6.Match(bookdata);
theme = string.Join(",", match6.Groups["key1"].Value.Trim().Split(' '));
comiclst.Add(new Comic()
{
comicname = comicname,
authorname = authorname,
bookurl = bk.bookurl,
comiccoversource = comiccover,
comiccoverlocal = "",
comicdesc = bookdesc,
comicid = (int)Source.Zymk + "_" + bk.bookurl.Split('/')[3],
isfinished = isfinished,
theme = theme,
isvip = "0",
source = Source.Zymk,
stopcrawer = false,
isoffline = false,
recrawer = false,
shortdate = shortdate,
modify = dt,
});
}
catch (Exception ex)
{
logger.Error(ex.Message);
Err_ComicJob err = new Err_ComicJob();
err.bookurl = bk.bookurl;
err.errtype = ErrComic.解析出错;
err.modify = dt;
err.shortdate = shortdate;
err.message = ex.Message;
err = dbcontext.Insert(err);
continue;
}
}
else if (bk.bookurl.StartsWith("http://www.manhuatai.com"))
{
try
{
HttpHelper _helper = new HttpHelper("http://www.manhuatai.com");
string bookurl = bk.bookurl.Replace("http://www.manhuatai.com/", "");
var bookdata = _helper.Get(null, bookurl);
Regex reg1 = new Regex("<meta property=\"og:novel:book_name\" content=\"(?<key1>.*?)\">");
Match match1 = reg1.Match(bookdata);
string comicname = match1.Groups["key1"].Value;
Regex reg2 = new Regex("<meta property=\"og:novel:author\" content=\"(?<key1>.*?)\">");
Match match2 = reg2.Match(bookdata);
string authorname = match2.Groups["key1"].Value;
Regex reg3 = new Regex("<meta property=\"og:image\" content=\"(?<key1>.*?)\">");
Match match3 = reg3.Match(bookdata);
string comiccover = match3.Groups["key1"].Value;
Regex reg4 = new Regex("<div class=\"wz clearfix t1\"><div>(?<key1>.*?)<a href=\"javascript:void(?<key2>.*?)\" target=\"_self\" class=\"wzrtitle\" onclick=\"openOrCloseSummary", RegexOptions.IgnoreCase | RegexOptions.Singleline);
Match match4 = reg4.Match(bookdata);
string bookdesc = match4.Groups["key1"].Value.Trim();
Regex reg5 = new Regex("<meta property=\"og:novel:status\" content=\"(?<key1>.*?)\">");
Match match5 = reg5.Match(bookdata);
string isfinished = match5.Groups["key1"].Value == "连载中" ? "连载中" : "已完结";
string theme = "";
Regex reg6 = new Regex("<meta property=\"og:novel:category\" content=\"(?<key1>.*?)\">");
Match match6 = reg6.Match(bookdata);
theme = string.Join(",", match6.Groups["key1"].Value.Trim().Split(' '));
comiclst.Add(new Comic()
{
comicname = comicname,
authorname = authorname,
bookurl = bk.bookurl,
comiccoversource = comiccover,
comiccoverlocal = "",
comicdesc = bookdesc,
comicid = (int)Source.Manhuatai + "_" + bk.bookurl.Split('/')[3],
isfinished = isfinished,
theme = theme,
isvip = "0",
source = Source.Manhuatai,
stopcrawer = false,
isoffline = false,
recrawer = false,
shortdate = shortdate,
modify = dt,
});
}
catch (Exception ex)
{
logger.Error(ex.Message);
Err_ComicJob err = new Err_ComicJob();
err.bookurl = bk.bookurl;
err.errtype = ErrComic.解析出错;
err.modify = dt;
err.shortdate = shortdate;
err.message = ex.Message;
err = dbcontext.Insert(err);
continue;
}
}
else if (bk.bookurl.StartsWith("https://manhua.163.com"))
{
try
{
HttpHelper _helper = new HttpHelper("https://manhua.163.com");
string bookurl = bk.bookurl.Replace("https://manhua.163.com/", "");
var bookdata = _helper.Get(null, bookurl);
Regex reg1 = new Regex("<meta name=\"keywords\" content=\"(?<key1>.*?)\">");
Match match1 = reg1.Match(bookdata);
var arry = match1.Groups["key1"].Value.Split(',').ToList();
string comicname = arry[0];
string authorname = arry[arry.Count - 2];
arry.Remove(arry[0]);
arry.Remove(arry[0]);
arry.Remove(arry[arry.Count - 1]);
arry.Remove(arry[arry.Count - 1]);
string theme = string.Join(",", arry.Where(x => x.Length == 2).ToArray());
Regex reg3 = new Regex("<img class=\"sr-bcover\" src=\"(?<key1>.*?)\"/>");
Match match3 = reg3.Match(bookdata);
string comiccover = match3.Groups["key1"].Value;
Regex reg4 = new Regex("<dt>简介</dt>(?<key2>.*?)<dd>(?<key1>.*?)</dd>", RegexOptions.IgnoreCase | RegexOptions.Singleline);
Match match4 = reg4.Match(bookdata);
string bookdesc = match4.Groups["key1"].Value.Trim();
Regex reg5 = new Regex("<dt>状态</dt>(?<key2>.*?)<dd><a href=\"(?<key3>.*?)\" target=\"_blank\">(?<key1>.*?)</a></dd>", RegexOptions.IgnoreCase | RegexOptions.Singleline);
Match match5 = reg5.Match(bookdata);
string isfinished = match5.Groups["key1"].Value == "连载中" ? "连载中" : "已完结";
comiclst.Add(new Comic()
{
comicname = comicname,
authorname = authorname,
bookurl = bk.bookurl,
comiccoversource = comiccover,
comiccoverlocal = "",
comicdesc = bookdesc,
comicid = (int)Source._163 + "_" + bk.bookurl.Split('/')[4],
isfinished = isfinished,
theme = theme,
isvip = "0",
source = Source._163,
stopcrawer = false,
isoffline = false,
recrawer = false,
shortdate = shortdate,
modify = dt,
});
}
catch (Exception ex)
{
logger.Error(ex.Message);
Err_ComicJob err = new Err_ComicJob();
err.bookurl = bk.bookurl;
err.errtype = ErrComic.解析出错;
err.modify = dt;
err.shortdate = shortdate;
err.message = ex.Message;
err = dbcontext.Insert(err);
continue;
}
}
else if (bk.bookurl.StartsWith("http://www.mh160.com"))
{
try
{
HttpWebHelper _helper = new HttpWebHelper();
var bookdata = _helper.Get(bk.bookurl, Encoding.GetEncoding("gb2312"));
Regex reg1 = new Regex("<meta property=\"og:novel:book_name\" content=\"(?<key1>.*?)\">");
Match match1 = reg1.Match(bookdata);
string comicname = match1.Groups["key1"].Value;
Regex reg2 = new Regex("<meta property=\"og:novel:author\" content=\"(?<key1>.*?)\">");
Match match2 = reg2.Match(bookdata);
string authorname = match2.Groups["key1"].Value;
Regex reg3 = new Regex("<meta property=\"og:image\" content=\"(?<key1>.*?)\">");
Match match3 = reg3.Match(bookdata);
string comiccover = match3.Groups["key1"].Value;
Regex reg4 = new Regex("<div class=\"introduction\" id=\"intro1\"><p>(?<key1>.*?)</p>", RegexOptions.IgnoreCase | RegexOptions.Singleline);
Match match4 = reg4.Match(bookdata);
string bookdesc = match4.Groups["key1"].Value.Trim();
Regex reg5 = new Regex("<meta property=\"og:novel:status\" content=\"(?<key1>.*?)\">");
Match match5 = reg5.Match(bookdata);
string isfinished = match5.Groups["key1"].Value == "连载中" ? "连载中" : "已完结";
string theme = "";
Regex reg6 = new Regex("<meta property=\"og:novel:category\" content=\"(?<key1>.*?)\">");
Match match6 = reg6.Match(bookdata);
theme = string.Join(",", match6.Groups["key1"].Value.Trim().Split(' '));
comiclst.Add(new Comic()
{
comicname = comicname,
authorname = authorname,
bookurl = bk.bookurl,
comiccoversource = comiccover,
comiccoverlocal = "",
comicdesc = bookdesc,
comicid = (int)Source.mh160 + "_" + bk.bookurl.Split('/')[4],
isfinished = isfinished,
theme = theme,
isvip = "0",
source = Source.mh160,
stopcrawer = false,
isoffline = false,
recrawer = false,
shortdate = shortdate,
modify = dt,
});
}
catch (Exception ex)
{
logger.Error(ex.Message);
Err_ComicJob err = new Err_ComicJob();
err.bookurl = bk.bookurl;
err.errtype = ErrComic.解析出错;
err.modify = dt;
err.shortdate = shortdate;
err.message = ex.Message;
err = dbcontext.Insert(err);
continue;
}
}
}
if (comiclst.Count > 0)
{
//foreach (var item in comiclst)
//{
// var comic = cq.Where(x => x.bookurl == item.bookurl).FirstOrDefault();
// comic.comicdesc = item.comicdesc;
// dbcontext.Update(comic);
//}
dbcontext.BulkInsert(comiclst);
}
}
public void Execute(IJobExecutionContext context)
{
DateTime dt = DateTime.Now;
string ticks = dt.Ticks.ToString();
string shortdate = dt.ToString("yyyy-MM-dd");
string updatedatetime = shortdate + " " + ((dt.Hour / 0.5 + 1) * 0.5).ToString();
string yesterday = dt.AddDays(-1).ToString("yyyy-MM-dd");
IQuery <Comic> q = dbcontext.Query <Comic>();
IQuery <Chapter> cpq = dbcontext.Query <Chapter>();
IQuery <PageHis> phisq = dbcontext.Query <PageHis>();
IQuery <Page> pq = dbcontext.Query <Page>();
IQuery <Notice> nq = dbcontext.Query <Notice>();
List <Comic> comiclst = q.Where(a => a.source == Source._163 && (a.updatedatetime == null || a.updatedatetime != updatedatetime)).Take(200).ToList();
List <int> ids = comiclst.Select(x => x.Id).ToList();
dbcontext.Update <Comic>(a => ids.Contains(a.Id), a => new Comic()
{
updatedatetime = updatedatetime,
modify = dt
});
foreach (var comic in comiclst)
{
List <Chapter> cplst = cpq.Where(a => a.comicid == comic.comicid && a.source == Source._163).ToList();
List <Chapter> chapterlst = new List <Chapter>();
if (cplst.Count > 0)
{
try
{
int sort = 0;
var bookdata = _helper.Get <_163_Chapter_Api>(null, string.Format(chapterurl, comic.comicid.Replace("6_", "")));
if (bookdata.code == 200)
{
bookdata.catalog.sections.ForEach(x =>
{
foreach (var item in x.sections)
{
sort = sort + 1;
chapterlst.Add(new Chapter()
{
chapterid = comic.comicid + "_" + item.sectionId,
chaptername = item.fullTitle,
chapterurl = $"https://manhua.163.com/reader/{item.bookId}/{item.sectionId}#imgIndex=0",
sort = sort,
comicid = comic.comicid,
retry = 0,
source = comic.source,
downstatus = DownChapter.待处理链接,
isvip = item.needPay == 0 ? "0" : "1",
chaptersource = "",
chapterlocal = "",
modify = dt,
shortdate = shortdate,
});
}
});
}
int delete = cplst.Except(chapterlst, new Chapter_Comparer()).Count(); // 删章
List <Chapter> add = chapterlst.Except(cplst, new Chapter_Comparer()).ToList(); // 新增
if (delete > 0)
{
List <string> idlst = cplst.Select(x => x.chapterid).ToList();
int phislstcount = phisq.Where(x => idlst.Contains(x.chapterid)).Count();
if (phislstcount == 0)
{
List <Page> pglst = pq.Where(x => idlst.Contains(x.chapterid)).ToList();
if (pglst.Count > 0)
{
List <PageHis> phislst = new List <PageHis>();
pglst.ForEach(x =>
{
phislst.Add(new PageHis()
{
chapterid = x.chapterid,
modify = x.modify,
pagelocal = x.pagelocal,
pagesource = x.pagesource,
pagestate = x.pagestate,
shortdate = x.shortdate,
sort = x.sort,
source = x.source,
ticks = x.ticks
});
});
dbcontext.BulkInsert(phislst);
}
}
dbcontext.Delete <Page>(x => idlst.Contains(x.chapterid));
dbcontext.Delete <Chapter>(x => idlst.Contains(x.chapterid));
if (chapterlst.Count > 0)
{
dbcontext.BulkInsert(chapterlst);
}
Notice notice = new Notice();
notice.noticeid = comic.comicid;
notice.noticestatus = NoticeStatus.等待处理;
notice.noticetype = NoticeType.目录变更;
notice.source = comic.source;
notice.shortdate = shortdate;
notice.modify = dt;
var nqwait = nq.Where(x => x.noticeid == comic.comicid && x.noticestatus == NoticeStatus.等待处理 && x.noticetype == NoticeType.目录变更).FirstOrDefault();
if (nqwait == null)
{
dbcontext.Insert(notice);
}
continue;
}
else
{
List <Chapter> mvadd = chapterlst.Except(add, new Chapter_Comparer()).ToList();
string cplststr = string.Join(",", cplst.Select(x => x.chapterid).ToArray());
string chapterlststr = string.Join(",", mvadd.Select(x => x.chapterid).ToArray());
if (cplststr != chapterlststr) // 调序
{
List <string> idlst = cplst.Select(x => x.chapterid).ToList();
int phislstcount = phisq.Where(x => idlst.Contains(x.chapterid)).Count();
if (phislstcount == 0)
{
List <Page> pglst = pq.Where(x => idlst.Contains(x.chapterid)).ToList();
if (pglst.Count > 0)
{
List <PageHis> phislst = new List <PageHis>();
pglst.ForEach(x =>
{
phislst.Add(new PageHis()
{
chapterid = x.chapterid,
modify = x.modify,
pagelocal = x.pagelocal,
pagesource = x.pagesource,
pagestate = x.pagestate,
shortdate = x.shortdate,
sort = x.sort,
source = x.source,
ticks = x.ticks
});
});
dbcontext.BulkInsert(phislst);
}
}
dbcontext.Delete <Page>(x => idlst.Contains(x.chapterid));
dbcontext.Delete <Chapter>(x => idlst.Contains(x.chapterid));
if (chapterlst.Count > 0)
{
dbcontext.BulkInsert(chapterlst);
}
Notice notice = new Notice();
notice.noticeid = comic.comicid;
notice.noticestatus = NoticeStatus.等待处理;
notice.noticetype = NoticeType.目录变更;
notice.source = comic.source;
notice.shortdate = shortdate;
notice.modify = dt;
var nqwait = nq.Where(x => x.noticeid == comic.comicid && x.noticestatus == NoticeStatus.等待处理 && x.noticetype == NoticeType.目录变更).FirstOrDefault();
if (nqwait == null)
{
dbcontext.Insert(notice);
}
continue;
}
}
if (add.Count > 0)
{
dbcontext.BulkInsert(add);
}
}
catch (Exception ex)
{
logger.Error(ex.Message);
comic.updatedatetime = "";
comic.modify = dt;
dbcontext.Update(comic);
Err_ChapterJob err = new Err_ChapterJob();
err.bookurl = comic.bookurl;
err.source = comic.source;
err.errtype = ErrChapter.解析出错;
err.modify = dt;
err.shortdate = shortdate;
err.message = ex.Message;
err = dbcontext.Insert(err);
continue;
}
}
}
}
public void Execute(IJobExecutionContext context)
{
DateTime dt = DateTime.Now;
string ticks = dt.Ticks.ToString();
string shortdate = dt.ToString("yyyy-MM-dd");
string updatedatetime = shortdate + " " + ((dt.Hour / 0.5 + 1) * 0.5).ToString();
string yesterday = dt.AddDays(-1).ToString("yyyy-MM-dd");
IQuery <Comic> q = dbcontext.Query <Comic>();
IQuery <Chapter> cpq = dbcontext.Query <Chapter>();
IQuery <PageHis> phisq = dbcontext.Query <PageHis>();
IQuery <Page> pq = dbcontext.Query <Page>();
IQuery <Notice> nq = dbcontext.Query <Notice>();
List <Comic> comiclst = q.Where(a => a.source == Source.Manhuatai && (a.updatedatetime == null || a.updatedatetime != updatedatetime)).Take(200).ToList();
List <int> ids = comiclst.Select(x => x.Id).ToList();
dbcontext.Update <Comic>(a => ids.Contains(a.Id), a => new Comic()
{
updatedatetime = updatedatetime,
modify = dt
});
foreach (var comic in comiclst)
{
List <Chapter> cplst = cpq.Where(a => a.comicid == comic.comicid && a.source == Source.Manhuatai).ToList();
List <Chapter> chapterlst = new List <Chapter>();
if (cplst.Count > 0)
{
try
{
string bookurl = comic.bookurl.Replace("http://www.manhuatai.com", "");
var bookdata = _helper.Get(null, bookurl);
string pattern = "<li><a title=\"(?<key1>.*?)\" href=\"(?<key2>.*?)\"><span>(?<key3>.*?)</span></a></li>";
MatchCollection matches = Regex.Matches(bookdata, pattern, RegexOptions.IgnoreCase | RegexOptions.Singleline);
for (int i = 0; i < matches.Count; i++)
{
string chapterid = comic.comicid + "_" + matches[i].Groups["key2"].Value.Replace(".html", "");
string chaptername = matches[i].Groups["key1"].Value;
string chapterurl = comic.bookurl + matches[i].Groups["key2"].Value;
int sort = matches.Count - i;
chapterlst.Add(new Chapter()
{
chapterid = chapterid,
chaptername = chaptername,
chapterurl = chapterurl,
sort = sort,
comicid = comic.comicid,
retry = 0,
source = comic.source,
downstatus = DownChapter.待处理链接,
isvip = "0",
chaptersource = "",
chapterlocal = "",
modify = dt,
shortdate = shortdate,
});
}
int delete = cplst.Except(chapterlst, new Chapter_Comparer()).Count(); // 删章
List <Chapter> add = chapterlst.Except(cplst, new Chapter_Comparer()).ToList(); // 新增
if (delete > 0)
{
List <string> idlst = cplst.Select(x => x.chapterid).ToList();
dbcontext.Delete <Page>(x => idlst.Contains(x.chapterid));
dbcontext.Delete <Chapter>(x => idlst.Contains(x.chapterid));
if (chapterlst.Count > 0)
{
dbcontext.BulkInsert(chapterlst);
}
Notice notice = new Notice();
notice.noticeid = comic.comicid;
notice.noticestatus = NoticeStatus.等待处理;
notice.noticetype = NoticeType.目录变更;
notice.source = comic.source;
notice.shortdate = shortdate;
notice.modify = dt;
var nqwait = nq.Where(x => x.noticeid == comic.comicid && x.noticestatus == NoticeStatus.等待处理 && x.noticetype == NoticeType.目录变更).FirstOrDefault();
if (nqwait == null)
{
dbcontext.Insert(notice);
}
continue;
}
else
{
List <Chapter> mvadd = chapterlst.Except(add, new Chapter_Comparer()).ToList();
string cplststr = string.Join(",", cplst.Select(x => x.chapterid).ToArray());
string chapterlststr = string.Join(",", mvadd.Select(x => x.chapterid).ToArray());
if (cplststr != chapterlststr) // 调序
{
List <string> idlst = cplst.Select(x => x.chapterid).ToList();
dbcontext.Delete <Page>(x => idlst.Contains(x.chapterid));
dbcontext.Delete <Chapter>(x => idlst.Contains(x.chapterid));
if (chapterlst.Count > 0)
{
dbcontext.BulkInsert(chapterlst);
}
Notice notice = new Notice();
notice.noticeid = comic.comicid;
notice.noticestatus = NoticeStatus.等待处理;
notice.noticetype = NoticeType.目录变更;
notice.source = comic.source;
notice.shortdate = shortdate;
notice.modify = dt;
var nqwait = nq.Where(x => x.noticeid == comic.comicid && x.noticestatus == NoticeStatus.等待处理 && x.noticetype == NoticeType.目录变更).FirstOrDefault();
if (nqwait == null)
{
dbcontext.Insert(notice);
}
continue;
}
}
if (add.Count > 0)
{
dbcontext.BulkInsert(add);
}
}
catch (Exception ex)
{
logger.Error(ex.Message);
comic.updatedatetime = "";
comic.modify = dt;
dbcontext.Update(comic);
Err_ChapterJob err = new Err_ChapterJob();
err.bookurl = comic.bookurl;
err.source = comic.source;
err.errtype = ErrChapter.解析出错;
err.modify = dt;
err.shortdate = shortdate;
err.message = ex.Message;
err = dbcontext.Insert(err);
continue;
}
}
}
}
public void Execute(IJobExecutionContext context)
{
DateTime dt = DateTime.Now;
string shortdate = dt.ToString("yyyy-MM-dd");
string yesterday = dt.AddDays(-1).ToString("yyyy-MM-dd");
IQuery <Comic> q = dbcontext.Query <Comic>();
IQuery <Chapter> cpq = dbcontext.Query <Chapter>();
List <Comic> comiclst = q.Where(a => a.source == Source.Zymk && a.shortdate == shortdate).Take(200).ToList();
List <int> ids = comiclst.Select(x => x.Id).ToList();
dbcontext.Update <Comic>(a => ids.Contains(a.Id), a => new Comic()
{
shortdate = yesterday,
modify = dt
});
List <Chapter> chapterlst = new List <Chapter>();
foreach (var comic in comiclst)
{
List <Chapter> cplst = cpq.Where(a => a.comicid == comic.comicid && a.source == Source.Zymk).ToList();
if (cplst.Count == 0)
{
try
{
string bookurl = comic.bookurl.Replace("http://www.zymk.cn", "");
var bookdata = _helper.Get(null, bookurl);
string pattern = "<li class=\"(?<key5>.*?)\" data-id=\"(?<key1>.*?)\"><a href=\"(?<key2>.*?)\" title=\"(?<key3>.*?)\">(?<key4>.*?)</a>";
MatchCollection matches = Regex.Matches(bookdata, pattern, RegexOptions.IgnoreCase | RegexOptions.Singleline);
for (int i = 0; i < matches.Count; i++)
{
string chapterid = comic.comicid + "_" + matches[i].Groups["key1"].Value;
string chaptername = matches[i].Groups["key4"].Value;
string chapterurl = comic.bookurl + matches[i].Groups["key2"].Value;
int sort = matches.Count - i;
chapterlst.Add(new Chapter()
{
chapterid = chapterid,
chaptername = chaptername,
chapterurl = chapterurl,
sort = sort,
comicid = comic.comicid,
retry = 0,
source = comic.source,
downstatus = DownChapter.待处理链接,
isvip = "0",
chaptersource = "",
chapterlocal = "",
modify = dt,
shortdate = shortdate,
});
}
}
catch (Exception ex)
{
logger.Error(ex.Message);
comic.shortdate = shortdate;
comic.modify = dt;
dbcontext.Update(comic);
Err_ChapterJob err = new Err_ChapterJob();
err.bookurl = comic.bookurl;
err.source = comic.source;
err.errtype = ErrChapter.解析出错;
err.modify = dt;
err.shortdate = shortdate;
err.message = ex.Message;
err = dbcontext.Insert(err);
continue;
}
}
}
if (chapterlst.Count > 0)
{
dbcontext.BulkInsert(chapterlst);
}
}
public void Execute(IJobExecutionContext context)
{
DateTime dt = DateTime.Now;
string shortdate = dt.ToString("yyyy-MM-dd");
IQuery <Chapter> cpq = dbcontext.Query <Chapter>();
List <Chapter> cplst = cpq.Where(x => x.source == Source._163 && x.downstatus == DownChapter.待处理链接 && x.isvip == "0").Take(200).ToList();
List <int> ids = cplst.Select(x => x.Id).ToList();
dbcontext.Update <Chapter>(a => ids.Contains(a.Id), a => new Chapter()
{
downstatus = DownChapter.处理完链接,
modify = dt
});
foreach (var cp in cplst)
{
try
{
string chapterpage = cp.chapterurl.Replace("https://manhua.163.com/", "");
var imgdata = _helper.Get(null, chapterpage);
Regex rex1 = new Regex("imageId:");
var match1 = rex1.Match(imgdata);
if (match1.Value.Trim() != "")
{
Regex rex2 = new Regex("url: window.IS_SUPPORT_WEBP (?<key1>.*?),");
var match2 = rex2.Matches(imgdata);
List <Page> pglst = new List <Page>();
int sort = 0;
foreach (var item in match2)
{
sort = sort + 1;
var tt = ((Match)item).Groups["key1"].Value;
var imgurl = tt.Split('"')[3].Substring(0, tt.Split('"')[3].IndexOf("%3D") + 3);
pglst.Add(new Page()
{
pagesource = imgurl,
chapterid = cp.chapterid,
modify = dt,
shortdate = shortdate,
sort = sort,
source = cp.source,
pagelocal = "",
});
}
if (pglst.Count > 0)
{
dbcontext.BulkInsert(pglst);
}
}
else
{
cp.downstatus = DownChapter.待处理链接;
cp.modify = dt;
dbcontext.Update(cp);
}
}
catch (Exception ex)
{
logger.Error(ex.Message);
cp.downstatus = DownChapter.待处理链接;
cp.modify = dt;
dbcontext.Update(cp);
Err_ChapterJob err = new Err_ChapterJob();
err.bookurl = cp.chapterurl;
err.source = cp.source;
err.errtype = ErrChapter.解析出错;
err.modify = dt;
err.shortdate = shortdate;
err.message = ex.Message;
err = dbcontext.Insert(err);
continue;
}
}
}
public void Execute(IJobExecutionContext context)
{
DateTime dt = DateTime.Now;
string shortdate = dt.ToString("yyyy-MM-dd");
string yesterday = dt.AddDays(-1).ToString("yyyy-MM-dd");
IQuery <Comic> q = dbcontext.Query <Comic>();
IQuery <Chapter> cpq = dbcontext.Query <Chapter>();
List <Comic> comiclst = q.Where(a => a.source == Source._163 && a.shortdate == shortdate).Take(200).ToList();
List <int> ids = comiclst.Select(x => x.Id).ToList();
dbcontext.Update <Comic>(a => ids.Contains(a.Id), a => new Comic()
{
shortdate = yesterday,
modify = dt
});
List <Chapter> chapterlst = new List <Chapter>();
foreach (var comic in comiclst)
{
List <Chapter> cplst = cpq.Where(a => a.comicid == comic.comicid && a.source == Source._163).ToList();
if (cplst.Count == 0)
{
try
{
int sort = 0;
var bookdata = _helper.Get <_163_Chapter_Api>(null, string.Format(chapterurl, comic.comicid.Replace("6_", "")));
if (bookdata.code == 200)
{
bookdata.catalog.sections.ForEach(x =>
{
foreach (var item in x.sections)
{
sort = sort + 1;
chapterlst.Add(new Chapter()
{
chapterid = comic.comicid + "_" + item.sectionId,
chaptername = item.fullTitle,
chapterurl = $"https://manhua.163.com/reader/{item.bookId}/{item.sectionId}#imgIndex=0",
sort = sort,
comicid = comic.comicid,
retry = 0,
source = comic.source,
downstatus = DownChapter.待处理链接,
isvip = item.needPay == 0 ?"0":"1",
chaptersource = "",
chapterlocal = "",
modify = dt,
shortdate = shortdate,
});
}
});
}
}
catch (Exception ex)
{
logger.Error(ex.Message);
comic.shortdate = shortdate;
comic.modify = dt;
dbcontext.Update(comic);
Err_ChapterJob err = new Err_ChapterJob();
err.bookurl = comic.bookurl;
err.source = comic.source;
err.errtype = ErrChapter.解析出错;
err.modify = dt;
err.shortdate = shortdate;
err.message = ex.Message;
err = dbcontext.Insert(err);
continue;
}
}
}
if (chapterlst.Count > 0)
{
dbcontext.BulkInsert(chapterlst);
}
}
public void Execute(IJobExecutionContext context)
{
DateTime dt = DateTime.Now;
string shortdate = dt.ToString("yyyy-MM-dd");
string yesterday = dt.AddDays(-1).ToString("yyyy-MM-dd");
IQuery <Chapter> cpq = dbcontext.Query <Chapter>();
List <Chapter> cplst = cpq.Where(x => x.source == Source.dongmanmanhua && x.downstatus == DownChapter.待处理链接).Take(200).ToList();
List <int> ids = cplst.Select(x => x.Id).ToList();
dbcontext.Update <Chapter>(a => ids.Contains(a.Id), a => new Chapter()
{
downstatus = DownChapter.处理完链接,
modify = dt
});
List <Chapter> chapterlst = new List <Chapter>();
foreach (var cp in cplst)
{
try
{
string chapterpage = cp.chapterurl.Replace("https://www.dongmanmanhua.cn", "");
var imgdata = _helper.Get(null, chapterpage);
string pattern = "<div class=\"(?<key1>.*?)\" id=\"_imageList\">(?<key2>.*?)</div>";
MatchCollection matches = Regex.Matches(imgdata, pattern, RegexOptions.IgnoreCase | RegexOptions.Singleline);
string imghtml = matches[0].Groups["key2"].Value;
string pattern2 = "data-url=\"(?<key1>.*?)\"";
MatchCollection matches2 = Regex.Matches(imghtml, pattern2);
List <Page> pglst = new List <Page>();
for (int i = 0; i < matches2.Count; i++)
{
pglst.Add(new Page()
{
chapterid = cp.Id,
modify = dt,
shortdate = shortdate,
sort = i + 1,
source = cp.source,
pagelocal = "",
pagesource = matches2[i].Groups["key1"].Value
});
}
dbcontext.BulkInsert(pglst);
}
catch (Exception ex)
{
logger.Error(ex.Message);
cp.downstatus = DownChapter.待处理链接;
cp.modify = dt;
dbcontext.Update(cp);
Err_ChapterJob err = new Err_ChapterJob();
err.bookurl = cp.chapterurl;
err.source = cp.source;
err.errtype = ErrChapter.解析出错;
err.modify = dt;
err.shortdate = shortdate;
err.message = ex.Message;
err = dbcontext.Insert(err);
continue;
}
}
}
public void Execute(IJobExecutionContext context)
{
DateTime dt = DateTime.Now;
string shortdate = dt.ToString("yyyy-MM-dd");
IQuery <Chapter> cpq = dbcontext.Query <Chapter>();
List <Chapter> cplst = cpq.Where(x => x.source == Source.U17 && x.downstatus == DownChapter.待处理链接 && x.isvip == "0").Take(200).ToList();
List <int> ids = cplst.Select(x => x.Id).ToList();
dbcontext.Update <Chapter>(a => ids.Contains(a.Id), a => new Chapter()
{
downstatus = DownChapter.处理完链接,
modify = dt
});
foreach (var cp in cplst)
{
try
{
var imgdata = _helper.Get <U17_Page_Api>(null, string.Format(pageurl, cp.chapterid.Replace(cp.comicid + "_", "")));
List <Page> pglst = new List <Page>();
int sort = 0;
imgdata.image_list.ForEach(x =>
{
sort = sort + 1;
pglst.Add(new Page()
{
pagesource = x.src,
chapterid = cp.chapterid,
modify = dt,
shortdate = shortdate,
sort = sort,
source = cp.source,
pagelocal = "",
});
});
if (pglst.Count > 0)
{
dbcontext.BulkInsert(pglst);
}
}
catch (Exception ex)
{
logger.Error(ex.Message);
cp.downstatus = DownChapter.待处理链接;
cp.modify = dt;
dbcontext.Update(cp);
Err_ChapterJob err = new Err_ChapterJob();
err.bookurl = cp.chapterurl;
err.source = cp.source;
err.errtype = ErrChapter.解析出错;
err.modify = dt;
err.shortdate = shortdate;
err.message = ex.Message;
err = dbcontext.Insert(err);
continue;
}
}
}
public void Execute(IJobExecutionContext context)
{
//logger.Info("QQ_page vip begin");
//string isStart = "IsStartBuyQQ".ValueOfAppSetting();
//if (isStart != null && isStart.Equals("1"))
{
DateTime dt = DateTime.Now;
string shortdate = dt.ToString("yyyy-MM-dd");
string yesterday = dt.AddDays(-1).ToString("yyyy-MM-dd");
IQuery <Chapter> cpq = dbcontext.Query <Chapter>();//x.comicid == "1_524356"
IQuery <PageHis> phisq = dbcontext.Query <PageHis>();
//List<Chapter> cplst = cpq.Where(x => x.source == Source.QQ && x.downstatus == DownChapter.待处理链接 && x.isvip.Equals("1")).Take(200).ToList();
List <Chapter> cplst = cpq.Where(x => x.source == Source.QQ && x.downstatus == DownChapter.完图片 && x.isvip.Equals("1")).Take(200).ToList();
List <int> ids = cplst.Select(x => x.Id).ToList();
dbcontext.Update <Chapter>(a => ids.Contains(a.Id), a => new Chapter()
{
downstatus = DownChapter.处理中,
modify = dt
});
List <Chapter> chapterlst = new List <Chapter>();
Regex rex = new Regex("var DATA = '(?<key1>.*?)',");
string errMsg;
foreach (var cp in cplst)
{
try
{
IQuery <PageHis> cpHis = phisq.Where(x => x.chapterid == cp.chapterid);
if (cpHis != null && cpHis.Count() > 0)
{
List <PageHis> cpHisList = cpHis.ToList();
List <Page> pglst = new List <Page>();
foreach (var page in cpHisList)
{
pglst.Add(new Page()
{
chapterid = page.chapterid,
modify = dt,
shortdate = shortdate,
sort = page.sort,
source = page.source,
pagelocal = "",
pagesource = page.pagesource
});
}
cp.downstatus = DownChapter.处理完链接;
cp.modify = dt;
dbcontext.Update(cp);
dbcontext.BulkInsert(pglst);
logger.Info("QQ_page vip syn history sucess:id=" + cp.Id);
}
else
{
errMsg = string.Empty;
selenium.Navigate().GoToUrl(cp.chapterurl);
IList <IWebElement> frames = selenium.FindElements(By.TagName("iframe"));
IWebElement controlPanelFrame = null;
foreach (var frame in frames)
{
if (frame.GetAttribute("id") == "iframeAll")
{
controlPanelFrame = frame;
break;
}
}
if (controlPanelFrame != null && controlPanelFrame.Displayed == true) //QQ登录
{
try
{
selenium.SwitchTo().Frame(controlPanelFrame);
IReadOnlyCollection <IWebElement> switchtoElement = selenium.FindElements(By.Id("switcher_plogin"));
if (switchtoElement != null && switchtoElement.Count > 0 && switchtoElement.First().Displayed == true)
{
switchtoElement.First().Click();
selenium.FindElement(By.Id("u")).Clear();
selenium.FindElement(By.Id("u")).SendKeys("3283360259");
selenium.FindElement(By.Id("p")).Clear();
selenium.FindElement(By.Id("p")).SendKeys("xxxttt5544");
//selenium.FindElement(By.Id("u")).Clear();
//selenium.FindElement(By.Id("u")).SendKeys("1434299101");
//selenium.FindElement(By.Id("p")).Clear();
//selenium.FindElement(By.Id("p")).SendKeys("zhangyin123");
selenium.FindElement(By.Id("login_button")).Click();
}
selenium.SwitchTo().DefaultContent();
}
catch (Exception ex)
{
errMsg = "QQ 登录失败：" + ex.Message;
logger.Error(errMsg);
}
}
selenium.Navigate().GoToUrl("http://ac.qq.com/Home/buyList");
ICookieJar listCookie = selenium.Manage().Cookies;
// IList<Cookie> listCookie = selenuim.Manage( ).Cookies.AllCookies;//只是显示 可以用Ilist对象
//显示初始Cookie的内容
Console.WriteLine("--------------------");
Console.WriteLine($"当前Cookie集合的数量：\t{listCookie.AllCookies.Count}");
for (int i = 0; i < listCookie.AllCookies.Count; i++)
{
Console.WriteLine($"Cookie的名称:{listCookie.AllCookies[i].Name}");
Console.WriteLine($"Cookie的值:{listCookie.AllCookies[i].Value}");
Console.WriteLine($"Cookie的所在域:{listCookie.AllCookies[i].Domain}");
Console.WriteLine($"Cookie的路径:{listCookie.AllCookies[i].Path}");
Console.WriteLine($"Cookie的过期时间:{listCookie.AllCookies[i].Expiry}");
Console.WriteLine("-----");
}
frames = selenium.FindElements(By.TagName("iframe"));
IWebElement checkVipFrame = null;
foreach (var frame in frames)
{
if (frame.GetAttribute("id") == "checkVipFrame")
{
checkVipFrame = frame;
break;
}
}
if (checkVipFrame != null && checkVipFrame.Displayed == true)
{
try
{
//自动购买
selenium.SwitchTo().Frame(checkVipFrame);
IReadOnlyCollection <IWebElement> checkAutoElement = selenium.FindElements(By.Id("check_auto_next"));
IReadOnlyCollection <IWebElement> singlBbuyElement = selenium.FindElements(By.ClassName("single_buy"));
if (checkAutoElement != null && singlBbuyElement != null && checkAutoElement.Count > 0 && singlBbuyElement.Count > 0 && checkAutoElement.First().Displayed == true)
{
if (singlBbuyElement.First().Text.IndexOf("点券不足") > -1)
{
//列表中未成功购买的数据还原成待处理
dbcontext.Update <Chapter>(a => ids.Contains(a.Id), a => new Chapter()
{
downstatus = DownChapter.待处理链接,
modify = dt
});
////关闭购买，等待修改配置
//"IsStartBuyQQ".SetAppSettingValue("0");
if (isHasMoney)
{
Err_ChapterJob err = new Err_ChapterJob();
err.bookurl = cp.chapterurl;
err.source = cp.source;
err.errtype = ErrChapter.解析出错;
err.modify = dt;
err.shortdate = shortdate;
err.message = "点券不足，请去充值！";
err = dbcontext.Insert(err);
}
isHasMoney = false;
//Thread.Sleep(3600000);
continue;
}
else
{
isHasMoney = true;
}
checkAutoElement.First().Click();
singlBbuyElement.First().Click();
}
selenium.SwitchTo().DefaultContent();
}
catch (Exception ex)
{
errMsg = "自动购买失败：" + ex.Message;
logger.Error(errMsg);
}
}
Match match1 = rex.Match(selenium.PageSource);
string key = match1.Groups["key1"].Value;
if (string.IsNullOrEmpty(key) || errMsg != string.Empty)
{
cp.downstatus = DownChapter.待处理链接;
cp.modify = dt;
dbcontext.Update(cp);
Err_ChapterJob err = new Err_ChapterJob();
err.bookurl = cp.chapterurl;
err.source = cp.source;
err.errtype = ErrChapter.解析出错;
err.modify = dt;
err.shortdate = shortdate;
err.message = errMsg != string.Empty ? errMsg : "DATA解析失败";
err = dbcontext.Insert(err);
continue;
}
string s = DecodeHelper.QQPageDecode(key.Substring(1));
var t = JsonHelper.DeserializeJsonToObject <QQ_Page_Api>(s);
if (t.picture.Count < 1)
{
cp.downstatus = DownChapter.待处理链接;
cp.modify = dt;
dbcontext.Update(cp);
Err_ChapterJob err = new Err_ChapterJob();
err.bookurl = cp.chapterurl;
err.source = cp.source;
err.errtype = ErrChapter.解析出错;
err.modify = dt;
err.shortdate = shortdate;
err.message = "访问付费章节内容时只存在一张图片";
err = dbcontext.Insert(err);
continue;
}
List <Page> pglst = new List <Page>();
for (int i = 0; i < t.picture.Count; i++)
{
if (pglst.Exists(x => x.pagesource == t.picture[i].url) == false)
{
pglst.Add(new Page()
{
chapterid = cp.chapterid,
modify = DateTime.Now,
shortdate = shortdate,
sort = i + 1,
source = cp.source,
pagelocal = "",
pagesource = t.picture[i].url
});
}
else
{
Err_ChapterJob err = new Err_ChapterJob();
err.bookurl = cp.chapterurl;
err.source = cp.source;
err.errtype = ErrChapter.解析出错;
err.modify = DateTime.Now;
err.shortdate = shortdate;
err.message = "存在重复图片";
err = dbcontext.Insert(err);
}
}
cp.downstatus = DownChapter.处理完链接;
cp.modify = dt;
dbcontext.Update(cp);
dbcontext.BulkInsert(pglst);
ids.Remove(cp.Id);
logger.Info("QQ_page vip buy sucess:id=" + cp.Id);
}
}
catch (Exception ex)
{
if (ex.Message.IndexOf("Unexpected error. System.Net.WebException:") > -1)
{
InitChromeDriver();
}
logger.Error(ex.Message);
cp.downstatus = DownChapter.待处理链接;
cp.modify = dt;
dbcontext.Update(cp);
Err_ChapterJob err = new Err_ChapterJob();
err.bookurl = cp.chapterurl;
err.source = cp.source;
err.errtype = ErrChapter.解析出错;
err.modify = dt;
err.shortdate = shortdate;
err.message = ex.Message;
err = dbcontext.Insert(err);
continue;
}
}
}
}
public void Execute(IJobExecutionContext context)
{
//string keystr = "meyJjb21pYyI6eyJpZCI6NTM4NzQzLCJ0aXRsZSI6Ilx1NGUwN1x1OTFjY1x1NjY3NFx1NWRkZCIsImNvbGxlY3QiOiI0NzA0MSIsImlzSmFwYW5Db21pYyI6ZmFsc2UsImlzTGlnaHROb3ZlbCI6ZmFsc2UsImlzTGlnaHRDb21pYyI6ZmFsc2UsImlzRmluaXNoIjpmYWxzZSwiaXNSb2FzdGFibGUiOnRydWUsImVJZCI6IktsQlBTVU5BVmxWV0Jnc2ZBUVlPQUF3S0hFUldOQT09In0sImNoYXB0ZXIiOnsiY2lkIjo0OSwiY1RpdGxlIjoiXHU3YjJjMjVcdTU2ZGUgXHU2NDk1XHU4OGMyMlx1ZmYwOFx1NGUwYVx1ZmYwOSIsImNTZXEiOiI0OSIsInZpcFN0YXR1cyI6MiwicHJldkNpZCI6NDgsIm5leHRDaWQiOjUwLCJibGFua0ZpcnN0IjoxLCJjYW5SZWFkIjp0cnVlfSwicGljdHVyZSI6W3sicGlkIjoiMjY1MCIsIndpZHRoIjo4MDAsImhlaWdodCI6MTIwMCwidXJsIjoiaHR0cDpcL1wvYWMudGMucXEuY29tXC9zdG9yZV9maWxlX2Rvd25sb2FkP2J1aWQ9MTUwMTcmdWluPTE1MTA3MDk2NTgmZGlyX3BhdGg9XC8mbmFtZT0xNV8wOV8zNF8wY2E3NjllMDk1YTU0NTYwYzQ5Nzk3ZmEwNGY1NzQ5N18yNjUwLmpwZyJ9LHsicGlkIjoiMjY1MSIsIndpZHRoIjo4MDAsImhlaWdodCI6MTIwMCwidXJsIjoiaHR0cDpcL1wvYWMudGMucXEuY29tXC9zdG9yZV9maWxlX2Rvd25sb2FkP2J1aWQ9MTUwMTcmdWluPTE1MTA3MDk2NzImZGlyX3BhdGg9XC8mbmFtZT0xNV8wOV8zNF8yZmVmYmQ0Mzk3ODY5YmU5MGI4OTQxOTdmYTEwMGZmNF8yNjUxLmpwZyJ9LHsicGlkIjoiMjY1MiIsIndpZHRoIjo4MDAsImhlaWdodCI6MTIwMCwidXJsIjoiaHR0cDpcL1wvYWMudGMucXEuY29tXC9zdG9yZV9maWxlX2Rvd25sb2FkP2J1aWQ9MTUwMTcmdWluPTE1MTA3MDk2NzImZGlyX3BhdGg9XC8mbmFtZT0xNV8wOV8zNF9jODBiMTE0ZDIwOGIzZWMyZDkxNWM3MmRhODIwZDRhYl8yNjUyLmpwZyJ9LHsicGlkIjoiMjY1MyIsIndpZHRoIjo4MDAsImhlaWdodCI6MTIwMCwidXJsIjoiaHR0cDpcL1wvYWMudGMucXEuY29tXC9zdG9yZV9maWxlX2Rvd25sb2FkP2J1aWQ9MTUwMTcmdWluPTE1MTA3MDk2NzImZGlyX3BhdGg9XC8mbmFtZT0xNV8wOV8zNF9kYjI5MjI5MmM4NmVkN2U0MDIwN2Q2MDg4YjU2ODliYl8yNjUzLmpwZyJ9LHsicGlkIjoiMjY1NCIsIndpZHRoIjo4MDAsImhlaWdodCI6MTIwMCwidXJsIjoiaHR0cDpcL1wvYWMudGMucXEuY29tXC9zdG9yZV9maWxlX2Rvd25sb2FkP2J1aWQ9MTUwMTcmdWluPTE1MTA3MDk2NzImZGlyX3BhdGg9XC8mbmFtZT0xNV8wOV8zNF81YjEyOWVmMWJjY2NlZDQ3OWU1NDE3NTliMTIzZWJkNF8yNjU0LmpwZyJ9LHsicGlkIjoiMjY1NSIsIndpZHRoIjo4MDAsImhlaWdodCI6MTIwMCwidXJsIjoiaHR0cDpcL1wvYWMudGMucXEuY29tXC9zdG9yZV9maWxlX2Rvd25sb2FkP2J1aWQ9MTUwMTcmdWluPTE1MTA3MDk2NzImZGlyX3BhdGg9XC8mbmFtZT0xNV8wOV8zNF85MGY2MTg2MjMyNzcxYTE5NzVmOTVlM2RiYTQ3ZGIxZV8yNjU1LmpwZyJ9LHsicGlkIjoiMjY1NiIsIndpZHRoIjo4MDAsImhlaWdodCI6MTIwMCwidXJsIjoiaHR0cDpcL1wvYWMudGMucXEuY29tXC9zdG9yZV9maWxlX2Rvd25sb2FkP2J1aWQ9MTUwMTcmdWluPTE1MTA3MDk2ODYmZGlyX3BhdGg9XC8mbmFtZT0xNV8wOV8zNF9mZGVmMjExZmIxZjRlYWIxNGQzNDE1MjM1ZDNjOWIxNV8yNjU2LmpwZyJ9LHsicGlkIjoiMjY1NyIsIndpZHRoIjo4MDAsImhlaWdodCI6MTIwMCwidXJsIjoiaHR0cDpcL1wvYWMudGMucXEuY29tXC9zdG9yZV9maWxlX2Rvd25sb2FkP2J1aWQ9MTUwMTcmdWluPTE1MTA3MDk2ODYmZGlyX3BhdGg9XC8mbmFtZT0xNV8wOV8zNF9lNmQxZjAwNWQ1OTkxNzRkMjJlODI1MDE3MGVhMTM4N18yNjU3LmpwZyJ9LHsicGlkIjoiMjY1OCIsIndpZHRoIjo4MDAsImhlaWdodCI6MTIwMCwidXJsIjoiaHR0cDpcL1wvYWMudGMucXEuY29tXC9zdG9yZV9maWxlX2Rvd25sb2FkP2J1aWQ9MTUwMTcmdWluPTE1MTA3MDk2ODYmZGlyX3BhdGg9XC8mbmFtZT0xNV8wOV8zNF81NmI3YjI1YjVkZDM5YzdhNmU1OGNhYjY1Njc1MGY3MV8yNjU4LmpwZyJ9LHsicGlkIjoiMjY1OSIsIndpZHRoIjo4MDAsImhlaWdodCI6MTIwMCwidXJsIjoiaHR0cDpcL1wvYWMudGMucXEuY29tXC9zdG9yZV9maWxlX2Rvd25sb2FkP2J1aWQ9MTUwMTcmdWluPTE1MTA3MDk2ODYmZGlyX3BhdGg9XC8mbmFtZT0xNV8wOV8zNF9jODEyNGJiYzk2NTVlODVhMTk0ZTRiNDQwMzA2YTFjYV8yNjU5LmpwZyJ9LHsicGlkIjoiMjY2MCIsIndpZHRoIjo4MDAsImhlaWdodCI6MTIwMCwidXJsIjoiaHR0cDpcL1wvYWMudGMucXEuY29tXC9zdG9yZV9maWxlX2Rvd25sb2FkP2J1aWQ9MTUwMTcmdWluPTE1MTA3MDk2ODYmZGlyX3BhdGg9XC8mbmFtZT0xNV8wOV8zNF85ZDE0ZDYzOTY1Mjg0ZDUzY2MwMDkwN2QxZGIzY2YzNV8yNjYwLmpwZyJ9LHsicGlkIjoiMjY2MSIsIndpZHRoIjo4MDAsImhlaWdodCI6MTIwMCwidXJsIjoiaHR0cDpcL1wvYWMudGMucXEuY29tXC9zdG9yZV9maWxlX2Rvd25sb2FkP2J1aWQ9MTUwMTcmdWluPTE1MTA3MDk2ODYmZGlyX3BhdGg9XC8mbmFtZT0xNV8wOV8zNF9lZmU5OThmYjU1ZDJlZjYyMGFlMmIzYmM3ZjYyY2IzOV8yNjYxLmpwZyJ9LHsicGlkIjoiMjY2MiIsIndpZHRoIjo4NjAsImhlaWdodCI6MTQ1MywidXJsIjoiaHR0cDpcL1wvYWMudGMucXEuY29tXC9zdG9yZV9maWxlX2Rvd25sb2FkP2J1aWQ9MTUwMTcmdWluPTE1MTA3MDk2ODcmZGlyX3BhdGg9XC8mbmFtZT0xNV8wOV8zNF84NmYzY2QzMzgzYjA2N2JhOTRjYWQ4YjZlOThkMGY2NV8yNjYyLmpwZyJ9XSwiYWRzIjp7InRvcCI6eyJ0aXRsZSI6Ilx1NjVmNlx1N2E3YVx1NjA0Ylx1NGViYSIsInBpYyI6Imh0dHBzOlwvXC9tYW5odWEucXBpYy5jblwvb3BlcmF0aW9uXC8wXC8xM18wOV81NF9kOTU4YWYyOGU4NmYzNjhmYjhiN2EwMGMzMDZmODQ4OV8xNTEwNTM4MDQxNTMwLmpwZ1wvMCIsInVybCI6Imh0dHA6XC9cL2FjLnFxLmNvbVwvQ29taWNWaWV3XC9pbmRleFwvaWRcLzU1MDkyMVwvY2lkXC80Iiwid2lkdGgiOiI2NTAiLCJoZWlnaHQiOiIxMTAifSwibGVmdCI6W10sImJvdHRvbSI6eyJ0aXRsZSI6Ilx1NzNiMFx1NGUxNlx1NjBjNVx1NGViYVx1NjYyZlx1NWMzZVx1NzJkMCIsInBpYyI6Imh0dHBzOlwvXC9tYW5odWEucXBpYy5jblwvb3BlcmF0aW9uXC8wXC8xM18wOV81Ml84ZWJjZTJkZmJhMzZhZDhlYTA2YjEyODRkNzY1Mzc0Nl8xNTEwNTM3OTM3NDY2LmpwZ1wvMCIsInVybCI6Imh0dHA6XC9cL2FjLnFxLmNvbVwvQ29taWNWaWV3XC9pbmRleFwvaWRcLzYyNjg1MFwvY2lkXC8yIiwid2lkdGgiOiI2NTAiLCJoZWlnaHQiOiIxMTAifX0sImFydGlzdCI6eyJhdmF0YXIiOiJodHRwOlwvXC9xNC5xbG9nby5jblwvZz9iPXFxJms9d2g1c0ZOazAzUDV6RmlhOGljdGQ4ZW13JnM9NjQwJnQ9MTQ4MzMzOTY3NSIsIm5pY2siOiJcdTc3ZTVcdTk3ZjNcdTc5ZmJcdTUyYThcdTY1NzBcdTViNTciLCJ1aW5DcnlwdCI6IlVUSk1kVkpCZDAxTlFrSkVibGhzWmpOWlJqRjBaejA5In19";
//var ss = DecodeHelper.QQPageDecode(keystr.Substring(1));
//var tt = JsonHelper.DeserializeJsonToObject<QQ_Page_Api>(ss);
//var redis = new RedisProxy();
//try
//{
// redis.HashSet("QQ_Chapter", "185158", "185157");
//}
//catch (Exception ex)
//{
// throw;
//}
//var login_key = redis.Get<QQ_Login_Key>("QQ_1434299101");
DateTime dt = DateTime.Now;
string ticks = dt.Ticks.ToString();
string shortdate = dt.ToString("yyyy-MM-dd");
string yesterday = dt.AddDays(-1).ToString("yyyy-MM-dd");
IQuery <Chapter> cpq = dbcontext.Query <Chapter>();//x.comicid == "1_622585"
IQuery <VIPFreeComic> vfcq = dbcontext.Query <VIPFreeComic>();
List <Chapter> cplst;
if (QQChapterTotal > 0)
{
cplst = cpq.Where(x => x.source == Source.QQ && x.downstatus == DownChapter.待处理链接 && x.isvip == "0" && x.Id % QQChapterTotal == QQChapterIndex).Take(200).ToList();
}
else
{
cplst = cpq.Where(x => x.source == Source.QQ && x.downstatus == DownChapter.待处理链接 && x.isvip == "0").Take(200).ToList();
}
List <int> ids = cplst.Select(x => x.Id).ToList();
dbcontext.Update <Chapter>(a => ids.Contains(a.Id), a => new Chapter()
{
ticks = ticks,
downstatus = DownChapter.处理完链接,
modify = dt
});
List <Chapter> chapterlst = new List <Chapter>();
foreach (var cp in cplst)
{
try
{
//Dictionary<string, string> headers = new Dictionary<string, string>()
//{
// {"Cookie","uin=o1434299101; skey="+login_key.skey+";" }
//};
var imgdata = _helper.Get(cp.chapterurl, Encoding.GetEncoding("UTF-8"), null, "", null, "", null, "application/x-www-form-urlencoded; charset=UTF-8");
Regex rex = new Regex("var DATA = '(?<key1>.*?)',");
Match match = rex.Match(imgdata);
string key = match.Groups["key1"].Value;
if (string.IsNullOrEmpty(key))
{
cp.downstatus = DownChapter.待处理链接;
cp.modify = dt;
cp.ticks = ticks;
dbcontext.Update(cp);
Err_ChapterJob err = new Err_ChapterJob();
err.bookurl = cp.chapterurl;
err.source = cp.source;
err.errtype = ErrChapter.解析出错;
err.modify = dt;
err.shortdate = shortdate;
err.message = "DATA解析失败";
err = dbcontext.Insert(err);
continue;
}
var s = DecodeHelper.QQPageDecode(key.Substring(1));
var t = JsonHelper.DeserializeJsonToObject <QQ_Page_Api>(s);
if (t.chapter.canRead)
{
List <Page> pglst = new List <Page>();
for (int i = 0; i < t.picture.Count; i++)
{
pglst.Add(new Page()
{
chapterid = cp.chapterid,
modify = dt,
shortdate = shortdate,
sort = i + 1,
source = cp.source,
pagelocal = "",
ticks = ticks,
pagesource = t.picture[i].url
});
}
cp.isvip = "0";
cp.ticks = ticks;
cp.downstatus = DownChapter.处理完链接;
cp.modify = dt;
dbcontext.Update(cp);
dbcontext.BulkInsert(pglst);
}
else
{
List <VIPFreeComic> vfclst = vfcq.Where(x => x.source == Source.QQ && x.comicid == cp.comicid).ToList();
cp.isvip = vfclst.Count == 0 ? "1" : "0";
cp.ticks = ticks;
cp.downstatus = DownChapter.待处理链接;
cp.modify = dt;
dbcontext.Update(cp);
}
}
catch (Exception ex)
{
cp.downstatus = DownChapter.待处理链接;
cp.modify = dt;
cp.ticks = ticks;
dbcontext.Update(cp);
Err_ChapterJob err = new Err_ChapterJob();
err.bookurl = cp.chapterurl;
err.source = cp.source;
err.errtype = ErrChapter.解析出错;
err.modify = dt;
err.shortdate = shortdate;
err.message = ex.Message;
err = dbcontext.Insert(err);
continue;
}
}
}
public void Execute(IJobExecutionContext context)
{
DateTime dt = DateTime.Now;
string ticks = dt.Ticks.ToString();
string shortdate = dt.ToString("yyyy-MM-dd");
string updatedatetime = shortdate + " " + ((dt.Hour / 0.5 + 1) * 0.5).ToString();
string yesterday = dt.AddDays(-1).ToString("yyyy-MM-dd");
IQuery <Comic> q = dbcontext.Query <Comic>();
IQuery <Chapter> cpq = dbcontext.Query <Chapter>();
IQuery <PageHis> phisq = dbcontext.Query <PageHis>();
IQuery <Page> pq = dbcontext.Query <Page>();
IQuery <Notice> nq = dbcontext.Query <Notice>();
List <Comic> comiclst = q.Where(a => a.source == Source.dongmanmanhua && (a.updatedatetime == null || a.updatedatetime != updatedatetime)).Take(200).ToList();
List <int> ids = comiclst.Select(x => x.Id).ToList();
dbcontext.Update <Comic>(a => ids.Contains(a.Id), a => new Comic()
{
updatedatetime = updatedatetime,
modify = dt
});
foreach (var comic in comiclst)
{
List <Chapter> cplst = cpq.Where(a => a.comicid == comic.comicid && a.source == Source.dongmanmanhua).ToList();
List <Chapter> chapterlst = new List <Chapter>();
if (cplst.Count > 0)
{
try
{
string bookurl = comic.bookurl.Replace("https://www.dongmanmanhua.cn/", "");
var bookdata = _helper.Get(null, bookurl);
string pattern = "<li id=\"episode_(?<key1>.*?)\" data-episode-no=\"(?<key2>.*?)\">(?<key3>.*?)</li>";
MatchCollection matches = Regex.Matches(bookdata, pattern, RegexOptions.IgnoreCase | RegexOptions.Singleline);
int pagecount = (int)Math.Ceiling(int.Parse(matches[0].Groups[1].Value) / 10.0);
for (int i = 0; i < matches.Count; i++)
{
var lihtml = matches[i].Groups["key3"].Value;
Regex reg1 = new Regex("href=\"(?<key1>.*?)\"");
Match match1 = reg1.Match(lihtml);
string chapterurl = match1.Groups["key1"].Value;
Regex reg2 = new Regex("<span class=\"tx\">#(?<key1>.*?)</span>");
Match match2 = reg2.Match(lihtml);
int sort = int.Parse(match2.Groups["key1"].Value);
//Regex reg3 = new Regex("<span class=\"subj\"><span>(?<key1>.*?)</span></span>");
//Match match3 = reg3.Match(lihtml);
//string chaptername = match3.Groups["key1"].Value;
Regex reg3 = new Regex("alt=\"(?<key1>.*?)\"");
Match match3 = reg3.Match(lihtml);
string chaptername = match3.Groups["key1"].Value;
Regex reg4 = new Regex("src=\"(?<key1>.*?)\"");
Match match4 = reg4.Match(lihtml);
string chaptersource = match4.Groups["key1"].Value;
chapterlst.Add(new Chapter()
{
chapterid = comic.comicid + "_" + sort,
chaptername = chaptername,
chapterurl = "https:" + chapterurl,
sort = sort,
comicid = comic.comicid,
retry = 0,
source = comic.source,
downstatus = DownChapter.待处理链接,
isvip = "0",
chaptersource = chaptersource,
chapterlocal = "",
modify = dt,
shortdate = shortdate,
});
}
for (int i = 2; i <= pagecount; i++)
{
var bookdata2 = _helper.Get(null, bookurl + "&page=" + i);
string pattern2 = "<li id=\"episode_(?<key1>.*?)\" data-episode-no=\"(?<key2>.*?)\">(?<key3>.*?)</li>";
MatchCollection matches2 = Regex.Matches(bookdata2, pattern2, RegexOptions.IgnoreCase | RegexOptions.Singleline);
for (int j = 0; j < matches2.Count; j++)
{
var lihtml = matches2[j].Groups["key3"].Value;
Regex reg1 = new Regex("href=\"(?<key1>.*?)\"");
Match match1 = reg1.Match(lihtml);
string chapterurl = match1.Groups["key1"].Value;
Regex reg2 = new Regex("<span class=\"tx\">#(?<key1>.*?)</span>");
Match match2 = reg2.Match(lihtml);
int sort = int.Parse(match2.Groups["key1"].Value);
//Regex reg3 = new Regex("<span class=\"subj\"><span>(?<key1>.*?)</span></span>");
//Match match3 = reg3.Match(lihtml);
//string chaptername = match3.Groups["key1"].Value;
Regex reg3 = new Regex("alt=\"(?<key1>.*?)\"");
Match match3 = reg3.Match(lihtml);
string chaptername = match3.Groups["key1"].Value;
Regex reg4 = new Regex("src=\"(?<key1>.*?)\"");
Match match4 = reg4.Match(lihtml);
string chaptersource = match4.Groups["key1"].Value;
chapterlst.Add(new Chapter()
{
chapterid = comic.comicid + "_" + sort,
chaptername = chaptername,
chapterurl = "https:" + chapterurl,
sort = sort,
comicid = comic.comicid,
retry = 0,
source = comic.source,
downstatus = DownChapter.待处理链接,
isvip = "0",
chaptersource = chaptersource,
chapterlocal = "",
modify = dt,
shortdate = shortdate,
});
}
}
bool contextChange = false;
if (chapterlst.Count > 0)
{
chapterlst = chapterlst.OrderBy(x => x.sort).ToList();
int cmpCount = cplst.Count >= chapterlst.Count ? chapterlst.Count : cplst.Count;
for (int i = 0; i < cmpCount; i++)
{
if (cplst[i].chapterid != chapterlst[i].chapterid)
{
contextChange = true;
}
}
}
else
{
continue;
}
if (contextChange)
{
dbcontext.Delete <Chapter>(x => x.comicid == comic.comicid);
if (chapterlst.Count > 0)
{
dbcontext.BulkInsert(chapterlst);
Notice notice = new Notice();
notice.noticeid = comic.comicid;
notice.noticestatus = NoticeStatus.等待处理;
notice.noticetype = NoticeType.目录变更;
notice.source = comic.source;
notice.shortdate = shortdate;
notice.modify = dt;
//var nqwait = nq.Where(x => x.noticeid == comic.comicid && x.noticestatus == NoticeStatus.等待处理 && x.noticetype == NoticeType.目录变更).FirstOrDefault();
//if (nqwait == null)
//{
dbcontext.Insert(notice);
//}
}
}
else
{
if (cplst.Count == chapterlst.Count)
{
continue;
}
if (cplst.Count > chapterlst.Count)
{
List <Chapter> needrmvlst = cplst.Except(chapterlst, new Chapter_Comparer()).ToList();
List <int> rmvidlst = needrmvlst.Select(x => x.Id).ToList();
logger.Error(string.Join(",", needrmvlst.Select(x => x.chapterid).ToArray()));
dbcontext.Delete <Chapter>(x => rmvidlst.Contains(x.Id));
}
if (cplst.Count < chapterlst.Count)
{
List <Chapter> needaddlst = chapterlst.Except(cplst, new Chapter_Comparer()).ToList();
dbcontext.BulkInsert(needaddlst);
}
}
//int delete = cplst.Except(chapterlst, new Chapter_Comparer()).Count(); // 删章
//List<Chapter> add = chapterlst.Except(cplst, new Chapter_Comparer()).ToList(); // 新增
//Chapter maxChapter = cplst.OrderBy(x => x.sort).Max();
//if (delete > 0)
//{
// List<string> idlst = cplst.Select(x => x.chapterid).ToList();
// dbcontext.Delete<Page>(x => idlst.Contains(x.chapterid));
// dbcontext.Delete<Chapter>(x => idlst.Contains(x.chapterid));
// if (chapterlst.Count > 0)
// {
// dbcontext.BulkInsert(chapterlst);
// }
// Notice notice = new Notice();
// notice.noticeid = comic.comicid;
// notice.noticestatus = NoticeStatus.等待处理;
// notice.noticetype = NoticeType.目录变更;
// notice.source = comic.source;
// notice.shortdate = shortdate;
// notice.modify = dt;
// var nqwait = nq.Where(x => x.noticeid == comic.comicid && x.noticestatus == NoticeStatus.等待处理 && x.noticetype == NoticeType.目录变更).FirstOrDefault();
// if (nqwait == null)
// {
// dbcontext.Insert(notice);
// }
// continue;
//}
//else
//{
// List<Chapter> mvadd = chapterlst.Except(add, new Chapter_Comparer()).ToList();
// string cplststr = string.Join(",", cplst.Select(x => x.chapterid).ToArray());
// string chapterlststr = string.Join(",", mvadd.Select(x => x.chapterid).ToArray());
// if (cplststr != chapterlststr) // 调序
// {
// List<string> idlst = cplst.Select(x => x.chapterid).ToList();
// dbcontext.Delete<Page>(x => idlst.Contains(x.chapterid));
// dbcontext.Delete<Chapter>(x => idlst.Contains(x.chapterid));
// if (chapterlst.Count > 0)
// {
// dbcontext.BulkInsert(chapterlst);
// }
// Notice notice = new Notice();
// notice.noticeid = comic.comicid;
// notice.noticestatus = NoticeStatus.等待处理;
// notice.noticetype = NoticeType.目录变更;
// notice.source = comic.source;
// notice.shortdate = shortdate;
// notice.modify = dt;
// var nqwait = nq.Where(x => x.noticeid == comic.comicid && x.noticestatus == NoticeStatus.等待处理 && x.noticetype == NoticeType.目录变更).FirstOrDefault();
// if (nqwait == null)
// {
// dbcontext.Insert(notice);
// }
// continue;
// }
//}
//if (add.Count > 0)
//{
// //int addsort = cplst.Count();
// //foreach (var item in add)
// //{
// // item.sort = item.sort + addsort;
// //}
// dbcontext.BulkInsert(add);
//}
}
catch (Exception ex)
{
logger.Error(ex.Message);
comic.updatedatetime = "";
comic.modify = dt;
dbcontext.Update(comic);
Err_ChapterJob err = new Err_ChapterJob();
err.bookurl = comic.bookurl;
err.source = comic.source;
err.errtype = ErrChapter.解析出错;
err.modify = dt;
err.shortdate = shortdate;
err.message = ex.Message;
err = dbcontext.Insert(err);
continue;
}
}
}
}
public void Execute(IJobExecutionContext context)
{
DateTime dt = DateTime.Now;
string ticks = dt.Ticks.ToString();
string shortdate = dt.ToString("yyyy-MM-dd");
string yesterday = dt.AddDays(-1).ToString("yyyy-MM-dd");
IQuery <Comic> q = dbcontext.Query <Comic>();
IQuery <Chapter> cpq = dbcontext.Query <Chapter>();
//List<Comic> comiclst = q.Where(a => a.source == Source.U17 && a.shortdate == shortdate).Take(200).ToList();
List <Comic> comiclst = q.Where(a => a.source == Source.U17).Take(200).ToList();
List <int> ids = comiclst.Select(x => x.Id).ToList();
dbcontext.Update <Comic>(a => ids.Contains(a.Id), a => new Comic()
{
shortdate = yesterday,
modify = dt
});
List <Chapter> chapterlst = new List <Chapter>();
foreach (var comic in comiclst)
{
List <Chapter> cplst = cpq.Where(a => a.comicid == comic.comicid && a.source == Source.U17).ToList();
if (cplst.Count == 0)
{
try
{
string bookurl = comic.bookurl.Replace("http://www.u17.com/", "");
var bookdata = _helper.Get(null, bookurl);
bookdata = StringHelper.MergeSpace(bookdata);
string pattern = "<li id='cpt_read_(?<key1>.*?)'> <a id=\"cpt_(?<key2>.*?)\" href=\"(?<key3>.*?)\" title=(?<key5>.*?)>(?<key4>.*?)</a>";
MatchCollection matches = Regex.Matches(bookdata, pattern, RegexOptions.IgnoreCase | RegexOptions.Singleline);
for (int i = 0; i < matches.Count; i++)
{
chapterlst.Add(new Chapter()
{
chapterid = comic.comicid + "_" + matches[i].Groups["key2"].Value,
chaptername = StringHelper.ReplaceHtmlTag(matches[i].Groups["key4"].Value.Trim()),
chapterurl = matches[i].Groups["key3"].Value,
sort = i + 1,
comicid = comic.comicid,
retry = 0,
source = comic.source,
downstatus = DownChapter.待处理链接,
isvip = matches[i].Groups["key5"].Value.IndexOf("_chapter") == -1? "0":"1",
chaptersource = "",
chapterlocal = "",
modify = dt,
shortdate = shortdate,
ticks = ticks
});
}
}
catch (Exception ex)
{
logger.Error(ex.Message);
comic.shortdate = shortdate;
comic.modify = dt;
dbcontext.Update(comic);
Err_ChapterJob err = new Err_ChapterJob();
err.bookurl = comic.bookurl;
err.source = comic.source;
err.errtype = ErrChapter.解析出错;
err.modify = dt;
err.shortdate = shortdate;
err.message = ex.Message;
err = dbcontext.Insert(err);
continue;
}
}
}
if (chapterlst.Count > 0)
{
dbcontext.BulkInsert(chapterlst);
}
}