public string HttpRequest(IDictionary <string, string> dicParams, string responseEncoding = "UTF-8")
{
var responseStr = "";
HttpWebHelper http = new HttpWebHelper();
if (RequestMethod.ToUpper() == "GET")
{
var _payUrl = HttpWebHelper.CreateParameter(dicParams);
responseStr = http.Get(_payUrl, Encoding.GetEncoding(responseEncoding));
}
else
{
responseStr = http.Post(PayUrl, dicParams, Encoding.GetEncoding(responseEncoding), Encoding.GetEncoding(responseEncoding));
}
return(responseStr);
}
public void Execute(IJobExecutionContext context)
{
Dictionary <string, string> qbr = new Dictionary <string, string>()
{
{ "tokenKey", "6po9oJ4vSkwBlc5QjdZ58aGlhZIgJaW0D6Nf5BCTYHTDCwesNCedNa/Mw6lNeqILzFXoF46zIJu6tv5mEt+HgRPd/2mqaM0pQtEHGjQb/yg=" },
{ "comic_id", "520392" },
{ "chapter_id", "99" },
{ "buy_type", "1" },
{ "pay_type", "1" },
//{"uin","o3283360259" },
//{"skey","@Yb26CdH3W" }
};
Dictionary <string, string> headers = new Dictionary <string, string>()
{
{ "Cookie", "uin=o3283360259; skey=@Yb26CdH3W;" }
};
var rs = _helper.Post("http://ac.qq.com/Buy/buyComic", qbr, Encoding.GetEncoding("UTF-8"), Encoding.GetEncoding("UTF-8"), null, "", null, "", headers, "application/x-www-form-urlencoded; charset=UTF-8");
}
public void Execute(IJobExecutionContext context)
{
var comiclst = new List <Comic>();
HttpWebHelper _helper = new HttpWebHelper();
int page = 0;
List <Comic> comicall = new List <Comic>();
List <Comic> arealst = new List <Comic>();
List <Comic> themelst = new List <Comic>();
#region area
while (true)
{
try
{
var count = page * 28;
var t = _helper.Post($"http://www.buka.cn/category/12053/%E6%97%A5%E9%9F%A9", $"start={count}", Encoding.GetEncoding("utf-8"), Encoding.GetEncoding("utf-8"), null, "", null, "http://www.buka.cn/category/12053/%E6%97%A5%E9%9F%A9.html");
var doc = new HtmlDocument();
doc.LoadHtml(t);
var obj = doc.DocumentNode.SelectNodes("//*[@id=\"mangawrap\"]/li");
if (obj != null)
{
foreach (var item in obj)
{
var alink = item.SelectNodes(".//div/a").First();
var bookurl = alink.Attributes["href"].Value;
var title = alink.Attributes["title"].Value;
var au = item.SelectNodes(".//*[@class=\"manga-author\"]").First().Attributes["title"].Value;
arealst.Add(new Comic()
{
comicname = title,
area = "日韩",
authorname = au,
bookurl = "http://www.buka.cn" + bookurl,
comiccoverlocal = "",
comiccoversource = "",
comicdesc = "",
comicid = "13_" + bookurl.Split('/').Last().Replace(".html", ""),
isfinished = "连载中",
isdelete = false,
isoffline = false,
isvip = "0",
recrawer = false,
source = 13,
stopcrawer = false,
shortdate = DateTime.Now.ToDateStr(),
modify = DateTime.Now,
comicsort = 1,
hot = "",
theme = "",
updatedatetime = ""
});
}
}
else
{
break;
}
page = page + 1;
}
catch (Exception ex)
{
break;
}
}
#endregion
#region theme
Dictionary <string, string> tags = new Dictionary <string, string>();
tags.Add("少女", "http://www.buka.cn/category/302/%E5%B0%91%E5%A5%B3");
tags.Add("百合", "http://www.buka.cn/category/206/%E7%99%BE%E5%90%88");
tags.Add("肥皂", "http://www.buka.cn/category/12009/%E8%82%A5%E7%9A%82");
tags.Add("恋爱", "http://www.buka.cn/category/404/%E6%81%8B%E7%88%B1");
tags.Add("玄幻", "http://www.buka.cn/category/12041/%E7%8E%84%E5%B9%BB");
tags.Add("游戏", "http://www.buka.cn/category/12018/%E6%B8%B8%E6%88%8F");
tags.Add("治愈", "http://www.buka.cn/category/202/%E6%B2%BB%E6%84%88");
tags.Add("恐怖", "http://www.buka.cn/category/12075/%E6%81%90%E6%80%96");
tags.Add("科幻", "http://www.buka.cn/category/403/%E7%A7%91%E5%B9%BB");
tags.Add("搞笑", "http://www.buka.cn/category/10008/%E6%90%9E%E7%AC%91");
tags.Add("鬼怪", "http://www.buka.cn/category/211/%E9%AC%BC%E6%80%AA");
tags.Add("励志", "http://www.buka.cn/category/12023/%E5%8A%B1%E5%BF%97");
tags.Add("格斗", "http://www.buka.cn/category/410/%E6%A0%BC%E6%96%97");
tags.Add("少女漫", "http://www.buka.cn/category/12103/%E5%B0%91%E5%A5%B3%E6%BC%AB");
tags.Add("少年漫", "http://www.buka.cn/category/12104/%E5%B0%91%E5%B9%B4%E6%BC%AB");
tags.Add("布卡娘", "http://www.buka.cn/category/10035/布卡娘");
foreach (var tag in tags)
{
page = 0;
while (true)
{
try
{
var count = page * 28;
var t = _helper.Post(tag.Value, $"start={count}", Encoding.GetEncoding("utf-8"), Encoding.GetEncoding("utf-8"));
var doc = new HtmlDocument();
doc.LoadHtml(t);
var obj = doc.DocumentNode.SelectNodes("//*[@id=\"mangawrap\"]/li");
if (obj != null)
{
foreach (var item in obj)
{
var alink = item.SelectNodes(".//div/a").First();
var bookurl = alink.Attributes["href"].Value;
var title = alink.Attributes["title"].Value;
var au = item.SelectNodes(".//*[@class=\"manga-author\"]").First().Attributes["title"].Value;
themelst.Add(new Comic()
{
comicname = title,
area = "",
authorname = au,
bookurl = "http://www.buka.cn" + bookurl,
comiccoverlocal = "",
comiccoversource = "",
comicdesc = "",
comicid = "13_" + bookurl.Split('/').Last().Replace(".html", ""),
isfinished = "连载中",
isdelete = false,
isoffline = false,
isvip = "0",
recrawer = false,
source = 13,
stopcrawer = false,
shortdate = DateTime.Now.ToDateStr(),
modify = DateTime.Now,
comicsort = 1,
hot = "",
theme = tag.Key,
updatedatetime = ""
});
}
}
else
{
break;
}
page = page + 1;
}
catch (Exception ex)
{
break;
}
}
}
#endregion
page = 0;
int sort = 0;
while (true)
{
var count = page * 28;
var url = $"http://www.buka.cn/all/{count}";
var web = new HtmlWeb();
var doc = web.Load(url);
var obj = doc.DocumentNode.SelectNodes("//*[@id=\"mangawrap\"]/li");
if (obj != null)
{
foreach (var item in obj)
{
sort = sort + 1;
var alink = item.SelectNodes(".//div/a").First();
var bookurl = alink.Attributes["href"].Value;
var title = alink.Attributes["title"].Value;
var au = item.SelectNodes(".//*[@class=\"manga-author\"]").First().Attributes["title"].Value.Substring(3);
comicall.Add(new Comic()
{
comicname = title,
area = "",
authorname = au,
bookurl = "http://www.buka.cn" + bookurl,
comiccoverlocal = "",
comicid = "13_" + bookurl.Split('/').Last().Replace(".html", ""),
isfinished = "连载中",
isdelete = false,
isoffline = false,
isvip = "0",
recrawer = false,
source = 13,
stopcrawer = false,
shortdate = DateTime.Now.ToDateStr(),
modify = DateTime.Now,
comicsort = sort,
hot = "",
comicdesc = "",
theme = "",
comiccoversource = "",
updatedatetime = ""
});
}
}
else
{
break;
}
page = page + 1;
}
comicall.ForEach(x =>
{
string area = "";
if (arealst.Where(y => y.comicid == x.comicid).FirstOrDefault() != null)
{
area = arealst.Where(y => y.comicid == x.comicid).FirstOrDefault().area;
}
string theme = "";
if (themelst.Where(t => t.comicid == x.comicid).Count() > 0)
{
theme = string.Join(",", themelst.Where(t => t.comicid == x.comicid).Select(t => t.theme).ToArray());
}
x.area = area;
x.theme = theme;
});
List <Comic> exlst = new List <Comic>();
foreach (var item in comicall)
{
try
{
var web1 = new HtmlWeb();
var doc1 = web1.Load(item.bookurl);
var img1 = doc1.DocumentNode.SelectNodes("/html/body/section/aside/div[1]/img").First().Attributes["src"].Value;
var rate = doc1.DocumentNode.SelectNodes("//*[@class=\"manga-grade-num\"]").First().InnerText.Trim();
var desc = doc1.DocumentNode.SelectNodes("//*[@class=\"manga-desc-font\"]").First().InnerText.Trim();
item.comiccoversource = img1;
item.hot = rate;
item.comicdesc = desc;
}
catch (Exception ex)
{
Console.WriteLine(item.comicid);
exlst.Add(item);
continue;
}
}
List <Comic> addlst = comicall.Except(exlst, new Comic_Comparer()).ToList();
service.ComicCompareBatchAdd(addlst, 13, DateTime.Now.ToDateStr());
}
public void Execute(IJobExecutionContext context)
{
List <Comic> comiclst = new List <Comic>();
DateTime dt = DateTime.Now;
string shortdate = dt.ToDateStr();
var web = new HtmlWeb();
HttpWebHelper _helper = new HttpWebHelper();
string cataurl = "http://www.u17.com/comic/ajax.php?mod=comic_list&act=comic_list_new_fun&a=get_comic_list";
string paras = "data%5Bgroup_id%5D=no&data%5Btheme_id%5D=no&data%5Bis_vip%5D=no&data%5Baccredit%5D=no&data%5Bcolor%5D=no&data%5Bcomic_type%5D=no&data%5Bseries_status%5D=no&data%5Border%5D=0&data%5Bpage_num%5D={0}&data%5Bread_mode%5D=no";
var data = _helper.Post <u17_comic>(cataurl, string.Format(paras, 1), Encoding.GetEncoding("utf-8"), Encoding.GetEncoding("utf-8"));
int pagecount = int.Parse(data.page_total);
int sort = 0;
for (int i = 1; i <= pagecount; i++)
{
var booklst = _helper.Post <u17_comic>(cataurl, string.Format(paras, i), Encoding.GetEncoding("utf-8"), Encoding.GetEncoding("utf-8"));
foreach (var x in booklst.comic_list)
{
try
{
sort = sort + 1;
string bookurl = $"http://www.u17.com/comic/{x.comic_id}.html";
var doc = web.Load(bookurl);
if (doc.DocumentNode.SelectSingleNode("//title") != null)
{
string title = doc.DocumentNode.SelectSingleNode("//title").InnerText.Trim();
string comicname = title.Split('_')[0];
string authorname = title.Split('_')[1];
//var scriptnode = doc.DocumentNode.SelectSingleNode("//script[contains(text(),'var cover_url =')]");
//Match match = Regex.Match(scriptnode.InnerText, "cover_url = \"(?<key1>.*?)\"");
string comiccover = doc.DocumentNode.SelectSingleNode("//div[@class=\"cover\"]").SelectSingleNode(".//img").Attributes["src"].Value;
string bookdesc = doc.DocumentNode.SelectSingleNode("//div[@class=\"info\"]").SelectNodes(".//p").Last().InnerText.Trim();
var line1 = doc.DocumentNode.SelectSingleNode("//div[@class=\"line1\"]");
string isfinished = "";
string hot = "";
string theme = "";
if (line1.InnerText.Contains("类别："))
{
isfinished = line1.InnerText.Contains("连载中") ? "连载中" : "已完结";
hot = line1.InnerText.Substring(line1.InnerText.IndexOf("总点击：")).Replace("总点击：", "").Trim();
theme = line1.SelectSingleNode(".//span").InnerText.Replace(" ", "").Replace("/", ",").Trim();
}
else
{
var line2 = doc.DocumentNode.SelectSingleNode("//div[@class=\"cf line2\"]").SelectNodes(".//span");
isfinished = line2[0].InnerText.Contains("连载中") ? "连载中" : "已完结";
hot = line2[1].InnerText.Trim();
var tags = line1.SelectNodes(".//a");
foreach (var tag in tags)
{
if (!tag.InnerText.Trim().IsNullOrWhiteSpace())
{
theme = theme + " " + tag.InnerText.Trim();
}
}
theme = theme.Trim().Replace(" ", ",");
}
var cpnode = doc.DocumentNode.SelectSingleNode("//div[@class=\"left_tag\"]").SelectSingleNode(".//img");
var copyright = cpnode == null?"":cpnode.Attributes["title"].Value.Replace("有", "").Replace("妖", "").Replace("气", "").Replace("作品", "").Replace("刊载", "");
comiclst.Add(new Comic()
{
comicname = comicname,
authorname = authorname,
bookurl = bookurl,
comiccoversource = comiccover,
comiccoverlocal = "",
comicdesc = bookdesc,
comicid = 3 + "_" + bookurl.Split('/').LastOrDefault().Replace(".html", ""),
isfinished = isfinished,
theme = theme,
isvip = "0",
source = 3,
stopcrawer = false,
isoffline = true,
recrawer = false,
shortdate = shortdate,
modify = dt,
comicsort = sort,
hot = hot,
copyright = copyright
});
}
}
catch (Exception ex)
{
continue;
}
}
}
service.ComicCompareBatchAdd(comiclst, 3, DateTime.Now.ToDateStr());
}