public void ErgodicNovels() { //遍历取出数据库记录,每次100条 var IDstr = 0; IDstr = int.Parse(ReadConfig("novelconfig.txt")); try { while (true) { var sql = "SELECT * FROM story a JOIN (select ID from story limit " + IDstr + ", 100) b ON a.ID = b.id";//大数据列表的分页查询 using (var dssource = MysqlHelper.GetDataSet(sql, CommandType.Text, null)) { if (dssource == null || dssource.Tables.Count == 0 || dssource.Tables[0].Rows.Count == 0) { continue; } for (int j = 0; j < dssource.Tables[0].Rows.Count; j++) { var ID = dssource.Tables[0].Rows[j]["ID"].ToString(); var Type = dssource.Tables[0].Rows[j]["Type2"].ToString(); var Title = dssource.Tables[0].Rows[j]["Title"].ToString(); var Content = dssource.Tables[0].Rows[j]["Content"].ToString(); Console.WriteLine("正在提取第" + ID + "个小说..."); Title = ReplaceNovelTitle(Title); var novel = GetNovel(Content); novel = ReplaceHtml(novel); //更新本条记录 var sqlupd = "UPDATE story SET Title='" + Title + "', col5='" + novel + "' WHERE ID=" + ID + ";"; MysqlHelper.ExecuteNonQuery(sqlupd, CommandType.Text, null); Console.WriteLine("存储成功!"); } } IDstr += 100; } } catch (Exception ex) { WriteConfig("novelconfig.txt", IDstr.ToString()); Console.WriteLine("程序出错,原因是:" + ex.Message); Console.ReadKey(); } }
/// <summary> /// 下载小说、图片、电影源码页面 /// </summary> public void DowonloadResource() { var urls = Makedic(); //Dictionary<string, string> cols = new Dictionary<string, string>(); //foreach (KeyValuePair<string, Dictionary<string, string>> pair in urls) //{ // foreach (var dic in pair.Value) // { // cols.Add(dic.Key, GetHtmlSource(dic.Value)); // } //} //网站根 var url = "http://www.yiren02.com/"; foreach (var url1 in urls) { if (url1.Key == "电影") { //逐个赋值起始页 foreach (var url2 in url1.Value) { if (url2.Key != "疯狂群交") //已下载过的去掉 { //找到文章目录页 var urlreal = url2.Value; //循环读取目录页 while (true) { //根据目录页地址读取目录页面 var cnt = GetWeb(urlreal); if (string.IsNullOrEmpty(cnt)) { Console.WriteLine("获取到了空列表网页!"); break; } //从目录页面获取所有文章链接数组(标题,URL) //Dictionary<string, string> dicUrlName = GetList(cnt); Dictionary <string, string> dicUrlName = GetListVideo(cnt);//电影列表 Console.WriteLine("获取到了" + dicUrlName.Count + "个项!——" + url2.Value); //若无目录则退出循环 if (dicUrlName.Count == 0) { break; } //循环数组,获取每一篇文章 foreach (var urlname in dicUrlName) { try { var Type1 = url1.Key; var Type2 = url2.Key; var Title = urlname.Key; var URL = url + urlname.Value; var Content = ReplaceCode(GetWeb(URL)); if (string.IsNullOrEmpty(Content)) { Console.WriteLine("获取到了空网页!"); break; } var Count = Content.Length.ToString(); var sql = "INSERT INTO video (Type1, Type2, Title, URL, Content, Count) VALUES ('" + Type1 + "','" + Type2 + "','" + Title + "','" + URL + "','" + Content + "','" + Count + "')"; MysqlHelper.ExecuteNonQuery(sql, CommandType.Text, null); Console.WriteLine(URL + ",获取成功!"); } catch (Exception ex) { try { var Type1 = url1.Key; var Type2 = url2.Key; var Title = urlname.Key; var URL = url + urlname.Value; var sql = "INSERT INTO video (Type1, Type2, Title, URL) VALUES ('" + Type1 + "','" + Type2 + "','" + Title + "','" + URL + "')"; MysqlHelper.ExecuteNonQuery(sql, CommandType.Text, null); } catch { } } } //更新目录页地址 urlreal = url + GetNextListUrl(cnt); } } } } } }
public void DowonloadMovie() { var IDstr = 35779;//初始ID //IDstr = int.Parse(ReadConfig("configmovie.txt")) + 1; while (true) { //从数据库获取所需分类的记录 var sql = "select * from video where ID='" + IDstr + "'"; using (var dssource = MysqlHelper.GetDataSet(sql, CommandType.Text, null)) { if (dssource == null || dssource.Tables.Count == 0 || dssource.Tables[0].Rows.Count == 0) { break;//若递增的记录不存在,则迭代完成 } try { int loopcout = 3;//重试3次 //取得存储的源码 D: var filename = dssource.Tables[0].Rows[0]["Title"].ToString(); filename = ReplaceFilename(filename); filename = IDstr + "-" + filename; Filename = filename; var source = dssource.Tables[0].Rows[0]["Content"].ToString(); //取得视频地址 string movieurl = GetMovieUrl(source); if (string.IsNullOrEmpty(movieurl)) { IDstr++; Console.WriteLine("没有找到视频文件!"); continue; } Console.WriteLine(); Console.WriteLine("正在获取——" + IDstr + "..." + movieurl); var file = movieurl.Split('/')[movieurl.Split('/').Length - 1];//原文件名 var arr = file.Split('.'); if (arr.Length < 1) { continue; } //倒计时强制退出本轮循环,防止死磕无法下载的图片 tCountdown = new System.Timers.Timer(); tCountdown.Elapsed += TCountdown_Elapsed; tCountdown.Interval = 5 * 60 * 1000;//5分钟没有更新进度则退出,启动下一个电影下载 tCountdown.Start(); eventX = new ManualResetEvent(false); //主线程获取子线程事件信号量 cts = new CancellationTokenSource(); //主线程告知子线程退出线程信号量 //下载文件并保存 //pBar = new ProgressBar(Console.CursorLeft, Console.CursorTop, 50, ProgressBarType.Character);//星号进度条 //pBar = new ProgressBar(Console.CursorLeft, Console.CursorTop, 50, ProgressBarType.Multicolor);//彩色进度条 i = 0; _k = 0; _K = 10; webclient.DownloadFile(movieurl, filename + "." + arr[1], _K);//多线程异步断点续传下载文件 eventX.WaitOne(Timeout.Infinite, true); tCountdown.Stop(); //打包图片文件夹成zip文件 Console.WriteLine("正在压缩文件..."); ProgressBar progressBar = new ProgressBar(Console.CursorLeft, Console.CursorTop, 50, ProgressBarType.Multicolor);//星号进度条 GZipHelper.Compress(filename + "." + arr[1], progressBar.Dispaly); Console.WriteLine(); Console.WriteLine("压缩完成!"); } catch { } } IDstr++; } E: Console.WriteLine("程序退出..."); }
/// <summary> /// 从图片页面获取图片地址并下载分类打包 /// </summary> public void DownloadPics() { var IDstr = 47582;//初始ID,真实自拍45542,美腿丝袜164314 IDstr = int.Parse(ReadConfig("config.txt")) + 1; while (true) { //从数据库获取所需分类的记录 var sql = "select * from picture where ID='" + IDstr + "'"; using (var dssource = MysqlHelper.GetDataSet(sql, CommandType.Text, null)) { if (dssource == null || dssource.Tables.Count == 0 || dssource.Tables[0].Rows.Count == 0) { break;//若递增的记录不存在,则迭代完成 } try { int loopcout = 3;//重试3次 //取得存储的源码 B: var filename = dssource.Tables[0].Rows[0]["Title"].ToString(); filename = ReplaceFilename(filename); filename = filename.Substring(7, filename.Length - 7); filename = IDstr + "-" + filename; Filename = filename; var source = dssource.Tables[0].Rows[0]["Content"].ToString(); //取得图片组 List <string> picurls = GetPicUrls(source); if (picurls.Count == 0) { IDstr++; continue; } //Console.WriteLine("正在获取——" + Filename + ",共计" + picurls.Count + "张图片..."); Console.WriteLine("正在获取——" + IDstr + ",共计" + picurls.Count + "张图片..."); //按名称创建下载文件夹,并下载图片组至入 if (!Directory.Exists(filename)) { Directory.CreateDirectory(filename); } iCount = 0; //重置计数器 iMaxCount = picurls.Count; //重置计数器上限 eventX = new ManualResetEvent(false); //重置事件信号量 cts = new CancellationTokenSource(); //强制退出线程 ThreadPool.SetMaxThreads(300, 300); //设置设置线程池线程最大最小 foreach (var url in picurls) { //ThreadPool.QueueUserWorkItem(new WaitCallback(Downloadpic), url); ThreadPool.QueueUserWorkItem(o => Downloadpic(cts.Token, url));//改用线程池管理 } //倒计时强制退出本轮循环,防止死磕无法下载的图片 tCountdown = new System.Timers.Timer(); tCountdown.Elapsed += TCountdown_Elapsed; tCountdown.Interval = picurls.Count * 1500;//设置倒计时 tCountdown.Start(); //等待事件的完成,即线程调用ManualResetEvent.Set()方法 //阻塞当前线程,直到当前 WaitHandle 收到信号为止。 eventX.WaitOne(Timeout.Infinite, true); tCountdown.Stop(); //检查文件夹内容是否为空,为空则删除文件夹并重试3次 if (Directory.GetFiles(filename).Length == 0) { new DirectoryInfo(filename).Delete(true);//true:删除里面所有的文件,包括文件夹和子文件夹 loopcout--; if (loopcout > 0) { //Console.WriteLine("正在重试..."); Console.WriteLine("获取失败..."); //记录当前失败ID,并退出程序,等待守护进程重新唤醒 WriteConfig("config.txt", IDstr.ToString()); goto C; } } Console.WriteLine("获取成功!"); //打包图片文件夹成zip文件 //ZipHelper.CreateZip(filename, filename + ".zip"); //创建压缩文件 //ZipHelper.ZipDirectory(filename, filename + ".zip","",false); //删除打包前的文件夹 } catch { } } IDstr++; } C: Console.WriteLine("程序退出..."); }