public void ErgodicNovels()
{
//遍历取出数据库记录，每次100条
var IDstr = 0;
IDstr = int.Parse(ReadConfig("novelconfig.txt"));
try
{
while (true)
{
var sql = "SELECT * FROM story a JOIN (select ID from story limit " + IDstr + ", 100) b ON a.ID = b.id";//大数据列表的分页查询
using (var dssource = MysqlHelper.GetDataSet(sql, CommandType.Text, null))
{
if (dssource == null || dssource.Tables.Count == 0 || dssource.Tables[0].Rows.Count == 0)
{
continue;
}
for (int j = 0; j < dssource.Tables[0].Rows.Count; j++)
{
var ID = dssource.Tables[0].Rows[j]["ID"].ToString();
var Type = dssource.Tables[0].Rows[j]["Type2"].ToString();
var Title = dssource.Tables[0].Rows[j]["Title"].ToString();
var Content = dssource.Tables[0].Rows[j]["Content"].ToString();
Console.WriteLine("正在提取第" + ID + "个小说...");
Title = ReplaceNovelTitle(Title);
var novel = GetNovel(Content);
novel = ReplaceHtml(novel);
//更新本条记录
var sqlupd = "UPDATE story SET Title='" + Title + "', col5='" + novel + "' WHERE ID=" + ID + ";";
MysqlHelper.ExecuteNonQuery(sqlupd, CommandType.Text, null);
Console.WriteLine("存储成功！");
}
}
IDstr += 100;
}
}
catch (Exception ex)
{
WriteConfig("novelconfig.txt", IDstr.ToString());
Console.WriteLine("程序出错，原因是：" + ex.Message);
Console.ReadKey();
}
}
/// <summary>
/// 下载小说、图片、电影源码页面
/// </summary>
public void DowonloadResource()
{
var urls = Makedic();
//Dictionary<string, string> cols = new Dictionary<string, string>();
//foreach (KeyValuePair<string, Dictionary<string, string>> pair in urls)
//{
// foreach (var dic in pair.Value)
// {
// cols.Add(dic.Key, GetHtmlSource(dic.Value));
// }
//}
//网站根
var url = "http://www.yiren02.com/";
foreach (var url1 in urls)
{
if (url1.Key == "电影")
{
//逐个赋值起始页
foreach (var url2 in url1.Value)
{
if (url2.Key != "疯狂群交") //已下载过的去掉
{
//找到文章目录页
var urlreal = url2.Value;
//循环读取目录页
while (true)
{
//根据目录页地址读取目录页面
var cnt = GetWeb(urlreal);
if (string.IsNullOrEmpty(cnt))
{
Console.WriteLine("获取到了空列表网页！");
break;
}
//从目录页面获取所有文章链接数组（标题，URL）
//Dictionary<string, string> dicUrlName = GetList(cnt);
Dictionary <string, string> dicUrlName = GetListVideo(cnt);//电影列表
Console.WriteLine("获取到了" + dicUrlName.Count + "个项！——" + url2.Value);
//若无目录则退出循环
if (dicUrlName.Count == 0)
{
break;
}
//循环数组，获取每一篇文章
foreach (var urlname in dicUrlName)
{
try
{
var Type1 = url1.Key;
var Type2 = url2.Key;
var Title = urlname.Key;
var URL = url + urlname.Value;
var Content = ReplaceCode(GetWeb(URL));
if (string.IsNullOrEmpty(Content))
{
Console.WriteLine("获取到了空网页！");
break;
}
var Count = Content.Length.ToString();
var sql = "INSERT INTO video (Type1, Type2, Title, URL, Content, Count) VALUES ('" + Type1 + "','" +
Type2 + "','" + Title + "','" + URL + "','" + Content + "','" + Count + "')";
MysqlHelper.ExecuteNonQuery(sql, CommandType.Text, null);
Console.WriteLine(URL + "，获取成功！");
}
catch (Exception ex)
{
try
{
var Type1 = url1.Key;
var Type2 = url2.Key;
var Title = urlname.Key;
var URL = url + urlname.Value;
var sql = "INSERT INTO video (Type1, Type2, Title, URL) VALUES ('" + Type1 + "','" +
Type2 + "','" + Title + "','" + URL + "')";
MysqlHelper.ExecuteNonQuery(sql, CommandType.Text, null);
}
catch
{
}
}
}
//更新目录页地址
urlreal = url + GetNextListUrl(cnt);
}
}
}
}
}
}
public void DowonloadMovie()
{
var IDstr = 35779;//初始ID
//IDstr = int.Parse(ReadConfig("configmovie.txt")) + 1;
while (true)
{
//从数据库获取所需分类的记录
var sql = "select * from video where ID='" + IDstr + "'";
using (var dssource = MysqlHelper.GetDataSet(sql, CommandType.Text, null))
{
if (dssource == null || dssource.Tables.Count == 0 || dssource.Tables[0].Rows.Count == 0)
{
break;//若递增的记录不存在，则迭代完成
}
try
{
int loopcout = 3;//重试3次
//取得存储的源码
D:
var filename = dssource.Tables[0].Rows[0]["Title"].ToString();
filename = ReplaceFilename(filename);
filename = IDstr + "-" + filename;
Filename = filename;
var source = dssource.Tables[0].Rows[0]["Content"].ToString();
//取得视频地址
string movieurl = GetMovieUrl(source);
if (string.IsNullOrEmpty(movieurl))
{
IDstr++;
Console.WriteLine("没有找到视频文件！");
continue;
}
Console.WriteLine();
Console.WriteLine("正在获取——" + IDstr + "..." + movieurl);
var file = movieurl.Split('/')[movieurl.Split('/').Length - 1];//原文件名
var arr = file.Split('.');
if (arr.Length < 1)
{
continue;
}
//倒计时强制退出本轮循环，防止死磕无法下载的图片
tCountdown = new System.Timers.Timer();
tCountdown.Elapsed += TCountdown_Elapsed;
tCountdown.Interval = 5 * 60 * 1000;//5分钟没有更新进度则退出，启动下一个电影下载
tCountdown.Start();
eventX = new ManualResetEvent(false); //主线程获取子线程事件信号量
cts = new CancellationTokenSource(); //主线程告知子线程退出线程信号量
//下载文件并保存
//pBar = new ProgressBar(Console.CursorLeft, Console.CursorTop, 50, ProgressBarType.Character);//星号进度条
//pBar = new ProgressBar(Console.CursorLeft, Console.CursorTop, 50, ProgressBarType.Multicolor);//彩色进度条
i = 0;
_k = 0;
_K = 10;
webclient.DownloadFile(movieurl, filename + "." + arr[1], _K);//多线程异步断点续传下载文件
eventX.WaitOne(Timeout.Infinite, true);
tCountdown.Stop();
//打包图片文件夹成zip文件
Console.WriteLine("正在压缩文件...");
ProgressBar progressBar = new ProgressBar(Console.CursorLeft, Console.CursorTop, 50, ProgressBarType.Multicolor);//星号进度条
GZipHelper.Compress(filename + "." + arr[1], progressBar.Dispaly);
Console.WriteLine();
Console.WriteLine("压缩完成！");
}
catch
{
}
}
IDstr++;
}
E:
Console.WriteLine("程序退出...");
}
/// <summary>
/// 从图片页面获取图片地址并下载分类打包
/// </summary>
public void DownloadPics()
{
var IDstr = 47582;//初始ID，真实自拍45542，美腿丝袜164314
IDstr = int.Parse(ReadConfig("config.txt")) + 1;
while (true)
{
//从数据库获取所需分类的记录
var sql = "select * from picture where ID='" + IDstr + "'";
using (var dssource = MysqlHelper.GetDataSet(sql, CommandType.Text, null))
{
if (dssource == null || dssource.Tables.Count == 0 || dssource.Tables[0].Rows.Count == 0)
{
break;//若递增的记录不存在，则迭代完成
}
try
{
int loopcout = 3;//重试3次
//取得存储的源码
B:
var filename = dssource.Tables[0].Rows[0]["Title"].ToString();
filename = ReplaceFilename(filename);
filename = filename.Substring(7, filename.Length - 7);
filename = IDstr + "-" + filename;
Filename = filename;
var source = dssource.Tables[0].Rows[0]["Content"].ToString();
//取得图片组
List <string> picurls = GetPicUrls(source);
if (picurls.Count == 0)
{
IDstr++;
continue;
}
//Console.WriteLine("正在获取——" + Filename + "，共计" + picurls.Count + "张图片...");
Console.WriteLine("正在获取——" + IDstr + "，共计" + picurls.Count + "张图片...");
//按名称创建下载文件夹，并下载图片组至入
if (!Directory.Exists(filename))
{
Directory.CreateDirectory(filename);
}
iCount = 0; //重置计数器
iMaxCount = picurls.Count; //重置计数器上限
eventX = new ManualResetEvent(false); //重置事件信号量
cts = new CancellationTokenSource(); //强制退出线程
ThreadPool.SetMaxThreads(300, 300); //设置设置线程池线程最大最小
foreach (var url in picurls)
{
//ThreadPool.QueueUserWorkItem(new WaitCallback(Downloadpic), url);
ThreadPool.QueueUserWorkItem(o => Downloadpic(cts.Token, url));//改用线程池管理
}
//倒计时强制退出本轮循环，防止死磕无法下载的图片
tCountdown = new System.Timers.Timer();
tCountdown.Elapsed += TCountdown_Elapsed;
tCountdown.Interval = picurls.Count * 1500;//设置倒计时
tCountdown.Start();
//等待事件的完成，即线程调用ManualResetEvent.Set()方法
//阻塞当前线程，直到当前 WaitHandle 收到信号为止。
eventX.WaitOne(Timeout.Infinite, true);
tCountdown.Stop();
//检查文件夹内容是否为空，为空则删除文件夹并重试3次
if (Directory.GetFiles(filename).Length == 0)
{
new DirectoryInfo(filename).Delete(true);//true:删除里面所有的文件，包括文件夹和子文件夹
loopcout--;
if (loopcout > 0)
{
//Console.WriteLine("正在重试...");
Console.WriteLine("获取失败...");
//记录当前失败ID，并退出程序，等待守护进程重新唤醒
WriteConfig("config.txt", IDstr.ToString());
goto C;
}
}
Console.WriteLine("获取成功！");
//打包图片文件夹成zip文件
//ZipHelper.CreateZip(filename, filename + ".zip"); //创建压缩文件
//ZipHelper.ZipDirectory(filename, filename + ".zip","",false);
//删除打包前的文件夹
}
catch
{
}
}
IDstr++;
}
C:
Console.WriteLine("程序退出...");
}