public void ErgodicNovels()
{
//遍历取出数据库记录，每次100条
var IDstr = 0;
IDstr = int.Parse(ReadConfig("novelconfig.txt"));
try
{
while (true)
{
var sql = "SELECT * FROM story a JOIN (select ID from story limit " + IDstr + ", 100) b ON a.ID = b.id";//大数据列表的分页查询
using (var dssource = MysqlHelper.GetDataSet(sql, CommandType.Text, null))
{
if (dssource == null || dssource.Tables.Count == 0 || dssource.Tables[0].Rows.Count == 0)
{
continue;
}
for (int j = 0; j < dssource.Tables[0].Rows.Count; j++)
{
var ID = dssource.Tables[0].Rows[j]["ID"].ToString();
var Type = dssource.Tables[0].Rows[j]["Type2"].ToString();
var Title = dssource.Tables[0].Rows[j]["Title"].ToString();
var Content = dssource.Tables[0].Rows[j]["Content"].ToString();
Console.WriteLine("正在提取第" + ID + "个小说...");
Title = ReplaceNovelTitle(Title);
var novel = GetNovel(Content);
novel = ReplaceHtml(novel);
//更新本条记录
var sqlupd = "UPDATE story SET Title='" + Title + "', col5='" + novel + "' WHERE ID=" + ID + ";";
MysqlHelper.ExecuteNonQuery(sqlupd, CommandType.Text, null);
Console.WriteLine("存储成功！");
}
}
IDstr += 100;
}
}
catch (Exception ex)
{
WriteConfig("novelconfig.txt", IDstr.ToString());
Console.WriteLine("程序出错，原因是：" + ex.Message);
Console.ReadKey();
}
}
/// <summary>
/// 下载小说、图片、电影源码页面
/// </summary>
public void DowonloadResource()
{
var urls = Makedic();
//Dictionary<string, string> cols = new Dictionary<string, string>();
//foreach (KeyValuePair<string, Dictionary<string, string>> pair in urls)
//{
// foreach (var dic in pair.Value)
// {
// cols.Add(dic.Key, GetHtmlSource(dic.Value));
// }
//}
//网站根
var url = "http://www.yiren02.com/";
foreach (var url1 in urls)
{
if (url1.Key == "电影")
{
//逐个赋值起始页
foreach (var url2 in url1.Value)
{
if (url2.Key != "疯狂群交") //已下载过的去掉
{
//找到文章目录页
var urlreal = url2.Value;
//循环读取目录页
while (true)
{
//根据目录页地址读取目录页面
var cnt = GetWeb(urlreal);
if (string.IsNullOrEmpty(cnt))
{
Console.WriteLine("获取到了空列表网页！");
break;
}
//从目录页面获取所有文章链接数组（标题，URL）
//Dictionary<string, string> dicUrlName = GetList(cnt);
Dictionary <string, string> dicUrlName = GetListVideo(cnt);//电影列表
Console.WriteLine("获取到了" + dicUrlName.Count + "个项！——" + url2.Value);
//若无目录则退出循环
if (dicUrlName.Count == 0)
{
break;
}
//循环数组，获取每一篇文章
foreach (var urlname in dicUrlName)
{
try
{
var Type1 = url1.Key;
var Type2 = url2.Key;
var Title = urlname.Key;
var URL = url + urlname.Value;
var Content = ReplaceCode(GetWeb(URL));
if (string.IsNullOrEmpty(Content))
{
Console.WriteLine("获取到了空网页！");
break;
}
var Count = Content.Length.ToString();
var sql = "INSERT INTO video (Type1, Type2, Title, URL, Content, Count) VALUES ('" + Type1 + "','" +
Type2 + "','" + Title + "','" + URL + "','" + Content + "','" + Count + "')";
MysqlHelper.ExecuteNonQuery(sql, CommandType.Text, null);
Console.WriteLine(URL + "，获取成功！");
}
catch (Exception ex)
{
try
{
var Type1 = url1.Key;
var Type2 = url2.Key;
var Title = urlname.Key;
var URL = url + urlname.Value;
var sql = "INSERT INTO video (Type1, Type2, Title, URL) VALUES ('" + Type1 + "','" +
Type2 + "','" + Title + "','" + URL + "')";
MysqlHelper.ExecuteNonQuery(sql, CommandType.Text, null);
}
catch
{
}
}
}
//更新目录页地址
urlreal = url + GetNextListUrl(cnt);
}
}
}
}
}
}