Ejemplo n.º 1
0
        public void ErgodicNovels()
        {
            //遍历取出数据库记录,每次100条
            var IDstr = 0;

            IDstr = int.Parse(ReadConfig("novelconfig.txt"));
            try
            {
                while (true)
                {
                    var sql = "SELECT * FROM story a JOIN (select ID from story limit " + IDstr + ", 100) b ON a.ID = b.id";//大数据列表的分页查询
                    using (var dssource = MysqlHelper.GetDataSet(sql, CommandType.Text, null))
                    {
                        if (dssource == null || dssource.Tables.Count == 0 || dssource.Tables[0].Rows.Count == 0)
                        {
                            continue;
                        }
                        for (int j = 0; j < dssource.Tables[0].Rows.Count; j++)
                        {
                            var ID      = dssource.Tables[0].Rows[j]["ID"].ToString();
                            var Type    = dssource.Tables[0].Rows[j]["Type2"].ToString();
                            var Title   = dssource.Tables[0].Rows[j]["Title"].ToString();
                            var Content = dssource.Tables[0].Rows[j]["Content"].ToString();
                            Console.WriteLine("正在提取第" + ID + "个小说...");
                            Title = ReplaceNovelTitle(Title);
                            var novel = GetNovel(Content);
                            novel = ReplaceHtml(novel);
                            //更新本条记录
                            var sqlupd = "UPDATE story SET Title='" + Title + "', col5='" + novel + "' WHERE ID=" + ID + ";";
                            MysqlHelper.ExecuteNonQuery(sqlupd, CommandType.Text, null);
                            Console.WriteLine("存储成功!");
                        }
                    }
                    IDstr += 100;
                }
            }
            catch (Exception ex)
            {
                WriteConfig("novelconfig.txt", IDstr.ToString());
                Console.WriteLine("程序出错,原因是:" + ex.Message);
                Console.ReadKey();
            }
        }
Ejemplo n.º 2
0
        /// <summary>
        /// 下载小说、图片、电影源码页面
        /// </summary>
        public void DowonloadResource()
        {
            var urls = Makedic();
            //Dictionary<string, string> cols = new Dictionary<string, string>();
            //foreach (KeyValuePair<string, Dictionary<string, string>> pair in urls)
            //{
            //    foreach (var dic in pair.Value)
            //    {
            //        cols.Add(dic.Key, GetHtmlSource(dic.Value));
            //    }
            //}

            //网站根
            var url = "http://www.yiren02.com/";

            foreach (var url1 in urls)
            {
                if (url1.Key == "电影")
                {
                    //逐个赋值起始页
                    foreach (var url2 in url1.Value)
                    {
                        if (url2.Key != "疯狂群交") //已下载过的去掉
                        {
                            //找到文章目录页
                            var urlreal = url2.Value;
                            //循环读取目录页
                            while (true)
                            {
                                //根据目录页地址读取目录页面
                                var cnt = GetWeb(urlreal);
                                if (string.IsNullOrEmpty(cnt))
                                {
                                    Console.WriteLine("获取到了空列表网页!");
                                    break;
                                }

                                //从目录页面获取所有文章链接数组(标题,URL)
                                //Dictionary<string, string> dicUrlName = GetList(cnt);
                                Dictionary <string, string> dicUrlName = GetListVideo(cnt);//电影列表
                                Console.WriteLine("获取到了" + dicUrlName.Count + "个项!——" + url2.Value);
                                //若无目录则退出循环
                                if (dicUrlName.Count == 0)
                                {
                                    break;
                                }
                                //循环数组,获取每一篇文章
                                foreach (var urlname in dicUrlName)
                                {
                                    try
                                    {
                                        var Type1   = url1.Key;
                                        var Type2   = url2.Key;
                                        var Title   = urlname.Key;
                                        var URL     = url + urlname.Value;
                                        var Content = ReplaceCode(GetWeb(URL));
                                        if (string.IsNullOrEmpty(Content))
                                        {
                                            Console.WriteLine("获取到了空网页!");
                                            break;
                                        }
                                        var Count = Content.Length.ToString();
                                        var sql   = "INSERT INTO video (Type1, Type2, Title, URL, Content, Count) VALUES ('" + Type1 + "','" +
                                                    Type2 + "','" + Title + "','" + URL + "','" + Content + "','" + Count + "')";
                                        MysqlHelper.ExecuteNonQuery(sql, CommandType.Text, null);
                                        Console.WriteLine(URL + ",获取成功!");
                                    }
                                    catch (Exception ex)
                                    {
                                        try
                                        {
                                            var Type1 = url1.Key;
                                            var Type2 = url2.Key;
                                            var Title = urlname.Key;
                                            var URL   = url + urlname.Value;
                                            var sql   = "INSERT INTO video (Type1, Type2, Title, URL) VALUES ('" + Type1 + "','" +
                                                        Type2 + "','" + Title + "','" + URL + "')";
                                            MysqlHelper.ExecuteNonQuery(sql, CommandType.Text, null);
                                        }
                                        catch
                                        {
                                        }
                                    }
                                }
                                //更新目录页地址
                                urlreal = url + GetNextListUrl(cnt);
                            }
                        }
                    }
                }
            }
        }