Example #1
0
        public void ErgodicNovels()
        {
            //遍历取出数据库记录,每次100条
            var IDstr = 0;

            IDstr = int.Parse(ReadConfig("novelconfig.txt"));
            try
            {
                while (true)
                {
                    var sql = "SELECT * FROM story a JOIN (select ID from story limit " + IDstr + ", 100) b ON a.ID = b.id";//大数据列表的分页查询
                    using (var dssource = MysqlHelper.GetDataSet(sql, CommandType.Text, null))
                    {
                        if (dssource == null || dssource.Tables.Count == 0 || dssource.Tables[0].Rows.Count == 0)
                        {
                            continue;
                        }
                        for (int j = 0; j < dssource.Tables[0].Rows.Count; j++)
                        {
                            var ID      = dssource.Tables[0].Rows[j]["ID"].ToString();
                            var Type    = dssource.Tables[0].Rows[j]["Type2"].ToString();
                            var Title   = dssource.Tables[0].Rows[j]["Title"].ToString();
                            var Content = dssource.Tables[0].Rows[j]["Content"].ToString();
                            Console.WriteLine("正在提取第" + ID + "个小说...");
                            Title = ReplaceNovelTitle(Title);
                            var novel = GetNovel(Content);
                            novel = ReplaceHtml(novel);
                            //更新本条记录
                            var sqlupd = "UPDATE story SET Title='" + Title + "', col5='" + novel + "' WHERE ID=" + ID + ";";
                            MysqlHelper.ExecuteNonQuery(sqlupd, CommandType.Text, null);
                            Console.WriteLine("存储成功!");
                        }
                    }
                    IDstr += 100;
                }
            }
            catch (Exception ex)
            {
                WriteConfig("novelconfig.txt", IDstr.ToString());
                Console.WriteLine("程序出错,原因是:" + ex.Message);
                Console.ReadKey();
            }
        }
Example #2
0
        /// <summary>
        /// 下载小说、图片、电影源码页面
        /// </summary>
        public void DowonloadResource()
        {
            var urls = Makedic();
            //Dictionary<string, string> cols = new Dictionary<string, string>();
            //foreach (KeyValuePair<string, Dictionary<string, string>> pair in urls)
            //{
            //    foreach (var dic in pair.Value)
            //    {
            //        cols.Add(dic.Key, GetHtmlSource(dic.Value));
            //    }
            //}

            //网站根
            var url = "http://www.yiren02.com/";

            foreach (var url1 in urls)
            {
                if (url1.Key == "电影")
                {
                    //逐个赋值起始页
                    foreach (var url2 in url1.Value)
                    {
                        if (url2.Key != "疯狂群交") //已下载过的去掉
                        {
                            //找到文章目录页
                            var urlreal = url2.Value;
                            //循环读取目录页
                            while (true)
                            {
                                //根据目录页地址读取目录页面
                                var cnt = GetWeb(urlreal);
                                if (string.IsNullOrEmpty(cnt))
                                {
                                    Console.WriteLine("获取到了空列表网页!");
                                    break;
                                }

                                //从目录页面获取所有文章链接数组(标题,URL)
                                //Dictionary<string, string> dicUrlName = GetList(cnt);
                                Dictionary <string, string> dicUrlName = GetListVideo(cnt);//电影列表
                                Console.WriteLine("获取到了" + dicUrlName.Count + "个项!——" + url2.Value);
                                //若无目录则退出循环
                                if (dicUrlName.Count == 0)
                                {
                                    break;
                                }
                                //循环数组,获取每一篇文章
                                foreach (var urlname in dicUrlName)
                                {
                                    try
                                    {
                                        var Type1   = url1.Key;
                                        var Type2   = url2.Key;
                                        var Title   = urlname.Key;
                                        var URL     = url + urlname.Value;
                                        var Content = ReplaceCode(GetWeb(URL));
                                        if (string.IsNullOrEmpty(Content))
                                        {
                                            Console.WriteLine("获取到了空网页!");
                                            break;
                                        }
                                        var Count = Content.Length.ToString();
                                        var sql   = "INSERT INTO video (Type1, Type2, Title, URL, Content, Count) VALUES ('" + Type1 + "','" +
                                                    Type2 + "','" + Title + "','" + URL + "','" + Content + "','" + Count + "')";
                                        MysqlHelper.ExecuteNonQuery(sql, CommandType.Text, null);
                                        Console.WriteLine(URL + ",获取成功!");
                                    }
                                    catch (Exception ex)
                                    {
                                        try
                                        {
                                            var Type1 = url1.Key;
                                            var Type2 = url2.Key;
                                            var Title = urlname.Key;
                                            var URL   = url + urlname.Value;
                                            var sql   = "INSERT INTO video (Type1, Type2, Title, URL) VALUES ('" + Type1 + "','" +
                                                        Type2 + "','" + Title + "','" + URL + "')";
                                            MysqlHelper.ExecuteNonQuery(sql, CommandType.Text, null);
                                        }
                                        catch
                                        {
                                        }
                                    }
                                }
                                //更新目录页地址
                                urlreal = url + GetNextListUrl(cnt);
                            }
                        }
                    }
                }
            }
        }
Example #3
0
        public void DowonloadMovie()
        {
            var IDstr = 35779;//初始ID

            //IDstr = int.Parse(ReadConfig("configmovie.txt")) + 1;
            while (true)
            {
                //从数据库获取所需分类的记录
                var sql = "select * from video where ID='" + IDstr + "'";
                using (var dssource = MysqlHelper.GetDataSet(sql, CommandType.Text, null))
                {
                    if (dssource == null || dssource.Tables.Count == 0 || dssource.Tables[0].Rows.Count == 0)
                    {
                        break;//若递增的记录不存在,则迭代完成
                    }
                    try
                    {
                        int loopcout = 3;//重试3次
                        //取得存储的源码
D:
                        var filename = dssource.Tables[0].Rows[0]["Title"].ToString();
                        filename     = ReplaceFilename(filename);
                        filename     = IDstr + "-" + filename;
                        Filename     = filename;
                        var source = dssource.Tables[0].Rows[0]["Content"].ToString();
                        //取得视频地址
                        string movieurl = GetMovieUrl(source);
                        if (string.IsNullOrEmpty(movieurl))
                        {
                            IDstr++;
                            Console.WriteLine("没有找到视频文件!");
                            continue;
                        }
                        Console.WriteLine();
                        Console.WriteLine("正在获取——" + IDstr + "..." + movieurl);

                        var file = movieurl.Split('/')[movieurl.Split('/').Length - 1];//原文件名
                        var arr  = file.Split('.');
                        if (arr.Length < 1)
                        {
                            continue;
                        }

                        //倒计时强制退出本轮循环,防止死磕无法下载的图片
                        tCountdown          = new System.Timers.Timer();
                        tCountdown.Elapsed += TCountdown_Elapsed;
                        tCountdown.Interval = 5 * 60 * 1000;//5分钟没有更新进度则退出,启动下一个电影下载
                        tCountdown.Start();

                        eventX = new ManualResetEvent(false);   //主线程获取子线程事件信号量
                        cts    = new CancellationTokenSource(); //主线程告知子线程退出线程信号量
                        //下载文件并保存
                        //pBar = new ProgressBar(Console.CursorLeft, Console.CursorTop, 50, ProgressBarType.Character);//星号进度条
                        //pBar = new ProgressBar(Console.CursorLeft, Console.CursorTop, 50, ProgressBarType.Multicolor);//彩色进度条
                        i  = 0;
                        _k = 0;
                        _K = 10;
                        webclient.DownloadFile(movieurl, filename + "." + arr[1], _K);//多线程异步断点续传下载文件
                        eventX.WaitOne(Timeout.Infinite, true);
                        tCountdown.Stop();


                        //打包图片文件夹成zip文件
                        Console.WriteLine("正在压缩文件...");
                        ProgressBar progressBar = new ProgressBar(Console.CursorLeft, Console.CursorTop, 50, ProgressBarType.Multicolor);//星号进度条
                        GZipHelper.Compress(filename + "." + arr[1], progressBar.Dispaly);
                        Console.WriteLine();
                        Console.WriteLine("压缩完成!");
                    }
                    catch
                    {
                    }
                }
                IDstr++;
            }
E:
            Console.WriteLine("程序退出...");
        }
Example #4
0
        /// <summary>
        /// 从图片页面获取图片地址并下载分类打包
        /// </summary>
        public void DownloadPics()
        {
            var IDstr = 47582;//初始ID,真实自拍45542,美腿丝袜164314

            IDstr = int.Parse(ReadConfig("config.txt")) + 1;
            while (true)
            {
                //从数据库获取所需分类的记录
                var sql = "select * from picture where ID='" + IDstr + "'";
                using (var dssource = MysqlHelper.GetDataSet(sql, CommandType.Text, null))
                {
                    if (dssource == null || dssource.Tables.Count == 0 || dssource.Tables[0].Rows.Count == 0)
                    {
                        break;//若递增的记录不存在,则迭代完成
                    }
                    try
                    {
                        int loopcout = 3;//重试3次
                        //取得存储的源码
B:
                        var filename = dssource.Tables[0].Rows[0]["Title"].ToString();
                        filename     = ReplaceFilename(filename);
                        filename     = filename.Substring(7, filename.Length - 7);
                        filename     = IDstr + "-" + filename;
                        Filename     = filename;
                        var source = dssource.Tables[0].Rows[0]["Content"].ToString();
                        //取得图片组
                        List <string> picurls = GetPicUrls(source);
                        if (picurls.Count == 0)
                        {
                            IDstr++;
                            continue;
                        }
                        //Console.WriteLine("正在获取——" + Filename + ",共计" + picurls.Count + "张图片...");
                        Console.WriteLine("正在获取——" + IDstr + ",共计" + picurls.Count + "张图片...");
                        //按名称创建下载文件夹,并下载图片组至入
                        if (!Directory.Exists(filename))
                        {
                            Directory.CreateDirectory(filename);
                        }

                        iCount    = 0;                             //重置计数器
                        iMaxCount = picurls.Count;                 //重置计数器上限
                        eventX    = new ManualResetEvent(false);   //重置事件信号量
                        cts       = new CancellationTokenSource(); //强制退出线程
                        ThreadPool.SetMaxThreads(300, 300);        //设置设置线程池线程最大最小
                        foreach (var url in picurls)
                        {
                            //ThreadPool.QueueUserWorkItem(new WaitCallback(Downloadpic), url);
                            ThreadPool.QueueUserWorkItem(o => Downloadpic(cts.Token, url));//改用线程池管理
                        }
                        //倒计时强制退出本轮循环,防止死磕无法下载的图片
                        tCountdown          = new System.Timers.Timer();
                        tCountdown.Elapsed += TCountdown_Elapsed;
                        tCountdown.Interval = picurls.Count * 1500;//设置倒计时
                        tCountdown.Start();
                        //等待事件的完成,即线程调用ManualResetEvent.Set()方法
                        //阻塞当前线程,直到当前 WaitHandle 收到信号为止。
                        eventX.WaitOne(Timeout.Infinite, true);
                        tCountdown.Stop();

                        //检查文件夹内容是否为空,为空则删除文件夹并重试3次
                        if (Directory.GetFiles(filename).Length == 0)
                        {
                            new DirectoryInfo(filename).Delete(true);//true:删除里面所有的文件,包括文件夹和子文件夹
                            loopcout--;
                            if (loopcout > 0)
                            {
                                //Console.WriteLine("正在重试...");
                                Console.WriteLine("获取失败...");

                                //记录当前失败ID,并退出程序,等待守护进程重新唤醒
                                WriteConfig("config.txt", IDstr.ToString());
                                goto C;
                            }
                        }

                        Console.WriteLine("获取成功!");

                        //打包图片文件夹成zip文件
                        //ZipHelper.CreateZip(filename, filename + ".zip"); //创建压缩文件
                        //ZipHelper.ZipDirectory(filename, filename + ".zip","",false);
                        //删除打包前的文件夹
                    }
                    catch
                    {
                    }
                }
                IDstr++;
            }
C:
            Console.WriteLine("程序退出...");
        }