コード例 #1
0
ファイル: MyHttp.cs プロジェクト: zyayoung/SDownloader
        public static string getHtml(string myUrl)
        {
            HttpHelper http = new HttpHelper();
            HttpItem   item = new HttpItem()
            {
                URL               = myUrl, //URL     必需项
                Method            = "get", //URL     可选项 默认为Get
                Expect100Continue = false,
                Allowautoredirect = false,
                KeepAlive         = true,
                UserAgent         = "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36",
                ContentType       = "application/x-www-form-urlencoded",//返回类型    可选项有默认值
                Timeout           = HTML_TIMEOUT,
                //Cookie = cookieString,
            };

            item.Header.Add("AcceptEncoding", "gzip,deflate");
            HttpResult result = http.GetHtml(item);

            NetworkSpeed.increment(Encoding.Default.GetBytes(result.Html).Length);
            string html = result.Html;

            html = html.Replace(@"\s", string.Empty).Replace(@"\r", string.Empty).Replace(@"\n", string.Empty).Replace(@"\f", string.Empty);
            return(html);
        }
コード例 #2
0
ファイル: MyHttp.cs プロジェクト: zyayoung/SDownloader
 public static async Task <string> getImg(string imgUrl, string path, string fileName, httpParameter MyHttpParameter)
 {
     return(await Task.Run(() => {
         try {
             var watch = new Stopwatch();
             watch.Start();
             if (path.Equals(""))
             {
                 throw new Exception("未指定保存文件的路径");
             }
             string imgName = imgUrl.ToString().Substring(imgUrl.ToString().LastIndexOf("/") + 1);
             HttpHelper http = new HttpHelper();
             HttpItem item = new HttpItem()
             {
                 URL = imgUrl,                                                                                       //URL     必需项
                 Method = "get",                                                                                     //URL     可选项 默认为Get
                 Timeout = IMG_TIMEOUT,                                                                              //连接超时时间     可选项默认为100000
                 ReadWriteTimeout = IMG_TIMEOUT,
                 UserAgent = "Mozilla / 5.0(Windows NT 10.0; Win64; x64; rv: 55.0) Gecko / 20100101 Firefox / 55.0", //用户的浏览器类型,版本,操作系统     可选项有默认值
                 ContentType = "text/html",                                                                          //返回类型    可选项有默认值
                 //Cookie = MyHttpParameter.cookie,
                 //Host = MyHttpParameter.host,
                 //Referer = MyHttpParameter.referer,
                 ResultType = ResultType.Byte
             };
             HttpResult result = http.GetHtml(item);
             watch.Stop();
             if (result.ResultByte == null)
             {
                 throw new Exception(result.Html);
             }
             if (result.ResultByte.Length < MIN_IMG_SIZE)
             {
                 throw new Exception("Invalid image size:" + result.ResultByte.Length);
             }
             NetworkSpeed.increment(result.ResultByte.Length);
             NetworkSpeed.addTotalSize(result.ResultByte.Length);
             var milliseconds = watch.ElapsedMilliseconds;//获取请求执行时间
             string saveResult = saveImg(result.ResultByte, path, fileName, imgName);
             if (!saveResult.StartsWith("Error:"))
             {
                 return "Name:" + saveResult + " Time:" + milliseconds + "ms";
             }
             else
             {
                 throw new Exception(saveResult);
             }
         } catch (Exception ex) {
             return (ex.Message.StartsWith("Error:") ? "" : "Error:") + ex.Message + " ImgUrl:" + imgUrl;
         }
     }));
 }
コード例 #3
0
ファイル: Form1.cs プロジェクト: zyayoung/SDownloader
 private void refreshUI()
 {
     while (true)
     {
         toolStripStatusLabel1.Text = DateTime.Now.ToString();
         if (MySpider != null)
         {
             toolStripStatusLabel2.Text = "Downloads:" + MySpider.finishImgCount.ToString();
             toolStripStatusLabel3.Text = MySpider.finishPageCount + "/" + MySpider.fetchPageCount;
             toolStripStatusLabel4.Text = NetworkSpeed.getSpeed();
         }
         Thread.Sleep(500);
     }
 }
コード例 #4
0
        private void end()
        {
            if (hasEnded)
            {
                return;
            }
            hasEnded       = true;
            workFinishFlag = true;
            watch.Stop();
            myWriteLine("All Tasks Finished!", ConsoleColor.Yellow);
            TimeSpan ts          = watch.Elapsed;
            string   elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}",
                                                 ts.Hours, ts.Minutes, ts.Seconds,
                                                 ts.Milliseconds / 10);

            myWriteLine("Total Time:" + elapsedTime, ConsoleColor.Yellow);
            myWriteLine("Total Images Count:" + finishImgCount + " Total Pages Count:" + finishPageCount, ConsoleColor.Yellow);
            myWriteLine("Total Download Size:" + NetworkSpeed.getTotalSizeText(), ConsoleColor.Yellow);
        }
コード例 #5
0
ファイル: Form1.cs プロジェクト: zyayoung/SDownloader
 private void HSButton_Click(object sender, EventArgs e)
 {
     if (HSButton.Text == "Start Download")
     {
         Task clickTask = new Task(() => {
             HSButton.Text           = "Finish Download";
             settingGroupBox.Enabled = false;
             WebsiteInfo.websiteList[WebsiteInfo.websiteList.FindIndex(s => s.siteName == siteComboBox.Text)] = new WebSiteSetting()
             {
                 siteName            = siteComboBox.Text,
                 domain              = urlTextBox.Text,
                 imgType             = imgTypeTextBox.Text,
                 imgKeys             = imgKeysTextBox.Text.Split('|'),
                 pageRegex           = pageRegTextBox.Text,
                 urlPattern          = urlPatTextBox.Text,
                 firstPageUrlPattern = _1stUrlTextBox.Text,
             };
             saveCurrentSettings();
             NetworkSpeed.init();
             MySettings = new SpiderSettings();
             MySettings.TextKeywords.Add("P");
             MySettings.imgType      = imgTypeTextBox.Text;
             MySettings.domain       = urlTextBox.Text;
             MySettings.siteName     = siteComboBox.Text;
             MySettings.savePath     = pathTextBox.Text;
             MySettings.startPage    = (long)startPageNUD.Value;
             MySettings.endPage      = (long)endPageNUD.Value;
             MySpider                = new GetSpider(MySettings);
             MySpider.OnPageFetched += (s, ex) => {
                 mylistBox.Items.Insert(0, ex.imgInfoResult.picIndex + " " + ex.imgInfoResult.title);
                 progressBar.Maximum = (int)MySpider.fetchPageCount;
             };
             MySpider.OnPageFinished += (s, ex) => {
                 if (ex != null && MySpider != null)
                 {
                     //if (mylistBox.Items.Contains(ex.imgInfoResult.picIndex + " " + ex.imgInfoResult.title)) {
                     mylistBox.Items.Remove(ex.imgInfoResult.picIndex + " " + ex.imgInfoResult.title);
                     //}
                     progressBar.Value = (int)MySpider.finishPageCount;
                 }
                 if (MySpider != null && MySpider.workFinishFlag)
                 {
                     progressBar.Value = 0;
                     mylistBox.Items.Clear();
                     settingGroupBox.Enabled = true;
                     HSButton.Enabled        = true;
                     HSButton.Text           = "Start Download";
                 }
             };
             MySpider.run();
             //MySpider.test();
         }
                                   );
         clickTask.Start();
     }
     else if (HSButton.Text == "Finish Download")
     {
         HSButton.Enabled      = false;
         MySpider.stopWorkFlag = true;
     }
 }