Beispiel #1
0
 public void DoCatchShops()
 {
     //CahchData cahchData = new CahchData(SHOP_SEARCH_PATH, "https://shopsearch.taobao.com/search?app=shopsearch&q=显卡&s={0}");
     CahchData cahchData = new CahchData("shops", "https://shop36826868.taobao.com/category.htm?pageNo={0}", 1, 1);
     //List<CatchUrl> urls = GetCommonURLs(SHOP_SEARCH_PATH, "https://shopsearch.taobao.com/search?app=shopsearch&q=显卡&s={0}");
     Task<string> ttt = Task<string>.Run(() => DownloadPageAsync(cahchData));
 }
Beispiel #2
0
        async Task<string> DownloadPageAsync(CahchData cahchData)
        {
            string proxyUri = string.Format("{0}:{1}", "127.0.0.1", 8888);
            WebProxy proxy = new WebProxy(proxyUri);

            CookieContainer cookies = new CookieContainer();
            HttpClientHandler httpClientHandler = new HttpClientHandler()
            {
                //Proxy = proxy,
                //UseProxy = true,
                UseCookies = true,
                CookieContainer = cookies
            };

            HttpClient client = new HttpClient(httpClientHandler);
            client.DefaultRequestHeaders.Add("user-agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36");
            client.DefaultRequestHeaders.Add("accept-encoding", "gzip, deflate, br");
            client.DefaultRequestHeaders.Add("accept-language", "zh-CN,zh;q=0.9,en;q=0.8");
            client.Timeout = new TimeSpan(0, 0, 0, 3, 0);
            try
            {
                foreach (var catchURL in cahchData.URLs)
                {
                    Uri url = new Uri(catchURL.url);
                    var request = new HttpRequestMessage(HttpMethod.Get, url);
                    //httpContent.Content = 
                    request.Version = new Version(2, 0);

                    HttpResponseMessage response = await client.SendAsync(request);

                    IEnumerable<string> contentEncodings;
                    if (response.Content.Headers.TryGetValues("content-encoding", out contentEncodings))
                    {
                        var encoding = contentEncodings.FirstOrDefault();
                        var compressor = this.Compressors.FirstOrDefault(c => c.EncodingType.Equals(encoding, StringComparison.InvariantCultureIgnoreCase));
                        response.Content = new DecompressedHttpContent(response.Content, compressor);
                    }

                    HttpContent content = response.Content;
                    byte[] responseBytes = await content.ReadAsByteArrayAsync();

                    string filePath = Path.Combine(cahchData.outPath, catchURL.index.ToString());
                    FileStream fs = new FileStream(filePath, FileMode.Create);
                    fs.Write(responseBytes, 0, responseBytes.Length);
                    fs.Flush();
                    fs.Close();

                    Console.WriteLine("finish write page {0}", catchURL.index);

                    await Task.Delay(3000);
                }
            }
            catch (Exception ex)
            {
                Console.WriteLine(ex);
            }

            Console.WriteLine("all done");
            return "all done";
        }