public void DoCatchShops()
{
//CahchData cahchData = new CahchData(SHOP_SEARCH_PATH, "https://shopsearch.taobao.com/search?app=shopsearch&q=显卡&s={0}");
CahchData cahchData = new CahchData("shops", "https://shop36826868.taobao.com/category.htm?pageNo={0}", 1, 1);
//List<CatchUrl> urls = GetCommonURLs(SHOP_SEARCH_PATH, "https://shopsearch.taobao.com/search?app=shopsearch&q=显卡&s={0}");
Task<string> ttt = Task<string>.Run(() => DownloadPageAsync(cahchData));
}
async Task<string> DownloadPageAsync(CahchData cahchData)
{
string proxyUri = string.Format("{0}:{1}", "127.0.0.1", 8888);
WebProxy proxy = new WebProxy(proxyUri);
CookieContainer cookies = new CookieContainer();
HttpClientHandler httpClientHandler = new HttpClientHandler()
{
//Proxy = proxy,
//UseProxy = true,
UseCookies = true,
CookieContainer = cookies
};
HttpClient client = new HttpClient(httpClientHandler);
client.DefaultRequestHeaders.Add("user-agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36");
client.DefaultRequestHeaders.Add("accept-encoding", "gzip, deflate, br");
client.DefaultRequestHeaders.Add("accept-language", "zh-CN,zh;q=0.9,en;q=0.8");
client.Timeout = new TimeSpan(0, 0, 0, 3, 0);
try
{
foreach (var catchURL in cahchData.URLs)
{
Uri url = new Uri(catchURL.url);
var request = new HttpRequestMessage(HttpMethod.Get, url);
//httpContent.Content =
request.Version = new Version(2, 0);
HttpResponseMessage response = await client.SendAsync(request);
IEnumerable<string> contentEncodings;
if (response.Content.Headers.TryGetValues("content-encoding", out contentEncodings))
{
var encoding = contentEncodings.FirstOrDefault();
var compressor = this.Compressors.FirstOrDefault(c => c.EncodingType.Equals(encoding, StringComparison.InvariantCultureIgnoreCase));
response.Content = new DecompressedHttpContent(response.Content, compressor);
}
HttpContent content = response.Content;
byte[] responseBytes = await content.ReadAsByteArrayAsync();
string filePath = Path.Combine(cahchData.outPath, catchURL.index.ToString());
FileStream fs = new FileStream(filePath, FileMode.Create);
fs.Write(responseBytes, 0, responseBytes.Length);
fs.Flush();
fs.Close();
Console.WriteLine("finish write page {0}", catchURL.index);
await Task.Delay(3000);
}
}
catch (Exception ex)
{
Console.WriteLine(ex);
}
Console.WriteLine("all done");
return "all done";
}
