private static async Task <string> GetResponse(string url, int pageIndex) { var proxy = PoolManageService.GetProxy(); try { HttpClientHandler httpClientHandler = new HttpClientHandler { UseProxy = true, Proxy = new WebProxy(proxy, false), PreAuthenticate = false, UseDefaultCredentials = false, }; var httpClient = new HttpClient(httpClientHandler) { Timeout = TimeSpan.FromSeconds(5), }; httpClient.DefaultRequestHeaders.TryAddWithoutValidation("Accept", "text/html,application/xhtml+xml,application/xml"); httpClient.DefaultRequestHeaders.TryAddWithoutValidation("Accept-Encoding", "gzip, deflate"); httpClient.DefaultRequestHeaders.TryAddWithoutValidation("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.66 Safari/537.36"); var ct = new { CategoryType = "Picked", ParentCategoryId = 0, CategoryId = -2, PageIndex = pageIndex, TotalPostCount = 4000, ItemListActionName = "PostList" }; var stringContent = new StringContent(JsonConvert.SerializeObject(ct), Encoding.UTF8, "application/json"); var response = await httpClient.PostAsync(url, stringContent).ConfigureAwait(false); response.EnsureSuccessStatusCode(); using (var responseStream = await response.Content.ReadAsStreamAsync().ConfigureAwait(false)) using (var decompressedStream = new GZipStream(responseStream, CompressionMode.Decompress)) using (var streamReader = new StreamReader(decompressedStream)) { var r = await streamReader.ReadToEndAsync().ConfigureAwait(false); return(r); } } catch (Exception e) { Console.WriteLine(e); return(""); } }
// 下载西刺代理的html页面 public void Downloadxicidaili() { try { List <string> list = new List <string>() { "http://www.xicidaili.com/nt/", "http://www.xicidaili.com/nn/", "http://www.xicidaili.com/wn/", "http://www.xicidaili.com/wt/" }; foreach (var utlitem in list) { for (int i = 1; i < 5; i++) { string url = utlitem + i; var ipProxy = PoolManageService.GetProxy(); if (string.IsNullOrEmpty(ipProxy)) { Log.Error().Message("Ip代理池暂无可用代理IP").Write(); return; } var ip = ipProxy; WebProxy webproxy; if (ipProxy.Contains(":")) { ip = ipProxy.Split(new[] { ':' })[0]; var port = int.Parse(ipProxy.Split(new[] { ':' })[1]); webproxy = new WebProxy(ip, port); } else { webproxy = new WebProxy(ip); } string html = HttpHelper.DownloadHtml(url, webproxy); if (string.IsNullOrEmpty(html)) { Log.Error().Message("代理地址:" + url + " 访问失败").Write(); continue; } var doc = new HtmlDocument(); doc.LoadHtml(html); HtmlNode node = doc.DocumentNode; string xpathstring = "//tr[@class='odd']"; HtmlNodeCollection collection = node.SelectNodes(xpathstring); foreach (var item in collection) { var proxy = new IpProxy(); string xpath = "td[2]"; proxy.Address = item.SelectSingleNode(xpath).InnerHtml; xpath = "td[3]"; proxy.Port = int.Parse(item.SelectSingleNode(xpath).InnerHtml); Task.Run(() => { PoolManageService.Add(proxy); }); } } } } catch (Exception e) { //LogUtils.ErrorLog(new Exception("下载西刺代理IP池出现故障", e)); } }