private static async Task <string> GetResponse(string url, int pageIndex)
{
var proxy = PoolManageService.GetProxy();
try
{
HttpClientHandler httpClientHandler = new HttpClientHandler
{
UseProxy = true,
Proxy = new WebProxy(proxy, false),
PreAuthenticate = false,
UseDefaultCredentials = false,
};
var httpClient = new HttpClient(httpClientHandler)
{
Timeout = TimeSpan.FromSeconds(5),
};
httpClient.DefaultRequestHeaders.TryAddWithoutValidation("Accept", "text/html,application/xhtml+xml,application/xml");
httpClient.DefaultRequestHeaders.TryAddWithoutValidation("Accept-Encoding", "gzip, deflate");
httpClient.DefaultRequestHeaders.TryAddWithoutValidation("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.66 Safari/537.36");
var ct = new { CategoryType = "Picked", ParentCategoryId = 0, CategoryId = -2, PageIndex = pageIndex, TotalPostCount = 4000, ItemListActionName = "PostList" };
var stringContent = new StringContent(JsonConvert.SerializeObject(ct), Encoding.UTF8, "application/json");
var response = await httpClient.PostAsync(url, stringContent).ConfigureAwait(false);
response.EnsureSuccessStatusCode();
using (var responseStream = await response.Content.ReadAsStreamAsync().ConfigureAwait(false))
using (var decompressedStream = new GZipStream(responseStream, CompressionMode.Decompress))
using (var streamReader = new StreamReader(decompressedStream))
{
var r = await streamReader.ReadToEndAsync().ConfigureAwait(false);
return(r);
}
}
catch (Exception e)
{
Console.WriteLine(e);
return("");
}
}
// 下载西刺代理的html页面
public void Downloadxicidaili()
{
try
{
List <string> list = new List <string>()
{
"http://www.xicidaili.com/nt/",
"http://www.xicidaili.com/nn/",
"http://www.xicidaili.com/wn/",
"http://www.xicidaili.com/wt/"
};
foreach (var utlitem in list)
{
for (int i = 1; i < 5; i++)
{
string url = utlitem + i;
var ipProxy = PoolManageService.GetProxy();
if (string.IsNullOrEmpty(ipProxy))
{
Log.Error().Message("Ip代理池暂无可用代理IP").Write();
return;
}
var ip = ipProxy;
WebProxy webproxy;
if (ipProxy.Contains(":"))
{
ip = ipProxy.Split(new[] { ':' })[0];
var port = int.Parse(ipProxy.Split(new[] { ':' })[1]);
webproxy = new WebProxy(ip, port);
}
else
{
webproxy = new WebProxy(ip);
}
string html = HttpHelper.DownloadHtml(url, webproxy);
if (string.IsNullOrEmpty(html))
{
Log.Error().Message("代理地址：" + url + " 访问失败").Write();
continue;
}
var doc = new HtmlDocument();
doc.LoadHtml(html);
HtmlNode node = doc.DocumentNode;
string xpathstring = "//tr[@class='odd']";
HtmlNodeCollection collection = node.SelectNodes(xpathstring);
foreach (var item in collection)
{
var proxy = new IpProxy();
string xpath = "td[2]";
proxy.Address = item.SelectSingleNode(xpath).InnerHtml;
xpath = "td[3]";
proxy.Port = int.Parse(item.SelectSingleNode(xpath).InnerHtml);
Task.Run(() =>
{
PoolManageService.Add(proxy);
});
}
}
}
}
catch (Exception e)
{
//LogUtils.ErrorLog(new Exception("下载西刺代理IP池出现故障", e));
}
}