public async Task <Response> DownloadAsync(Request request)
{
Response response = null;
HttpClientEntry httpClientEntry = null;
HttpResponseMessage httpResponseMessage = null;
try
{
var httpRequest = request.ToHttpRequestMessage();
httpClientEntry = await CreateAsync(request);
var stopwatch = new Stopwatch();
stopwatch.Start();
httpResponseMessage = await httpClientEntry.HttpClient.SendAsync(httpRequest);
stopwatch.Stop();
response = await HandleAsync(request, httpResponseMessage);
if (response != null)
{
return(response);
}
response = await httpResponseMessage.ToResponseAsync();
response.ElapsedMilliseconds = (int)stopwatch.ElapsedMilliseconds;
response.RequestHash = request.Hash;
return(response);
}
catch (Exception e)
{
Logger.LogError($"{request.RequestUri} download failed: {e}");
response = new Response
{
RequestHash = request.Hash, StatusCode = HttpStatusCode.Gone, ReasonPhrase = e.ToString()
};
return(response);
}
finally
{
DisposeSafely(httpResponseMessage);
Release(response, httpClientEntry);
}
}
/// <summary>
/// Return same <see cref="HttpClientEntry"/> instance when <paramref name="hash"/> is same.
/// This can ensure some pages have same CookieContainer.
/// </summary>
/// <summary xml:lang="zh-CN">
/// 通过不同的Hash分组, 返回对应的HttpClient
/// 设计初衷: 某些网站会对COOKIE某部分做承上启下的检测, 因此必须保证: www.a.com/keyword=xxxx&page=1 www.a.com/keyword=xxxx&page=2 在同一个HttpClient里访问
/// </summary>
/// <param name="hash">分组的哈希 Hashcode to identify different group.</param>
/// <param name="proxy">代理</param>
/// <returns>HttpClientItem</returns>
private HttpClientEntry GetHttpClientEntry(string hash, IWebProxy proxy)
{
if (string.IsNullOrWhiteSpace(hash))
{
hash = string.Empty;
}
Interlocked.Increment(ref _getHttpClientCount);
if (_getHttpClientCount % 100 == 0)
{
CleanupPool();
}
if (_httpClients.ContainsKey(hash))
{
_httpClients[hash].LastUseTime = DateTime.Now;
return(_httpClients[hash]);
}
else
{
var handler = new HttpClientHandler
{
AutomaticDecompression = DecompressionMethods.Deflate | DecompressionMethods.GZip,
UseProxy = UseProxy,
UseCookies = UseCookies,
AllowAutoRedirect = false,
Proxy = proxy,
CookieContainer = _cookieContainer
};
var item = new HttpClientEntry(handler, AllowAutoRedirect)
{
LastUseTime = DateTime.Now
};
item.HttpClient.Timeout = new TimeSpan(0, 0, 0, Timeout);
_httpClients.TryAdd(hash, item);
return(item);
}
}
protected override void Release(Response response, HttpClientEntry httpClientEntry)
{
_proxyService.ReturnAsync(httpClientEntry.Resource,
response?.StatusCode ?? HttpStatusCode.NotFound);
}
protected virtual void Release(Response response, HttpClientEntry httpClientEntry)
{
}
public async Task <Response> DownloadAsync(Request request)
{
Response response = null;
HttpClientEntry httpClientEntry = null;
try
{
var httpRequest = GenerateHttpRequestMessage(request);
httpClientEntry = await CreateAsync(request);
var stopwatch = new Stopwatch();
stopwatch.Start();
var httpResponseMessage = await httpClientEntry.HttpClient.SendAsync(httpRequest);
stopwatch.Stop();
response = await HandleAsync(request, httpResponseMessage);
if (response != null)
{
return(response);
}
response = new Response
{
ElapsedMilliseconds = (int)stopwatch.ElapsedMilliseconds,
StatusCode = httpResponseMessage.StatusCode
};
foreach (var header in httpResponseMessage.Headers)
{
response.Headers.Add(header.Key, new HashSet <string>(header.Value));
}
response.RequestHash = request.Hash;
response.Content = new ResponseContent
{
Data = await httpResponseMessage.Content.ReadAsByteArrayAsync()
};
foreach (var header in httpResponseMessage.Content.Headers)
{
response.Content.Headers.Add(header.Key, new HashSet <string>(header.Value));
}
return(response);
}
catch (Exception e)
{
Logger.LogError($"{request.Url} download failed: {e}");
response = new Response
{
RequestHash = request.Hash,
StatusCode = HttpStatusCode.BadGateway,
Content = new ResponseContent {
Data = Encoding.UTF8.GetBytes(e.ToString())
}
};
return(response);
}
finally
{
Release(response, httpClientEntry);
}
}
