public override Page Download(Request request, ITask task)
{
if (task.Site == null)
{
return null;
}
Site site = task.Site;
ICollection<int> acceptStatCode;
Encoding charset = null;
IDictionary headers = null;
if (site != null)
{
acceptStatCode = site.AcceptStatCode;
charset = site.Encoding;
headers = site.GetHeaders();
}
else
{
acceptStatCode = new HashSet<int> { 200 };
}
//Logger.InfoFormat("Downloading page {0}", request.Url);
int statusCode = 0;
HttpWebResponse response = null;
try
{
HttpWebRequest httpWebRequest = GetHttpWebRequest(request, site, headers);
response = (HttpWebResponse)httpWebRequest.GetResponse();
statusCode = (int)response.StatusCode;
request.PutExtra(Request.StatusCode, statusCode);
if (StatusAccept(acceptStatCode, statusCode))
{
Page page = HandleResponse(request, charset, response, statusCode, task);
// 结束后要置空, 这个值存到Redis会导置无限循环跑单个任务
request.PutExtra(Request.CycleTriedTimes, null);
httpWebRequest.ServicePoint.ConnectionLimit = int.MaxValue;
OnSuccess(request);
return page;
}
else
{
throw new SpiderExceptoin("Download failed.");
}
}
catch (Exception e)
{
if (site.CycleRetryTimes > 0)
{
return AddToCycleRetry(request, site);
}
OnError(request, e);
return null;
}
finally
{
request.PutExtra(Request.StatusCode, statusCode);
try
{
//ensure the connection is released back to pool
//check:
//EntityUtils.consume(httpResponse.getEntity());
response?.Close();
}
catch (Exception e)
{
Logger.Warn("Close response fail.", e);
}
}
}
public override Page Download(Request request, ISpider spider)
{
if (spider.Site == null)
{
return null;
}
Site site = spider.Site;
var acceptStatCodes = site.AcceptStatCode;
//Logger.InfoFormat("Downloading page {0}", request.Url);
HttpResponseMessage response = null;
var proxy = site.GetHttpProxyFromPool();
request.PutExtra(Request.Proxy, proxy);
int statusCode = 200;
try
{
if (GeneratePostBody != null)
{
SingleExecutor.Execute(() =>
{
GeneratePostBody(spider.Site, request);
});
}
var httpMessage = GenerateHttpRequestMessage(request, site);
response = RedialManagerUtils.Execute("downloader-download", (m) =>
{
var message = (HttpRequestMessage)m;
return httpClient.SendAsync(message).Result;
}, httpMessage);
AddRequestCount();
response.EnsureSuccessStatusCode();
if (!site.AcceptStatCode.Contains(response.StatusCode))
{
throw new DownloadException($"下载 {request.Url} 失败. Code: {response.StatusCode}");
}
statusCode = (int)response.StatusCode;
request.PutExtra(Request.StatusCode, statusCode);
Page page = HandleResponse(request, response, statusCode, site);
// need update
page.TargetUrl = request.Url.ToString();
//page.SetRawText(File.ReadAllText(@"C:\Users\Lewis\Desktop\taobao.html"));
// 这里只要是遇上登录的, 则在拨号成功之后, 全部抛异常在Spider中加入Scheduler调度
// 因此如果使用多线程遇上多个Warning Custom Validate Failed不需要紧张, 可以考虑用自定义Exception分开
ValidatePage(page, spider);
// 结束后要置空, 这个值存到Redis会导致无限循环跑单个任务
//request.PutExtra(Request.CycleTriedTimes, null);
//#if !NET_CORE
// httpWebRequest.ServicePoint.ConnectionLimit = int.MaxValue;
//#endif
return page;
//正常结果在上面已经Return了, 到此处必然是下载失败的值.
//throw new SpiderExceptoin("Download failed.");
}
catch (RedialException)
{
throw;
}
catch (Exception e)
{
Page page = new Page(request, site.ContentType) { Exception = e };
ValidatePage(page, spider);
throw;
}
finally
{
// 先Close Response, 避免前面语句异常导致没有关闭.
try
{
//ensure the connection is released back to pool
//check:
//EntityUtils.consume(httpResponse.getEntity());
response?.Dispose();
}
catch (Exception e)
{
var logger = LogUtils.GetLogger(spider);
logger.Warn("Close response fail.", e);
}
}
}
protected HttpWebRequest GetHttpWebRequest(Request request, Site site, IDictionary headers)
{
if (site == null) return null;
HttpWebRequest httpWebRequest = SelectRequestMethod(request);
httpWebRequest.UserAgent = site.UserAgent ??
"Mozilla/5.0 (Windows NT 10.0; WOW64; rv:39.0) Gecko/20100101 Firefox/39.0Mozilla/5.0 (Windows NT 10.0; WOW64; rv:39.0) Gecko/20100101 Firefox/39.0";
if (site.IsUseGzip)
{
httpWebRequest.Headers.Add("Accept-Encoding", "gzip");
}
// headers
if (headers != null)
{
var enumerator = headers.GetEnumerator();
while (enumerator.MoveNext())
{
var key = enumerator.Key;
var value = enumerator.Value;
httpWebRequest.Headers.Add(key.ToString(), value.ToString());
}
}
// cookie
httpWebRequest = GeneratorCookie(httpWebRequest, site);
//check:
httpWebRequest.Timeout = site.Timeout;
httpWebRequest.ContinueTimeout = site.Timeout;
httpWebRequest.ReadWriteTimeout = site.Timeout;
httpWebRequest.AllowAutoRedirect = true;
if (headers != null)
{
foreach (DictionaryEntry entry in headers)
{
httpWebRequest.Headers.Add(entry.Key.ToString(), entry.Value.ToString());
}
}
if (site.GetHttpProxyPool().Enable)
{
HttpHost host = site.GetHttpProxyFromPool();
httpWebRequest.Proxy = new WebProxy(host.Host, host.Port);
request.PutExtra(Request.Proxy, host);
}
return httpWebRequest;
}