public Response Request(Request request)
{
var httpResponse = GetHttpWebResponse(request);
var buff = GetResponseBuff(httpResponse);
var response = new Response();
response.StatusCode = httpResponse.StatusCode;
response.Headers = httpResponse.Headers;
response.RequestUri = request.Uri;
response.ResponseUri = httpResponse.ResponseUri;
response.IsRaw = request.IsRaw;
response.Method = request.Method;
if (request.IsRaw)
{
response.Data = buff;
}
else
{
var result = Decoding.GetStringFromBuff(buff, httpResponse, request.Charset);
response.Charset = result.CharSet;
response.Data = result.Body;
}
httpResponse.Close();
return(response);
}
public Response Request(Request request)
{
if (!string.IsNullOrEmpty(request.Ip))
{
if (!IPHelper.IsHostIPAddress(IPAddress.Parse(request.Ip)))
{
return(new Response {
IsRaw = false,
StatusCode = HttpStatusCode.BadRequest,
Data = "specified Ip is invalid!"
});
}
}
var httpResponse = GetHttpWebResponse(request);
var buff = GetResponseBuff(httpResponse);
var response = new Response();
response.StatusCode = httpResponse.StatusCode;
response.Headers = WebHeader.FromWebHeader(httpResponse.Headers);
response.RequestUri = request.Uri;
response.ResponseUri = httpResponse.ResponseUri;
response.IsRaw = request.IsRaw;
response.Method = request.Method;
if (request.UseCookie)
{
SetCookie(request, httpResponse.Headers.Get("Set-Cookie"));
}
if (request.IsRaw)
{
response.Data = buff;
}
else
{
var result = Decoding.GetStringFromBuff(buff, httpResponse, request.Charset);
response.Charset = result.CharSet;
response.Data = result.Body;
response.Cookie = GetCookie(request);
}
httpResponse.Close();
return(response);
}
/// <summary>
/// request
/// </summary>
/// <param name="request">crawl request</param>
/// <returns>crawl response</returns>
public Response Request(Request request)
{
Logger.GetLogger(request.Elect).Info("request " + request.Uri.ToString() + " with ip:" + request.Ip + (request.Proxy != null ? (" proxy:" + request.Proxy.Ip + ":" + request.Proxy.Port) : ""));
SimulateBrowser(request);
try
{
if (request.RunJS)
{
var p = new PhantomCrawler();
var res = p.Request(request);
if (request.UseCookie && res.Headers != null)
{
var cookies = res.Headers.Where(m => m.Name == "Set-Cookie").Select(m => m.Value).ToList();
if (cookies.Count > 0)
{
var c = string.Join("", cookies).Replace("\n", ",");
SetCookie(request, c);
res.Cookie = GetCookie(request);
}
}
res.ElectInfo = request.Elect;
res.Request = request;
Logger.GetLogger(request.Elect).Info(request.Uri.ToString() + " response status is " + res.StatusCode.ToString());
return(res);
}
if (!string.IsNullOrEmpty(request.Ip))
{
if (!IPHelper.IsHostIPAddress(IPAddress.Parse(request.Ip)))
{
return(new Response
{
IsRaw = false,
StatusCode = HttpStatusCode.BadRequest,
Data = "specified Ip is invalid!"
});
}
}
var httpResponse = GetHttpWebResponse(request);
if (httpResponse == null)
{
var r = new Response();
r.StatusCode = HttpStatusCode.BadRequest;
r.Data = "httpResponse is null";
return(r);
}
var buff = GetResponseBuff(httpResponse);
var response = new Response();
response.StatusCode = httpResponse.StatusCode;
response.Headers = WebHeader.FromWebHeader(httpResponse.Headers);
response.Request = request;
response.ResponseUri = httpResponse.ResponseUri;
//response.Method = request.Method;
if (!string.IsNullOrEmpty(httpResponse.ContentType))
{
response.IsRaw = MimeDetect.IsRaw(httpResponse.ContentType);
}
else
{
response.IsRaw = MimeDetect.IsRaw(httpResponse.ResponseUri);
}
if (request.UseCookie)
{
SetCookie(request, httpResponse.Headers.Get("Set-Cookie"));
}
if (response.IsRaw)
{
response.Data = buff;
}
else
{
var result = Decoding.GetStringFromBuff(buff, httpResponse, request.Charset);
response.Charset = result.CharSet;
response.Data = result.Body;
response.Cookie = GetCookie(request);
}
httpResponse.Close();
Logger.GetLogger(request.Elect).Info(request.Uri.ToString() + " response status is " + response.StatusCode.ToString());
return(response);
}
catch (Exception ex)
{
var r = new Response();
r.StatusCode = HttpStatusCode.BadRequest;
r.Data = "response error " + ex.Message;
return(r);
}
}