public ResponseModel(IRestResponse restResponse, RequestModel requestModel, ISettings settings)
{
ReferrerUrl = requestModel.ReferrerUrl;
StatusCode = restResponse.StatusCode;
RequestedUrl = requestModel.Url;
IsSuccess = settings.IsSuccess(StatusCode);
if (!IsSuccess)
return;
Markup = restResponse.Content;
ShouldCrawl = IsSuccess && requestModel.IsInternalUrl && restResponse.IsHtmlDocument();
}
public void SendRequest(string crawlUrl, string referrerUrl = "")
{
var requestModel = new RequestModel(crawlUrl, referrerUrl, BaseUrl);
var restClient = new RestClient(new Uri(crawlUrl)) { FollowRedirects = false };
restClient.ExecuteAsync(RestRequest, response =>
{
if (response == null)
return;
var responseModel = new ResponseModel(response, requestModel, _settings);
ProcessResponse(responseModel);
});
}
