public ResponseModel(IRestResponse restResponse, RequestModel requestModel, ISettings settings) { ReferrerUrl = requestModel.ReferrerUrl; StatusCode = restResponse.StatusCode; RequestedUrl = requestModel.Url; IsSuccess = settings.IsSuccess(StatusCode); if (!IsSuccess) return; Markup = restResponse.Content; ShouldCrawl = IsSuccess && requestModel.IsInternalUrl && restResponse.IsHtmlDocument(); }
public void SendRequest(string crawlUrl, string referrerUrl = "") { var requestModel = new RequestModel(crawlUrl, referrerUrl, BaseUrl); var restClient = new RestClient(new Uri(crawlUrl)) { FollowRedirects = false }; restClient.ExecuteAsync(RestRequest, response => { if (response == null) return; var responseModel = new ResponseModel(response, requestModel, _settings); ProcessResponse(responseModel); }); }