private void GetResponsesFromLinks(IEnumerable <string> links)
{
using var client = new HttpClient();
foreach (var link in links)
{
var url = link;
// If link is internal concatenate IP and link
if (!UrlFinder.IsExternalLink(link))
{
url = _url + link;
}
var response = client.GetAsync(url).Result;
var log = $"{url} {response.StatusCode.GetHashCode()} {response.ReasonPhrase}";
if (response.IsSuccessStatusCode)
{
Valid.Add(log);
}
else
{
Invalid.Add(log);
}
}
}
public void ScrapUrl()
{
var links = UrlFinder.GetUniqueLinksFromSite(_url).ToList();
GetResponsesFromLinks(links);
}
