private void ProcessUrl(HttpClient httpClient, Uri url, int depthLevel)
{
if (depthLevel > _maxDepthLevel ||
_downloadedUrls.Contains(url) ||
(!url.Scheme.Equals("http", StringComparison.OrdinalIgnoreCase) &&
!url.Scheme.Equals("https", StringComparison.OrdinalIgnoreCase))
)
{
return;
}
_downloadedUrls.Add(url);
HttpResponseMessage response = httpClient.GetAsync(url).Result;
if (response.Content.Headers.ContentType.MediaType.Equals("text/html", StringComparison.OrdinalIgnoreCase))
{
if (_restrictionHelper.IsDomainRestricted(url))
{
return;
}
_logger?.Log($"Html page founded: {url}");
this.ProcessHtmlPage(httpClient, response, url, depthLevel);
}
else
{
if (_restrictionHelper.IsExtensionRestricted(url))
{
return;
}
_logger?.Log($"Resource founded: {url}");
Stream stream = response.Content.ReadAsStreamAsync().Result;
_saver.SaveResource(url, stream);
stream.Close();
}
}
