private async Task DownloadLinkAsync(ParentLink nextLink)
{
DownloadResult result = new DownloadResult(nextLink);
bool isDomainResource = result.Uri.StartsWith(_seed);
bool shouldAddLink(ContentType x) => (x == ContentType.Html && (_followExternalLinks || isDomainResource)) ||
(x != ContentType.Html && (_downloadExternalContent || isDomainResource));
try
{
Stopwatch watch = new Stopwatch();
watch.Start();
var getResult = await _client.GetAsync(result.Uri);
watch.Stop();
result.Status = ((int)getResult.StatusCode).ToString();
if (getResult.IsSuccessStatusCode)
{
result.IsSuccessCode = true;
var resultContent = getResult.Content;
result.ContentType = GetContentType(resultContent.Headers.ContentType?.MediaType);
if (shouldAddLink(result.ContentType))
{
var bytes = await resultContent.ReadAsByteArrayAsync();
result.ContentLengthBytes = bytes.Length;
result.Content = Encoding.ASCII.GetString(bytes);
result.DownloadTime = watch.ElapsedMilliseconds;
result.Redirectedto = getResult.RequestMessage.RequestUri.AbsoluteUri;
}
}
}
catch (HttpRequestException e)
{
result.Exception = e;
Console.WriteLine($"ERROR: {nextLink.Link}, {e.Message}");
}
finally
{
if (shouldAddLink(result.ContentType))
{
_progress.Add(result.ToViewdownloadResult());
_downloadResults.Add(result);
}
}
}
public DownloadResult(ParentLink parentLink)
{
Uri = parentLink.Link;
ParentId = parentLink.ParentId;
}
