private Task <Exception> GetLink(Url link, SemaphoreSlim semaphore)
{
return(Task.Run(async() =>
{
await semaphore.WaitAsync();
try
{
using (var client = _client.Create())
{
var result = await client.Download(link).ConfigureAwait(false);
if (result.Data.IsEmpty() || !result.Data.ToLowerInvariant().Contains("<html"))
{
if (result.ErrorCode != 404)
{
throw new ApplicationException($"Empty data. Page {link}. Code {result.ErrorCode}");
}
return null;
}
result.Data = _simplifier.Simplify(result.Data);
_queue.AddRange(
HtmlHelpers.GetAllLinks(result.Data, result.Url)
.Select(x => x.Fix())
.Where(x => x != null && x.Domain == _domain.Domain)
);
_writer.Write(result.Data, result.Url.ToString());
return null;
}
}
catch (Exception ex)
{
return ex;
}
finally
{
semaphore.Release();
}
}));
}
