public async Task Download(List <ICrawledUrl> crawledUrls, string downloadDirectory, CancellationToken cancellationToken)
{
if (crawledUrls == null)
{
throw new ArgumentNullException(nameof(crawledUrls));
}
if (string.IsNullOrEmpty(downloadDirectory))
{
throw new ArgumentException("Argument cannot be null or empty", nameof(downloadDirectory));
}
using (SemaphoreSlim concurrencySemaphore = new SemaphoreSlim(4)) //todo: allow setting the count here (issue #4)
{
List <Task> tasks = new List <Task>();
for (int i = 0; i < crawledUrls.Count; i++)
{
concurrencySemaphore.Wait();
cancellationToken.ThrowIfCancellationRequested();
int entryPos = i;
Task task = Task.Run(async() =>
{
try
{
ICrawledUrl entry = crawledUrls[entryPos];
if (!_urlChecker.IsValidUrl(entry.Url))
{
_logger.Error($"Invalid url: {entry.Url}");
return;
}
if (_urlChecker.IsBlacklistedUrl(entry.Url))
{
_logger.Warn($"Url is blacklisted: {entry.Url}");
return;
}
_logger.Debug($"Downloading {entryPos + 1}/{crawledUrls.Count}: {entry.Url}");
try
{
_logger.Debug($"Calling url processor for: {entry.Url}");
bool isDownloadAllowed = await _crawledUrlProcessor.ProcessCrawledUrl(entry, downloadDirectory);
if (isDownloadAllowed)
{
if (string.IsNullOrWhiteSpace(entry.DownloadPath))
{
throw new DownloadException($"Download path is not filled for {entry.Url}");
}
await _pluginManager.DownloadCrawledUrl(entry, downloadDirectory);
}
else
{
_logger.Debug($"ProcessCrawledUrl returned false, {entry.Url} will be skipped");
}
//TODO: mark isDownloadAllowed = false entries as skipped
entry.IsDownloaded = true;
OnFileDownloaded(new FileDownloadedEventArgs(entry.Url, crawledUrls.Count));
}
catch (DownloadException ex)
{
string logMessage = $"Error while downloading {entry.Url}: {ex.Message}";
if (ex.InnerException != null)
{
logMessage += $". Inner Exception: {ex.InnerException}";
}
_logger.Error(logMessage);
OnFileDownloaded(new FileDownloadedEventArgs(entry.Url, crawledUrls.Count,
false, logMessage));
}
catch (Exception ex)
{
throw new UniversalDownloaderPlatformException(
$"Error while downloading {entry.Url}: {ex.Message}", ex);
}
}
finally
{
concurrencySemaphore.Release();
}
}, cancellationToken);
tasks.Add(task);
}
await Task.WhenAll(tasks);
_logger.Debug("Finished all tasks");
}
}