public async Task Download(List <ICrawledUrl> crawledUrls, string downloadDirectory, CancellationToken cancellationToken) { if (crawledUrls == null) { throw new ArgumentNullException(nameof(crawledUrls)); } if (string.IsNullOrEmpty(downloadDirectory)) { throw new ArgumentException("Argument cannot be null or empty", nameof(downloadDirectory)); } using (SemaphoreSlim concurrencySemaphore = new SemaphoreSlim(4)) //todo: allow setting the count here (issue #4) { List <Task> tasks = new List <Task>(); for (int i = 0; i < crawledUrls.Count; i++) { concurrencySemaphore.Wait(); cancellationToken.ThrowIfCancellationRequested(); int entryPos = i; Task task = Task.Run(async() => { try { ICrawledUrl entry = crawledUrls[entryPos]; if (!_urlChecker.IsValidUrl(entry.Url)) { _logger.Error($"Invalid url: {entry.Url}"); return; } if (_urlChecker.IsBlacklistedUrl(entry.Url)) { _logger.Warn($"Url is blacklisted: {entry.Url}"); return; } _logger.Debug($"Downloading {entryPos + 1}/{crawledUrls.Count}: {entry.Url}"); try { _logger.Debug($"Calling url processor for: {entry.Url}"); bool isDownloadAllowed = await _crawledUrlProcessor.ProcessCrawledUrl(entry, downloadDirectory); if (isDownloadAllowed) { if (string.IsNullOrWhiteSpace(entry.DownloadPath)) { throw new DownloadException($"Download path is not filled for {entry.Url}"); } await _pluginManager.DownloadCrawledUrl(entry, downloadDirectory); } else { _logger.Debug($"ProcessCrawledUrl returned false, {entry.Url} will be skipped"); } //TODO: mark isDownloadAllowed = false entries as skipped entry.IsDownloaded = true; OnFileDownloaded(new FileDownloadedEventArgs(entry.Url, crawledUrls.Count)); } catch (DownloadException ex) { string logMessage = $"Error while downloading {entry.Url}: {ex.Message}"; if (ex.InnerException != null) { logMessage += $". Inner Exception: {ex.InnerException}"; } _logger.Error(logMessage); OnFileDownloaded(new FileDownloadedEventArgs(entry.Url, crawledUrls.Count, false, logMessage)); } catch (Exception ex) { throw new UniversalDownloaderPlatformException( $"Error while downloading {entry.Url}: {ex.Message}", ex); } } finally { concurrencySemaphore.Release(); } }, cancellationToken); tasks.Add(task); } await Task.WhenAll(tasks); _logger.Debug("Finished all tasks"); } }