/// <summary> /// Downloads to the specified folder the provided file references using cancellation and pause tokens. /// </summary> /// <param name="cancelToken">A token used to cancel the task.</param> /// <param name="pauseToken">A token used to pause the task.</param> /// <param name="folder">The fully-qualified path of the folder in which the downloaded files will be saved. If the directory does not exist, an attempt to create it will be made.</param> /// <param name="links">An array of URLs referencing the files to download.</param> /// <returns>A task that returns an integer that represents the number of files processed (not necessarily downloaded).</returns> public virtual async Task <int> DownloadFiles(CancellationToken cancelToken, PauseToken pauseToken, string folder, params string[] links) { if (NewDownload(folder, links, links.Length)) { try { int saved = 0; int result = await DownloadCore(cancelToken, pauseToken, folder, links, (data, url, computedName) => { bool processed = false; try { int ioutcome = OnFileSaving(computedName, url, data); if (ioutcome == 0) { File.WriteAllBytes(computedName, data); _lastBytesSaved += data.Length; this.OnFileSaved(computedName, url, data); } if (ioutcome == 0 || ioutcome == 1) { processed = true; saved++; _skippedFiles.Add(computedName); } } catch (Exception ex) { _errors.Add(ex); } return(processed); }); _filesSaved += saved; return(result); } catch (AggregateException ex) { foreach (var e in ex.InnerExceptions) { if (!(e is ThreadAbortException)) { LogError(e); } } return(-3); } catch (HttpRequestException ex) { this.LogError(ex.InnerException ?? ex); return(-4); } catch (Exception ex) { this.LogError(ex); return(-1); } finally { if (_cancel) { this.OnCancelled(); } _busy = false; } } return(-2); }
/// <summary> /// Loops through all provided links and, based on the given folder, determines for each /// URL found in the <paramref name="links"/> array whether a download is required. /// </summary> /// <param name="cancelToken">A token used to cancel the task.</param> /// <param name="pauseToken">A token used to pause the task.</param> /// <param name="folder">The fully-qualified path of the folder in which the downloaded files will presumably be saved. This parameter is used only to compute a unique file name for each provided URL.</param> /// <param name="links">An array of URLs referencing the files to download.</param> /// <param name="saveDataCallback">A callback delegate that executes the persistence logic for the data downloaded.</param> /// <param name="skipRequiredCallback"> /// An optional callback delegate that peforms custom checks on whether to skip the resource identified /// by the computed name or not. If this argument is omitted, the default behavior is to skip any file /// that matches the computed name either in the current list of skipped files, or in the local file system. /// </param> /// <returns></returns> protected virtual async Task <int> DownloadCore(CancellationToken cancelToken, PauseToken pauseToken, string folder, string[] links, Func <byte[], string, string, bool> saveDataCallback, Func <string, string, bool> skipRequiredCallback = null) { int counter = 0, index = 1, max = links.Length; foreach (string url in links) { await pauseToken.WaitWhilePausedAsync(); this.OnProgressStep(string.Format("Downloading file {0} of {1}...", index++, max)); string name = ComputeUniqueName(url, folder); if (skipRequiredCallback != null) { if (skipRequiredCallback(url, name)) { if (!_skippedFiles.Contains(name)) { _skippedFiles.Add(name); } counter++; continue; } } else if (_skippedFiles.Contains(name)) { counter++; continue; } else if (File.Exists(name)) { counter++; _skippedFiles.Add(name); continue; } // download the resource as a byte array var data = await _client.GetByteArrayAsync(url); _totalBytesDownloaded += data.Length; // invoke persistence policy if (saveDataCallback(data, url, name)) { counter++; } if (_cancel || cancelToken.IsCancellationRequested) { break; } } // endforeach return(counter); }
/// <summary> /// Downloads to the specified folder the provided image references using cancellation and pause tokens. /// </summary> /// <param name="cancelToken">A token used to cancel the task.</param> /// <param name="pauseToken">A token used to pause the task.</param> /// <param name="folder">The fully-qualified path of the folder in which the downloaded images will be saved. If the directory does not exist, an attempt to create it will be made.</param> /// <param name="links">An array of URLs referencing the images to download.</param> /// <returns>A task that returns an integer that represents the number of images processed (not necessarily downloaded).</returns> /// <remarks> /// <para>This method skips images that have already been downloaded and saved to the specified destination folder.</para> /// <para>It also skips images that are not compliant with the current minimum size and, eventually, minimum aspect ratio (if stricly positive).</para> /// <para>The return value of the method is an integer that indicates the number of links processed, including any skipped files.</para> /// </remarks> public virtual async Task <int> DownloadImages(CancellationToken cancelToken, PauseToken pauseToken, string folder, params string[] links) { if (NewDownload(folder, links, links.Length)) { try { int saved = 0, minW = _minImageSize.Width, minH = _minImageSize.Height; int result = await DownloadCore(cancelToken, pauseToken, folder, links, (data, url, name) => { bool processed = false; using (var img = Image.FromStream(new MemoryStream(data))) { int w = img.Width, h = img.Height; bool shouldSave = w >= minW || h >= minH; if (shouldSave && _minImageRatio > 0) { shouldSave = this.GetImageAspectRatio(w, h) >= _minImageRatio; } if (shouldSave) { try { int ioutcome = OnFileSaving(name, url, data); if (ioutcome == 0) { img.Save(name, img.RawFormat); _lastBytesSaved += data.Length; this.OnFileSaved(name, url, data); } if (ioutcome == 0 || ioutcome == 1) { processed = true; saved++; _skippedFiles.Add(name); } } catch (Exception ex) { _errors.Add(ex); } } else { processed = true; _skippedFiles.Add(name); } // endif } // endusing return(processed); }); _filesSaved += saved; return(result); } catch (AggregateException ex) { foreach (var e in ex.InnerExceptions) { if (!(e is ThreadAbortException)) { LogError(e); } } return(-3); } catch (HttpRequestException ex) { this.LogError(ex.InnerException ?? ex); return(-4); } catch (Exception ex) { this.LogError(ex); return(-1); } finally { if (_cancel) { this.OnCancelled(); } _busy = false; } } return(-2); }