static TaskResult DownloadUserImage(string sourcePath, PersonDownloadData downloadData, IDictionary <string, string> files, IOHttpDownloadFileWebRequestSettings settings, ILogger logger, CancellationToken token) { if (token.IsCancellationRequested) { return(TaskResult.Canceled); } string url = string.Format(IMAGES_URL, downloadData.Gender == Genders.Female ? IMAGES_GENDER_FEMALE : IMAGES_GENDER_MALE, downloadData.Number); FileStream stream = null; string fileName = Path.Combine(sourcePath, downloadData.Id, $"{(downloadData.Gender == Genders.Female ? IMAGES_PREFIX_FEMALE : IMAGES_PREFIX_MALE)}{downloadData.Number:D2}.jpg"); try { // since this is already a thread/task, no need to use any async version. // This will automatically retry to download the file. I love my library!! stream = UriHelper.DownloadFile(url, fileName, settings); if (token.IsCancellationRequested) { return(TaskResult.Canceled); } files[downloadData.Id] = Path.GetFileName(fileName); return(TaskResult.Success); } catch (Exception e) { logger?.LogError(e.CollectMessages()); return(TaskResult.Error); } finally { ObjectHelper.Dispose(stream); } }
static async Task <IDictionary <Genders, IDictionary <string, string> > > DownloadUserImages(IList <User> users, IConfiguration configuration, IHostEnvironment environment, ILogger logger) { if (users.Count == 0) { return(null); } string imagesUrl = UriHelper.ToUri(configuration.GetValue <string>("images:users:url"), UriKind.Relative).String() ?? IMAGES_FOLDER_DEF; string imagesPath = PathHelper.Trim(environment.ContentRootPath); if (string.IsNullOrEmpty(imagesPath) || !Directory.Exists(imagesPath)) { imagesPath = Directory.GetCurrentDirectory(); } imagesPath = Path.Combine(imagesPath, imagesUrl.Replace('/', '\\').TrimStart('\\')); logger?.LogInformation($"Initialized images directory as '{imagesPath}'."); if (!DirectoryHelper.Ensure(imagesPath)) { logger?.LogError($"Failed to create images directory '{imagesPath}'."); return(null); } IDictionary <string, string> femalesNeeded = new Dictionary <string, string>(StringComparer.OrdinalIgnoreCase); IDictionary <string, string> malesNeeded = new Dictionary <string, string>(StringComparer.OrdinalIgnoreCase); IDictionary <string, string> females = new Dictionary <string, string>(StringComparer.OrdinalIgnoreCase); IDictionary <string, string> males = new Dictionary <string, string>(StringComparer.OrdinalIgnoreCase); string femalePattern = $"{IMAGES_PREFIX_FEMALE}??.jpg"; string malePattern = $"{IMAGES_PREFIX_MALE}??.jpg"; foreach (User user in users.Where(e => e.Gender == Genders.Female || e.Gender == Genders.Male)) { string pattern; IDictionary <string, string> needed, queue; if (user.Gender == Genders.Female) { pattern = femalePattern; needed = femalesNeeded; queue = females; } else { pattern = malePattern; needed = malesNeeded; queue = males; } string path = Path.Combine(imagesPath, user.Id); if (!Directory.Exists(path)) { needed[user.Id] = null; continue; } string file = Directory.EnumerateFiles(path, pattern, SearchOption.TopDirectoryOnly) .FirstOrDefault(); if (string.IsNullOrEmpty(file)) { needed[user.Id] = null; continue; } queue[user.Id] = file; } IDictionary <Genders, IDictionary <string, string> > result = new Dictionary <Genders, IDictionary <string, string> > { [Genders.Female] = females, [Genders.Male] = males }; if (femalesNeeded.Count == 0 && malesNeeded.Count == 0) { return(result); } logger?.LogInformation($"Will download {femalesNeeded.Count} female images and {malesNeeded.Count} male images."); Regex regex = new Regex("auto_[fm]_(?<x>\\d+)\\.jpg$", RegexHelper.OPTIONS_I); HashSet <int> usedFemales = new HashSet <int>(females.Select(e => { Match match = regex.Match(e.Value); return(!match.Success ? -1 : int.Parse(match.Groups["x"].Value)); }).Where(e => e > -1)); HashSet <int> usedMales = new HashSet <int>(males.Select(e => { Match match = regex.Match(e.Value); return(!match.Success ? -1 : int.Parse(match.Groups["x"].Value)); }).Where(e => e > -1)); // download will timeout in x minutes where x is a number between 0 and 10 from the configuration int timeout = configuration.GetValue("images:users:downloadTimeout", 5).Within(0, 10); // use multi-thread to download the images using (CancellationTokenSource cts = timeout > 0 ? new CancellationTokenSource(TimeSpan.FromMinutes(timeout)) : null) { CancellationToken token = cts?.Token ?? CancellationToken.None; IOHttpDownloadFileWebRequestSettings downloadSettings = new IOHttpDownloadFileWebRequestSettings { BufferSize = Constants.BUFFER_256_KB, Overwrite = true, Timeout = TimeSpan.FromSeconds(configuration.GetValue("images:users:requestTimeout", 30).Within(0, 180)).TotalIntMilliseconds() }; #if DEBUG int threads = configuration.GetValue <bool>("limitThreads") ? 1 : TaskHelper.ProcessMaximum; #else int threads = TaskHelper.ProcessMaximum; #endif ProducerConsumerThreadQueueOptions <PersonDownloadData> options = new ProducerConsumerThreadQueueOptions <PersonDownloadData>(threads, (_, pdd) => { // copy to local vars for threading issues IDictionary <string, string> queue = result[pdd.Gender]; return(DownloadUserImage(imagesPath, pdd, queue, downloadSettings, logger, token)); }) { WorkStartedCallback = _ => logger?.LogInformation($"Download started using {threads} threads..."), WorkCompletedCallback = _ => logger?.LogInformation("Download completed.") }; using (IProducerConsumer <PersonDownloadData> requests = ProducerConsumerQueue.Create(ThreadQueueMode.Task, options, token)) { int number; foreach (string id in femalesNeeded.Keys) { do { number = RNGRandomHelper.Next(0, 99); }while (usedFemales.Contains(number)); usedFemales.Add(number); requests.Enqueue(new PersonDownloadData { Id = id, Gender = Genders.Female, Number = number }); } foreach (string id in malesNeeded.Keys) { do { number = RNGRandomHelper.Next(0, 99); }while (usedMales.Contains(number)); usedMales.Add(number); requests.Enqueue(new PersonDownloadData { Id = id, Gender = Genders.Male, Number = number }); } requests.Complete(); await requests.WaitAsync(); } } return(result); }