/// <inheritdoc />
public SynchronizedProducerQueue(ThreadQueueMode mode, [NotNull] ProducerConsumerQueueOptions <T> options, CancellationToken token = default(CancellationToken))
{
_queue = ProducerConsumerQueue.Create(mode, options, token);
_producerQueue = _queue as IProducerQueue <TQueue, T> ?? throw new NotSupportedException();
}
static async Task <IDictionary <Genders, IDictionary <string, string> > > DownloadUserImages(IList <User> users, IConfiguration configuration, IHostEnvironment environment, ILogger logger)
{
if (users.Count == 0)
{
return(null);
}
string imagesUrl = UriHelper.ToUri(configuration.GetValue <string>("images:users:url"), UriKind.Relative).String() ?? IMAGES_FOLDER_DEF;
string imagesPath = PathHelper.Trim(environment.ContentRootPath);
if (string.IsNullOrEmpty(imagesPath) || !Directory.Exists(imagesPath))
{
imagesPath = Directory.GetCurrentDirectory();
}
imagesPath = Path.Combine(imagesPath, imagesUrl.Replace('/', '\\').TrimStart('\\'));
logger?.LogInformation($"Initialized images directory as '{imagesPath}'.");
if (!DirectoryHelper.Ensure(imagesPath))
{
logger?.LogError($"Failed to create images directory '{imagesPath}'.");
return(null);
}
IDictionary <string, string> femalesNeeded = new Dictionary <string, string>(StringComparer.OrdinalIgnoreCase);
IDictionary <string, string> malesNeeded = new Dictionary <string, string>(StringComparer.OrdinalIgnoreCase);
IDictionary <string, string> females = new Dictionary <string, string>(StringComparer.OrdinalIgnoreCase);
IDictionary <string, string> males = new Dictionary <string, string>(StringComparer.OrdinalIgnoreCase);
string femalePattern = $"{IMAGES_PREFIX_FEMALE}??.jpg";
string malePattern = $"{IMAGES_PREFIX_MALE}??.jpg";
foreach (User user in users.Where(e => e.Gender == Genders.Female || e.Gender == Genders.Male))
{
string pattern;
IDictionary <string, string> needed, queue;
if (user.Gender == Genders.Female)
{
pattern = femalePattern;
needed = femalesNeeded;
queue = females;
}
else
{
pattern = malePattern;
needed = malesNeeded;
queue = males;
}
string path = Path.Combine(imagesPath, user.Id);
if (!Directory.Exists(path))
{
needed[user.Id] = null;
continue;
}
string file = Directory.EnumerateFiles(path, pattern, SearchOption.TopDirectoryOnly)
.FirstOrDefault();
if (string.IsNullOrEmpty(file))
{
needed[user.Id] = null;
continue;
}
queue[user.Id] = file;
}
IDictionary <Genders, IDictionary <string, string> > result = new Dictionary <Genders, IDictionary <string, string> >
{
[Genders.Female] = females,
[Genders.Male] = males
};
if (femalesNeeded.Count == 0 && malesNeeded.Count == 0)
{
return(result);
}
logger?.LogInformation($"Will download {femalesNeeded.Count} female images and {malesNeeded.Count} male images.");
Regex regex = new Regex("auto_[fm]_(?<x>\\d+)\\.jpg$", RegexHelper.OPTIONS_I);
HashSet <int> usedFemales = new HashSet <int>(females.Select(e =>
{
Match match = regex.Match(e.Value);
return(!match.Success
? -1
: int.Parse(match.Groups["x"].Value));
}).Where(e => e > -1));
HashSet <int> usedMales = new HashSet <int>(males.Select(e =>
{
Match match = regex.Match(e.Value);
return(!match.Success
? -1
: int.Parse(match.Groups["x"].Value));
}).Where(e => e > -1));
// download will timeout in x minutes where x is a number between 0 and 10 from the configuration
int timeout = configuration.GetValue("images:users:downloadTimeout", 5).Within(0, 10);
// use multi-thread to download the images
using (CancellationTokenSource cts = timeout > 0 ? new CancellationTokenSource(TimeSpan.FromMinutes(timeout)) : null)
{
CancellationToken token = cts?.Token ?? CancellationToken.None;
IOHttpDownloadFileWebRequestSettings downloadSettings = new IOHttpDownloadFileWebRequestSettings
{
BufferSize = Constants.BUFFER_256_KB,
Overwrite = true,
Timeout = TimeSpan.FromSeconds(configuration.GetValue("images:users:requestTimeout", 30).Within(0, 180)).TotalIntMilliseconds()
};
#if DEBUG
int threads = configuration.GetValue <bool>("limitThreads")
? 1
: TaskHelper.ProcessMaximum;
#else
int threads = TaskHelper.ProcessMaximum;
#endif
ProducerConsumerThreadQueueOptions <PersonDownloadData> options = new ProducerConsumerThreadQueueOptions <PersonDownloadData>(threads, (_, pdd) =>
{
// copy to local vars for threading issues
IDictionary <string, string> queue = result[pdd.Gender];
return(DownloadUserImage(imagesPath, pdd, queue, downloadSettings, logger, token));
})
{
WorkStartedCallback = _ => logger?.LogInformation($"Download started using {threads} threads..."),
WorkCompletedCallback = _ => logger?.LogInformation("Download completed.")
};
using (IProducerConsumer <PersonDownloadData> requests = ProducerConsumerQueue.Create(ThreadQueueMode.Task, options, token))
{
int number;
foreach (string id in femalesNeeded.Keys)
{
do
{
number = RNGRandomHelper.Next(0, 99);
}while (usedFemales.Contains(number));
usedFemales.Add(number);
requests.Enqueue(new PersonDownloadData
{
Id = id,
Gender = Genders.Female,
Number = number
});
}
foreach (string id in malesNeeded.Keys)
{
do
{
number = RNGRandomHelper.Next(0, 99);
}while (usedMales.Contains(number));
usedMales.Add(number);
requests.Enqueue(new PersonDownloadData
{
Id = id,
Gender = Genders.Male,
Number = number
});
}
requests.Complete();
await requests.WaitAsync();
}
}
return(result);
}
