private async Task ProcessQueueAsync()
{
isBusy = true;
try
{
var nextWorkerId = 0;
var locker = new KeyedSemaphore();
var counter = new ExtractCounter();
var start = DateTime.Now;
//note that multiple workers can result in the working status (status bar text)
//being inaccurate - each tag will only set the status once so if a slow tag
//sets the status and a fast tag replaces it then the status will not update
//back to the slow tag after the fast one finishes.
Func <Task> process = async() =>
{
var prefix = Settings.BatchWorkerCount > 1 ? $"[Worker {nextWorkerId++}] " : string.Empty;
while (extractionQueue.Count > 0)
{
if (tokenSource.IsCancellationRequested)
{
break;
}
IExtractable item;
if (!extractionQueue.TryDequeue(out item))
{
break;
}
using (await locker.WaitAsync(item.ItemKey))
{
SetWorkingStatus($"{prefix}Extracting {item.DisplayName}");
Extract(item, counter);
}
}
};
var processors = Enumerable.Range(0, Settings.BatchWorkerCount).Select(i => Task.Run(process)).ToList();
await Task.WhenAll(processors);
var span = DateTime.Now - start;
LogOutput($"Extracted {counter.Extracted} tags in {Math.Round(span.TotalSeconds)} seconds with {counter.Errors} errors.");
}
catch (Exception ex)
{
Substrate.LogError("Error during batch extraction", ex);
}
finally
{
tokenSource.Dispose();
tokenSource = null;
}
isBusy = false;
ClearWorkingStatus();
}
public async Task <Size> GetImageSizeCached(string path)
{
if (string.IsNullOrEmpty(path))
{
return(new Size(0, 0));
}
using (var key = await KeyedSemaphore.LockAsync(path + "size"))
{
Size size;
if (imagesSizeCache.TryGet(path, out size))
{
return(size);
}
else
{
var image = await GetImageCached(path);
if (image == null)
{
return(Size.Empty);
}
size = await ImageProcessing.GetImageSize(image);
imagesSizeCache.AddReplace(path, size);
return(size);
}
}
}
public async Task <byte[]?> GetImageCached(string path, bool forced = false)
{
if (string.IsNullOrEmpty(path))
{
return(null);
}
using (var key = await KeyedSemaphore.LockAsync(path))
{
byte[]? image;
if (imagesCache.TryGet(path, out image) && !forced)
{
return(image);
}
else
{
image = await ArchivesProvider.GetImage(path);
if (image == null)
{
return(null);
}
imagesCache.AddReplace(path, image);
return(image);
}
}
}
static async Task SayHelloWorldAsync(int i)
{
string key = "Key" + Math.Ceiling((double)i / 2);
Log($"Task {i:0}: I am waiting for key '{key}'");
using (await KeyedSemaphore.LockAsync(key))
{
Log($"Task {i:0}: Hello world! I have key '{key}' now!");
await Task.Delay(50);
}
Log($"Task {i:0}: I have released '{key}'");
}
public void ShouldRunThreadsWithDistinctKeysInParallel()
{
// Arrange
var currentParallelism = 0;
var maxParallelism = 0;
var parallelismLock = new object();
// 100 threads, 100 keys
var threads = Enumerable.Range(0, 100)
.Select(i => new Thread(() => OccupyTheLockALittleBit(i)))
.ToList();
// Act
foreach (var thread in threads)
{
thread.Start();
}
foreach (var thread in threads)
{
thread.Join();
}
maxParallelism.Should().BeGreaterThan(10);
void OccupyTheLockALittleBit(int key)
{
using (KeyedSemaphore.Lock(key.ToString()))
{
var incrementedCurrentParallelism = Interlocked.Increment(ref currentParallelism);
lock (parallelismLock)
{
maxParallelism = Math.Max(incrementedCurrentParallelism, maxParallelism);
}
const int delay = 250;
Thread.Sleep(TimeSpan.FromMilliseconds(delay));
Interlocked.Decrement(ref currentParallelism);
}
}
}
public async Task ShouldRunThreadsWithDistinctKeysInParallel()
{
// Arrange
var currentParallelism = 0;
var maxParallelism = 0;
var parallelismLock = new object();
// 100 threads, 100 keys
var threads = Enumerable.Range(0, 100)
.Select(i => Task.Run(async() => await OccupyTheLockALittleBit(i).ConfigureAwait(false)))
.ToList();
// Act
await Task.WhenAll(threads).ConfigureAwait(false);
maxParallelism.Should().BeGreaterThan(10);
async Task OccupyTheLockALittleBit(int key)
{
using (await KeyedSemaphore.LockAsync(key.ToString()))
{
var incrementedCurrentParallelism = Interlocked.Increment(ref currentParallelism);
lock (parallelismLock)
{
maxParallelism = Math.Max(incrementedCurrentParallelism, maxParallelism);
}
const int delay = 250;
await Task.Delay(TimeSpan.FromMilliseconds(delay)).ConfigureAwait(false);
Interlocked.Decrement(ref currentParallelism);
}
}
}
private async Task OrganizeFileAsync(FileInfo file, DicomOrganizerOptions options, CancellationToken cancellationToken)
{
var directory = options.Directory;
var pattern = options.Pattern;
var action = options.Action;
DicomFile dicomFile;
try
{
await using var fileStream = new FileStream(file.FullName, FileMode.Open, FileAccess.Read, FileShare.Read, 4096);
dicomFile = await DicomFile.OpenAsync(fileStream, FileReadOption.SkipLargeTags);
}
catch (DicomFileException e)
{
throw new DicomOrganizeException("Not a DICOM file: " + file.FullName, e);
}
string fileName;
try
{
fileName = _patternApplier.Apply(dicomFile.Dataset, pattern);
}
catch (PatternException e)
{
throw new DicomOrganizeException($"Failed to apply pattern to file {file}", e);
}
var targetFile = new FileInfo(Path.Join(directory.FullName, fileName));
if (!targetFile.Directory !.Exists)
{
var highestDirectoryName = HighestDirectoryNameDeterminer.Determine(fileName !);
using (await KeyedSemaphore.LockAsync(highestDirectoryName, cancellationToken))
{
try
{
if (!targetFile.Directory.Exists)
{
var directoryToCreate = targetFile.Directory !;
_ioPolicy.Execute(() => directoryToCreate.Create());
}
}
catch (IOException exception)
{
throw new DicomOrganizeException($"Failed to create directory {targetFile.Directory.FullName}", exception);
}
}
}
if (file.FullName == targetFile.FullName)
{
_logger.WriteLine(targetFile.FullName);
return;
}
if (targetFile.Exists)
{
var counter = 1;
var targetFileName = targetFile.Name;
var targetFileDirectoryName = targetFile.Directory.FullName;
var targetFileExtension = targetFile.Extension;
var targetFileNameWithoutExtension = targetFileName.Substring(0, targetFileName.Length - targetFileExtension.Length);
while (targetFile.Exists)
{
targetFileName = $"{targetFileNameWithoutExtension} ({counter++}){targetFile.Extension}";
targetFile = new FileInfo(Path.Join(targetFileDirectoryName, targetFileName));
if (file.FullName == targetFile.FullName)
{
_logger.WriteLine(targetFile.FullName);
return;
}
}
}
try
{
switch (action)
{
case Action.Move:
{
_ioPolicy.Execute(() => File.Move(file.FullName, targetFile.FullName));
_logger.WriteLine(targetFile.FullName);
break;
}
case Action.Copy:
{
_ioPolicy.Execute(() => File.Copy(file.FullName, targetFile.FullName));
_logger.WriteLine(targetFile.FullName);
break;
}
}
}
catch (IOException e)
{
_errorHandler.Handle(new DicomOrganizeException($"Failed to {action.ToString().ToLowerInvariant()} {file}", e));
}
}
public async Task <byte[]?> GetThumbnailCached(string id, int page = 0, bool forced = false, bool ignoreCache = false)
{
if (string.IsNullOrEmpty(id))
{
return(null);
}
var thumbKey = $"{id}.{page}";
if (ignoreCache)
{
return(await GetThumbnailRaw(id, page));
}
using (var key = await KeyedSemaphore.LockAsync(thumbKey))
{
byte[]? data;
if (thumbnailsCache.TryGet(thumbKey, out data) && !forced)
{
return(data);
}
else
{
var path = $"{thumbnailCacheDirectory.FullName}/{id.Substring(0, 2)}/{id}/{page}.cache";
if (File.Exists(path) && !forced)
{
data = await Files.GetFileBytes(path);
if (data.Length == 55876)
{
using (var md5 = System.Security.Cryptography.MD5.Create())
if (NoThumbHash.Equals(string.Concat(md5.ComputeHash(data).Select(x => x.ToString("X2")))))
{
File.Delete(path);
data = await GetThumbnailRaw(id, page);
if (data == null)
{
return(null);
}
Directory.CreateDirectory(Path.GetDirectoryName(path));
await Files.StoreFile(path, data);
}
}
}
else
{
data = await GetThumbnailRaw(id, page);
if (data == null)
{
return(null);
}
if (data.Length == 55876)
{
using (var md5 = System.Security.Cryptography.MD5.Create())
if (NoThumbHash.Equals(string.Concat(md5.ComputeHash(data).Select(x => x.ToString("X2")))))
{
return(null);
}
}
Directory.CreateDirectory(Path.GetDirectoryName(path));
await Files.StoreFile(path, data);
}
thumbnailsCache.AddReplace(thumbKey, data);
return(data);
}
}
}
public async Task ShouldRunThreadsWithSameKeysLinearly()
{
// Arrange
var runningTasksIndex = new ConcurrentDictionary <int, int>();
var parallelismLock = new object();
var currentParallelism = 0;
var maxParallelism = 0;
// 100 threads, 10 keys
var threads = Enumerable.Range(0, 100)
.Select(i => Task.Run(async() => await OccupyTheLockALittleBit(i % 10).ConfigureAwait(false)))
.ToList();
// Act + Assert
await Task.WhenAll(threads).ConfigureAwait(false);
maxParallelism.Should().BeLessOrEqualTo(10);
async Task OccupyTheLockALittleBit(int key)
{
using (await KeyedSemaphore.LockAsync(key.ToString()))
{
var incrementedCurrentParallelism = Interlocked.Increment(ref currentParallelism);
lock (parallelismLock)
{
maxParallelism = Math.Max(incrementedCurrentParallelism, maxParallelism);
}
var currentTaskId = Task.CurrentId ?? -1;
if (runningTasksIndex.TryGetValue(key, out var otherThread))
{
throw new Exception($"Thread #{currentTaskId} acquired a lock using key ${key} " +
$"but another thread #{otherThread} is also still running using this key!");
}
runningTasksIndex[key] = currentTaskId;
const int delay = 10;
await Task.Delay(TimeSpan.FromMilliseconds(delay)).ConfigureAwait(false);
if (!runningTasksIndex.TryRemove(key, out var value))
{
var ex = new Exception($"Thread #{currentTaskId} has finished " +
$"but when trying to cleanup the running threads index, the value is already gone");
throw ex;
}
if (value != currentTaskId)
{
var ex = new Exception($"Thread #{currentTaskId} has finished and has removed itself from the running threads index," +
$" but that index contained an incorrect value: #{value}!");
throw ex;
}
Interlocked.Decrement(ref currentParallelism);
}
}
}
public void ShouldRunThreadsWithSameKeysLinearly()
{
// Arrange
var runningThreadsIndex = new ConcurrentDictionary <int, int>();
var parallelismLock = new object();
var currentParallelism = 0;
var maxParallelism = 0;
// 100 threads, 10 keys
var threads = Enumerable.Range(0, 100)
.Select(i => new Thread(() => OccupyTheLockALittleBit(i % 10)))
.ToList();
// Act
foreach (var thread in threads)
{
thread.Start();
}
foreach (var thread in threads)
{
thread.Join();
}
// Assert
maxParallelism.Should().BeLessOrEqualTo(10);
void OccupyTheLockALittleBit(int key)
{
using (KeyedSemaphore.Lock(key.ToString()))
{
var incrementedCurrentParallelism = Interlocked.Increment(ref currentParallelism);
lock (parallelismLock)
{
maxParallelism = Math.Max(incrementedCurrentParallelism, maxParallelism);
}
var currentThreadId = Thread.CurrentThread.ManagedThreadId;
if (runningThreadsIndex.TryGetValue(key, out var otherThread))
{
throw new Exception($"Thread #{currentThreadId} acquired a lock using key ${key} " +
$"but another thread #{otherThread} is also still running using this key!");
}
runningThreadsIndex[key] = currentThreadId;
const int delay = 10;
Thread.Sleep(TimeSpan.FromMilliseconds(delay));
if (!runningThreadsIndex.TryRemove(key, out var value))
{
var ex = new Exception($"Thread #{currentThreadId} has finished " +
$"but when trying to cleanup the running threads index, the value is already gone");
throw ex;
}
if (value != currentThreadId)
{
var ex = new Exception($"Thread #{currentThreadId} has finished and has removed itself from the running threads index," +
$" but that index contained an incorrect value: #{value}!");
throw ex;
}
Interlocked.Decrement(ref currentParallelism);
}
}
}
