protected override async Task <ToolResult <List <ArchiveHit>, List <string> > > Process(DeduplicatorParams @params, int threads)
{
var factory = new TaskFactory(new LimitedConcurrencyLevelTaskScheduler(threads));
int pixelThreshold = @params.PixelThreshold;
float percentDifference = @params.PercentDifference;
int width = @params.Width;
float aspectRatioLimit = @params.AspectRatioLimit;
int delay = @params.Delay;
bool skipMissing = @params.SkipMissing;
var archives = Archives.Archives;
var thumbnailJob = await ArchivesProvider.RegenerateThumbnails();
if (thumbnailJob is null)
{
return(EarlyExit(Platform.GetLocalizedString("Tools/Deduplicator/NoThumbTask/Title"), Platform.GetLocalizedString("Tools/Deduplicator/NoThumbTask/Message")));
}
UpdateProgress(DeduplicatorStatus.GenerateThumbnails, archives.Count, -2, 3, 0);
while (true)
{
await Task.Delay(1000);
if ((await ServerProvider.GetMinionStatus(thumbnailJob.job))?.state?.Equals("finished") ?? true)
{
break;
}
}
UpdateProgress(DeduplicatorStatus.PreloadAndDecode, archives.Count, 0, 3, 1);
await Task.Delay(1000);
int count = 0;
var tmp = (await Task.WhenAll(archives.Select(pair => factory.StartNew(() =>
{
int tries = 5;
Image <Rgba32>?image = null;
while (tries > 0)
{
Thread.Sleep(delay * (6 - tries)); // TODO Good ol' Thread.Sleep
var bytes = Task.Run(async() => await Images.GetThumbnailCached(pair.Key)).GetAwaiter().GetResult();
if (bytes != null)
{
image = Image.Load(bytes);
image.Mutate(i => i.Resize(width, 0));
break;
}
else
{
tries--;
}
}
int itemCount = Interlocked.Increment(ref count);
UpdateProgress(DeduplicatorStatus.PreloadAndDecode, archives.Count, itemCount);
return(new Tuple <string, Image <Rgba32>?>(pair.Key, image));
})))).AsEnumerable().ToList();
List <string> removed = new List <string>();
tmp.RemoveAll(pair =>
{
if (pair.Item2 == null)
{
removed.Add(pair.Item1);
}
return(pair.Item2 == null);
});
if (removed.Count > 0 && !skipMissing)
{
return(EarlyExit(Platform.GetLocalizedString("Tools/Deduplicator/InvalidThumb/Title"), Platform.GetLocalizedString("Tools/Deduplicator/InvalidThumb/Message"), removed));
}
var decodedThumbnails = tmp.ToDictionary(pair => pair.Item1, pair => pair.Item2);
tmp.Clear();
UpdateProgress(DeduplicatorStatus.PreloadAndDecode, archives.Count, count);
await Task.Delay(1000);
count = 0;
UpdateProgress(DeduplicatorStatus.Comparing, decodedThumbnails.Count, 0, 3, 2);
await Task.Delay(1000);
var markedNonDuplicated = Settings.Profile.MarkedAsNonDuplicated;
var hits = new ConcurrentBag <ArchiveHit>();
int maxItems = decodedThumbnails.Count;
await Task.Run(async() =>
{
while (decodedThumbnails.Count != 0)
{
var start = DateTime.Now;
var sourcePair = decodedThumbnails.First();
decodedThumbnails.Remove(sourcePair.Key);
using (var source = sourcePair.Value)
{
await Task.WhenAll(decodedThumbnails.Select(targetPair => factory.StartNew(() =>
{
var target = targetPair.Value;
if (Math.Abs((float)source !.Height / source.Width - (float)target !.Height / target.Width) > aspectRatioLimit)
{
return;
}
var hit = new ArchiveHit {
Left = sourcePair.Key, Right = targetPair.Key
};
if (markedNonDuplicated.Contains(hit))
{
return;
}
int differences = 0;
for (int y = 0; y < Math.Min(source.Height, target.Height); y++)
{
Span <Rgba32> sourcePixelRow = source.GetPixelRowSpan(y);
Span <Rgba32> targetPixelRow = target.GetPixelRowSpan(y);
for (int x = 0; x < source.Width; x++)
{
float diff = GetManhattanDistanceInRgbSpace(ref sourcePixelRow[x], ref targetPixelRow[x]) / 765f; //255+255+255
if (diff > pixelThreshold / 765f)
{
differences++;
}
}
}
float diffPixels = differences;
diffPixels /= source.Width *source.Height;
if (diffPixels < percentDifference)
{
hits.Add(hit);
}
})));
}
int itemCount = Interlocked.Increment(ref count);
var delta = DateTime.Now.Subtract(start).Ticks;
long time = (maxItems - itemCount) * delta;
UpdateProgress(DeduplicatorStatus.Comparing, maxItems, itemCount, time: time);
}
});
UpdateProgress(DeduplicatorStatus.Comparing, maxItems, count, time: 0);
await Task.Delay(1000);
UpdateProgress(DeduplicatorStatus.Completed, 0, 0, 3, 3);
await Task.Delay(1000);
UpdateProgress(DeduplicatorStatus.Completed, 0, 0, 0, 0);
return(new ToolResult <List <ArchiveHit>, List <string> > {
Data = hits.ToList(), Ok = true, Error = removed.Count > 0 ? removed : null
});
}