protected override async Task <ToolResult <List <ArchiveHit>, List <string> > > Process(DeduplicatorParams @params, int threads) { var factory = new TaskFactory(new LimitedConcurrencyLevelTaskScheduler(threads)); int pixelThreshold = @params.PixelThreshold; float percentDifference = @params.PercentDifference; int width = @params.Width; float aspectRatioLimit = @params.AspectRatioLimit; int delay = @params.Delay; bool skipMissing = @params.SkipMissing; var archives = Archives.Archives; var thumbnailJob = await ArchivesProvider.RegenerateThumbnails(); if (thumbnailJob is null) { return(EarlyExit(Platform.GetLocalizedString("Tools/Deduplicator/NoThumbTask/Title"), Platform.GetLocalizedString("Tools/Deduplicator/NoThumbTask/Message"))); } UpdateProgress(DeduplicatorStatus.GenerateThumbnails, archives.Count, -2, 3, 0); while (true) { await Task.Delay(1000); if ((await ServerProvider.GetMinionStatus(thumbnailJob.job))?.state?.Equals("finished") ?? true) { break; } } UpdateProgress(DeduplicatorStatus.PreloadAndDecode, archives.Count, 0, 3, 1); await Task.Delay(1000); int count = 0; var tmp = (await Task.WhenAll(archives.Select(pair => factory.StartNew(() => { int tries = 5; Image <Rgba32>?image = null; while (tries > 0) { Thread.Sleep(delay * (6 - tries)); // TODO Good ol' Thread.Sleep var bytes = Task.Run(async() => await Images.GetThumbnailCached(pair.Key)).GetAwaiter().GetResult(); if (bytes != null) { image = Image.Load(bytes); image.Mutate(i => i.Resize(width, 0)); break; } else { tries--; } } int itemCount = Interlocked.Increment(ref count); UpdateProgress(DeduplicatorStatus.PreloadAndDecode, archives.Count, itemCount); return(new Tuple <string, Image <Rgba32>?>(pair.Key, image)); })))).AsEnumerable().ToList(); List <string> removed = new List <string>(); tmp.RemoveAll(pair => { if (pair.Item2 == null) { removed.Add(pair.Item1); } return(pair.Item2 == null); }); if (removed.Count > 0 && !skipMissing) { return(EarlyExit(Platform.GetLocalizedString("Tools/Deduplicator/InvalidThumb/Title"), Platform.GetLocalizedString("Tools/Deduplicator/InvalidThumb/Message"), removed)); } var decodedThumbnails = tmp.ToDictionary(pair => pair.Item1, pair => pair.Item2); tmp.Clear(); UpdateProgress(DeduplicatorStatus.PreloadAndDecode, archives.Count, count); await Task.Delay(1000); count = 0; UpdateProgress(DeduplicatorStatus.Comparing, decodedThumbnails.Count, 0, 3, 2); await Task.Delay(1000); var markedNonDuplicated = Settings.Profile.MarkedAsNonDuplicated; var hits = new ConcurrentBag <ArchiveHit>(); int maxItems = decodedThumbnails.Count; await Task.Run(async() => { while (decodedThumbnails.Count != 0) { var start = DateTime.Now; var sourcePair = decodedThumbnails.First(); decodedThumbnails.Remove(sourcePair.Key); using (var source = sourcePair.Value) { await Task.WhenAll(decodedThumbnails.Select(targetPair => factory.StartNew(() => { var target = targetPair.Value; if (Math.Abs((float)source !.Height / source.Width - (float)target !.Height / target.Width) > aspectRatioLimit) { return; } var hit = new ArchiveHit { Left = sourcePair.Key, Right = targetPair.Key }; if (markedNonDuplicated.Contains(hit)) { return; } int differences = 0; for (int y = 0; y < Math.Min(source.Height, target.Height); y++) { Span <Rgba32> sourcePixelRow = source.GetPixelRowSpan(y); Span <Rgba32> targetPixelRow = target.GetPixelRowSpan(y); for (int x = 0; x < source.Width; x++) { float diff = GetManhattanDistanceInRgbSpace(ref sourcePixelRow[x], ref targetPixelRow[x]) / 765f; //255+255+255 if (diff > pixelThreshold / 765f) { differences++; } } } float diffPixels = differences; diffPixels /= source.Width *source.Height; if (diffPixels < percentDifference) { hits.Add(hit); } }))); } int itemCount = Interlocked.Increment(ref count); var delta = DateTime.Now.Subtract(start).Ticks; long time = (maxItems - itemCount) * delta; UpdateProgress(DeduplicatorStatus.Comparing, maxItems, itemCount, time: time); } }); UpdateProgress(DeduplicatorStatus.Comparing, maxItems, count, time: 0); await Task.Delay(1000); UpdateProgress(DeduplicatorStatus.Completed, 0, 0, 3, 3); await Task.Delay(1000); UpdateProgress(DeduplicatorStatus.Completed, 0, 0, 0, 0); return(new ToolResult <List <ArchiveHit>, List <string> > { Data = hits.ToList(), Ok = true, Error = removed.Count > 0 ? removed : null }); }