public static async Task RemoveDupeFrames(string path, float threshold, string ext, bool testRun = false, bool debugLog = false, bool skipIfNoDupes = false) { Stopwatch sw = new Stopwatch(); sw.Restart(); Logger.Log("Removing duplicate frames - Threshold: " + threshold.ToString("0.00")); FileInfo[] framePaths = IOUtils.GetFileInfosSorted(path, false, "*." + ext); List <string> framesToDelete = new List <string>(); int bufferSize = await GetBufferSize(); int currentOutFrame = 1; int currentDupeCount = 0; int statsFramesKept = 0; int statsFramesDeleted = 0; int skipAfterNoDupesFrames = Config.GetInt("autoDedupFrames"); bool hasEncounteredAnyDupes = false; bool skipped = false; bool hasReachedEnd = false; string fileContent = ""; for (int i = 0; i < framePaths.Length; i++) // Loop through frames { if (hasReachedEnd) { break; } string frame1 = framePaths[i].FullName; int compareWithIndex = i + 1; while (true) // Loop dupes { //compareWithIndex++; if (compareWithIndex >= framePaths.Length) { hasReachedEnd = true; break; } if (framesToDelete.Contains(framePaths[compareWithIndex].FullName) || !File.Exists(framePaths[compareWithIndex].FullName)) { //Logger.Log($"Frame {compareWithIndex} was already deleted - skipping"); compareWithIndex++; } else { string frame2 = framePaths[compareWithIndex].FullName; float diff = GetDifference(frame1, frame2); if (diff < threshold) // Is a duped frame. { if (!testRun) { framesToDelete.Add(frame2); if (debugLog) { Logger.Log("[Deduplication] Deleted " + Path.GetFileName(frame2)); } hasEncounteredAnyDupes = true; } statsFramesDeleted++; currentDupeCount++; } else { fileContent += $"{Path.GetFileNameWithoutExtension(framePaths[i].Name)}:{currentDupeCount}\n"; statsFramesKept++; currentOutFrame++; currentDupeCount = 0; break; } } } if (sw.ElapsedMilliseconds >= 500 || (i + 1) == framePaths.Length) // Print every 0.5s (or when done) { sw.Restart(); Logger.Log($"[Deduplication] Running de-duplication ({i}/{framePaths.Length}), deleted {statsFramesDeleted} ({(((float)statsFramesDeleted / framePaths.Length) * 100f).ToString("0")}%) duplicate frames so far...", false, true); Program.mainForm.SetProgress((int)Math.Round(((float)i / framePaths.Length) * 100f)); if (imageCache.Count > bufferSize || (imageCache.Count > 50 && OSUtils.GetFreeRamMb() < 3500)) { ClearCache(); } } // int oldIndex = -1; // TODO: Compare with 1st to fix loops? // if (i >= framePaths.Length) // If this is the last frame, compare with 1st to avoid OutOfRange error // { // oldIndex = i; // i = 0; // } if (i % 3 == 0) { await Task.Delay(1); } if (Interpolate.canceled) { return; } if (!testRun && skipIfNoDupes && !hasEncounteredAnyDupes && skipAfterNoDupesFrames > 0 && i >= skipAfterNoDupesFrames) { skipped = true; break; } } foreach (string frame in framesToDelete) { IOUtils.TryDeleteIfExists(frame); } string testStr = testRun ? " [TestRun]" : ""; if (Interpolate.canceled) { return; } int framesLeft = IOUtils.GetAmountOfFiles(path, false, $"*.png"); int framesDeleted = framePaths.Length - framesLeft; float percentDeleted = ((float)framesDeleted / framePaths.Length) * 100f; string keptPercent = $"{(100f - percentDeleted).ToString("0.0")}%"; if (skipped) { Logger.Log($"[Deduplication] First {skipAfterNoDupesFrames} frames did not have any duplicates - Skipping the rest!", false, true); } else { Logger.Log($"[Deduplication]{testStr} Done. Kept {framesLeft} ({keptPercent}) frames, deleted {framesDeleted} frames.", false, true); } if (statsFramesKept <= 0) { Interpolate.Cancel("No frames were left after de-duplication!\n\nTry decreasing the de-duplication threshold."); } }