public static async Task RemoveDupeFrames(string path, float threshold, string ext, bool testRun = false, bool debugLog = false, bool skipIfNoDupes = false)
{
Stopwatch sw = new Stopwatch();
sw.Restart();
Logger.Log("Removing duplicate frames - Threshold: " + threshold.ToString("0.00"));
FileInfo[] framePaths = IoUtils.GetFileInfosSorted(path, false, "*." + ext);
List <string> framesToDelete = new List <string>();
int bufferSize = await GetBufferSize();
int currentOutFrame = 1;
int currentDupeCount = 0;
int statsFramesKept = 0;
int statsFramesDeleted = 0;
bool hasReachedEnd = false;
string fileContent = "";
for (int i = 0; i < framePaths.Length; i++) // Loop through frames
{
if (hasReachedEnd)
{
break;
}
string frame1 = framePaths[i].FullName;
int compareWithIndex = i + 1;
while (true) // Loop dupes
{
//compareWithIndex++;
if (compareWithIndex >= framePaths.Length)
{
hasReachedEnd = true;
break;
}
if (framesToDelete.Contains(framePaths[compareWithIndex].FullName) || !File.Exists(framePaths[compareWithIndex].FullName))
{
//Logger.Log($"Frame {compareWithIndex} was already deleted - skipping");
compareWithIndex++;
}
else
{
string frame2 = framePaths[compareWithIndex].FullName;
float diff = GetDifference(frame1, frame2);
if (diff < threshold) // Is a duped frame.
{
if (!testRun)
{
framesToDelete.Add(frame2);
if (debugLog)
{
Logger.Log("Deduplication: Deleted " + Path.GetFileName(frame2));
}
}
statsFramesDeleted++;
currentDupeCount++;
}
else
{
fileContent += $"{Path.GetFileNameWithoutExtension(framePaths[i].Name)}:{currentDupeCount}\n";
statsFramesKept++;
currentOutFrame++;
currentDupeCount = 0;
break;
}
}
}
if (sw.ElapsedMilliseconds >= 500 || (i + 1) == framePaths.Length) // Print every 0.5s (or when done)
{
sw.Restart();
Logger.Log($"Deduplication: Running de-duplication ({i}/{framePaths.Length}), deleted {statsFramesDeleted} ({(((float)statsFramesDeleted / framePaths.Length) * 100f).ToString("0")}%) duplicate frames so far...", false, true);
Program.mainForm.SetProgress((int)Math.Round(((float)i / framePaths.Length) * 100f));
if (imageCache.Count > bufferSize || (imageCache.Count > 50 && OsUtils.GetFreeRamMb() < 3500))
{
ClearCache();
}
}
// int oldIndex = -1; // TODO: Compare with 1st to fix loops?
// if (i >= framePaths.Length) // If this is the last frame, compare with 1st to avoid OutOfRange error
// {
// oldIndex = i;
// i = 0;
// }
if (i % 3 == 0)
{
await Task.Delay(1);
}
if (Interpolate.canceled)
{
return;
}
}
foreach (string frame in framesToDelete)
{
IoUtils.TryDeleteIfExists(frame);
}
string testStr = testRun ? " [TestRun]" : "";
if (Interpolate.canceled)
{
return;
}
int framesLeft = IoUtils.GetAmountOfFiles(path, false, "*" + Interpolate.current.framesExt);
int framesDeleted = framePaths.Length - framesLeft;
float percentDeleted = ((float)framesDeleted / framePaths.Length) * 100f;
string keptPercent = $"{(100f - percentDeleted).ToString("0.0")}%";
Logger.Log($"[Deduplication]{testStr} Done. Kept {framesLeft} ({keptPercent}) frames, deleted {framesDeleted} frames.", false, true);
if (statsFramesKept <= 0)
{
Interpolate.Cancel("No frames were left after de-duplication!\n\nTry decreasing the de-duplication threshold.");
}
}