private void InternalSearch(CancellationToken cancelToken, PauseTokenSource pauseTokenSource)
{
ElapsedTimer.Start();
SearchSW.Start();
var duplicateDict = new Dictionary <string, DuplicateItem>();
try {
var parallelOpts = new ParallelOptions {
MaxDegreeOfParallelism = Environment.ProcessorCount,
CancellationToken = cancelToken
};
var reScanList = ScanFileList
.Where(vf => !vf.Flags.Any(EntryFlags.ManuallyExcluded | EntryFlags.AllErrors))
.Where(vf => (vf.mediaInfo == null && !vf.IsImage) || vf.grayBytes == null || vf.grayBytes.Count != Settings.ThumbnailCount)
.ToList();
InitProgress(reScanList.Count);
Parallel.For(0, reScanList.Count, parallelOpts, i => {
while (pauseTokenSource.IsPaused)
{
Thread.Sleep(50);
}
var entry = reScanList[i];
if (entry.mediaInfo == null && !entry.IsImage)
{
var ffProbe = new FFProbeWrapper.FFProbeWrapper();
var info = ffProbe.GetMediaInfo(entry.Path);
if (info == null)
{
entry.Flags.Set(EntryFlags.MetadataError);
return;
}
entry.mediaInfo = info;
}
if (entry.grayBytes == null)
{
var(error, grayBytes) = entry.IsImage ? GetImageAsBitmaps(entry, positionList.Count) : GetVideoThumbnailAsBitmaps(entry, positionList);
if (error > 0)
{
entry.Flags.Set(error);
}
else
{
entry.grayBytes = grayBytes;
}
}
IncrementProgress(entry.Path);
});
SearchSW.Stop();
Logger.Instance.Info(string.Format(Properties.Resources.ThumbnailsFinished, SearchSW.Elapsed, processedFiles));
SearchSW.Restart();
var percentageDifference = 1.0f - Settings.Percent / 100f;
var dupeScanList = ScanFileList.Where(vf => !vf.Flags.Any(EntryFlags.AllErrors | EntryFlags.ManuallyExcluded)).ToList();
InitProgress(dupeScanList.Count);
Parallel.For(0, dupeScanList.Count, parallelOpts, i => {
while (pauseTokenSource.IsPaused)
{
Thread.Sleep(50);
}
var baseItem = dupeScanList[i];
if (baseItem.grayBytes == null || baseItem.grayBytes.Count == 0)
{
IncrementProgress(baseItem.Path);
return;
}
for (var n = i + 1; n < dupeScanList.Count; n++)
{
var compItem = dupeScanList[n];
if (baseItem.IsImage && !compItem.IsImage)
{
continue;
}
if (compItem.grayBytes == null || compItem.grayBytes.Count == 0)
{
continue;
}
if (baseItem.grayBytes.Count != compItem.grayBytes.Count)
{
continue;
}
var duplicateCounter = 0;
var percent = new float[baseItem.grayBytes.Count];
for (var j = 0; j < baseItem.grayBytes.Count; j++)
{
Debug.Assert(baseItem.grayBytes[j].Length == compItem.grayBytes[j].Length, "Images must be of same length");
percent[j] = ExtensionMethods.PercentageDifference(baseItem.grayBytes[j], compItem.grayBytes[j]);
if (percent[j] < percentageDifference)
{
duplicateCounter++;
}
else
{
break;
}
}
if (duplicateCounter != baseItem.grayBytes.Count)
{
continue;
}
var percSame = percent.Average();
lock (duplicateDict) {
var foundBase = duplicateDict.TryGetValue(baseItem.Path, out var existingBase);
var foundComp = duplicateDict.TryGetValue(compItem.Path, out var existingComp);
if (foundBase && foundComp)
{
//this happens with 4+ identical items:
//first, 2+ duplicate groups are found independently, they are merged in this branch
if (existingBase.GroupId != existingComp.GroupId)
{
foreach (var dup in duplicateDict.Values.Where(c => c.GroupId == existingComp.GroupId))
{
dup.GroupId = existingBase.GroupId;
}
}
}
else if (foundBase)
{
duplicateDict.Add(compItem.Path, new DuplicateItem(compItem, percSame)
{
GroupId = existingBase.GroupId
});
}
else if (foundComp)
{
duplicateDict.Add(baseItem.Path, new DuplicateItem(baseItem, percSame)
{
GroupId = existingComp.GroupId
});
}
else
{
var groupId = Guid.NewGuid();
duplicateDict.Add(compItem.Path, new DuplicateItem(compItem, percSame)
{
GroupId = groupId
});
duplicateDict.Add(baseItem.Path, new DuplicateItem(baseItem, percSame)
{
GroupId = groupId
});
}
}
}
IncrementProgress(baseItem.Path);
});
SearchSW.Stop();
Logger.Instance.Info(string.Format(Properties.Resources.DuplicatesCheckFinishedIn, SearchSW.Elapsed));
Duplicates = new HashSet <DuplicateItem>(duplicateDict.Values);
}
catch (OperationCanceledException) {
Logger.Instance.Info(Properties.Resources.CancellationExceptionCaught);
}
}
private void InternalSearch(CancellationToken cancelToken, PauseTokenSource pauseTokenSource)
{
ElapsedTimer.Start();
var startTime = DateTime.Now;
processedFiles = 0;
var lastProgressUpdate = DateTime.MinValue;
var progressUpdateItvl = TimeSpan.FromMilliseconds(300);
void IncrementProgress(Action <TimeSpan> fn)
{
Interlocked.Increment(ref processedFiles);
var pushUpdate = processedFiles == ScanProgressMaxValue ||
lastProgressUpdate + progressUpdateItvl < DateTime.Now;
if (!pushUpdate)
{
return;
}
lastProgressUpdate = DateTime.Now;
var timeRemaining = TimeSpan.FromTicks(DateTime.Now.Subtract(startTime).Ticks *
(ScanProgressMaxValue - (processedFiles + 1)) / (processedFiles + 1));
fn(timeRemaining);
}
try {
var st = Stopwatch.StartNew();
var reScanList = ScanFileList.Where(vf => (vf.mediaInfo == null && !vf.IsImage) || vf.grayBytes == null).ToList();
ScanProgressMaxValue = reScanList.Count;
Parallel.For(0, reScanList.Count, new ParallelOptions {
MaxDegreeOfParallelism = Environment.ProcessorCount, CancellationToken = cancelToken
}, i => {
while (pauseTokenSource.IsPaused)
{
Thread.Sleep(50);
}
var entry = reScanList[i];
if (entry.mediaInfo == null && !entry.IsImage)
{
var ffProbe = new FFProbeWrapper.FFProbeWrapper();
var info = ffProbe.GetMediaInfo(entry.Path);
if (info == null)
{
return;
}
entry.mediaInfo = info;
}
if (entry.grayBytes == null)
{
entry.grayBytes = entry.IsImage ? GetImageAsBitmaps(entry, positionList.Count) : GetVideoThumbnailAsBitmaps(entry, positionList);
if (entry.grayBytes == null)
{
return;
}
}
//report progress
IncrementProgress(remaining =>
Progress?.Invoke(this,
new OwnScanProgress {
CurrentPosition = processedFiles,
CurrentFile = entry.Path,
Elapsed = ElapsedTimer.Elapsed,
Remaining = remaining
}));
});
st.Stop();
Logger.Instance.Info(string.Format(Properties.Resources.ThumbnailsFinished, st.Elapsed, processedFiles));
processedFiles = 0;
ScanProgressMaxValue = ScanFileList.Count;
st.Restart();
startTime = DateTime.Now;
var percentageDifference = 1.0f - Settings.Percent / 100f;
Parallel.For(0, ScanFileList.Count,
new ParallelOptions {
MaxDegreeOfParallelism = Environment.ProcessorCount,
CancellationToken = cancelToken
},
i => {
while (pauseTokenSource.IsPaused)
{
Thread.Sleep(50);
}
foreach (var itm in ScanFileList)
{
if (itm == ScanFileList[i] || itm.IsImage && !ScanFileList[i].IsImage)
{
continue;
}
if (itm.grayBytes == null || itm.grayBytes.Count == 0)
{
continue;
}
if (ScanFileList[i].grayBytes == null || ScanFileList[i].grayBytes.Count == 0)
{
continue;
}
if (itm.grayBytes.Count != ScanFileList[i].grayBytes.Count)
{
continue;
}
var duplicateCounter = 0;
var percent = new float[itm.grayBytes.Count];
for (var j = 0; j < itm.grayBytes.Count; j++)
{
percent[j] = ExtensionMethods.PercentageDifference2(itm.grayBytes[j],
ScanFileList[i].grayBytes[j]);
if (percent[j] < percentageDifference)
{
duplicateCounter++;
}
else
{
break;
}
}
if (duplicateCounter != itm.grayBytes.Count)
{
continue;
}
lock (AddDuplicateLock) {
var firstInList = false;
var secondInList = false;
var groupId = Guid.NewGuid();
foreach (var v in Duplicates)
{
if (v.Path == itm.Path)
{
groupId = v.GroupId;
firstInList = true;
}
else if (v.Path == ScanFileList[i].Path)
{
secondInList = true;
}
}
if (!firstInList)
{
var origDup = new DuplicateItem(itm, percent.Average())
{
GroupId = groupId
};
var origImages = itm.IsImage ? GetImageThumbnail(origDup, positionList.Count) : GetVideoThumbnail(origDup, positionList);
if (origImages == null)
{
continue;
}
origDup.Thumbnail = origImages;
Duplicates.Add(origDup);
}
if (!secondInList)
{
var dup = new DuplicateItem(ScanFileList[i], percent.Average())
{
GroupId = groupId
};
var images = ScanFileList[i].IsImage ? GetImageThumbnail(dup, positionList.Count) : GetVideoThumbnail(dup, positionList);
if (images == null)
{
continue;
}
dup.Thumbnail = images;
Duplicates.Add(dup);
}
}
//we found a matching source then duplicate was added no need to go deeper
break;
}
//report progress
IncrementProgress(remaining =>
Progress?.Invoke(this,
new OwnScanProgress {
CurrentPosition = processedFiles,
CurrentFile = ScanFileList[i].Path,
Elapsed = ElapsedTimer.Elapsed,
Remaining = remaining
}));
});
st.Stop();
Logger.Instance.Info(string.Format(Properties.Resources.DuplicatesCheckFinishedIn, st.Elapsed));
}
catch (OperationCanceledException) {
Logger.Instance.Info(Properties.Resources.CancellationExceptionCaught);
}
}