public void Process()
{
_log.Info(Name);
foreach (var root in ZapperProcessor.Settings.RootFolders)
{
Console.WriteLine("{0}: Parsing folder {1}", DateTime.Now.ToString("HH:mm:ss.fff"), root.FullPath);
var filepaths = Directory.EnumerateFiles(root.FullPath, "*.*", System.IO.SearchOption.AllDirectories);
try
{
Parallel.ForEach(filepaths, filepath =>
{
if (filepath.Length >= 260)
{
Console.WriteLine("{0}: Path too long - {1}", DateTime.Now.ToString("HH:mm:ss.fff"), filepath);
}
else
{
var zfile = new ZapperFile(filepath);
if (!zfile.IsSystem)
{
if (ZapperProcessor.Settings.UnwantedExtensions.Contains(zfile.Extension))
{
FileSystem.DeleteFile(filepath, UIOption.OnlyErrorDialogs, RecycleOption.SendToRecycleBin);
ZapperFileDeleted zfiledeleted = new ZapperFileDeleted(zfile, ZapperProcessor.ZapperSession.Id);
if (!ZapperProcessor.ZapperFilesDeleted.TryAdd(zfiledeleted.FullPath, zfiledeleted))
{
throw new FileZapperAddToDictionaryFailureException("ZapperFilesDeleted", zfiledeleted.FullPath);
}
}
else if (!ZapperProcessor.Settings.SkippedExtensions.Contains(zfile.Extension)
&& zfile.Size > ZapperProcessor.Settings.IgnoreFilesBelowBytes
&& (ZapperProcessor.Settings.IgnoreFilesOverBytes <= 0 || zfile.Size < ZapperProcessor.Settings.IgnoreFilesOverBytes))
{
if (!ZapperProcessor.ZapperFiles.TryAdd(zfile.FullPath, zfile))
{
throw new FileZapperAddToDictionaryFailureException("ZapperFiles", zfile.FullPath);
}
}
}
}
});
}
catch (AggregateException ae)
{
ae.Handle(e =>
{
Exceptioneer.Log(_log, e);
return true;
});
}
}
}
public void Process()
{
_log.Info(Name);
var files = ZapperProcessor.ZapperFiles.Values.Where(x => !string.IsNullOrWhiteSpace(x.ContentHash) && !x.ContentHash.Equals("invalid", StringComparison.InvariantCultureIgnoreCase));
Console.WriteLine("{0}: Calc file scores", DateTime.Now.ToString("HH:mm:ss.fff"));
try
{
Parallel.ForEach(files, new ParallelOptions { MaxDegreeOfParallelism = Environment.ProcessorCount }, zfile =>
{
CalculateScore(zfile);
});
}
catch (AggregateException ae)
{
ae.Handle(e =>
{
Exceptioneer.Log(_log, e);
return true;
});
}
Console.WriteLine("{0}: Delete losers", DateTime.Now.ToString("HH:mm:ss.fff"));
var dupes = (from z in files
group z by z.ContentHash into g
select new { ContentHash = g.Key, Count = g.Count(), Files = g })
.Where(x => x.Count > 1);
try
{
Parallel.ForEach(dupes, new ParallelOptions { MaxDegreeOfParallelism = Environment.ProcessorCount }, dupe =>
{
var deadmenwalking = (from z in dupe.Files
orderby z.Score descending
select z).Skip(1);
// Not parallel; for large workloads this spammed the threadpool, even when limited to logical processor count
foreach (var dead in deadmenwalking)
{
if (File.Exists(dead.FullPath))
{
FileSystem.DeleteFile(dead.FullPath, UIOption.OnlyErrorDialogs, RecycleOption.SendToRecycleBin);
}
ZapperFileDeleted zfiledeleted = new ZapperFileDeleted(dead, ZapperProcessor.ZapperSession.Id);
if (!ZapperProcessor.ZapperFilesDeleted.TryAdd(zfiledeleted.FullPath, zfiledeleted))
{
throw new FileZapperAddToDictionaryFailureException("ZapperFilesDeleted", zfiledeleted.FullPath);
}
ZapperFile killed;
if (!ZapperProcessor.ZapperFiles.TryRemove(dead.FullPath, out killed))
{
throw new FileZapperRemoveFromDictionaryFailureException("ZapperFiles", dead.FullPath);
}
}
});
}
catch (AggregateException ae)
{
ae.Handle(e =>
{
Exceptioneer.Log(_log, e);
return true;
});
}
}