public void Process() { _log.Info(Name); foreach (var root in ZapperProcessor.Settings.RootFolders) { Console.WriteLine("{0}: Parsing folder {1}", DateTime.Now.ToString("HH:mm:ss.fff"), root.FullPath); var filepaths = Directory.EnumerateFiles(root.FullPath, "*.*", System.IO.SearchOption.AllDirectories); try { Parallel.ForEach(filepaths, filepath => { if (filepath.Length >= 260) { Console.WriteLine("{0}: Path too long - {1}", DateTime.Now.ToString("HH:mm:ss.fff"), filepath); } else { var zfile = new ZapperFile(filepath); if (!zfile.IsSystem) { if (ZapperProcessor.Settings.UnwantedExtensions.Contains(zfile.Extension)) { FileSystem.DeleteFile(filepath, UIOption.OnlyErrorDialogs, RecycleOption.SendToRecycleBin); ZapperFileDeleted zfiledeleted = new ZapperFileDeleted(zfile, ZapperProcessor.ZapperSession.Id); if (!ZapperProcessor.ZapperFilesDeleted.TryAdd(zfiledeleted.FullPath, zfiledeleted)) { throw new FileZapperAddToDictionaryFailureException("ZapperFilesDeleted", zfiledeleted.FullPath); } } else if (!ZapperProcessor.Settings.SkippedExtensions.Contains(zfile.Extension) && zfile.Size > ZapperProcessor.Settings.IgnoreFilesBelowBytes && (ZapperProcessor.Settings.IgnoreFilesOverBytes <= 0 || zfile.Size < ZapperProcessor.Settings.IgnoreFilesOverBytes)) { if (!ZapperProcessor.ZapperFiles.TryAdd(zfile.FullPath, zfile)) { throw new FileZapperAddToDictionaryFailureException("ZapperFiles", zfile.FullPath); } } } } }); } catch (AggregateException ae) { ae.Handle(e => { Exceptioneer.Log(_log, e); return true; }); } } }
public void Process() { _log.Info(Name); var files = ZapperProcessor.ZapperFiles.Values.Where(x => !string.IsNullOrWhiteSpace(x.ContentHash) && !x.ContentHash.Equals("invalid", StringComparison.InvariantCultureIgnoreCase)); Console.WriteLine("{0}: Calc file scores", DateTime.Now.ToString("HH:mm:ss.fff")); try { Parallel.ForEach(files, new ParallelOptions { MaxDegreeOfParallelism = Environment.ProcessorCount }, zfile => { CalculateScore(zfile); }); } catch (AggregateException ae) { ae.Handle(e => { Exceptioneer.Log(_log, e); return true; }); } Console.WriteLine("{0}: Delete losers", DateTime.Now.ToString("HH:mm:ss.fff")); var dupes = (from z in files group z by z.ContentHash into g select new { ContentHash = g.Key, Count = g.Count(), Files = g }) .Where(x => x.Count > 1); try { Parallel.ForEach(dupes, new ParallelOptions { MaxDegreeOfParallelism = Environment.ProcessorCount }, dupe => { var deadmenwalking = (from z in dupe.Files orderby z.Score descending select z).Skip(1); // Not parallel; for large workloads this spammed the threadpool, even when limited to logical processor count foreach (var dead in deadmenwalking) { if (File.Exists(dead.FullPath)) { FileSystem.DeleteFile(dead.FullPath, UIOption.OnlyErrorDialogs, RecycleOption.SendToRecycleBin); } ZapperFileDeleted zfiledeleted = new ZapperFileDeleted(dead, ZapperProcessor.ZapperSession.Id); if (!ZapperProcessor.ZapperFilesDeleted.TryAdd(zfiledeleted.FullPath, zfiledeleted)) { throw new FileZapperAddToDictionaryFailureException("ZapperFilesDeleted", zfiledeleted.FullPath); } ZapperFile killed; if (!ZapperProcessor.ZapperFiles.TryRemove(dead.FullPath, out killed)) { throw new FileZapperRemoveFromDictionaryFailureException("ZapperFiles", dead.FullPath); } } }); } catch (AggregateException ae) { ae.Handle(e => { Exceptioneer.Log(_log, e); return true; }); } }