/// <summary>
/// Executes the delete command, and logs the deleted files if requested
/// </summary>
/// <param name="options">The command options</param>
public static void Run(DeleteOptions options)
{
if (string.IsNullOrEmpty(options.LogDirectory))
{
Run(options, File.Delete);
}
else
{
ClupStatisticsManager statistics = Run(options, File.Delete);
WriteLog(options.LogDirectory, options, statistics);
}
}
/// <summary>
/// Executes the list command
/// </summary>
/// <param name="options">The command options</param>
public static void Run(ListOptions options)
{
// Find the duplicate files
ClupStatisticsManager statistics = Run(options, null);
// Write the log to disk
if (options.LogDirectoryRoot)
{
WriteLog(options.SourceDirectoryCurrent ? Directory.GetCurrentDirectory() : options.SourceDirectory, options, statistics);
}
else if (!string.IsNullOrEmpty(options.LogDirectory))
{
Directory.CreateDirectory(options.LogDirectory);
WriteLog(options.LogDirectory, options, statistics);
}
}
private static ClupStatisticsManager Run([NotNull] ClupOptionsBase options, [CanBeNull] Action <string> handler)
{
// Stats
ClupStatisticsManager statistics = new ClupStatisticsManager();
Console.ForegroundColor = ConsoleColor.White;
// Initial arguments validation
options.Validate();
// Prepare the files query
ConsoleHelper.WriteLine("Querying files...");
List <string> files = new List <string>();
IReadOnlyList <string> extensions = options.Preset.HasValue
? ExtensionsPresetConverter.Convert(options.Preset.Value)
: options.FileInclusions.Select(ext => ext.ToLowerInvariant()).ToArray();
// Local functions to manually explore the source directory (to be able to handle errors)
void ExploreDirectory(string path)
{
try
{
IEnumerable <string> query = extensions.Count == 0
? Directory.EnumerateFiles(path, "*")
: extensions.SelectMany(extension => Directory.EnumerateFiles(path, $"*.{extension}"));
files.AddRange(query);
foreach (string subdirectory in Directory.EnumerateDirectories(path))
{
ExploreDirectory(subdirectory);
}
}
catch (Exception e) when(e is UnauthorizedAccessException || e is PathTooLongException || e is DirectoryNotFoundException)
{
// Just ignore and carry on
if (options.Verbose)
{
ConsoleHelper.WriteTaggedMessage(MessageType.Error, path);
}
}
}
// Execute the query and check the results
ExploreDirectory(options.SourceDirectoryCurrent ? Directory.GetCurrentDirectory() : options.SourceDirectory);
if (files.Count < 2)
{
statistics.StopTracking();
ConsoleHelper.WriteLine("No files were found in the source directory");
return(statistics);
}
ConsoleHelper.WriteTaggedMessage(MessageType.Info, $"Identified {files.Count} files");
// Look for files that have at least another file with the same size
ConsoleHelper.Write("Filtering files... ");
ConcurrentDictionary <long, List <string> > sizeMap = new ConcurrentDictionary <long, List <string> >();
Console.ForegroundColor = ConsoleColor.Gray;
HashSet <string> exclusions = new HashSet <string>(options.FileExclusions.Select(entry => $".{entry.ToLowerInvariant()}"));
using (AsciiProgressBar progressBar = new AsciiProgressBar())
{
int i = 0;
Parallel.ForEach(files, file =>
{
if (exclusions.Count > 0 && exclusions.Contains(Path.GetExtension(file).ToLowerInvariant()))
{
return;
}
try
{
long size = new FileInfo(file).Length;
sizeMap.AddOrUpdate(size, new List <string> {
file
}, (_, list) =>
{
list.Add(file);
return(list);
});
progressBar.Report((double)Interlocked.Increment(ref i) / files.Count);
}
catch (Exception e) when(e is IOException)
{
// Carry on
}
});
}
IReadOnlyCollection <string> filtered = sizeMap.Values.Where(group => group.Count > 1).SelectMany(l => l).ToArray();
ConsoleHelper.WriteTaggedMessage(MessageType.Info, $"Found {sizeMap.Values.Sum(l => l.Count > 1 ? l.Count - 1 : 0)} potential duplicate(s)");
// Initialize the mapping between each target file and its MD5 hash
Console.ForegroundColor = ConsoleColor.White;
ConsoleHelper.Write("Preprocessing filtered files... ");
ConcurrentDictionary <string, List <string> > hashMap = new ConcurrentDictionary <string, List <string> >();
Console.ForegroundColor = ConsoleColor.Gray;
using (AsciiProgressBar progressBar = new AsciiProgressBar())
{
int i = 0;
Parallel.ForEach(filtered, file =>
{
// Compute the MD5 hash
if (exclusions.Count > 0 && exclusions.Contains(Path.GetExtension(file).ToLowerInvariant()))
{
return;
}
try
{
using (FileStream stream = File.OpenRead(file))
using (MD5 md5 = MD5.Create())
{
byte[] hash = md5.ComputeHash(stream);
string base64 = Convert.ToBase64String(hash);
// Get the actual key for the current file
string key;
switch (options.Hash)
{
case HashMode.MD5AndExtension: key = $"{base64}|{Path.GetExtension(file).ToLowerInvariant()}"; break;
case HashMode.MD5AndFilename: key = $"{base64}|{Path.GetFileName(file)}"; break;
default: key = base64; break;
}
// Update the mapping
hashMap.AddOrUpdate(key, new List <string> {
file
}, (_, list) =>
{
list.Add(file);
return(list);
});
progressBar.Report((double)Interlocked.Increment(ref i) / filtered.Count);
}
}
catch (Exception e) when(e is IOException || e is UnauthorizedAccessException)
{
// Why u do dis?
}
});
}
// Process each duplicate file that has been found
Console.ForegroundColor = ConsoleColor.White;
ConsoleHelper.Write("Processing duplicates... ");
Console.ForegroundColor = ConsoleColor.Gray;
using (AsciiProgressBar progressBar = new AsciiProgressBar())
{
int i = 0, count = hashMap.Values.Count;
Parallel.ForEach(hashMap, pair =>
{
// Only keep the original file in each group
if (pair.Value.Count < 2)
{
return;
}
statistics.AddDuplicates(pair.Value);
(long ticks, string path) = (File.GetCreationTimeUtc(pair.Value[0]).Ticks, pair.Value[0]);
foreach (string duplicate in pair.Value.Skip(1))
{
try
{
long creation = File.GetCreationTimeUtc(duplicate).Ticks;
if (creation >= ticks)
{
handler?.Invoke(duplicate);
}
else
{
handler?.Invoke(path);
(ticks, path) = (creation, duplicate);
}
}
catch
{
// Whops!
if (options.Verbose)
{
ConsoleHelper.WriteTaggedMessage(MessageType.Error, path);
}
}
}
// Update the progress bar and the statistics
statistics.AddDuplicates(pair.Key, path, pair.Value);
progressBar.Report((double)Interlocked.Increment(ref i) / count);
});