public void ApplyMd5Checksum_CheckDupesAndCompleteFullHash_DoesItEnsureAllPartialDupesAreFullHashed_Exercise()
{
var rootEntries = RootEntry.LoadCurrentDirCache();
var d = new Duplication(_logger, _configuration, _applicationDiagnostics);
d.ApplyMd5Checksum(rootEntries);
}
public static void CreateMd5OnCache()
{
var logger = Container.Resolve <ILogger>();
var diagnostics = Container.Resolve <IApplicationDiagnostics>();
logger.LogInfo("Memory pre-catload: {0}", diagnostics.GetMemoryAllocated().FormatAsBytes());
var rootEntries = RootEntry.LoadCurrentDirCache();
logger.LogInfo("Memory post-catload: {0}", diagnostics.GetMemoryAllocated().FormatAsBytes());
var duplication = Container.Resolve <Duplication>();
var sw = new Stopwatch();
sw.Start();
duplication.ApplyMd5Checksum(rootEntries);
foreach (var rootEntry in rootEntries)
{
logger.LogDebug("Saving {0}", rootEntry.DefaultFileName);
rootEntry.SaveRootEntry();
}
sw.Stop();
var ts = sw.Elapsed;
var elapsedTime = $"{ts.Hours:00}:{ts.Minutes:00}:{ts.Seconds:00}.{ts.Milliseconds/10:00}";
Console.WriteLine($"Hash took : {elapsedTime}");
}
private static void FindDupes()
{
var rootEntries = RootEntry.LoadCurrentDirCache();
var duplication = Container.Resolve <Duplication>();
duplication.FindDuplicates(rootEntries);
}
private void LoadRoots()
{
var rootEntries = RootEntry.LoadCurrentDirCache();
RootEntries = new BindableCollection <RootEntry>();
RootEntries.AddRange(rootEntries);
RootEntries2 = new BindableCollection <RootEntry>();
RootEntries2.AddRange(rootEntries);
}
public void PerformanceTest_Compare_TraverseTree_With_DirEntryEnumerator()
{
var rootEntries = RootEntry.LoadCurrentDirCache();
System.Threading.Thread.Sleep(200); // msec
//_num = 0;
_fileCount = 0;
var sw = new Stopwatch();
sw.Start();
for (var i = 0; i < 100; ++i)
{
var deEnumerator = CommonEntry.GetDirEntries(rootEntries);
foreach (var dirEntry in deEnumerator)
{
//_num += (ulong)dirEntry.FullPath.Length;
++_fileCount;
//if (Hack.BreakConsoleFlag)
//{
// Console.WriteLine("\nBreak key detected exiting full TraverseTree inner.");
// break;
//}
}
}
sw.Stop();
var ts = sw.Elapsed;
var elapsedTime = $"{ts.Hours:00}:{ts.Minutes:00}:{ts.Seconds:00}.{ts.Milliseconds/10:00}";
Console.WriteLine($"Enumerator took : {elapsedTime}");
Console.WriteLine($"Total files enumerated : {_fileCount}");
//Console.WriteLine("Total path length : {0}", _num);
var re = rootEntries.First();
sw.Reset();
sw.Start();
//_num = 0;
_fileCount = 0;
for (var i = 0; i < 100; ++i)
{
re.TraverseTreePair((p, d) => { ++_fileCount; return(true); });
}
sw.Stop();
ts = sw.Elapsed;
elapsedTime = $"{ts.Hours:00}:{ts.Minutes:00}:{ts.Seconds:00}.{ts.Milliseconds/10:00}";
Console.WriteLine($"TraverseTreePair took : {elapsedTime}");
Console.WriteLine($"Total files enumerated : {_fileCount}");
//Console.WriteLine("Total path length : {0}", _num);
}
private static void PrintPathsHaveHashEnumerator()
{
var rootEntries = RootEntry.LoadCurrentDirCache();
var pdee = CommonEntry.GetPairDirEntries(rootEntries);
foreach (var pairDirEntry in pdee)
{
var hash = pairDirEntry.ChildDE.IsHashDone ? "#" : " ";
var bang = pairDirEntry.PathProblem ? "!" : " ";
Console.WriteLine($"{hash}{bang}{pairDirEntry.FullPath}");
if (Hack.BreakConsoleFlag)
{
break;
}
}
}
public static void GetDirCache()
{
if (_rootEntries == null)
{
var start = DateTime.UtcNow;
_rootEntries = RootEntry.LoadCurrentDirCache();
var end = DateTime.UtcNow;
var loadTimeSpan = end - start;
Console.WriteLine($"Loaded {_rootEntries.Count} file(s) in {loadTimeSpan.TotalMilliseconds:0.00} msecs");
foreach (var rootEntry in _rootEntries)
{
Console.WriteLine(
$"Loaded File {rootEntry.DefaultFileName} with {rootEntry.DirEntryCount + rootEntry.FileEntryCount} entries.");
}
}
}
public void GetSizePairs_CheckSanityOfDupeSizeCountandDupeFileCount_Exercise()
{
int dupeCount = 10;
var testPath = this.AssemblyPathLocation();
// Create some dummy duplicate data.
// create a catalog
var random = FileHelper.RandomString(4096 * 16);
for (int i = 1; i <= dupeCount; i++)
{
File.WriteAllText($"{testPath}\\CDE_testFile{i}.txt", random);
}
//hacky creating catalog.
Program.Container = BootStrapper.Components();
Program.CreateCache($"{testPath}.\\");
Program.CreateMd5OnCache();
//run tests.
Console.WriteLine($"0 Directory.GetCurrentDirectory() {Directory.GetCurrentDirectory()}");
var rootEntries = RootEntry.LoadCurrentDirCache();
if (rootEntries.Count == 0)
{
Console.WriteLine("No Catalogs found.");
Assert.Fail("No catalogs found.");
}
foreach (var r in rootEntries)
{
Console.WriteLine($"loaded {r.DefaultFileName}");
}
var d = new Duplication(_logger, _configuration, _applicationDiagnostics);
var sizePairDictionary = d.GetSizePairs(rootEntries);
Console.WriteLine($"Number of Size Pairs {sizePairDictionary.Count}");
var sumOfUniqueHashesForEachSize = GetSumOfUniqueHashesForEachSize_ExcludePartialHash(sizePairDictionary);
Console.WriteLine($"Sum of total unique hashes (split on filesize to) {sumOfUniqueHashesForEachSize}");
var dupePairEnum = d.GetDupePairs(rootEntries);
var itemToVerify = dupePairEnum.SingleOrDefault(x => x.Key.Path.Contains("CDE_testFile"));
//Assert we have at least a count of the number of dupe files we originally created.
Assert.GreaterOrEqual(itemToVerify.Value.Count, dupeCount);
//TODO: Cleanup test.
}
public void GetDupePairs_CheckAllDupeFilesHaveFullHash_OK()
{
var rootEntries = RootEntry.LoadCurrentDirCache();
var d = new Duplication(_logger, _configuration, _applicationDiagnostics);
var dupePairEnum = d.GetDupePairs(rootEntries);
foreach (var dupe in dupePairEnum)
{
foreach (var de in dupe.Value)
{
if (de.ChildDE.IsPartialHash)
{
Console.WriteLine($"Trouble partial hash {de.FullPath}");
Assert.Fail();
}
}
}
}
private static void FindPouplous(int minimumCount)
{
var rootEntries = RootEntry.LoadCurrentDirCache();
var entries = CommonEntry.GetDirEntries(rootEntries);
var largeEntries = entries
.Where(e => e.Children != null && e.Children.Count > minimumCount)
.ToList();
largeEntries.Sort(CompareDirEntries);
foreach (var e in largeEntries.Where(e => e.Children != null && e.Children.Count > minimumCount))
{
Console.WriteLine($"{e.FullPath} {e.Children.Count}");
if (Hack.BreakConsoleFlag)
{
break;
}
}
}
private static void Main(string[] args)
{
Container = BootStrapper.Components();
Console.CancelKeyPress += BreakConsole;
if (args.Length == 0)
{
ShowHelp();
return;
}
var param0 = args[0].ToLowerInvariant();
if (args.Length == 2 && param0 == "--scan")
{
CreateCache(args[1]);
}
else if (args.Length == 2 && param0 == "--scan2")
{
CreateCache2(args[1]);
}
else if (args.Length == 2 && Find.FindParams.Contains(param0))
{
Find.StaticFind(args[1], param0);
}
else if (args.Length == 2 && param0 == "--replgreppath")
{
FindRepl(Find.ParamGreppath, args[1]);
}
else if (args.Length == 2 && param0 == "--replgrep")
{
FindRepl(Find.ParamGrep, args[1]);
}
else if (args.Length == 2 && param0 == "--replfind")
{
FindRepl(Find.ParamFind, args[1]);
}
else if (args.Length == 1 && param0 == "--hash")
{
CreateMd5OnCache();
}
else if (args.Length == 1 && param0 == "--hash2")
{
CreateMd5OnCache2();
}
else if (args.Length == 1 && param0 == "--dupes")
{
FindDupes();
}
else if (args.Length == 1 && param0 == "--treedump1")
{
PrintPathsHaveHashEnumerator();
}
else if (args.Length == 1 && param0 == "--treedump2")
{
EntryStore.PrintPathsHaveHash();
}
else if (args.Length == 1 && param0 == "--version")
{
Console.WriteLine(Version);
}
else if (args.Length == 1 && param0 == "--loadwait")
{
Console.WriteLine(Version);
RootEntry.LoadCurrentDirCache();
Console.ReadLine();
}
else if (args.Length == 1 && param0 == "--loadwait2")
{
Console.WriteLine(Version);
EntryStore.LoadCurrentDirCache();
Console.ReadLine();
}
else if (args.Length == 1 && param0 == "--repl")
{
var le = new LineEditor(null);
string s;
var running = true;
while (running && (s = le.Edit("shell> ", string.Empty)) != null)
{
Console.WriteLine($"----> [{s}]");
switch (s)
{
case "quit":
running = false;
break;
case "history":
case "!":
le.CmdHistoryDump();
break;
case "help":
case "?":
Console.WriteLine("Builtin Commands:");
Console.WriteLine(" quit - quit, ");
Console.WriteLine(" help - show help, ? - show help");
Console.WriteLine(" history - show history, ! - show history");
Console.WriteLine("Keystrokes:");
Console.WriteLine(" Home, End, Left, Right, Up, Down, Back, Del, Tab");
Console.WriteLine(" C-a, C-e, C-b, C-f, C-p, C-n, C-d");
Console.WriteLine(" C-l - clear console to top");
Console.WriteLine(" C-r - reverse seach history");
Console.WriteLine(" A-b - move backward word");
Console.WriteLine(" A-f - move forward word");
Console.WriteLine(" A-d - delete word forward");
Console.WriteLine(" A-Backspace - delete word backward");
break;
}
}
}
else if (args.Length == 2 && param0 == "--populousfolders")
{
int count;
if (int.TryParse(args[1], out count))
{
FindPouplous(count);
}
else
{
Console.WriteLine("Populous folders option requires an integer as second parameter");
}
}
else
{
ShowHelp();
}
}
public void LoadData()
{
var rootEntries = RootEntry.LoadCurrentDirCache();
Response = string.Format("Data Loaded: {0}", rootEntries.Count);
}
