static void CheckStringsForHashCollisions(List <string> strings, HashType hash) { var makeHash = HashManager.GetHashFunction(hash); var lineComparer = HashManager.GetStringComparer(hash); var hashMap = new Dictionary <UInt32, string>(); var buffer = new List <string>(); foreach (var line in strings) { var lineHash = makeHash(line); if (hashMap.ContainsKey(lineHash)) { var testLine = hashMap[lineHash]; if (!lineComparer.Equals(line, testLine)) { buffer.Add(string.Format("INPUT_COLLISION\t0x{0:X8}\t{1}\t{2}", lineHash, testLine, line)); } } else { hashMap.Add(lineHash, line); } } if (buffer.Any()) { Console.WriteLine("#INPUT_COLLISION\tHash\tLine_1\tLine_2"); buffer.Sort(); buffer.ForEach(s => Console.WriteLine(s)); Console.WriteLine(""); } }
static void SortFolder(string inputFolder, HashType hash) { var makeHash = HashManager.GetHashFunction(hash); var files = Directory.GetFiles(inputFolder, "*.txt", SearchOption.AllDirectories); var sw = Stopwatch.StartNew(); foreach (var fileName in files) { Console.WriteLine(fileName + " - Processing file..."); var fileData = new Dictionary <UInt32, List <string> >(); var fileLines = File.ReadAllLines(fileName); foreach (var line in fileLines) { var hashValue = makeHash(line); List <string> hashLine; if (!fileData.TryGetValue(hashValue, out hashLine)) { hashLine = new List <string>(); fileData.Add(hashValue, hashLine); } hashLine.Add(line); } var soretedLines = new List <string>(); foreach (var line in fileData) { if (line.Value.Count == 1) { soretedLines.Add(line.Value.First()); } else { var test = line.Value.Distinct(HashManager.GetStringComparer(hash)).ToList(); if (test.Count == 1) { soretedLines.Add(test.First()); } else { Console.WriteLine("Hash Collision found, hash = 0x" + line.Key.ToString("X8")); foreach (var s in test) { Console.WriteLine(s); } Console.WriteLine(""); return; } } } soretedLines.Sort(); File.WriteAllLines(fileName, soretedLines); } sw.Stop(); Console.WriteLine("Files processing done in " + sw.Elapsed.ToString()); }
static void CheckStrings(string inputFolder, HashType hash, List <string> strings) { var converter = new UInt32Converter(); var makeHash = HashManager.GetHashFunction(hash); var lineComparer = HashManager.GetStringComparer(hash); var files = Directory.GetFiles(inputFolder, "*.hashes", SearchOption.AllDirectories); var testData = strings.Select(s => new KeyValuePair <UInt32, string>(makeHash(s), s)).ToList(); var bufferCollisions = new List <string>(); var bufferNew = new List <string>(); foreach (var fileName in files) { var hashList = File.ReadAllLines(fileName).Select(s => (UInt32)converter.ConvertFromString(s)).ToDictionary(k => k, v => v); var lines = File.ReadAllLines(Path.ChangeExtension(fileName, ".txt")).ToDictionary(k => makeHash(k), v => v); var localFileName = fileName.Remove(0, inputFolder.Length + 1); foreach (var test in testData) { if (hashList.ContainsKey(test.Key)) { string currentValue; if (lines.TryGetValue(test.Key, out currentValue)) { if (!lineComparer.Equals(currentValue, test.Value)) { bufferCollisions.Add(string.Format("COLLISION\t{0}\t{1}\t{2}", localFileName, currentValue, test.Value)); } } else { bufferNew.Add(string.Format("NEW\t{0}\t{1}", localFileName, test.Value)); } } } } if (bufferCollisions.Any()) { Console.WriteLine("#COLLISION\tFile_name\tCurrent_line\tInput_line"); bufferCollisions.Sort(); bufferCollisions.ForEach(s => Console.WriteLine(s)); Console.WriteLine(""); } if (bufferNew.Any()) { Console.WriteLine("#NEW\tFile_name\tInput_line"); bufferNew.Sort(); bufferNew.ForEach(s => Console.WriteLine(s)); Console.WriteLine(""); } }
static void BuildForOpenIV(string databaseName, HashType hash, string inputFolder) { var makeHash = HashManager.GetHashFunction(hash); var files = Directory.GetFiles(inputFolder, "*.txt", SearchOption.AllDirectories); var sw = Stopwatch.StartNew(); var linesData = new Dictionary <string, List <string> >(); foreach (var fileName in files) { Console.WriteLine(fileName + " - Reading file..."); var containerName = Path.GetFileNameWithoutExtension(fileName).ToUpper(); var fileLines = File.ReadAllLines(fileName); List <string> containerLines; if (linesData.TryGetValue(containerName, out containerLines)) { var buffer = new List <string>(containerLines); buffer.AddRange(fileLines); containerLines.Clear(); containerLines.AddRange(buffer.Distinct()); } else { containerLines = new List <string>(fileLines); linesData.Add(containerName, containerLines); } } sw.Stop(); Console.WriteLine("Files reading done in " + sw.Elapsed.ToString()); sw = Stopwatch.StartNew(); var database = new Dictionary <UInt32, List <StringLine> >(); foreach (var container in linesData) { Console.WriteLine(container.Key + " - Processing container..."); foreach (var stringLine in container.Value) { var hashValue = makeHash(stringLine); List <StringLine> hashLine; if (!database.TryGetValue(hashValue, out hashLine)) { hashLine = new List <StringLine>(); database.Add(hashValue, hashLine); } hashLine.Add(new StringLine() { Container = container.Key, Value = stringLine }); } } sw.Stop(); Console.WriteLine("Containers processing done in " + sw.Elapsed.ToString()); sw = Stopwatch.StartNew(); var databaseLines = new List <string>(); databaseLines.Add(string.Empty); foreach (var hashData in database) { var hashLines = hashData.Value; if (hashLines.Count > 1) { hashLines = hashLines.Distinct(HashManager.GetStringLineComparer(hash)).ToList(); } if (hashLines.Count > 1) { hashLines = hashData.Value; foreach (var line in hashLines) { databaseLines.Add(line.Container + "\\" + line.Value); } } else { databaseLines.Add(hashLines.First().Value); } } sw.Stop(); Console.WriteLine("Strings processing done in " + sw.Elapsed.ToString()); sw = Stopwatch.StartNew(); databaseLines.Sort(); File.WriteAllLines(databaseName + ".txt", databaseLines); Console.WriteLine("File saved in " + sw.Elapsed.ToString()); }