static void CheckStringsForHashCollisions(List <string> strings, HashType hash)
        {
            var makeHash     = HashManager.GetHashFunction(hash);
            var lineComparer = HashManager.GetStringComparer(hash);
            var hashMap      = new Dictionary <UInt32, string>();
            var buffer       = new List <string>();

            foreach (var line in strings)
            {
                var lineHash = makeHash(line);
                if (hashMap.ContainsKey(lineHash))
                {
                    var testLine = hashMap[lineHash];
                    if (!lineComparer.Equals(line, testLine))
                    {
                        buffer.Add(string.Format("INPUT_COLLISION\t0x{0:X8}\t{1}\t{2}", lineHash, testLine, line));
                    }
                }
                else
                {
                    hashMap.Add(lineHash, line);
                }
            }

            if (buffer.Any())
            {
                Console.WriteLine("#INPUT_COLLISION\tHash\tLine_1\tLine_2");
                buffer.Sort();
                buffer.ForEach(s => Console.WriteLine(s));
                Console.WriteLine("");
            }
        }
        static void SortFolder(string inputFolder, HashType hash)
        {
            var makeHash = HashManager.GetHashFunction(hash);
            var files    = Directory.GetFiles(inputFolder, "*.txt", SearchOption.AllDirectories);

            var sw = Stopwatch.StartNew();

            foreach (var fileName in files)
            {
                Console.WriteLine(fileName + " - Processing file...");
                var fileData  = new Dictionary <UInt32, List <string> >();
                var fileLines = File.ReadAllLines(fileName);
                foreach (var line in fileLines)
                {
                    var hashValue = makeHash(line);

                    List <string> hashLine;
                    if (!fileData.TryGetValue(hashValue, out hashLine))
                    {
                        hashLine = new List <string>();
                        fileData.Add(hashValue, hashLine);
                    }

                    hashLine.Add(line);
                }

                var soretedLines = new List <string>();
                foreach (var line in fileData)
                {
                    if (line.Value.Count == 1)
                    {
                        soretedLines.Add(line.Value.First());
                    }
                    else
                    {
                        var test = line.Value.Distinct(HashManager.GetStringComparer(hash)).ToList();
                        if (test.Count == 1)
                        {
                            soretedLines.Add(test.First());
                        }
                        else
                        {
                            Console.WriteLine("Hash Collision found, hash = 0x" + line.Key.ToString("X8"));
                            foreach (var s in test)
                            {
                                Console.WriteLine(s);
                            }
                            Console.WriteLine("");
                            return;
                        }
                    }
                }

                soretedLines.Sort();
                File.WriteAllLines(fileName, soretedLines);
            }
            sw.Stop();
            Console.WriteLine("Files processing done in " + sw.Elapsed.ToString());
        }
        static void CheckStrings(string inputFolder, HashType hash, List <string> strings)
        {
            var converter    = new UInt32Converter();
            var makeHash     = HashManager.GetHashFunction(hash);
            var lineComparer = HashManager.GetStringComparer(hash);
            var files        = Directory.GetFiles(inputFolder, "*.hashes", SearchOption.AllDirectories);
            var testData     = strings.Select(s => new KeyValuePair <UInt32, string>(makeHash(s), s)).ToList();

            var bufferCollisions = new List <string>();
            var bufferNew        = new List <string>();

            foreach (var fileName in files)
            {
                var hashList      = File.ReadAllLines(fileName).Select(s => (UInt32)converter.ConvertFromString(s)).ToDictionary(k => k, v => v);
                var lines         = File.ReadAllLines(Path.ChangeExtension(fileName, ".txt")).ToDictionary(k => makeHash(k), v => v);
                var localFileName = fileName.Remove(0, inputFolder.Length + 1);

                foreach (var test in testData)
                {
                    if (hashList.ContainsKey(test.Key))
                    {
                        string currentValue;
                        if (lines.TryGetValue(test.Key, out currentValue))
                        {
                            if (!lineComparer.Equals(currentValue, test.Value))
                            {
                                bufferCollisions.Add(string.Format("COLLISION\t{0}\t{1}\t{2}", localFileName, currentValue, test.Value));
                            }
                        }
                        else
                        {
                            bufferNew.Add(string.Format("NEW\t{0}\t{1}", localFileName, test.Value));
                        }
                    }
                }
            }

            if (bufferCollisions.Any())
            {
                Console.WriteLine("#COLLISION\tFile_name\tCurrent_line\tInput_line");
                bufferCollisions.Sort();
                bufferCollisions.ForEach(s => Console.WriteLine(s));
                Console.WriteLine("");
            }

            if (bufferNew.Any())
            {
                Console.WriteLine("#NEW\tFile_name\tInput_line");
                bufferNew.Sort();
                bufferNew.ForEach(s => Console.WriteLine(s));
                Console.WriteLine("");
            }
        }
Ejemplo n.º 4
0
        static void BuildForOpenIV(string databaseName, HashType hash, string inputFolder)
        {
            var makeHash = HashManager.GetHashFunction(hash);
            var files    = Directory.GetFiles(inputFolder, "*.txt", SearchOption.AllDirectories);

            var sw        = Stopwatch.StartNew();
            var linesData = new Dictionary <string, List <string> >();

            foreach (var fileName in files)
            {
                Console.WriteLine(fileName + " - Reading file...");
                var           containerName = Path.GetFileNameWithoutExtension(fileName).ToUpper();
                var           fileLines     = File.ReadAllLines(fileName);
                List <string> containerLines;
                if (linesData.TryGetValue(containerName, out containerLines))
                {
                    var buffer = new List <string>(containerLines);
                    buffer.AddRange(fileLines);

                    containerLines.Clear();
                    containerLines.AddRange(buffer.Distinct());
                }
                else
                {
                    containerLines = new List <string>(fileLines);
                    linesData.Add(containerName, containerLines);
                }
            }
            sw.Stop();
            Console.WriteLine("Files reading done in " + sw.Elapsed.ToString());

            sw = Stopwatch.StartNew();
            var database = new Dictionary <UInt32, List <StringLine> >();

            foreach (var container in linesData)
            {
                Console.WriteLine(container.Key + " - Processing container...");

                foreach (var stringLine in container.Value)
                {
                    var hashValue = makeHash(stringLine);

                    List <StringLine> hashLine;
                    if (!database.TryGetValue(hashValue, out hashLine))
                    {
                        hashLine = new List <StringLine>();
                        database.Add(hashValue, hashLine);
                    }

                    hashLine.Add(new StringLine()
                    {
                        Container = container.Key, Value = stringLine
                    });
                }
            }
            sw.Stop();
            Console.WriteLine("Containers processing done in " + sw.Elapsed.ToString());

            sw = Stopwatch.StartNew();
            var databaseLines = new List <string>();

            databaseLines.Add(string.Empty);
            foreach (var hashData in database)
            {
                var hashLines = hashData.Value;
                if (hashLines.Count > 1)
                {
                    hashLines = hashLines.Distinct(HashManager.GetStringLineComparer(hash)).ToList();
                }

                if (hashLines.Count > 1)
                {
                    hashLines = hashData.Value;
                    foreach (var line in hashLines)
                    {
                        databaseLines.Add(line.Container + "\\" + line.Value);
                    }
                }
                else
                {
                    databaseLines.Add(hashLines.First().Value);
                }
            }
            sw.Stop();
            Console.WriteLine("Strings processing done in " + sw.Elapsed.ToString());

            sw = Stopwatch.StartNew();
            databaseLines.Sort();
            File.WriteAllLines(databaseName + ".txt", databaseLines);
            Console.WriteLine("File saved in " + sw.Elapsed.ToString());
        }