static void CheckStringsForHashCollisions(List <string> strings, HashType hash)
{
var makeHash = HashManager.GetHashFunction(hash);
var lineComparer = HashManager.GetStringComparer(hash);
var hashMap = new Dictionary <UInt32, string>();
var buffer = new List <string>();
foreach (var line in strings)
{
var lineHash = makeHash(line);
if (hashMap.ContainsKey(lineHash))
{
var testLine = hashMap[lineHash];
if (!lineComparer.Equals(line, testLine))
{
buffer.Add(string.Format("INPUT_COLLISION\t0x{0:X8}\t{1}\t{2}", lineHash, testLine, line));
}
}
else
{
hashMap.Add(lineHash, line);
}
}
if (buffer.Any())
{
Console.WriteLine("#INPUT_COLLISION\tHash\tLine_1\tLine_2");
buffer.Sort();
buffer.ForEach(s => Console.WriteLine(s));
Console.WriteLine("");
}
}
static void SortFolder(string inputFolder, HashType hash)
{
var makeHash = HashManager.GetHashFunction(hash);
var files = Directory.GetFiles(inputFolder, "*.txt", SearchOption.AllDirectories);
var sw = Stopwatch.StartNew();
foreach (var fileName in files)
{
Console.WriteLine(fileName + " - Processing file...");
var fileData = new Dictionary <UInt32, List <string> >();
var fileLines = File.ReadAllLines(fileName);
foreach (var line in fileLines)
{
var hashValue = makeHash(line);
List <string> hashLine;
if (!fileData.TryGetValue(hashValue, out hashLine))
{
hashLine = new List <string>();
fileData.Add(hashValue, hashLine);
}
hashLine.Add(line);
}
var soretedLines = new List <string>();
foreach (var line in fileData)
{
if (line.Value.Count == 1)
{
soretedLines.Add(line.Value.First());
}
else
{
var test = line.Value.Distinct(HashManager.GetStringComparer(hash)).ToList();
if (test.Count == 1)
{
soretedLines.Add(test.First());
}
else
{
Console.WriteLine("Hash Collision found, hash = 0x" + line.Key.ToString("X8"));
foreach (var s in test)
{
Console.WriteLine(s);
}
Console.WriteLine("");
return;
}
}
}
soretedLines.Sort();
File.WriteAllLines(fileName, soretedLines);
}
sw.Stop();
Console.WriteLine("Files processing done in " + sw.Elapsed.ToString());
}
static void CheckStrings(string inputFolder, HashType hash, List <string> strings)
{
var converter = new UInt32Converter();
var makeHash = HashManager.GetHashFunction(hash);
var lineComparer = HashManager.GetStringComparer(hash);
var files = Directory.GetFiles(inputFolder, "*.hashes", SearchOption.AllDirectories);
var testData = strings.Select(s => new KeyValuePair <UInt32, string>(makeHash(s), s)).ToList();
var bufferCollisions = new List <string>();
var bufferNew = new List <string>();
foreach (var fileName in files)
{
var hashList = File.ReadAllLines(fileName).Select(s => (UInt32)converter.ConvertFromString(s)).ToDictionary(k => k, v => v);
var lines = File.ReadAllLines(Path.ChangeExtension(fileName, ".txt")).ToDictionary(k => makeHash(k), v => v);
var localFileName = fileName.Remove(0, inputFolder.Length + 1);
foreach (var test in testData)
{
if (hashList.ContainsKey(test.Key))
{
string currentValue;
if (lines.TryGetValue(test.Key, out currentValue))
{
if (!lineComparer.Equals(currentValue, test.Value))
{
bufferCollisions.Add(string.Format("COLLISION\t{0}\t{1}\t{2}", localFileName, currentValue, test.Value));
}
}
else
{
bufferNew.Add(string.Format("NEW\t{0}\t{1}", localFileName, test.Value));
}
}
}
}
if (bufferCollisions.Any())
{
Console.WriteLine("#COLLISION\tFile_name\tCurrent_line\tInput_line");
bufferCollisions.Sort();
bufferCollisions.ForEach(s => Console.WriteLine(s));
Console.WriteLine("");
}
if (bufferNew.Any())
{
Console.WriteLine("#NEW\tFile_name\tInput_line");
bufferNew.Sort();
bufferNew.ForEach(s => Console.WriteLine(s));
Console.WriteLine("");
}
}
static void BuildForOpenIV(string databaseName, HashType hash, string inputFolder)
{
var makeHash = HashManager.GetHashFunction(hash);
var files = Directory.GetFiles(inputFolder, "*.txt", SearchOption.AllDirectories);
var sw = Stopwatch.StartNew();
var linesData = new Dictionary <string, List <string> >();
foreach (var fileName in files)
{
Console.WriteLine(fileName + " - Reading file...");
var containerName = Path.GetFileNameWithoutExtension(fileName).ToUpper();
var fileLines = File.ReadAllLines(fileName);
List <string> containerLines;
if (linesData.TryGetValue(containerName, out containerLines))
{
var buffer = new List <string>(containerLines);
buffer.AddRange(fileLines);
containerLines.Clear();
containerLines.AddRange(buffer.Distinct());
}
else
{
containerLines = new List <string>(fileLines);
linesData.Add(containerName, containerLines);
}
}
sw.Stop();
Console.WriteLine("Files reading done in " + sw.Elapsed.ToString());
sw = Stopwatch.StartNew();
var database = new Dictionary <UInt32, List <StringLine> >();
foreach (var container in linesData)
{
Console.WriteLine(container.Key + " - Processing container...");
foreach (var stringLine in container.Value)
{
var hashValue = makeHash(stringLine);
List <StringLine> hashLine;
if (!database.TryGetValue(hashValue, out hashLine))
{
hashLine = new List <StringLine>();
database.Add(hashValue, hashLine);
}
hashLine.Add(new StringLine()
{
Container = container.Key, Value = stringLine
});
}
}
sw.Stop();
Console.WriteLine("Containers processing done in " + sw.Elapsed.ToString());
sw = Stopwatch.StartNew();
var databaseLines = new List <string>();
databaseLines.Add(string.Empty);
foreach (var hashData in database)
{
var hashLines = hashData.Value;
if (hashLines.Count > 1)
{
hashLines = hashLines.Distinct(HashManager.GetStringLineComparer(hash)).ToList();
}
if (hashLines.Count > 1)
{
hashLines = hashData.Value;
foreach (var line in hashLines)
{
databaseLines.Add(line.Container + "\\" + line.Value);
}
}
else
{
databaseLines.Add(hashLines.First().Value);
}
}
sw.Stop();
Console.WriteLine("Strings processing done in " + sw.Elapsed.ToString());
sw = Stopwatch.StartNew();
databaseLines.Sort();
File.WriteAllLines(databaseName + ".txt", databaseLines);
Console.WriteLine("File saved in " + sw.Elapsed.ToString());
}