private static void Main() { var groundTruth = DuplicateRates.FromLines(File.ReadLines(GroundTruthFile)); var documents = DirectoryTools.ReadStringDictionary(InputFile) //.MapValues(SourceCodeNormalizer.NormalizeContent) //.MapValues(SourceCodeNormalizer.NormalizeSpaces) //.MapValues(SourceCodeNormalizer.NormalizeWords) .MapValues(SourceCodeNormalizer.NormalizeSpaces); var hashes = HashTools.HashValues(documents); var duplicates = FindDuplicates(hashes, groundTruth); var result = documents.RemoveKeys(duplicates); DirectoryTools.SaveAsJson(result, OutputFile); }