private static void Main()
{
var documents = DirectoryTools.ReadStringDictionary(InputFile);
var allText = string.Join(' ', documents.Values);
var chars = allText
.Distinct()
.Except(TextTools.SourceCodeChars)
.ToArray();
var words = allText
.Split()
.GroupBy(x => x)
.ToDictionary(x => x.Key, x => x.Count())
.OrderByDescending(x => x.Value)
.ToDictionary(x => x.Key, x => x.Value);
var result = new
{
chars,
words,
};
DirectoryTools.SaveAsJson(result, OutputFile);
}
private static void Main()
{
var groundTruth = DuplicateRates.FromLines(File.ReadLines(GroundTruthFile));
var documents = DirectoryTools.ReadStringDictionary(InputFile)
//.MapValues(SourceCodeNormalizer.NormalizeContent)
//.MapValues(SourceCodeNormalizer.NormalizeSpaces)
//.MapValues(SourceCodeNormalizer.NormalizeWords)
.MapValues(SourceCodeNormalizer.NormalizeSpaces);
var hashes = HashTools.HashValues(documents);
var duplicates = FindDuplicates(hashes, groundTruth);
var result = documents.RemoveKeys(duplicates);
DirectoryTools.SaveAsJson(result, OutputFile);
}
private static void Main()
{
var documents = DirectoryTools.ReadStringDictionary(InputFile);
var groundTruth = DuplicateRates.FromLines(File.ReadLines(GroundTruthFile), documents.Keys);
var result = new
{
Min1 = ProcessMinHash(groundTruth, documents, 1),
Min5 = ProcessMinHash(groundTruth, documents, 5),
Min10 = ProcessMinHash(groundTruth, documents, 10),
Min30 = ProcessMinHash(groundTruth, documents, 30),
Min50 = ProcessMinHash(groundTruth, documents, 50),
Min100 = ProcessMinHash(groundTruth, documents, 100),
Sim8 = ProcessSimHash(groundTruth, documents, 8 / 8),
Sim16 = ProcessSimHash(groundTruth, documents, 16 / 8),
Sim64 = ProcessSimHash(groundTruth, documents, 64 / 8),
Sim128 = ProcessSimHash(groundTruth, documents, 128 / 8),
Sim256 = ProcessSimHash(groundTruth, documents, 256 / 8),
};
DirectoryTools.SaveAsJson(result, OutputFile);
}
