private void MergeIds(string region, string outputFile, string[] inputFiles)
{
var output = new MatchIdContainer(outputFile, EnumStrong.Parse <Region>(region));
var files = new[] { outputFile }.Concat(inputFiles).Select(file => new { file, count = new CountResult() }).ToList();
var ids = files.SelectMany(f => new MatchIdContainer(f.file).ReadItems().PassthroughCount(f.count)).ToHashSet();
foreach (var f in files)
{
Console.WriteLine($"Read input {f.file}: {f.count.Count:#,0} items");
}
File.Delete(outputFile);
output.AppendItems(ids.Order(), LosChunkFormat.LZ4HC);
output.Rewrite();
}
public void Merge(string outputPath)
{
Console.WriteLine("Non-existent IDs:");
var nonexistent = new HashSet <long>();
nonexistent.AddRange(ReadContainersWithLogging(NonexistentIdsFiles.Select(fi => new MatchIdContainer(fi.FullName))));
Console.WriteLine($" Total unique IDs: {nonexistent.Count:#,0}");
var nonexistentNew = new MatchIdContainer(Path.Combine(outputPath, $"{Region}-match-id-nonexistent.losmid"), Region);
nonexistentNew.AppendItems(nonexistent.Order(), LosChunkFormat.LZ4HC);
NonexistentCount = nonexistent.Count;
Console.WriteLine();
Console.WriteLine("Matches:");
var existingHave = new HashSet <long>();
foreach (var group in MatchFiles.GroupBy(x => x.queueId).OrderBy(grp => grp.Key))
{
var queueId = group.Key;
Console.WriteLine($" Queue {queueId}");
var newMatches = new JsonContainer(Path.Combine(outputPath, $"{Region}-matches-{queueId}.losjs"));
var count = new CountResult();
newMatches.AppendItems(
ReadContainersWithLogging(group.Select(x => new JsonContainer(x.fi.FullName)))
.Where(js => existingHave.Add(js["gameId"].GetLong()))
.PassthroughCount(count),
LosChunkFormat.LZ4HC);
Console.WriteLine($" Total unique: {count.Count:#,0}");
HaveCounts.Add(queueId, count.Count);
}
Console.WriteLine();
Console.WriteLine("Existing IDs:");
var existing = new HashSet <long>();
existing.AddRange(ReadContainersWithLogging(ExistingIdsFiles.Select(fi => new MatchIdContainer(fi.FullName))));
var existingWasCount = existing.Count;
existing.AddRange(existingHave);
Console.WriteLine($" IDs which were only in match files and not in match-id files: {existing.Count - existingWasCount:#,0}");
Console.WriteLine($" Total unique IDs: {existing.Count:#,0}");
var existingNew = new MatchIdContainer(Path.Combine(outputPath, $"{Region}-match-id-existing.losmid"), Region);
existingNew.AppendItems(existing.Order(), LosChunkFormat.LZ4HC);
RedownloadIds = existing.Except(existingHave).Order().ToList();
Console.WriteLine($" Known IDs we don't have; re-download: {RedownloadIds.Count:#,0}");
}
