private void MergeIds(string region, string outputFile, string[] inputFiles) { var output = new MatchIdContainer(outputFile, EnumStrong.Parse <Region>(region)); var files = new[] { outputFile }.Concat(inputFiles).Select(file => new { file, count = new CountResult() }).ToList(); var ids = files.SelectMany(f => new MatchIdContainer(f.file).ReadItems().PassthroughCount(f.count)).ToHashSet(); foreach (var f in files) { Console.WriteLine($"Read input {f.file}: {f.count.Count:#,0} items"); } File.Delete(outputFile); output.AppendItems(ids.Order(), LosChunkFormat.LZ4HC); output.Rewrite(); }
public void Merge(string outputPath) { Console.WriteLine("Non-existent IDs:"); var nonexistent = new HashSet <long>(); nonexistent.AddRange(ReadContainersWithLogging(NonexistentIdsFiles.Select(fi => new MatchIdContainer(fi.FullName)))); Console.WriteLine($" Total unique IDs: {nonexistent.Count:#,0}"); var nonexistentNew = new MatchIdContainer(Path.Combine(outputPath, $"{Region}-match-id-nonexistent.losmid"), Region); nonexistentNew.AppendItems(nonexistent.Order(), LosChunkFormat.LZ4HC); NonexistentCount = nonexistent.Count; Console.WriteLine(); Console.WriteLine("Matches:"); var existingHave = new HashSet <long>(); foreach (var group in MatchFiles.GroupBy(x => x.queueId).OrderBy(grp => grp.Key)) { var queueId = group.Key; Console.WriteLine($" Queue {queueId}"); var newMatches = new JsonContainer(Path.Combine(outputPath, $"{Region}-matches-{queueId}.losjs")); var count = new CountResult(); newMatches.AppendItems( ReadContainersWithLogging(group.Select(x => new JsonContainer(x.fi.FullName))) .Where(js => existingHave.Add(js["gameId"].GetLong())) .PassthroughCount(count), LosChunkFormat.LZ4HC); Console.WriteLine($" Total unique: {count.Count:#,0}"); HaveCounts.Add(queueId, count.Count); } Console.WriteLine(); Console.WriteLine("Existing IDs:"); var existing = new HashSet <long>(); existing.AddRange(ReadContainersWithLogging(ExistingIdsFiles.Select(fi => new MatchIdContainer(fi.FullName)))); var existingWasCount = existing.Count; existing.AddRange(existingHave); Console.WriteLine($" IDs which were only in match files and not in match-id files: {existing.Count - existingWasCount:#,0}"); Console.WriteLine($" Total unique IDs: {existing.Count:#,0}"); var existingNew = new MatchIdContainer(Path.Combine(outputPath, $"{Region}-match-id-existing.losmid"), Region); existingNew.AppendItems(existing.Order(), LosChunkFormat.LZ4HC); RedownloadIds = existing.Except(existingHave).Order().ToList(); Console.WriteLine($" Known IDs we don't have; re-download: {RedownloadIds.Count:#,0}"); }