public async Task FinishAsync() { if (!_urls.Any()) { return; } // Download all of the leaves. var urlsToDownload = new ConcurrentBag <string>(_urls); var leaves = new ConcurrentBag <CatalogLeaf>(); await ParallelAsync.Repeat( async() => { await Task.Yield(); while (urlsToDownload.TryTake(out var url)) { var leaf = await _catalogClient.GetLeafAsync(url); leaves.Add(leaf); } }); // Build the input to hive updaters. var entries = new List <CatalogCommitItem>(); var entryToLeaf = new Dictionary <CatalogCommitItem, PackageDetailsCatalogLeaf>( ReferenceEqualityComparer <CatalogCommitItem> .Default); foreach (var leaf in leaves) { if (leaf.IsPackageDelete() == leaf.IsPackageDetails()) { throw new InvalidOperationException("A catalog leaf must be either a package delete or a package details leaf."); } var typeUri = leaf.IsPackageDetails() ? Schema.DataTypes.PackageDetails : Schema.DataTypes.PackageDelete; var catalogCommitItem = new CatalogCommitItem( new Uri(leaf.Url), leaf.CommitId, leaf.CommitTimestamp.UtcDateTime, types: null, typeUris: new[] { typeUri }, packageIdentity: new PackageIdentity(_packageId, leaf.ParsePackageVersion())); entries.Add(catalogCommitItem); if (leaf.IsPackageDetails()) { entryToLeaf.Add(catalogCommitItem, (PackageDetailsCatalogLeaf)leaf); } } // Update the hives. await _updater.UpdateAsync(_packageId, entries, entryToLeaf); _urls.Clear(); }
public async Task OnProcessBatchAsync(IEnumerable <CatalogCommitItem> items) { var itemList = items.ToList(); _logger.LogInformation("Got {Count} catalog commit items to process.", itemList.Count); var latestItems = _utility.GetLatestPerIdentity(itemList); _logger.LogInformation("Got {Count} unique package identities.", latestItems.Count); var allWork = _utility.GroupById(latestItems); _logger.LogInformation("Got {Count} unique IDs.", allWork.Count); var allEntryToLeaf = await _utility.GetEntryToDetailsLeafAsync(latestItems); _logger.LogInformation("Fetched {Count} package details leaves.", allEntryToLeaf.Count); _logger.LogInformation("Starting {Count} workers processing each package ID batch.", _options.Value.MaxConcurrentIds); await ParallelAsync.Repeat( async() => { await Task.Yield(); while (allWork.TryTake(out var work)) { var entryToLeaf = work .Value .Where(CommitCollectorUtility.IsOnlyPackageDetails) .ToDictionary(e => e, e => allEntryToLeaf[e], ReferenceEqualityComparer <CatalogCommitItem> .Default); await _updater.UpdateAsync(work.Id, work.Value, entryToLeaf); } }, _options.Value.MaxConcurrentIds); _logger.LogInformation("All workers have completed successfully."); }