public async Task FinishAsync()
{
if (!_urls.Any())
{
return;
}
// Download all of the leaves.
var urlsToDownload = new ConcurrentBag <string>(_urls);
var leaves = new ConcurrentBag <CatalogLeaf>();
await ParallelAsync.Repeat(
async() =>
{
await Task.Yield();
while (urlsToDownload.TryTake(out var url))
{
var leaf = await _catalogClient.GetLeafAsync(url);
leaves.Add(leaf);
}
});
// Build the input to hive updaters.
var entries = new List <CatalogCommitItem>();
var entryToLeaf = new Dictionary <CatalogCommitItem, PackageDetailsCatalogLeaf>(
ReferenceEqualityComparer <CatalogCommitItem> .Default);
foreach (var leaf in leaves)
{
if (leaf.IsPackageDelete() == leaf.IsPackageDetails())
{
throw new InvalidOperationException("A catalog leaf must be either a package delete or a package details leaf.");
}
var typeUri = leaf.IsPackageDetails() ? Schema.DataTypes.PackageDetails : Schema.DataTypes.PackageDelete;
var catalogCommitItem = new CatalogCommitItem(
new Uri(leaf.Url),
leaf.CommitId,
leaf.CommitTimestamp.UtcDateTime,
types: null,
typeUris: new[] { typeUri },
packageIdentity: new PackageIdentity(_packageId, leaf.ParsePackageVersion()));
entries.Add(catalogCommitItem);
if (leaf.IsPackageDetails())
{
entryToLeaf.Add(catalogCommitItem, (PackageDetailsCatalogLeaf)leaf);
}
}
// Update the hives.
await _updater.UpdateAsync(_packageId, entries, entryToLeaf);
_urls.Clear();
}
public async Task OnProcessBatchAsync(IEnumerable <CatalogCommitItem> items)
{
var itemList = items.ToList();
_logger.LogInformation("Got {Count} catalog commit items to process.", itemList.Count);
var latestItems = _utility.GetLatestPerIdentity(itemList);
_logger.LogInformation("Got {Count} unique package identities.", latestItems.Count);
var allWork = _utility.GroupById(latestItems);
_logger.LogInformation("Got {Count} unique IDs.", allWork.Count);
var allEntryToLeaf = await _utility.GetEntryToDetailsLeafAsync(latestItems);
_logger.LogInformation("Fetched {Count} package details leaves.", allEntryToLeaf.Count);
_logger.LogInformation("Starting {Count} workers processing each package ID batch.", _options.Value.MaxConcurrentIds);
await ParallelAsync.Repeat(
async() =>
{
await Task.Yield();
while (allWork.TryTake(out var work))
{
var entryToLeaf = work
.Value
.Where(CommitCollectorUtility.IsOnlyPackageDetails)
.ToDictionary(e => e, e => allEntryToLeaf[e], ReferenceEqualityComparer <CatalogCommitItem> .Default);
await _updater.UpdateAsync(work.Id, work.Value, entryToLeaf);
}
},
_options.Value.MaxConcurrentIds);
_logger.LogInformation("All workers have completed successfully.");
}
