private async Task <bool> ProcessLeafAsync(CatalogLeafItem leafItem, CancellationToken cancellationToken)
{
bool success;
try
{
await _throttle.WaitAsync(cancellationToken);
if (cancellationToken.IsCancellationRequested)
{
return(false);
}
switch (leafItem.Type)
{
case CatalogLeafType.PackageDelete:
var packageDelete = await _client.GetPackageDeleteLeafAsync(leafItem.Url);
success = await _leafProcessor.ProcessPackageDeleteAsync(packageDelete);
break;
case CatalogLeafType.PackageDetails:
var packageDetails = await _client.GetPackageDetailsLeafAsync(leafItem.Url);
success = await _leafProcessor.ProcessPackageDetailsAsync(packageDetails);
break;
default:
throw new NotSupportedException($"The catalog leaf type '{leafItem.Type}' is not supported.");
}
}
catch (Exception exception)
{
_logger.LogError(
0,
exception,
"An exception was thrown while processing leaf {leafUrl}.",
leafItem.Url);
success = false;
}
finally
{
_throttle.Release();
}
if (!success)
{
_logger.LogWarning(
"Failed to process leaf {leafUrl} ({packageId} {packageVersion}, {leafType}).",
leafItem.Url,
leafItem.PackageId,
leafItem.PackageVersion,
leafItem.Type);
}
return(success);
}
public async Task <IReadOnlyDictionary <CatalogCommitItem, PackageDetailsCatalogLeaf> > GetEntryToDetailsLeafAsync(
IEnumerable <CatalogCommitItem> entries)
{
var packageDetailsEntries = entries.Where(IsOnlyPackageDetails);
var allWork = new ConcurrentBag <CatalogCommitItem>(packageDetailsEntries);
var output = new ConcurrentBag <KeyValuePair <CatalogCommitItem, PackageDetailsCatalogLeaf> >();
using (_telemetryService.TrackCatalogLeafDownloadBatch(allWork.Count))
{
var tasks = Enumerable
.Range(0, _options.Value.MaxConcurrentCatalogLeafDownloads)
.Select(async x =>
{
await Task.Yield();
while (allWork.TryTake(out var work))
{
try
{
_logger.LogInformation(
"Downloading catalog leaf for {PackageId} {Version}: {Url}",
work.PackageIdentity.Id,
work.PackageIdentity.Version.ToNormalizedString(),
work.Uri.AbsoluteUri);
var leaf = await _catalogClient.GetPackageDetailsLeafAsync(work.Uri.AbsoluteUri);
output.Add(KeyValuePair.Create(work, leaf));
}
catch (Exception ex)
{
_logger.LogError(
ex,
"An exception was thrown when fetching the package details leaf for {Id} {Version}. " +
"The URL is {Url}",
work.PackageIdentity.Id,
work.PackageIdentity.Version,
work.Uri.AbsoluteUri);
throw;
}
}
})
.ToList();
await Task.WhenAll(tasks);
return(output.ToDictionary(
x => x.Key,
x => x.Value,
ReferenceEqualityComparer <CatalogCommitItem> .Default));
}
}
private Task <CatalogLeaf> ProcessLeafAsync(CatalogLeafItem leafItem, CancellationToken token)
{
switch (leafItem.Type)
{
case CatalogLeafType.PackageDelete:
return(_client.GetPackageDeleteLeafAsync(leafItem.Url, token));
case CatalogLeafType.PackageDetails:
return(_client.GetPackageDetailsLeafAsync(leafItem.Url, token));
default:
throw new NotSupportedException($"The catalog leaf type '{leafItem.Type}' is not supported.");
}
}
private async Task <bool> ProcessLeafAsync(CatalogLeafItem leafItem, CancellationToken cancellationToken)
{
bool success;
try
{
if (leafItem.IsPackageDelete())
{
var packageDelete = await _client.GetPackageDeleteLeafAsync(leafItem.CatalogLeafUrl);
success = await _leafProcessor.ProcessPackageDeleteAsync(packageDelete, cancellationToken);
}
else if (leafItem.IsPackageDetails())
{
var packageDetails = await _client.GetPackageDetailsLeafAsync(leafItem.CatalogLeafUrl);
success = await _leafProcessor.ProcessPackageDetailsAsync(packageDetails, cancellationToken);
}
else
{
throw new NotSupportedException($"The catalog leaf type '{leafItem.Type}' is not supported.");
}
}
catch (Exception exception)
{
_logger.LogError(
0,
exception,
"An exception was thrown while processing leaf {leafUrl}.",
leafItem.CatalogLeafUrl);
success = false;
}
if (!success)
{
_logger.LogWarning(
"Failed to process leaf {leafUrl} ({packageId} {packageVersion}, {leafType}).",
leafItem.CatalogLeafUrl,
leafItem.PackageId,
leafItem.PackageVersion,
leafItem.Type);
}
return(success);
}
private async Task ProcessIconsAsync(
ConcurrentBag <IReadOnlyCollection <CatalogCommitItem> > items,
CancellationToken cancellationToken)
{
await Task.Yield();
var targetStorage = _targetStorageFactory.Create();
var iconCacheStorage = _iconCacheStorageFactory.Create();
using (_logger.BeginScope("{CallGuid}", Guid.NewGuid()))
while (items.TryTake(out var entries))
{
var firstItem = entries.First();
using (_logger.BeginScope("Processing commits for {PackageId} {PackageVersion}", firstItem.PackageIdentity.Id, firstItem.PackageIdentity.Version))
{
foreach (var item in entries)
{
if (item.IsPackageDetails)
{
PackageDetailsCatalogLeaf leaf;
try
{
leaf = await _catalogClient.GetPackageDetailsLeafAsync(item.Uri.AbsoluteUri);
}
catch (Exception e)
{
_logger.LogError(0, e, "Error while trying to retrieve catalog leaf {LeafUrl}", item.Uri.AbsoluteUri);
throw;
}
await _catalogLeafDataProcessor.ProcessPackageDetailsLeafAsync(targetStorage, iconCacheStorage, item, leaf.IconUrl, leaf.IconFile, cancellationToken);
}
else if (item.IsPackageDelete)
{
await _catalogLeafDataProcessor.ProcessPackageDeleteLeafAsync(targetStorage, item, cancellationToken);
}
}
}
}
}