private async Task <bool> ProcessLeafAsync(CatalogLeafItem leafItem, CancellationToken cancellationToken)
{
bool success;
try
{
await _throttle.WaitAsync(cancellationToken);
if (cancellationToken.IsCancellationRequested)
{
return(false);
}
switch (leafItem.Type)
{
case CatalogLeafType.PackageDelete:
var packageDelete = await _client.GetPackageDeleteLeafAsync(leafItem.Url);
success = await _leafProcessor.ProcessPackageDeleteAsync(packageDelete);
break;
case CatalogLeafType.PackageDetails:
var packageDetails = await _client.GetPackageDetailsLeafAsync(leafItem.Url);
success = await _leafProcessor.ProcessPackageDetailsAsync(packageDetails);
break;
default:
throw new NotSupportedException($"The catalog leaf type '{leafItem.Type}' is not supported.");
}
}
catch (Exception exception)
{
_logger.LogError(
0,
exception,
"An exception was thrown while processing leaf {leafUrl}.",
leafItem.Url);
success = false;
}
finally
{
_throttle.Release();
}
if (!success)
{
_logger.LogWarning(
"Failed to process leaf {leafUrl} ({packageId} {packageVersion}, {leafType}).",
leafItem.Url,
leafItem.PackageId,
leafItem.PackageVersion,
leafItem.Type);
}
return(success);
}
private async Task ProcessPageAsync(DateTimeOffset minCommitTimestamp, CatalogPageItem pageItem, CancellationToken token)
{
var page = await _client.GetPageAsync(pageItem.Url, token);
var leafItems = page.GetLeavesInBounds(
minCommitTimestamp,
_settings.MaxCommitTimestamp,
_settings.ExcludeRedundantLeaves);
_logger.LogInformation(
"On page {page}, {leaves} out of {totalLeaves} were in the time bounds.",
pageItem.Url,
leafItems.Count,
page.Items.Count);
DateTimeOffset?newCursor = null;
var tasks = new List <Task <CatalogLeaf> >();
foreach (var batch in leafItems
.Select((v, i) => new { Index = i, Value = v })
.GroupBy(v => v.Index / 25)
.Select(v => v.Select(p => p.Value)))
{
foreach (var leafItem in batch)
{
newCursor = leafItem.CommitTimestamp;
tasks.Add(ProcessLeafAsync(leafItem, token));
}
await Task.WhenAll(tasks);
foreach (var task in tasks)
{
if (task.Result is PackageDeleteCatalogLeaf del)
{
await _leafProcessor.ProcessPackageDeleteAsync(del, token);
}
else if (task.Result is PackageDetailsCatalogLeaf detail)
{
await _leafProcessor.ProcessPackageDetailsAsync(detail, token);
}
else
{
_logger.LogError("Unsupported leaf type: {type}.", task.Result?.GetType());
}
}
tasks.Clear();
}
if (newCursor.HasValue)
{
await _cursor.SetAsync(newCursor.Value, token);
}
}
private async Task <bool> ProcessLeafAsync(CatalogLeafItem leafItem, CancellationToken cancellationToken)
{
bool success;
try
{
if (leafItem.IsPackageDelete())
{
var packageDelete = await _client.GetPackageDeleteLeafAsync(leafItem.CatalogLeafUrl);
success = await _leafProcessor.ProcessPackageDeleteAsync(packageDelete, cancellationToken);
}
else if (leafItem.IsPackageDetails())
{
var packageDetails = await _client.GetPackageDetailsLeafAsync(leafItem.CatalogLeafUrl);
success = await _leafProcessor.ProcessPackageDetailsAsync(packageDetails, cancellationToken);
}
else
{
throw new NotSupportedException($"The catalog leaf type '{leafItem.Type}' is not supported.");
}
}
catch (Exception exception)
{
_logger.LogError(
0,
exception,
"An exception was thrown while processing leaf {leafUrl}.",
leafItem.CatalogLeafUrl);
success = false;
}
if (!success)
{
_logger.LogWarning(
"Failed to process leaf {leafUrl} ({packageId} {packageVersion}, {leafType}).",
leafItem.CatalogLeafUrl,
leafItem.PackageId,
leafItem.PackageVersion,
leafItem.Type);
}
return(success);
}