private async Task <bool> ProcessNextMessageAsync(PerWorkerContext workerContext, int batchSize)
{
var batchContext = new PerBatchContext(workerContext, UniqueName.New("batch"));
var packageIdentities = new HashSet <string>(StringComparer.OrdinalIgnoreCase);
var messages = new List <StorageQueueMessage <PackageMessage> >();
var packageContexts = new List <PerPackageContext>();
StorageQueueMessage <PackageMessage> lastMessage;
do
{
lastMessage = await _queue.GetNextAsync(CancellationToken.None);
if (lastMessage != null)
{
var packageId = lastMessage.Contents.PackageId.Trim();
var packageVersion = NuGetVersion.Parse(lastMessage.Contents.PackageVersion.Trim()).ToNormalizedString();
var packageIdentity = $"{packageId}/{packageVersion}";
// If this is a duplicate package, complete it and skip it.
if (!packageIdentities.Add(packageIdentity))
{
await _queue.RemoveAsync(lastMessage, CancellationToken.None);
continue;
}
messages.Add(lastMessage);
packageContexts.Add(new PerPackageContext(batchContext, packageId, packageVersion));
}
}while (messages.Count < batchSize && lastMessage != null && lastMessage.DequeueCount < 10);
if (packageContexts.Count == 0)
{
return(false);
}
var complete = await ProcessPackagesAsync(batchContext, packageContexts);
if (complete)
{
foreach (var message in messages)
{
try
{
await _queue.RemoveAsync(message, CancellationToken.None);
}
catch (StorageException ex) when(ex.RequestInformation?.HttpStatusCode == (int)HttpStatusCode.NotFound)
{
// Ignore this error. The message has already been removed.
}
}
}
return(true);
}
private async Task HandleQueueMessageAsync(
StorageQueueMessage <PackageValidatorContext> queueMessage,
CancellationToken token)
{
if (queueMessage == null)
{
return;
}
var queuedContext = queueMessage.Contents;
var messageWasProcessed = false;
try
{
await RunPackageValidatorAsync(queuedContext, token);
// The validations ran successfully and were saved to storage.
// We can remove the message from the queue because it was processed.
messageWasProcessed = true;
}
catch (Exception e)
{
// Validations failed to run! Save this failed status to storage.
await SaveFailedPackageMonitoringStatusAsync(queuedContext, e, token);
// We can then remove the message from the queue because this failed status can be used to requeue the message.
messageWasProcessed = true;
}
// Note that if both validations fail and saving the failure status fail, we cannot remove the message from the queue.
if (messageWasProcessed)
{
await _queue.RemoveAsync(queueMessage, token);
}
}
private async Task HandleQueueMessageAsync(
StorageQueueMessage <PackageValidatorContext> queueMessage,
CancellationToken token)
{
if (queueMessage == null)
{
return;
}
var queuedContext = queueMessage.Contents;
var messageWasProcessed = false;
try
{
await RunPackageValidatorAsync(queuedContext, token);
// The validations ran successfully and were saved to storage.
// We can remove the message from the queue because it was processed.
messageWasProcessed = true;
}
catch (Exception validationFailedToRunException)
{
try
{
// Validations failed to run! Save this failed status to storage.
await SaveFailedPackageMonitoringStatusAsync(queuedContext, validationFailedToRunException, token);
// We can then remove the message from the queue because this failed status can be used to requeue the message.
messageWasProcessed = true;
}
catch (Exception failedValidationSaveFailureException)
{
// We failed to run validations and failed to save the failed validation!
// We were not able to process this message. We need to log the exceptions so we can debug the issue.
var aggregateException = new AggregateException(
"Validations failed to run and saving unsuccessful validation failed!",
new[] { validationFailedToRunException, failedValidationSaveFailureException });
Logger.LogCritical(
NuGet.Services.Metadata.Catalog.Monitoring.LogEvents.QueueMessageFatalFailure,
aggregateException,
"Failed to process queue message");
}
}
// If we failed to run validations and failed to save the failed validation, we cannot remove the message from the queue.
if (messageWasProcessed)
{
await _queue.RemoveAsync(queueMessage, token);
}
}
private async Task HandleQueueMessageAsync(
StorageQueueMessage <PackageValidatorContext> queueMessage,
CancellationToken token)
{
if (queueMessage == null)
{
return;
}
var queuedContext = queueMessage.Contents;
var messageWasProcessed = false;
try
{
await RunPackageValidatorAsync(queuedContext, token);
// The validations ran successfully and were saved to storage.
// We can remove the message from the queue because it was processed.
messageWasProcessed = true;
}
catch (Exception validationFailedToRunException)
{
try
{
// Validations failed to run! Save this failed status to storage.
await SaveFailedPackageMonitoringStatusAsync(queuedContext, validationFailedToRunException, token);
// We can then remove the message from the queue because this failed status can be used to requeue the message.
messageWasProcessed = true;
}
catch (Exception failedValidationSaveFailureException)
{
// We failed to run validations and failed to save the failed validation!
// We were not able to process this message. We need to log the exceptions so we can debug the issue.
throw new AggregateException(
"Validations failed to run and saving unsuccessful validation failed!",
new[] { validationFailedToRunException, failedValidationSaveFailureException });
}
}
// If we failed to run validations and failed to save the failed validation, we cannot remove the message from the queue.
if (messageWasProcessed)
{
try
{
await _queue.RemoveAsync(queueMessage, token);
}
catch (StorageException storageException)
{
if (storageException.RequestInformation.ExtendedErrorInformation.ErrorCode == QueueErrorCodeStrings.MessageNotFound)
{
Logger.LogWarning(
NuGet.Services.Metadata.Catalog.Monitoring.LogEvents.QueueMessageRemovalFailure,
storageException,
"Queue message for {PackageId} {PackageVersion} no longer exists. Message was likely already processed.",
queuedContext.Package.Id, queuedContext.Package.Version);
}
else
{
Logger.LogCritical(
NuGet.Services.Metadata.Catalog.Monitoring.LogEvents.QueueMessageRemovalFailure,
storageException,
"Failed to remove queue message.");
}
}
}
}
public Task RemoveAsync(StorageQueueMessage <T> message, CancellationToken token)
{
return(_queue.RemoveAsync(SerializeMessage(message), token));
}