private async Task <bool> ProcessNextMessageAsync(PerWorkerContext workerContext, int batchSize) { var batchContext = new PerBatchContext(workerContext, UniqueName.New("batch")); var packageIdentities = new HashSet <string>(StringComparer.OrdinalIgnoreCase); var messages = new List <StorageQueueMessage <PackageMessage> >(); var packageContexts = new List <PerPackageContext>(); StorageQueueMessage <PackageMessage> lastMessage; do { lastMessage = await _queue.GetNextAsync(CancellationToken.None); if (lastMessage != null) { var packageId = lastMessage.Contents.PackageId.Trim(); var packageVersion = NuGetVersion.Parse(lastMessage.Contents.PackageVersion.Trim()).ToNormalizedString(); var packageIdentity = $"{packageId}/{packageVersion}"; // If this is a duplicate package, complete it and skip it. if (!packageIdentities.Add(packageIdentity)) { await _queue.RemoveAsync(lastMessage, CancellationToken.None); continue; } messages.Add(lastMessage); packageContexts.Add(new PerPackageContext(batchContext, packageId, packageVersion)); } }while (messages.Count < batchSize && lastMessage != null && lastMessage.DequeueCount < 10); if (packageContexts.Count == 0) { return(false); } var complete = await ProcessPackagesAsync(batchContext, packageContexts); if (complete) { foreach (var message in messages) { try { await _queue.RemoveAsync(message, CancellationToken.None); } catch (StorageException ex) when(ex.RequestInformation?.HttpStatusCode == (int)HttpStatusCode.NotFound) { // Ignore this error. The message has already been removed. } } } return(true); }
private async Task ProcessPackagesAsync(CancellationToken token) { StorageQueueMessage <PackageValidatorContext> queueMessage = null; do { Logger.LogInformation("Fetching next queue message."); queueMessage = await _queue.GetNextAsync(token); await HandleQueueMessageAsync(queueMessage, token); } while (queueMessage != null); Logger.LogInformation("No messages left in queue."); }
private async Task HandleQueueMessagesAsync( CancellationToken combinedQueueLoopCancellationToken, CancellationToken queueMessageCancellationToken) { Logger.LogInformation("Beginning fetching queue messages."); do { var shouldWaitBeforeNextMessage = false; try { Logger.LogInformation("Fetching next queue message."); var queueMessage = await _queue.GetNextAsync(queueMessageCancellationToken); await HandleQueueMessageAsync(queueMessage, queueMessageCancellationToken); if (queueMessage == null) { Logger.LogInformation( "Failed to fetch last message or no messages left in queue."); shouldWaitBeforeNextMessage = true; } } catch (Exception e) { Logger.LogCritical( NuGet.Services.Metadata.Catalog.Monitoring.LogEvents.QueueMessageFatalFailure, e, "Failed to process queue message."); shouldWaitBeforeNextMessage = true; } if (shouldWaitBeforeNextMessage && !combinedQueueLoopCancellationToken.IsCancellationRequested) { Logger.LogInformation( "Waiting {QueueDelaySeconds} seconds before polling again.", _queueDelay.TotalSeconds); try { await Task.Delay(_queueDelay, combinedQueueLoopCancellationToken); } catch (TaskCanceledException) { Logger.LogInformation("Stopped waiting before polling because task was cancelled."); } } } while (!combinedQueueLoopCancellationToken.IsCancellationRequested); Logger.LogInformation("Finished fetching queue messages."); }
public async Task <StorageQueueMessage <T> > GetNextAsync(CancellationToken token) { return(DeserializeMessage(await _queue.GetNextAsync(token))); }