public QueueListenerTests()
{
CloudQueue queue = new CloudQueue(new Uri("https://test.queue.core.windows.net/testqueue"));
_mockQueue = new Mock <IStorageQueue>(MockBehavior.Strict);
_mockQueue.Setup(p => p.SdkObject).Returns(queue);
_mockTriggerExecutor = new Mock <ITriggerExecutor <IStorageQueueMessage> >(MockBehavior.Strict);
Mock <IWebJobsExceptionHandler> mockExceptionDispatcher = new Mock <IWebJobsExceptionHandler>(MockBehavior.Strict);
_loggerFactory = new LoggerFactory();
_loggerFactory.AddProvider(new TestLoggerProvider());
Mock <IQueueProcessorFactory> mockQueueProcessorFactory = new Mock <IQueueProcessorFactory>(MockBehavior.Strict);
JobHostQueuesConfiguration queuesConfig = new JobHostQueuesConfiguration();
QueueProcessorFactoryContext context = new QueueProcessorFactoryContext(queue, _loggerFactory, queuesConfig);
_mockQueueProcessor = new Mock <QueueProcessor>(MockBehavior.Strict, context);
JobHostQueuesConfiguration queueConfig = new JobHostQueuesConfiguration
{
MaxDequeueCount = 5,
QueueProcessorFactory = mockQueueProcessorFactory.Object
};
mockQueueProcessorFactory.Setup(p => p.Create(It.IsAny <QueueProcessorFactoryContext>())).Returns(_mockQueueProcessor.Object);
_listener = new QueueListener(_mockQueue.Object, null, _mockTriggerExecutor.Object, mockExceptionDispatcher.Object, _loggerFactory, null, queueConfig);
CloudQueueMessage cloudMessage = new CloudQueueMessage("TestMessage");
_storageMessage = new StorageQueueMessage(cloudMessage);
}
private async Task HandleQueueMessageAsync(
StorageQueueMessage <PackageValidatorContext> queueMessage,
CancellationToken token)
{
if (queueMessage == null)
{
return;
}
var queuedContext = queueMessage.Contents;
var messageWasProcessed = false;
try
{
await RunPackageValidatorAsync(queuedContext, token);
// The validations ran successfully and were saved to storage.
// We can remove the message from the queue because it was processed.
messageWasProcessed = true;
}
catch (Exception e)
{
// Validations failed to run! Save this failed status to storage.
await SaveFailedPackageMonitoringStatusAsync(queuedContext, e, token);
// We can then remove the message from the queue because this failed status can be used to requeue the message.
messageWasProcessed = true;
}
// Note that if both validations fail and saving the failure status fail, we cannot remove the message from the queue.
if (messageWasProcessed)
{
await _queue.RemoveAsync(queueMessage, token);
}
}
private static Queue <CloudQueueMessage> GetAzureScheduledMessagesQueue(int count)
{
var queue = new Queue <CloudQueueMessage>();
var serializer = new XmlSerializer(typeof(StorageQueueMessage));
for (var i = count; i > 0; i--)
{
var storageScheduleMessage = new StorageQueueMessage
{
ClientRequestId = Guid.NewGuid(),
ExpectedExecutionTime = DateTime.UtcNow,
SchedulerJobId = i.ToString()
};
string message;
using (var ms = new MemoryStream())
{
serializer.Serialize(ms, storageScheduleMessage);
ms.Position = 0;
var sr = new StreamReader(ms);
message = sr.ReadToEnd();
}
var cloudMessage = new CloudQueueMessage(message);
queue.Enqueue(cloudMessage);
}
queue.Enqueue(null);
return(queue);
}
public async Task Should_not_renew_lock_after_message_timeout()
{
//arrange
var pipeline = new MyPipeline();
var settings = new MySettings
{
ExtensionDuration = TimeSpan.FromMilliseconds(1000),
ExtensionInterval = TimeSpan.FromMilliseconds(100),
MessageLockTimeout = TimeSpan.FromMilliseconds(1000)
};
pipeline.ProcessingSettings = settings;
pipeline.HostConfiguration = new Mock <IHostConfiguration>().Object;
var middleware = new ExtendMessageLockDurationMiddleware();
var storageQueueClient = new Mock <IStorageQueueClient>();
var message = new StorageQueueMessage();
var storageQueueMessageStateHandler = new StorageQueueMessageStateHandler <MyMessage>(storageQueueClient.Object, message, 0, _dependencyInjection.Object);
var next = new Mock <IMessageProcessor>();
next.Setup(x => x.ProcessAsync(It.IsAny <IMessageStateHandler <MyMessage> >(), It.IsAny <CancellationToken>()))
.Returns(() => Task.Delay(2000));
//act
await middleware.ProcessAsync(storageQueueMessageStateHandler, pipeline, next.Object, new CancellationTokenSource(settings.MessageLockTimeout).Token);
//assert
storageQueueClient.Verify(x => x.SetVisibilityTimeout(It.IsAny <StorageQueueMessage>(), TimeSpan.FromMilliseconds(1000), It.IsAny <CancellationToken>()), Times.Between(5, 10, Range.Inclusive));
}
public QueueListenerTests()
{
CloudQueue queue = new CloudQueue(new Uri("https://test.queue.core.windows.net/testqueue"));
Mock <IStorageQueue> mockQueue = new Mock <IStorageQueue>(MockBehavior.Strict);
mockQueue.Setup(p => p.SdkObject).Returns(queue);
_mockTriggerExecutor = new Mock <ITriggerExecutor <IStorageQueueMessage> >(MockBehavior.Strict);
Mock <IDelayStrategy> mockDelayStrategy = new Mock <IDelayStrategy>(MockBehavior.Strict);
Mock <IBackgroundExceptionDispatcher> mockExceptionDispatcher = new Mock <IBackgroundExceptionDispatcher>(MockBehavior.Strict);
TestTraceWriter log = new TestTraceWriter(TraceLevel.Verbose);
Mock <IQueueProcessorFactory> mockQueueProcessorFactory = new Mock <IQueueProcessorFactory>(MockBehavior.Strict);
JobHostQueuesConfiguration queuesConfig = new JobHostQueuesConfiguration();
QueueProcessorFactoryContext context = new QueueProcessorFactoryContext(queue, log, queuesConfig);
_mockQueueProcessor = new Mock <QueueProcessor>(MockBehavior.Strict, context);
JobHostQueuesConfiguration queueConfig = new JobHostQueuesConfiguration
{
MaxDequeueCount = 5,
QueueProcessorFactory = mockQueueProcessorFactory.Object
};
mockQueueProcessorFactory.Setup(p => p.Create(It.IsAny <QueueProcessorFactoryContext>())).Returns(_mockQueueProcessor.Object);
_listener = new QueueListener(mockQueue.Object, null, _mockTriggerExecutor.Object, mockDelayStrategy.Object, mockExceptionDispatcher.Object, log, null, queueConfig);
CloudQueueMessage cloudMessage = new CloudQueueMessage("TestMessage");
_storageMessage = new StorageQueueMessage(cloudMessage);
}
/// <summary>
/// Get job action.
/// </summary>
/// <param name="updateJobActionParams">Job action properties specified via PowerShell.</param>
/// <param name="existingJobAction">Job action properties from existing job.</param>
/// <returns>JobAction object.</returns>
private JobAction GetExistingJobAction(PSJobActionParams updateJobActionParams, JobAction existingJobAction)
{
if (updateJobActionParams != null)
{
if (existingJobAction != null &&
(existingJobAction.Type == updateJobActionParams.JobActionType ||
((existingJobAction.Type == JobActionType.Http || existingJobAction.Type == JobActionType.Https) &&
(updateJobActionParams.JobActionType == JobActionType.Http || updateJobActionParams.JobActionType == JobActionType.Https))))
{
switch (updateJobActionParams.JobActionType)
{
case JobActionType.Http:
case JobActionType.Https:
PSHttpJobActionParams httpJobAction = updateJobActionParams.HttpJobAction;
HttpRequest existinghHttpRequest = existingJobAction.Request;
if (httpJobAction.Uri != null)
{
existinghHttpRequest.Uri = httpJobAction.Uri.OriginalString;
existingJobAction.Type = updateJobActionParams.JobActionType;
}
existinghHttpRequest.Method = httpJobAction.RequestMethod.GetValueOrDefault(defaultValue: existinghHttpRequest.Method);
existinghHttpRequest.Body = httpJobAction.RequestBody.GetValueOrDefault(defaultValue: existinghHttpRequest.Body);
existinghHttpRequest.Headers = httpJobAction.RequestHeaders != null?httpJobAction.RequestHeaders.ToDictionary() : existinghHttpRequest.Headers;
existinghHttpRequest.Authentication = this.GetExistingAuthentication(httpJobAction.RequestAuthentication, existinghHttpRequest.Authentication);
break;
case JobActionType.StorageQueue:
PSStorageJobActionParams storageJobAction = updateJobActionParams.StorageJobAction;
StorageQueueMessage existingStorageQueue = existingJobAction.QueueMessage;
storageJobAction.StorageAccount = storageJobAction.StorageAccount.GetValueOrDefault(defaultValue: existingStorageQueue.StorageAccount);
storageJobAction.StorageQueueMessage = storageJobAction.StorageQueueMessage.GetValueOrDefault(defaultValue: existingStorageQueue.Message);
storageJobAction.StorageQueueName = storageJobAction.StorageQueueName.GetValueOrDefault(defaultValue: existingStorageQueue.QueueName);
storageJobAction.StorageSasToken = storageJobAction.StorageSasToken.GetValueOrDefault(defaultValue: existingStorageQueue.SasToken);
break;
case JobActionType.ServiceBusQueue:
PSServiceBusParams serviceBusQueueParams = updateJobActionParams.ServiceBusAction;
ServiceBusQueueMessage existingServiceBusQueueMessage = existingJobAction.ServiceBusQueueMessage;
this.UpdateServiceBus(serviceBusQueueParams, existingServiceBusQueueMessage);
existingServiceBusQueueMessage.QueueName = serviceBusQueueParams.QueueName.GetValueOrDefault(defaultValue: existingServiceBusQueueMessage.QueueName);
break;
case JobActionType.ServiceBusTopic:
PSServiceBusParams serviceBusTopicParams = updateJobActionParams.ServiceBusAction;
ServiceBusTopicMessage existingServiceBusTopicMessage = existingJobAction.ServiceBusTopicMessage;
this.UpdateServiceBus(serviceBusTopicParams, existingServiceBusTopicMessage);
existingServiceBusTopicMessage.TopicPath = serviceBusTopicParams.TopicPath.GetValueOrDefault(defaultValue: existingServiceBusTopicMessage.TopicPath);
break;
}
}
else
{
this.PopulateJobAction(updateJobActionParams, ref existingJobAction);
}
}
return(existingJobAction);
}
public async Task Should_allow_regular_processing_settings_to_bypass_the_renewal()
{
//arrange
var pipeline = new MyPipeline
{
ProcessingSettings = new Mock <IProcessingSettings>().Object,
HostConfiguration = new Mock <IHostConfiguration>().Object
};
var middleware = new ExtendMessageLockDurationMiddleware();
var storageQueueClient = new Mock <IStorageQueueClient>();
var message = new StorageQueueMessage();
var storageQueueMessageStateHandler = new StorageQueueMessageStateHandler <MyMessage>(storageQueueClient.Object, message, 0, _dependencyInjection.Object);
var next = new Mock <IMessageProcessor>();
next.Setup(x => x.ProcessAsync(It.IsAny <IMessageStateHandler <MyMessage> >(), It.IsAny <CancellationToken>()))
.Returns(() => Task.Delay(2000));
//act
await middleware.ProcessAsync(storageQueueMessageStateHandler, pipeline, next.Object, CancellationToken.None);
//assert
storageQueueClient.Verify(x => x.SetVisibilityTimeout(It.IsAny <StorageQueueMessage>(), TimeSpan.FromMilliseconds(1000), It.IsAny <CancellationToken>()), Times.Never);
next.Verify(x => x.ProcessAsync(storageQueueMessageStateHandler, It.IsAny <CancellationToken>()));
}
public async Task Should_not_renew_after_completion()
{
//arrange
var pipeline = new MyPipeline();
var settings = new MySettings
{
ExtensionDuration = TimeSpan.FromMilliseconds(1000),
ExtensionInterval = TimeSpan.FromMilliseconds(100),
MessageLockTimeout = TimeSpan.FromMilliseconds(1000)
};
pipeline.ProcessingSettings = settings;
pipeline.HostConfiguration = new Mock <IHostConfiguration>().Object;
var middleware = new ExtendMessageLockDurationMiddleware();
var storageQueueClient = new Mock <IStorageQueueClient>();
var message = new StorageQueueMessage();
var storageQueueMessageStateHandler = new StorageQueueMessageStateHandler <MyMessage>(storageQueueClient.Object, message, 0, _dependencyInjection.Object);
var cancellationToken = new CancellationTokenSource(settings.MessageLockTimeout).Token;
var next = new Mock <IMessageProcessor>();
next.Setup(x => x.ProcessAsync(It.IsAny <IMessageStateHandler <MyMessage> >(), It.IsAny <CancellationToken>()))
.Returns(Task.CompletedTask);
//act
await middleware.ProcessAsync(storageQueueMessageStateHandler, pipeline, next.Object, cancellationToken);
// for good measure, make sure the test not passes only because we immediately return ProcessAsync
await Task.Delay(1000);
//assert
storageQueueClient.Verify(x => x.SetVisibilityTimeout(It.IsAny <StorageQueueMessage>(), TimeSpan.FromMilliseconds(1000), It.IsAny <CancellationToken>()), Times.Never);
next.Verify(x => x.ProcessAsync(storageQueueMessageStateHandler, cancellationToken), Times.Once());
}
public QueueListenerTests()
{
CloudQueue queue = new CloudQueue(new Uri("https://test.queue.core.windows.net/testqueue"));
Mock<IStorageQueue> mockQueue = new Mock<IStorageQueue>(MockBehavior.Strict);
mockQueue.Setup(p => p.SdkObject).Returns(queue);
_mockTriggerExecutor = new Mock<ITriggerExecutor<IStorageQueueMessage>>(MockBehavior.Strict);
Mock<IDelayStrategy> mockDelayStrategy = new Mock<IDelayStrategy>(MockBehavior.Strict);
Mock<IBackgroundExceptionDispatcher> mockExceptionDispatcher = new Mock<IBackgroundExceptionDispatcher>(MockBehavior.Strict);
TextWriter log = new StringWriter();
Mock<IQueueProcessorFactory> mockQueueProcessorFactory = new Mock<IQueueProcessorFactory>(MockBehavior.Strict);
JobHostQueuesConfiguration queuesConfig = new JobHostQueuesConfiguration();
QueueProcessorFactoryContext context = new QueueProcessorFactoryContext(queue, log, queuesConfig);
_mockQueueProcessor = new Mock<QueueProcessor>(MockBehavior.Strict, context);
JobHostQueuesConfiguration queueConfig = new JobHostQueuesConfiguration
{
MaxDequeueCount = 5,
QueueProcessorFactory = mockQueueProcessorFactory.Object
};
mockQueueProcessorFactory.Setup(p => p.Create(It.IsAny<QueueProcessorFactoryContext>())).Returns(_mockQueueProcessor.Object);
_listener = new QueueListener(mockQueue.Object, null, _mockTriggerExecutor.Object, mockDelayStrategy.Object, mockExceptionDispatcher.Object, log, null, queueConfig);
CloudQueueMessage cloudMessage = new CloudQueueMessage("TestMessage");
_storageMessage = new StorageQueueMessage(cloudMessage);
}
public void QueueVacancyPages(StorageQueueMessage scheduledQueueMessage)
{
_logger.Debug("Retrieving vacancy summary page count");
var vacancyPageCount = _vacancyIndexDataProvider.GetVacancyPageCount();
_logger.Info("Retrieved vacancy summary page count of {0}", vacancyPageCount);
if (vacancyPageCount == 0)
{
_logger.Warn("Expected vacancy page count to be greater than zero. Indexes will not be created successfully");
_processControlQueue.DeleteMessage(scheduledQueueMessage.MessageId, scheduledQueueMessage.PopReceipt);
return;
}
var vacancySumaries = BuildVacancySummaryPages(scheduledQueueMessage.ExpectedExecutionTime, vacancyPageCount).ToList();
foreach (var vacancySummaryPage in vacancySumaries)
{
_messageBus.PublishMessage(vacancySummaryPage);
}
// Only delete from queue once we have all vacancies from the service without error.
_processControlQueue.DeleteMessage(scheduledQueueMessage.MessageId, scheduledQueueMessage.PopReceipt);
_logger.Info("Queued {0} vacancy summary pages", vacancySumaries.Count());
}
public async Task Should_send_message_to_secondary_if_primary_queue_not_available()
{
var message = new StorageQueueMessage(new CloudQueueMessage("{}"));
await cluster.AddMessageAsync(message);
var messages = await secondary.GetMessagesAsync(32);
Assert.Single(messages);
}
private async Task ProcessPackagesAsync(CancellationToken token)
{
StorageQueueMessage <PackageValidatorContext> queueMessage = null;
do
{
Logger.LogInformation("Fetching next queue message.");
queueMessage = await _queue.GetNextAsync(token);
await HandleQueueMessageAsync(queueMessage, token);
} while (queueMessage != null);
Logger.LogInformation("No messages left in queue.");
}
private async Task HandleQueueMessageAsync(
StorageQueueMessage <PackageValidatorContext> queueMessage,
CancellationToken token)
{
if (queueMessage == null)
{
return;
}
var queuedContext = queueMessage.Contents;
var messageWasProcessed = false;
try
{
await RunPackageValidatorAsync(queuedContext, token);
// The validations ran successfully and were saved to storage.
// We can remove the message from the queue because it was processed.
messageWasProcessed = true;
}
catch (Exception validationFailedToRunException)
{
try
{
// Validations failed to run! Save this failed status to storage.
await SaveFailedPackageMonitoringStatusAsync(queuedContext, validationFailedToRunException, token);
// We can then remove the message from the queue because this failed status can be used to requeue the message.
messageWasProcessed = true;
}
catch (Exception failedValidationSaveFailureException)
{
// We failed to run validations and failed to save the failed validation!
// We were not able to process this message. We need to log the exceptions so we can debug the issue.
var aggregateException = new AggregateException(
"Validations failed to run and saving unsuccessful validation failed!",
new[] { validationFailedToRunException, failedValidationSaveFailureException });
Logger.LogCritical(
NuGet.Services.Metadata.Catalog.Monitoring.LogEvents.QueueMessageFatalFailure,
aggregateException,
"Failed to process queue message");
}
}
// If we failed to run validations and failed to save the failed validation, we cannot remove the message from the queue.
if (messageWasProcessed)
{
await _queue.RemoveAsync(queueMessage, token);
}
}
private static Queue <StorageQueueMessage> GetScheduledMessagesQueue()
{
var queue = new Queue <StorageQueueMessage>();
var storageScheduleMessage = new StorageQueueMessage
{
ClientRequestId = Guid.NewGuid(),
ExpectedExecutionTime = DateTime.Today,
SchedulerJobId = Guid.NewGuid().ToString()
};
queue.Enqueue(storageScheduleMessage);
queue.Enqueue(null);
return(queue);
}
private static Queue <StorageQueueMessage> GetScheduledMessagesQueue(int count)
{
var queue = new Queue <StorageQueueMessage>();
for (var i = count; i > 0; i--)
{
var storageScheduleMessage = new StorageQueueMessage
{
ClientRequestId = Guid.NewGuid(),
ExpectedExecutionTime = DateTime.Today,
SchedulerJobId = i.ToString(CultureInfo.InvariantCulture)
};
queue.Enqueue(storageScheduleMessage);
}
queue.Enqueue(null);
return(queue);
}
/// <summary>
/// Gets storage queue.
/// </summary>
/// <param name="storageActionParams">Storage queue action propertis specified via PowerShell.</param>
/// <returns>StorageQueueMessage object.</returns>
private StorageQueueMessage GetStorageQueue(PSStorageJobActionParams storageActionParams)
{
if (string.IsNullOrWhiteSpace(storageActionParams.StorageAccount) ||
string.IsNullOrWhiteSpace(storageActionParams.StorageQueueMessage) ||
string.IsNullOrWhiteSpace(storageActionParams.StorageQueueName) ||
string.IsNullOrWhiteSpace(storageActionParams.StorageSasToken))
{
throw new PSManagement.PSArgumentException(Resources.SchedulerInvalidStorageQueue);
}
var storageQueue = new StorageQueueMessage()
{
Message = storageActionParams.StorageQueueMessage,
QueueName = storageActionParams.StorageQueueName,
SasToken = storageActionParams.StorageSasToken,
StorageAccount = storageActionParams.StorageAccount,
};
return(storageQueue);
}
public void ProcessVacancyPages(StorageQueueMessage scheduledQueueMessage)
{
_logger.Debug("Retrieving vacancy summary page count");
var vacancyPageCount = _vacancyIndexDataProvider.GetVacancyPageCount();
_logger.Info("Retrieved vacancy summary page count of {0}", vacancyPageCount);
if (vacancyPageCount == 0)
{
_logger.Warn("Expected vacancy page count to be greater than zero. Indexes will not be created successfully");
_jobControlQueue.DeleteMessage(ScheduledJobQueues.VacancyEtl, scheduledQueueMessage.MessageId, scheduledQueueMessage.PopReceipt);
return;
}
var vacancySummaries = BuildVacancySummaryPages(scheduledQueueMessage.ExpectedExecutionTime, vacancyPageCount).ToList();
// Only delete from queue once we have all vacancies from the service without error.
_jobControlQueue.DeleteMessage(ScheduledJobQueues.VacancyEtl, scheduledQueueMessage.MessageId, scheduledQueueMessage.PopReceipt);
//Process pages
Parallel.ForEach(vacancySummaries, new ParallelOptions {
MaxDegreeOfParallelism = 1
}, ProcessVacancySummaryPage);
var lastVacancySummaryPage = vacancySummaries.Last();
_logger.Info("Vacancy ETL Queue completed: {0} vacancy summary pages processed ", lastVacancySummaryPage.TotalPages);
_logger.Info("Publishing VacancySummaryUpdateComplete message to queue");
var vsuc = new VacancySummaryUpdateComplete
{
ScheduledRefreshDateTime = lastVacancySummaryPage.ScheduledRefreshDateTime
};
_serviceBus.PublishMessage(vsuc);
_logger.Info("Published VacancySummaryUpdateComplete message published to queue");
}
public void ShouldQueueCorrectNumberOfVacancyPages(int vancanyPages)
{
_vacancyProviderMock.Setup(x => x.GetVacancyPageCount()).Returns(vancanyPages);
var vacancyConsumer = GetGatewayVacancySummaryProcessor();
var scheduledMessage = new StorageQueueMessage
{
ClientRequestId = Guid.NewGuid(),
ExpectedExecutionTime = DateTime.Today,
MessageId = "456",
PopReceipt = "789"
};
vacancyConsumer.QueueVacancyPages(scheduledMessage);
Thread.Sleep(100); //Slight delay to ensure parallel foreach has completed before assertions are made
_messagingServiceMock.Verify(
x =>
x.DeleteMessage(It.Is <string>(mid => mid == scheduledMessage.MessageId),
It.Is <string>(pr => pr == scheduledMessage.PopReceipt), null), Times.Once);
_busMock.Verify(x => x.PublishMessage(It.IsAny <VacancySummaryPage>()), Times.Exactly(vancanyPages));
}
/// <summary>
/// Gets storage queue job action details.
/// </summary>
/// <param name="jobActionType">Job action type.</param>
/// <param name="storageQueue">Storage queue message.</param>
/// <returns>PSStorageJobActionDetails.</returns>
internal static PSStorageJobActionDetails GetSchedulerStorageJobActionDetails(JobActionType jobActionType, StorageQueueMessage storageQueue)
{
if (storageQueue == null)
{
throw new ArgumentNullException(paramName: "storageQueue");
}
var psStorageJobActionDetails = new PSStorageJobActionDetails(jobActionType)
{
StorageAccount = storageQueue.StorageAccount,
StorageQueueMessage = storageQueue.Message,
StorageQueueName = storageQueue.QueueName,
StorageSasToken = storageQueue.SasToken
};
return(psStorageJobActionDetails);
}
private async Task HandleQueueMessageAsync(
StorageQueueMessage <PackageValidatorContext> queueMessage,
CancellationToken token)
{
if (queueMessage == null)
{
return;
}
var queuedContext = queueMessage.Contents;
var messageWasProcessed = false;
try
{
await RunPackageValidatorAsync(queuedContext, token);
// The validations ran successfully and were saved to storage.
// We can remove the message from the queue because it was processed.
messageWasProcessed = true;
}
catch (Exception validationFailedToRunException)
{
try
{
// Validations failed to run! Save this failed status to storage.
await SaveFailedPackageMonitoringStatusAsync(queuedContext, validationFailedToRunException, token);
// We can then remove the message from the queue because this failed status can be used to requeue the message.
messageWasProcessed = true;
}
catch (Exception failedValidationSaveFailureException)
{
// We failed to run validations and failed to save the failed validation!
// We were not able to process this message. We need to log the exceptions so we can debug the issue.
throw new AggregateException(
"Validations failed to run and saving unsuccessful validation failed!",
new[] { validationFailedToRunException, failedValidationSaveFailureException });
}
}
// If we failed to run validations and failed to save the failed validation, we cannot remove the message from the queue.
if (messageWasProcessed)
{
try
{
await _queue.RemoveAsync(queueMessage, token);
}
catch (StorageException storageException)
{
if (storageException.RequestInformation.ExtendedErrorInformation.ErrorCode == QueueErrorCodeStrings.MessageNotFound)
{
Logger.LogWarning(
NuGet.Services.Metadata.Catalog.Monitoring.LogEvents.QueueMessageRemovalFailure,
storageException,
"Queue message for {PackageId} {PackageVersion} no longer exists. Message was likely already processed.",
queuedContext.Package.Id, queuedContext.Package.Version);
}
else
{
Logger.LogCritical(
NuGet.Services.Metadata.Catalog.Monitoring.LogEvents.QueueMessageRemovalFailure,
storageException,
"Failed to remove queue message.");
}
}
}
}
public DeserializedStorageQueueMessage(T contents, StorageQueueMessage message)
: base(contents, message.DequeueCount)
{
Message = message;
}
public Task SendAsync(StorageQueueMessage message, CancellationToken cancelationToken = default(CancellationToken))
{
var nextQueue = roundRobbinNumberResolver.GetNextNumber();
return(cloudQueues[nextQueue].AddMessageAsync(message.Data));
}
/// <summary>
/// Gets storage queue job action details.
/// </summary>
/// <param name="jobActionType">Job action type.</param>
/// <param name="storageQueue">Storage queue message.</param>
/// <returns>PSStorageJobActionDetails.</returns>
internal static PSStorageJobActionDetails GetSchedulerStorageJobActionDetails(JobActionType jobActionType, StorageQueueMessage storageQueue)
{
if (storageQueue == null)
{
throw new ArgumentNullException(paramName: "storageQueue");
}
var psStorageJobActionDetails = new PSStorageJobActionDetails(jobActionType)
{
StorageAccount = storageQueue.StorageAccount,
StorageQueueMessage = storageQueue.Message,
StorageQueueName = storageQueue.QueueName,
StorageSasToken = storageQueue.SasToken
};
return psStorageJobActionDetails;
}
public Task SendAsync(StorageQueueMessage message, CancellationToken cancelationToken = default(CancellationToken))
{
return(AddMessageRecusivelyAsync(cloudQueues, message, 0, 0, cancelationToken));
}
private Task AddMessageRecusivelyAsync(IReadOnlyList <CloudQueue> cloudQueues, StorageQueueMessage message, int index, int failed, CancellationToken cancelationToken)
{
if (index >= cloudQueues.Count)
{
return(Task.CompletedTask);
}
var next = index + 1;
var cloudQueue = cloudQueues[index];
try
{
return(cloudQueue.AddMessageAsync(message.Data, message.TimeToLive, message.InitialVisibilityDelay, message.Options, message.OperationContext, cancelationToken));
}
catch (Exception ex) when(ex.IsTransilient())
{
if (++failed == cloudQueues.Count)
{
throw;
}
return(AddMessageRecusivelyAsync(cloudQueues, message, next, failed, cancelationToken));
}
}
private System.Linq.Expressions.Expression <Func <CloudQueue, Task> > AddMessageAsync(StorageQueueMessage message)
{
return(self => self.AddMessageAsync(message.Data, message.TimeToLive, message.InitialVisibilityDelay, message.Options, message.OperationContext, It.IsAny <CancellationToken>()));
}
/// <summary>
/// Gets storage queue.
/// </summary>
/// <param name="storageActionParams">Storage queue action propertis specified via PowerShell.</param>
/// <returns>StorageQueueMessage object.</returns>
private StorageQueueMessage GetStorageQueue(PSStorageJobActionParams storageActionParams)
{
if (string.IsNullOrWhiteSpace(storageActionParams.StorageAccount) ||
string.IsNullOrWhiteSpace(storageActionParams.StorageQueueMessage) ||
string.IsNullOrWhiteSpace(storageActionParams.StorageQueueName) ||
string.IsNullOrWhiteSpace(storageActionParams.StorageSasToken))
{
throw new PSManagement.PSArgumentException(Resources.SchedulerInvalidStorageQueue);
}
var storageQueue = new StorageQueueMessage()
{
Message = storageActionParams.StorageQueueMessage,
QueueName = storageActionParams.StorageQueueName,
SasToken = storageActionParams.StorageSasToken,
StorageAccount = storageActionParams.StorageAccount,
};
return storageQueue;
}
