private ResultState ProcessQueueMessages(int queueId, int batchSize, bool archiveMessages) { var state = new ResultState { IsFailed = false }; var messages = Repository.FetchQueueMessages(queueId, batchSize); if (messages != null) { foreach (var message in messages) { var timer = Stopwatch.StartNew(); var messageId = message.MessageId; state.LastMessageId = messageId; try { var messageContext = GetMessageContext(message.Context); state.MaxRetries = messageContext.Settings.MaxRetriesBeforeFail.GetValueOrDefault(); state.RecoveryMode = messageContext.Settings.RecoveryMode; PublishSettings settings = messageContext.Settings; using (var transaction = Repository.CreateProcessingTransaction(settings.JobIsolationLevel, settings.JobTimeout)) { var executor = new ExpressionExecutor(_serializer, _jobActivator); if (!string.IsNullOrWhiteSpace(message.ContextFactory)) { using (var context = executor.Execute<ExecutionContext>(message.ContextFactory)) { if (context != null) { context.Invoke(executor, message.Content); } else { executor.Execute(message.Content); } } } else { executor.Execute(message.Content); } if (transaction.TransactionStatus.HasValue) { var transactionStatus = transaction.TransactionStatus.Value; if (transactionStatus == TransactionStatus.Aborted || transactionStatus == TransactionStatus.InDoubt) { throw new Exception($"Invalid transaction status: [{transactionStatus}]! Unable to commit!"); } } Repository.RemoveMessage(messageId, archiveMessages && !settings.DiscardWhenComplete, transaction); transaction.Complete(); } state.ProcessedMessages += 1; } catch (Exception e) { var error = e.GetFormattedError(messageId); Logger.Error(error); state.Error = error; state.IsFailed = true; } finally { timer.Stop(); Logger.Trace("Message {0} processed in {1}", messageId, timer.Elapsed); } if (Canceled || state.IsFailed) { break; } } } return state; }
private void ValidateResultState(ResultState state, QueueInfo queueInfo) { if (state.IsFailed) { // message failed, increment retry count and set error information // if there was at least one message successfull reset retry count var retries = (state.ProcessedMessages > 0 ? 0 : queueInfo.Retries.GetValueOrDefault()) + 1; Logger.Debug("Retries: {0}. State max retries: {1}. Recovery mode: {2}", retries, state.MaxRetries, state.RecoveryMode); if (retries >= state.MaxRetries && state.RecoveryMode != RecoveryMode.Block) { RecoverQueue(queueInfo.QueueId, state.LastMessageId, state.RecoveryMode); } else { var nextTryTime = DateTime.UtcNow.AddSeconds(ServerSettings.NextTryAfterFailInSeconds); Repository.SetQueueFailure(queueInfo.QueueId, retries, state.Error, nextTryTime); } } else { Repository.ReleaseQueue(queueInfo.QueueId); } }