private int _getRecoveryCount(Message message, RecoverabilityContext context) { var count = message.UserProperties.ContainsKey("RecoveryCount") ? (int)message.UserProperties["RecoveryCount"] : 0; return(count); }
public async Task <RecoverabilityContext> OnPreHandle(RecoverabilityContext context) { if (IsControlMessage(context.Message))//never try to reschedule the control messages themselves { return(await _handleControlMessage(context.Message, context)); } return(context); }
public async Task <RecoverabilityContext> OnPreHandle(RecoverabilityContext context) { var count = _getRecoveryCount(context.Message, context); if (await _handleMaxRetry(count, context.Message, context)) { context.SkipMessage = true; } return(context); }
public async Task OnPostHandle(RecoverabilityContext context)// int timesQueued, Endpoint endpoint, IMessageReceiver receiver, Message controlMessage = null, string description = null, Exception exc = null) { try { if (context.TempData.ContainsKey("ControlMessage")) { await _completeImmediateRetryPolicy.ExecuteAsync(() => context.Receiver.CompleteAsync(((Message)context.TempData["ControlMessage"]).SystemProperties.LockToken) ); } } catch (Exception exc) { ///TODO log that control message couldn't be completed } }
private async Task _handleMessage(RecoverabilityContext context, CancellationToken cancellationToken) { bool isHandled = false; try { context = await _recoverability.OnPreHandle(context); if (context.SkipMessage) { return; } await _handleMessageFunc( new TransportMessage(context.Message.Label, context.Message.Body, _metaDataMapper.ExtractMetaData(context.Message)), _endpoint ); isHandled = true; } catch (NonTransientException nte) { _logger.LogError(nte, "AzureServiceBusMessagePump encountered a NonTransient Exception handling a message. Sending to DLQ. Endpoint: {EndpointName}", context.Endpoint.Name); await _receiver.DeadLetterAsync(context.Message.SystemProperties.LockToken); } catch (Exception exc) { //backoff retry _logger.LogError(exc, "AzureServiceBusMessagePump encountered an exception handling a message. Attempting to recover. Endpoint: {EndpointName}. Label: {messageLabel}", context.Endpoint.Name, context.Message?.Label); await _recoverability.Recover(context); } //if an exception occurs completing the message then it will stay on the queue and be handled again //we don't want to trigger recoverability/retry if (isHandled) { await _completeImmediateRetryPolicy.ExecuteAsync(() => _receiver.CompleteAsync(context.Message.SystemProperties.LockToken) ); await _recoverability.OnPostHandle(context); } }
public async Task Recover(RecoverabilityContext context) { var sender = AzureServiceBusClientCache.GetSender(context.Endpoint.Settings.ConnectionString); var count = _getRecoveryCount(context.Message, context); var retryMessage = context.Message.Clone(); retryMessage.MessageId = Guid.NewGuid().ToString(); retryMessage.UserProperties.Add("OriginalMessageId", context.Message.MessageId); retryMessage.UserProperties.Add("RecoveryCount", ++count); await sender.ScheduleMessageAsync( retryMessage, this._retryStrategy.GetNextDateUtc(++count) ); //complete the original message - we've already scheduled a clone await _completeImmediateRetryPolicy.ExecuteAsync(() => context.Receiver.CompleteAsync(context.Message.SystemProperties.LockToken) ); }
public async Task Recover(RecoverabilityContext context) // int timesQueued, Endpoint endpoint, IMessageReceiver receiver, Message controlMessage = null, string description = null, Exception exc = null) { if (IsControlMessage(context.Message)) //never try to reschedule the control messages themselves { return; } var count = context.TempData.ContainsKey("ControlMessageContent") ? ((RecoverabilityControlMessage)context.TempData["ControlMessageContent"]).RecoveryCount : 0; var sender = AzureServiceBusClientCache.GetSender(context.Endpoint.Settings.ConnectionString); var retryMessage = new RecoverabilityControlMessage(context.Message.SystemProperties.SequenceNumber, ++count); //schedule a special control message to be delivered in the future to tell the the deferred message to be retrieved and re processed await sender.ScheduleMessageAsync( _createServiceBusMessage(retryMessage), this._retryStrategy.GetNextDateUtc(retryMessage.RecoveryCount) ); //defer the current message / first time through if (!context.TempData.ContainsKey("ControlMessage")) { await context.Receiver.DeferAsync(context.Message.SystemProperties.LockToken); } else //already deferred. complete the control message / we just sent a new one above { await _completeImmediateRetryPolicy.ExecuteAsync(() => context.Receiver.CompleteAsync(((Message)context.TempData["ControlMessage"]).SystemProperties.LockToken) ); //release the lock on the current deferred message await _completeImmediateRetryPolicy.ExecuteAsync(() => context.Receiver.AbandonAsync(context.Message.SystemProperties.LockToken) ); } }
private async Task <bool> _handleMaxRetry(int recoveryCount, Message message, RecoverabilityContext context) { if (message.SystemProperties.DeliveryCount > 1 || recoveryCount > _maxDeliveryCount) { await context.Receiver.DeadLetterAsync(message.SystemProperties.LockToken); return(true); } return(false); }
public Task OnPostHandle(RecoverabilityContext context) // int timesQueued, Endpoint endpoint, IMessageReceiver receiver, Message controlMessage = null, string description = null, Exception exc = null) { //no cleanup to do here return(Task.CompletedTask); }
private async Task <RecoverabilityContext> _handleControlMessage(Message serviceBusControlMessage, RecoverabilityContext context) { try { var json = Encoding.UTF8.GetString(serviceBusControlMessage.Body); var controlMessage = (RecoverabilityControlMessage)JsonConvert.DeserializeObject <RecoverabilityControlMessage>(json); context.Message = await _receiveDeferredImmediateRetryPolicy.ExecuteAsync(() => context.Receiver.ReceiveDeferredMessageAsync(controlMessage.SequenceNumber) ); context.TempData["ControlMessage"] = serviceBusControlMessage; context.TempData["ControlMessageContent"] = controlMessage; return(context); } catch (Exception exc) { //wrap all exceptions in a nontransient exception because retry won't help us now ///TODO: if deferred message times-out then it will throw exceptions until the control message retries expire ///need to use poly to retry and circuit break throw new NonTransientException("Error handling Recoverability control message", exc); } }