void MainLoop() { SynchronizationContext.SetSynchronizationContext(continuations); while (!shouldExit) { if (!shouldWork) { Thread.Sleep(20); continue; } try { try { continuations.Run(); TryProcessIncomingMessage(); errorThatBlocksOurAbilityToDoUsefulWorkBackoffHelper.Reset(); } catch (MessageHandleException exception) { UserException(this, exception); } catch (UnitOfWorkCommitException exception) { UserException(this, exception); } catch (QueueCommitException exception) { UserException(this, exception); // when the queue cannot commit, we can't get the message into the error queue either - basically, // there's no way we can do useful work if we end up in here, so we just procrastinate for a while errorThatBlocksOurAbilityToDoUsefulWorkBackoffHelper .Wait(waitTime => log.Warn( "Caught an exception when interacting with the queue system - there's basically no meaningful" + " work we can do as long as we can't successfully commit a queue transaction, so instead we" + " wait and hope that the situation gets better... current wait time is {0}", waitTime)); } catch (Exception e) { SystemException(this, e); } } catch (Exception ex) { errorThatBlocksOurAbilityToDoUsefulWorkBackoffHelper .Wait(waitTime => log.Error(ex, "An unhandled exception occurred while iterating the main worker loop - this is a critical error," + " and there's a probability that we cannot do any useful work, which is why we'll wait for" + " {0} and hope that the error goes away", waitTime)); } } }
void DoTry() { var transportMessage = receiveMessages.ReceiveMessage(TransactionContext.Current); if (transportMessage == null) { // to back off and relax when there's no messages to process, we do this nullMessageReceivedBackoffHelper.Wait(); return; } nullMessageReceivedBackoffHelper.Reset(); var id = transportMessage.Id; var label = transportMessage.Label; MessageContext context = null; if (id == null) { HandlePoisonMessage(id, transportMessage); return; } if (errorTracker.MessageHasFailedMaximumNumberOfTimes(id)) { HandlePoisonMessage(id, transportMessage); errorTracker.StopTracking(id); return; } Exception transportMessageExceptionOrNull = null; try { BeforeTransportMessage(transportMessage); // Populate rebus-msg-id, if not set, from transport-level-id if (!transportMessage.Headers.ContainsKey(Headers.MessageId)) { transportMessage.Headers[Headers.MessageId] = transportMessage.Id; } using (var scope = BeginTransaction()) { var message = serializeMessages.Deserialize(transportMessage); // successfully deserialized the transport message, let's enter a message context context = MessageContext.Establish(message.Headers); MessageContextEstablished(context); var unitsOfWork = unitOfWorkManagers.Select(u => u.Create()) .Where(u => !ReferenceEquals(null, u)) .ToArray(); //< remember to invoke the chain here :) try { foreach (var logicalMessage in message.Messages.Select(MutateIncoming)) { context.SetLogicalMessage(logicalMessage); Exception logicalMessageExceptionOrNull = null; try { BeforeMessage(logicalMessage); var typeToDispatch = logicalMessage.GetType(); messageLogger.LogReceive(id, logicalMessage); try { var dispatchMethod = GetDispatchMethod(typeToDispatch); var parameters = new[] { logicalMessage }; dispatchMethod.Invoke(this, parameters); } catch (TargetInvocationException tie) { var exception = tie.InnerException; exception.PreserveStackTrace(); throw exception; } } catch (Exception exception) { logicalMessageExceptionOrNull = exception; throw; } finally { try { AfterMessage(logicalMessageExceptionOrNull, logicalMessage); } catch (Exception exceptionWhileRaisingEvent) { if (logicalMessageExceptionOrNull != null) { log.Error( "An exception occurred while raising the AfterMessage event, and an exception occurred some" + " time before that as well. The first exception was this: {0}. And then, when raising the" + " AfterMessage event (including the details of the first error), this exception occurred: {1}", logicalMessageExceptionOrNull, exceptionWhileRaisingEvent); } else { log.Error("An exception occurred while raising the AfterMessage event: {0}", exceptionWhileRaisingEvent); } } context.ClearLogicalMessage(); } } foreach (var unitOfWork in unitsOfWork) { try { unitOfWork.Commit(); } catch (Exception exception) { throw new UnitOfWorkCommitException(exception, unitOfWork); } } } catch { foreach (var unitOfWork in unitsOfWork) { try { unitOfWork.Abort(); } catch (Exception abortException) { log.Warn("An error occurred while aborting the unit of work {0}: {1}", unitOfWork, abortException); } } throw; } finally { foreach (var unitOfWork in unitsOfWork) { unitOfWork.Dispose(); } } if (scope != null) { scope.Complete(); } } } catch (Exception exception) { transportMessageExceptionOrNull = exception; log.Debug("Handling message {0} with ID {1} has failed", label, id); errorTracker.TrackDeliveryFail(id, exception); throw new MessageHandleException(id, exception); } finally { try { AfterTransportMessage(transportMessageExceptionOrNull, transportMessage); } catch (Exception exceptionWhileRaisingEvent) { if (transportMessageExceptionOrNull != null) { log.Error( "An exception occurred while raising the AfterTransportMessage event, and an exception occurred some" + " time before that as well. The first exception was this: {0}. And then, when raising the" + " AfterTransportMessage event (including the details of the first error), this exception occurred: {1}", transportMessageExceptionOrNull, exceptionWhileRaisingEvent); } else { log.Error("An exception occurred while raising the AfterTransportMessage event: {0}", exceptionWhileRaisingEvent); } } if (context != null) { context.Dispose(); //< dispose it if we entered } } errorTracker.StopTracking(id); }
public void CanBeReset() { var timesWaited = new List<TimeSpan>(); var backoffTimes = new[] { TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(2), }; var helper = new BackoffHelper(backoffTimes) { waitAction = timesWaited.Add }; helper.Wait(); helper.Wait(); helper.Reset(); helper.Wait(); helper.Wait(); timesWaited.Count.ShouldBe(4); timesWaited[0].ShouldBe(TimeSpan.FromSeconds(1)); timesWaited[1].ShouldBe(TimeSpan.FromSeconds(2)); timesWaited[2].ShouldBe(TimeSpan.FromSeconds(1)); timesWaited[3].ShouldBe(TimeSpan.FromSeconds(2)); }