protected override Task OnStart(IMessageSession session)
{
stopTokenSource = new CancellationTokenSource();
monitorTask = Task.Run(async() =>
{
while (!stopTokenSource.IsCancellationRequested)
{
try
{
var routesChanged = routeState.Values.Count(x => x.Tick(DateTime.UtcNow));
if (routesChanged > 0)
{
UpdateRoutes();
}
}
catch (Exception e)
{
criticalError.Raise("Unexpected error while updating routing table", e);
}
await Task.Delay(1000).ConfigureAwait(false);
}
});
return(Task.CompletedTask);
}
static RepeatedFailuresOverTimeCircuitBreaker SetupCircuitBreaker(CriticalError criticalError, TimeSpan timeToWaitBeforeTriggering, TimeSpan delayAfterFailure)
{
return(new RepeatedFailuresOverTimeCircuitBreaker("SqlTransportConnectivity",
timeToWaitBeforeTriggering,
ex => criticalError.Raise("Repeated failures when communicating with SQL",
ex), delayAfterFailure));
}
static RepeatedFailuresOverTimeCircuitBreaker SetupCircuitBreaker(CriticalError criticalError, TimeSpan timeToWaitBeforeTriggering, TimeSpan delayAfterFailure)
{
return new RepeatedFailuresOverTimeCircuitBreaker("SqlTransportConnectivity",
timeToWaitBeforeTriggering,
ex => criticalError.Raise("Repeated failures when communicating with SQL",
ex), delayAfterFailure);
}
public void Increment(Exception lastException)
{
var result = Interlocked.Increment(ref failureCount);
if (result > 50)
{
criticalError.Raise("Failed to import too many times", lastException);
}
}
void InvokeCriticalError(CriticalError criticalError, string errorMessage, Exception exception)
{
#region InvokeCriticalError
// 'criticalError' is an instance of the NServiceBus.CriticalError class
// This instance can be resolved from the container.
criticalError.Raise(errorMessage, exception);
#endregion
}
void InvokeCriticalError(CriticalError criticalError, string errorMessage, Exception exception)
{
#region InvokeCriticalError
// 'criticalError' is an instance of NServiceBus.CriticalError
// This instance can be resolved from dependency injection.
criticalError.Raise(errorMessage, exception);
#endregion
}
public RoutingInfoCommunicator(SqlDataAccess dataAccess, CriticalError criticalError)
{
this.dataAccess = dataAccess;
circuitBreaker = new RepeatedFailuresOverTimeCircuitBreaker(
name: "Refresh",
timeToWaitBeforeTriggering: TimeSpan.FromMinutes(2),
triggerAction: exception =>
{
criticalError.Raise("Failed to refresh routing info.", exception);
});
}
protected override void OnStart()
{
subscription = store.Changes().ForDocumentsStartingWith("ExternalIntegrationDispatchRequests").Where(c => c.Type == DocumentChangeTypes.Put).Subscribe(OnNext);
tokenSource = new CancellationTokenSource();
circuitBreaker = new RepeatedFailuresOverTimeCircuitBreaker("EventDispatcher",
TimeSpan.FromMinutes(5),
ex => criticalError.Raise("Repeated failures when dispatching external integration events.", ex),
TimeSpan.FromSeconds(20));
StartDispatcher();
}
private void OnError(Exception exception)
{
Logger.Error("Failed to cleanup client outbox messages", exception);
_failures++;
if (_failures >= 10)
{
_criticalError.Raise("Failed to cleanup client outbox messages after 10 consecutive unsuccessful attempts", exception);
_failures = 0;
}
}
void HandleTimerCallback()
{
var age = DateTime.UtcNow - last;
var remaining = NoActivityDuration - age;
if (remaining.Ticks < 0)
{
var ex = new Exception($"No activity for {NoActivityDuration}, last activity {last}");
CriticalError.Raise(ex.Message, ex);
remaining = NoActivityDuration;
}
InactivityTimer.Change(remaining, Timeout.InfiniteTimeSpan);
}
public void Start()
{
circuitBreaker = new RepeatedFailuresOverTimeCircuitBreaker("TimeoutStorageConnectivity", TimeSpan.FromMinutes(2),
ex =>
CriticalError.Raise("Repeated failures when fetching timeouts from storage, endpoint will be terminated.", ex));
TimeoutManager.TimeoutPushed = TimeoutsManagerOnTimeoutPushed;
SecondsToSleepBetweenPolls = 1;
tokenSource = new CancellationTokenSource();
StartPoller();
}
public Task Init(Func <MessageContext, Task> pump, Func <ErrorContext, Task <ErrorHandleResult> > onError, CriticalError criticalError, PushSettings pushSettings)
{
topologyOperator = DetermineTopologyOperator(pushSettings.InputQueue);
messagePump = pump;
var name = $"MessagePump on the queue `{pushSettings.InputQueue}`";
circuitBreaker = new RepeatedFailuresOverTimeCircuitBreaker(name, timeToWaitBeforeTriggering, ex => criticalError.Raise("Failed to receive message from Azure Service Bus.", ex));
if (pushSettings.PurgeOnStartup)
{
throw new InvalidOperationException("Azure Service Bus transport doesn't support PurgeOnStartup behavior");
}
inputQueue = pushSettings.InputQueue;
topologyOperator.OnIncomingMessage(async(incoming, receiveContext) =>
{
if (circuitBreaker == null || throttler == null) /* can be disposed by fody injected logic, in theory */
{
return;
}
var tokenSource = new CancellationTokenSource();
receiveContext.CancellationToken = tokenSource.Token;
circuitBreaker.Success();
var transportTransaction = new TransportTransaction();
transportTransaction.Set(receiveContext);
await throttler.WaitAsync(receiveContext.CancellationToken).ConfigureAwait(false);
try
{
await messagePump(new MessageContext(incoming.MessageId, incoming.Headers, incoming.Body, transportTransaction, tokenSource, new ContextBag())).ConfigureAwait(false);
}
finally
{
throttler.Release();
}
});
topologyOperator.OnError(exception => circuitBreaker?.Failure(exception));
topologyOperator.OnProcessingFailure(onError);
topologyOperator.OnCritical(exception => criticalError.Raise("Failed to receive message from Azure Service Bus.", exception));
return(TaskEx.Completed);
}
protected override Task OnStart(IMessageSession session)
{
timer = new Timer(_ =>
{
try
{
routeTable.AddOrReplaceRoutes("MySource", LoadRoutes());
}
catch (Exception ex)
{
criticalError.Raise("Ambiguous route detected", ex);
}
}, null, TimeSpan.FromSeconds(30), TimeSpan.FromSeconds(30));
return(Task.FromResult(0));
}
void TryInvokeFaultManager(TransportMessage message, Exception exception)
{
try
{
message.RevertToOriginalBodyIfNeeded();
failureManager.ProcessingAlwaysFailsForMessage(message, exception);
}
catch (Exception ex)
{
criticalError.Raise(String.Format("Fault manager failed to process the failed message with id {0}", message.Id), ex);
throw;
}
}
void TryInvokeFaultManager(TransportMessage message, Exception exception, int numberOfAttempts)
{
try
{
message.RevertToOriginalBodyIfNeeded();
var numberOfRetries = numberOfAttempts - 1;
message.Headers[Headers.FLRetries] = numberOfRetries.ToString();
failureManager.ProcessingAlwaysFailsForMessage(message, exception);
}
catch (Exception ex)
{
criticalError.Raise(String.Format("Fault manager failed to process the failed message with id {0}", message.Id), ex);
throw;
}
}
public ServiceControlBackend(IDispatchMessages messageSender, ReadOnlySettings settings, CriticalError criticalError)
{
this.settings = settings;
this.messageSender = messageSender;
serializer = new DataContractJsonSerializer(typeof(ReportCustomCheckResult), new DataContractJsonSerializerSettings
{
DateTimeFormat = new DateTimeFormat("o"),
EmitTypeInformation = EmitTypeInformation.Always,
});
serviceControlBackendAddress = GetServiceControlAddress();
circuitBreaker =
new RepeatedFailuresOverTimeCircuitBreaker("ServiceControlConnectivity", TimeSpan.FromMinutes(2),
ex =>
criticalError.Raise("You have ServiceControl plugins installed in your endpoint, however, this endpoint is repeatedly unable to contact the ServiceControl backend to report endpoint information.", ex));
}
/// <summary>
/// <see cref="IStartableBus.Start()"/>
/// </summary>
public IBus Start()
{
LicenseManager.PromptUserForLicenseIfTrialHasExpired();
if (started)
{
return(this);
}
lock (startLocker)
{
if (started)
{
return(this);
}
AppDomain.CurrentDomain.SetPrincipalPolicy(PrincipalPolicy.WindowsPrincipal);
if (!DoNotStartTransport)
{
Transport.StartedMessageProcessing += TransportStartedMessageProcessing;
Transport.TransportMessageReceived += TransportMessageReceived;
Transport.FinishedMessageProcessing += TransportFinishedMessageProcessing;
Transport.Start(InputAddress);
}
started = true;
}
satelliteLauncher = new SatelliteLauncher(Builder);
satelliteLauncher.Start();
ProcessStartupItems(
Builder.BuildAll <IWantToRunWhenBusStartsAndStops>().ToList(),
toRun =>
{
toRun.Start();
thingsRanAtStartup.Add(toRun);
Log.DebugFormat("Started {0}.", toRun.GetType().AssemblyQualifiedName);
},
ex => CriticalError.Raise("Startup task failed to complete.", ex),
startCompletedEvent);
return(this);
}
public ServiceControlBackend(ISendMessages messageSender, Configure configure, CriticalError criticalError)
{
this.configure = configure;
this.criticalError = criticalError;
this.messageSender = messageSender;
serializer = new JsonMessageSerializer(new SimpleMessageMapper());
serviceControlBackendAddress = GetServiceControlAddress();
VerifyIfServiceControlQueueExists();
circuitBreaker =
new RepeatedFailuresOverTimeCircuitBreaker("ServiceControlConnectivity", TimeSpan.FromMinutes(2),
ex =>
criticalError.Raise(
"This endpoint is repeatedly unable to contact the ServiceControl backend to report endpoint information. You have the ServiceControl plugins installed in your endpoint. However, please ensure that the Particular ServiceControl service is installed on this machine, " +
"or if running ServiceControl on a different machine, then ensure that your endpoint's app.config / web.config, AppSettings has the following key set appropriately: ServiceControl/Queue. \r\n" +
@"For example: <add key=""ServiceControl/Queue"" value=""particular.servicecontrol@machine""/>" +
"\r\n", ex));
}
protected override Task OnStart(IMessageSession session)
{
timer = new Timer(
callback: _ =>
{
try
{
routeTable.AddOrReplaceRoutes("MySource", LoadRoutes());
}
catch (Exception exception)
{
criticalError.Raise("Ambiguous route detected", exception);
}
},
state: null,
dueTime: TimeSpan.FromSeconds(30),
period: TimeSpan.FromSeconds(30));
return(Task.CompletedTask);
}
private bool TerminateIfForwardingQueueNotWritable()
{
if (!settings.ForwardErrorMessages)
{
return(false);
}
try
{
//Send a message to test the forwarding queue
var testMessage = new TransportMessage(Guid.Empty.ToString("N"), new Dictionary <string, string>());
forwarder.Send(testMessage, new SendOptions(settings.ErrorLogQueue));
return(false);
}
catch (Exception messageForwardingException)
{
criticalError.Raise("Error Import cannot start", messageForwardingException);
return(true);
}
}
private async Task HandleError(
byte[] messageBody,
int retryAttempt,
Exception exception,
Dictionary <string, string> headers,
string messageId)
{
var transportTransaction = new TransportTransaction();
InMemoryTransaction errorHandlingTransaction = null;
if (this.pushSettings.RequiredTransactionMode == TransportTransactionMode.SendsAtomicWithReceive)
{
transportTransaction.Set(errorHandlingTransaction = new InMemoryTransaction());
}
try
{
var immediateProcessingFailures = ++retryAttempt;
var handleResult = await onError(new ErrorContext(
exception,
headers,
messageId,
messageBody,
transportTransaction,
immediateProcessingFailures));
if (handleResult == ErrorHandleResult.RetryRequired)
{
await PushInternal(messageBody, messageId, headers, immediateProcessingFailures);
}
else
{
errorHandlingTransaction?.Complete();
}
}
catch (Exception e)
{
criticalError.Raise("Exception while handling pipeline error.", e);
await HandleError(messageBody, ++retryAttempt, exception, headers, messageId);
}
}
private async Task <bool> HandleMessageWithRetries(SerializedMessage message, TransportTransaction transportTransaction, int processingAttempt)
{
try
{
var receiveCancellationTokenSource = new CancellationTokenSource();
var(messageId, headers, body) = message.Deserialize();
var pushContext = new MessageContext(messageId, headers, body, transportTransaction, receiveCancellationTokenSource, new ContextBag());
await _onMessage(pushContext).ConfigureAwait(false);
return(!receiveCancellationTokenSource.IsCancellationRequested);
}
catch (Exception e)
{
var(messageId, headers, body) = message.Deserialize();
var errorContext = new ErrorContext(e, headers, messageId, body, transportTransaction, processingAttempt);
processingAttempt++;
try
{
var errorHandlingResult = await _onError(errorContext).ConfigureAwait(false);
if (errorHandlingResult == ErrorHandleResult.RetryRequired)
{
return(await HandleMessageWithRetries(message, transportTransaction, processingAttempt).ConfigureAwait(false));
}
}
catch (Exception exception)
{
_criticalError.Raise($"Failed to execute recoverability policy for message with native ID: `{messageId}`", exception);
return(await HandleMessageWithRetries(message, transportTransaction, processingAttempt).ConfigureAwait(false));
}
return(true);
}
}
void VerifyIfServiceControlQueueExists()
{
try
{
// In order to verify if the queue exists, we are sending a control message to SC.
// If we are unable to send a message because the queue doesn't exist, then we can fail fast.
// We currently don't have a way to check if Queue exists in a transport agnostic way,
// hence the send.
messageSender.Send(ControlMessage.Create(), new SendOptions(serviceControlBackendAddress)
{
ReplyToAddress = configure.LocalAddress
});
}
catch (Exception ex)
{
const string errMsg = "This endpoint is unable to contact the ServiceControl Backend to report endpoint information. You have the ServiceControl plugins installed in your endpoint. However, please ensure that the Particular ServiceControl service is installed on this machine, " +
"or if running ServiceControl on a different machine, then ensure that your endpoint's app.config / web.config, AppSettings has the following key set appropriately: ServiceControl/Queue. \r\n" +
@"For example: <add key=""ServiceControl/Queue"" value=""particular.servicecontrol@machine""/>" +
"\r\n Additional details: {0}";
criticalError.Raise(errMsg, ex);
}
}
static RepeatedFailuresOverTimeCircuitBreaker SetupCircuitBreaker(CriticalError criticalError)
{
var timeToWaitBeforeTriggering = TimeSpan.FromMinutes(2);
var timeToWaitBeforeTriggeringOverride = ConfigurationManager.AppSettings["NServiceBus/KafkaDequeueStrategy/TimeToWaitBeforeTriggering"];
if (!string.IsNullOrEmpty(timeToWaitBeforeTriggeringOverride))
{
timeToWaitBeforeTriggering = TimeSpan.Parse(timeToWaitBeforeTriggeringOverride);
}
var delayAfterFailure = TimeSpan.FromSeconds(5);
var delayAfterFailureOverride = ConfigurationManager.AppSettings["NServiceBus/KafkaDequeueStrategy/DelayAfterFailure"];
if (!string.IsNullOrEmpty(delayAfterFailureOverride))
{
delayAfterFailure = TimeSpan.Parse(delayAfterFailureOverride);
}
return(new RepeatedFailuresOverTimeCircuitBreaker("KafkaConnectivity",
timeToWaitBeforeTriggering,
ex => criticalError.Raise("Repeated failures when communicating with the broker",
ex), delayAfterFailure));
}
public Task Init(Func <MessageContext, Task> onMessage, Func <ErrorContext, Task <ErrorHandleResult> > onError, CriticalError criticalError, PushSettings settings)
{
circuitBreaker = new RepeatedFailuresOverTimeCircuitBreaker("AzureStorageQueue-MessagePump", TimeToWaitBeforeTriggering, ex => criticalError.Raise("Failed to receive message from Azure Storage Queue.", ex));
messageReceiver.PurgeOnStartup = settings.PurgeOnStartup;
receiveStrategy = ReceiveStrategy.BuildReceiveStrategy(onMessage, onError, settings.RequiredTransactionMode, criticalError);
return(messageReceiver.Init(settings.InputQueue, settings.ErrorQueue));
}
public async Task Init(Func <MessageContext, Task> onMessage, Func <ErrorContext, Task <ErrorHandleResult> > onError, CriticalError criticalError, PushSettings settings)
{
receiveStrategy = receiveStrategyFactory(settings.RequiredTransactionMode);
peekCircuitBreaker = new RepeatedFailuresOverTimeCircuitBreaker("SqlPeek", waitTimeCircuitBreaker, ex => criticalError.Raise("Failed to peek " + settings.InputQueue, ex));
receiveCircuitBreaker = new RepeatedFailuresOverTimeCircuitBreaker("ReceiveText", waitTimeCircuitBreaker, ex => criticalError.Raise("Failed to receive from " + settings.InputQueue, ex));
inputQueue = queueFactory(settings.InputQueue);
errorQueue = queueFactory(settings.ErrorQueue);
receiveStrategy.Init(inputQueue, errorQueue, onMessage, onError, criticalError);
if (settings.PurgeOnStartup)
{
try
{
var purgedRowsCount = await queuePurger.Purge(inputQueue).ConfigureAwait(false);
Logger.InfoFormat("{0:N} messages purged from queue {1}", purgedRowsCount, settings.InputQueue);
}
catch (Exception ex)
{
Logger.Warn("Failed to purge input queue on startup.", ex);
}
}
await schemaInspector.PerformInspection(inputQueue).ConfigureAwait(false);
}
// this would be called in some case where you want the CriticalErrorAction executed
public void RaiseCriticalErrorAction(Exception theException)
{
criticalError.Raise("The message", theException);
}
public ConsumerHolder(string connectionString, string endpointName, PushSettings settings, SettingsHolder settingsHolder, Func <MessageContext, Task> onMessage, Func <ErrorContext, Task <ErrorHandleResult> > onError, CriticalError criticalError, bool doNotSubscribeToEndPointQueue = false)
{
this.onMessage = onMessage;
this.onError = onError;
this.settings = settings;
this.settingsHolder = settingsHolder;
this.connectionString = connectionString;
this.endpointName = endpointName;
this.doNotSubscribeToEndPointQueue = doNotSubscribeToEndPointQueue;
circuitBreaker = new RepeatedFailuresOverTimeCircuitBreaker("kafka circuit breaker", new TimeSpan(30), ex => criticalError.Raise("Failed to receive message from Kafka.", ex));
if (consumer == null)
{
lock (o)
{
if (consumer == null)
{
CreateConsumer();
}
}
}
}
public Task <HttpResponseMessage> Trigger(string message)
{
criticalError.Raise(message, new Exception());
return(Task.FromResult(Request.CreateResponse(HttpStatusCode.OK)));
}
static RepeatedFailuresOverTimeCircuitBreaker SetupCircuitBreaker(CriticalError criticalError)
{
var timeToWaitBeforeTriggering = TimeSpan.FromMinutes(2);
var timeToWaitBeforeTriggeringOverride = ConfigurationManager.AppSettings["NServiceBus/RabbitMqDequeueStrategy/TimeToWaitBeforeTriggering"];
if (!string.IsNullOrEmpty(timeToWaitBeforeTriggeringOverride))
{
timeToWaitBeforeTriggering = TimeSpan.Parse(timeToWaitBeforeTriggeringOverride);
}
var delayAfterFailure = TimeSpan.FromSeconds(5);
var delayAfterFailureOverride = ConfigurationManager.AppSettings["NServiceBus/RabbitMqDequeueStrategy/DelayAfterFailure"];
if (!string.IsNullOrEmpty(delayAfterFailureOverride))
{
delayAfterFailure = TimeSpan.Parse(delayAfterFailureOverride);
}
return new RepeatedFailuresOverTimeCircuitBreaker("RabbitMqConnectivity",
timeToWaitBeforeTriggering,
ex => criticalError.Raise("Repeated failures when communicating with the broker",
ex), delayAfterFailure);
}
protected override Task Terminate(ISatelliteProcessingContext context)
{
// To raise a critical error
criticalError.Raise("Something bad happened - trigger critical error", new Exception("CriticalError occured!!"));
return(Task.FromResult(true));
}
public AzureServiceBusDequeueStrategy(ITopology topology, CriticalError criticalError)
{
this.topology = topology;
this.criticalError = criticalError;
circuitBreaker = new RepeatedFailuresOverTimeCircuitBreaker("AzureServiceBusDequeueStrategy", TimeSpan.FromSeconds(30), ex => criticalError.Raise("Failed to receive message from Azure ServiceBus.", ex));
}
async Task InnerProcessMessage(Task <Message> receiveTask)
{
Message message = null;
try
{
// Workaround for ASB MessageReceiver.Receive() that has a timeout and doesn't take a CancellationToken.
// We want to track how many receives are waiting and could be ignored when endpoint is stopping.
// TODO: remove workaround when https://github.com/Azure/azure-service-bus-dotnet/issues/439 is fixed
Interlocked.Increment(ref numberOfExecutingReceives);
message = await receiveTask.ConfigureAwait(false);
circuitBreaker.Success();
}
catch (ServiceBusException sbe) when(sbe.IsTransient)
{
}
catch (ObjectDisposedException)
{
// Can happen during endpoint shutdown
}
catch (Exception exception)
{
logger.WarnFormat("Failed to receive a message. Exception: {0}", exception.Message);
await circuitBreaker.Failure(exception).ConfigureAwait(false);
}
finally
{
// TODO: remove workaround when https://github.com/Azure/azure-service-bus-dotnet/issues/439 is fixed
Interlocked.Decrement(ref numberOfExecutingReceives);
}
// By default, ASB client long polls for a minute and returns null if it times out
if (message == null || messageProcessing.IsCancellationRequested)
{
return;
}
var lockToken = message.SystemProperties.LockToken;
string messageId;
Dictionary <string, string> headers;
byte[] body;
try
{
messageId = message.GetMessageId();
headers = message.GetNServiceBusHeaders();
body = message.GetBody();
}
catch (Exception exception)
{
try
{
await receiver.SafeDeadLetterAsync(pushSettings.RequiredTransactionMode, lockToken, deadLetterReason : "Poisoned message", deadLetterErrorDescription : exception.Message).ConfigureAwait(false);
}
catch (Exception)
{
// nothing we can do about it, message will be retried
}
return;
}
try
{
using (var receiveCancellationTokenSource = new CancellationTokenSource())
{
var transportTransaction = CreateTransportTransaction(message.PartitionKey);
var contextBag = new ContextBag();
contextBag.Set(message);
var messageContext = new MessageContext(messageId, headers, body, transportTransaction, receiveCancellationTokenSource, contextBag);
using (var scope = CreateTransactionScope())
{
await onMessage(messageContext).ConfigureAwait(false);
if (receiveCancellationTokenSource.IsCancellationRequested == false)
{
await receiver.SafeCompleteAsync(pushSettings.RequiredTransactionMode, lockToken).ConfigureAwait(false);
scope?.Complete();
}
}
if (receiveCancellationTokenSource.IsCancellationRequested)
{
await receiver.SafeAbandonAsync(pushSettings.RequiredTransactionMode, lockToken).ConfigureAwait(false);
}
}
}
catch (Exception exception)
{
try
{
ErrorHandleResult result;
using (var scope = CreateTransactionScope())
{
var transportTransaction = CreateTransportTransaction(message.PartitionKey);
var errorContext = new ErrorContext(exception, message.GetNServiceBusHeaders(), messageId, body, transportTransaction, message.SystemProperties.DeliveryCount);
result = await onError(errorContext).ConfigureAwait(false);
if (result == ErrorHandleResult.Handled)
{
await receiver.SafeCompleteAsync(pushSettings.RequiredTransactionMode, lockToken).ConfigureAwait(false);
}
scope?.Complete();
}
if (result == ErrorHandleResult.RetryRequired)
{
await receiver.SafeAbandonAsync(pushSettings.RequiredTransactionMode, lockToken).ConfigureAwait(false);
}
}
catch (Exception onErrorException) when(onErrorException is MessageLockLostException || onErrorException is ServiceBusTimeoutException)
{
logger.Debug("Failed to execute recoverability.", onErrorException);
}
catch (Exception onErrorException)
{
criticalError.Raise($"Failed to execute recoverability policy for message with native ID: `{message.MessageId}`", onErrorException);
await receiver.SafeAbandonAsync(pushSettings.RequiredTransactionMode, lockToken).ConfigureAwait(false);
}
}
}