async Task ProcessFileAndComplete(ILearningTransportTransaction transaction, string filePath, string messageId, CancellationToken messageProcessingCancellationToken)
{
try
{
await ProcessFile(transaction, messageId, messageProcessingCancellationToken)
.ConfigureAwait(false);
}
finally
{
if (log.IsDebugEnabled)
{
log.Debug($"Completing processing for {filePath}({transaction.FileToProcess}).");
}
try
{
if (transaction.Complete())
{
File.Delete(Path.Combine(bodyDir, messageId + BodyFileSuffix));
}
}
catch (Exception ex)
{
log.Debug($"Failure while trying to complete receive transaction for {filePath}({transaction.FileToProcess})", ex);
}
concurrencyLimiter.Release();
}
}
async Task ProcessFileAndComplete(ILearningTransportTransaction transaction, string filePath, string messageId, CancellationToken messageProcessingCancellationToken)
{
var startedAt = DateTimeOffset.UtcNow;
Dictionary <string, string> headers = null;
var onMessageFailed = false;
var processingContext = new ContextBag();
try
{
(headers, onMessageFailed) = await ProcessFile(transaction, messageId, processingContext, messageProcessingCancellationToken)
.ConfigureAwait(false);
}
finally
{
if (log.IsDebugEnabled)
{
log.Debug($"Completing processing for {filePath}({transaction.FileToProcess}).");
}
var wasAcknowledged = false;
try
{
wasAcknowledged = transaction.Complete();
if (wasAcknowledged)
{
File.Delete(Path.Combine(bodyDir, messageId + BodyFileSuffix));
}
}
catch (Exception ex)
{
log.Debug($"Failure while trying to complete receive transaction for {filePath}({transaction.FileToProcess})" + filePath, ex);
}
try
{
await onCompleted(new CompleteContext(messageId, wasAcknowledged, headers ?? new Dictionary <string, string>(), startedAt, DateTimeOffset.UtcNow, onMessageFailed, processingContext), messageProcessingCancellationToken).ConfigureAwait(false);
}
catch (Exception ex)
{
log.Warn($"Failure when invoking {nameof(OnCompleted)} for {filePath}({transaction.FileToProcess})" + filePath, ex);
}
concurrencyLimiter.Release();
}
}
async Task ProcessMessageSwallowExceptionsAndReleaseConcurrencyLimiter(ILearningTransportTransaction transaction, string filePath, string messageId, CancellationToken messageProcessingCancellationToken)
{
try
{
await ProcessFileAndComplete(transaction, filePath, messageId, messageProcessingCancellationToken).ConfigureAwait(false);
}
catch (Exception ex) when(ex.IsCausedBy(messageProcessingCancellationToken))
{
log.Debug("Message processing canceled.", ex);
}
catch (Exception ex)
{
log.Error("Message processing failed.", ex);
}
finally
{
concurrencyLimiter.Release();
}
}
async Task ProcessFile(ILearningTransportTransaction transaction, string messageId, CancellationToken messageProcessingCancellationToken)
{
var message = await AsyncFile.ReadText(transaction.FileToProcess, messageProcessingCancellationToken)
.ConfigureAwait(false);
var bodyPath = Path.Combine(bodyDir, $"{messageId}{BodyFileSuffix}");
var headers = HeaderSerializer.Deserialize(message);
if (headers.TryGetValue(LearningTransportHeaders.TimeToBeReceived, out var ttbrString))
{
headers.Remove(LearningTransportHeaders.TimeToBeReceived);
var ttbr = TimeSpan.Parse(ttbrString);
//file.move preserves create time
var sentTime = File.GetCreationTimeUtc(transaction.FileToProcess);
var utcNow = DateTime.UtcNow;
if (sentTime + ttbr < utcNow)
{
await transaction.Commit(messageProcessingCancellationToken)
.ConfigureAwait(false);
log.InfoFormat("Dropping message '{0}' as the specified TimeToBeReceived of '{1}' expired since sending the message at '{2:O}'. Current UTC time is '{3:O}'", messageId, ttbrString, sentTime, utcNow);
return;
}
}
var body = await AsyncFile.ReadBytes(bodyPath, messageProcessingCancellationToken)
.ConfigureAwait(false);
var transportTransaction = new TransportTransaction();
if (transactionMode == TransportTransactionMode.SendsAtomicWithReceive)
{
transportTransaction.Set(transaction);
}
var processingContext = new ContextBag();
var messageContext = new MessageContext(messageId, headers, body, transportTransaction, processingContext);
try
{
await onMessage(messageContext, messageProcessingCancellationToken)
.ConfigureAwait(false);
}
catch (OperationCanceledException ex) when(messageProcessingCancellationToken.IsCancellationRequested)
{
log.Info("Message processing cancelled. Rolling back transaction.", ex);
transaction.Rollback();
return;
}
catch (Exception exception)
{
transaction.ClearPendingOutgoingOperations();
var processingFailures = retryCounts.AddOrUpdate(messageId, id => 1, (id, currentCount) => currentCount + 1);
headers = HeaderSerializer.Deserialize(message);
headers.Remove(LearningTransportHeaders.TimeToBeReceived);
var errorContext = new ErrorContext(exception, headers, messageId, body, transportTransaction, processingFailures, processingContext);
ErrorHandleResult result;
try
{
result = await onError(errorContext, messageProcessingCancellationToken).ConfigureAwait(false);
}
catch (OperationCanceledException ex) when(messageProcessingCancellationToken.IsCancellationRequested)
{
log.Info("Message processing cancelled. Rolling back transaction.", ex);
transaction.Rollback();
return;
}
catch (Exception ex)
{
criticalErrorAction($"Failed to execute recoverability policy for message with native ID: `{messageContext.NativeMessageId}`", ex, messageProcessingCancellationToken);
result = ErrorHandleResult.RetryRequired;
}
if (result == ErrorHandleResult.RetryRequired)
{
transaction.Rollback();
return;
}
}
await transaction.Commit(messageProcessingCancellationToken).ConfigureAwait(false);
}
async Task ProcessFile(ILearningTransportTransaction transaction, string messageId)
{
var message = await AsyncFile.ReadText(transaction.FileToProcess)
.ConfigureAwait(false);
var bodyPath = Path.Combine(bodyDir, $"{messageId}{BodyFileSuffix}");
var headers = HeaderSerializer.Deserialize(message);
if (headers.TryGetValue(LearningTransportHeaders.TimeToBeReceived, out var ttbrString))
{
headers.Remove(LearningTransportHeaders.TimeToBeReceived);
var ttbr = TimeSpan.Parse(ttbrString);
//file.move preserves create time
var sentTime = File.GetCreationTimeUtc(transaction.FileToProcess);
var utcNow = DateTime.UtcNow;
if (sentTime + ttbr < utcNow)
{
await transaction.Commit()
.ConfigureAwait(false);
log.InfoFormat("Dropping message '{0}' as the specified TimeToBeReceived of '{1}' expired since sending the message at '{2:O}'. Current UTC time is '{3:O}'", messageId, ttbrString, sentTime, utcNow);
return;
}
}
var tokenSource = new CancellationTokenSource();
var body = await AsyncFile.ReadBytes(bodyPath, cancellationToken)
.ConfigureAwait(false);
var transportTransaction = new TransportTransaction();
if (transactionMode == TransportTransactionMode.SendsAtomicWithReceive)
{
transportTransaction.Set(transaction);
}
var messageContext = new MessageContext(messageId, headers, body, transportTransaction, tokenSource, new ContextBag());
try
{
await onMessage(messageContext)
.ConfigureAwait(false);
}
catch (Exception exception)
{
transaction.ClearPendingOutgoingOperations();
var processingFailures = retryCounts.AddOrUpdate(messageId, id => 1, (id, currentCount) => currentCount + 1);
var errorContext = new ErrorContext(exception, headers, messageId, body, transportTransaction, processingFailures);
// the transport tests assume that all transports use a circuit breaker to be resilient against exceptions
// in onError. Since we don't need that robustness, we just retry onError once should it fail.
ErrorHandleResult actionToTake;
try
{
actionToTake = await onError(errorContext)
.ConfigureAwait(false);
}
catch (Exception)
{
actionToTake = await onError(errorContext)
.ConfigureAwait(false);
}
if (actionToTake == ErrorHandleResult.RetryRequired)
{
transaction.Rollback();
return;
}
}
if (tokenSource.IsCancellationRequested)
{
transaction.Rollback();
return;
}
await transaction.Commit()
.ConfigureAwait(false);
}
