public void LogWithMessageInfo( LogLevel logLevel, EventId eventId, Exception?exception, string logMessage, ConsumerPipelineContext context) => LogWithMessageInfo(logLevel, eventId, exception, logMessage, context.Envelope, context.Sequence);
protected virtual async Task HandleMessageAsync( byte[]?message, IReadOnlyCollection <MessageHeader> headers, string sourceEndpointName, IOffset offset, IDictionary <string, string>?additionalLogData) { var envelope = new RawInboundEnvelope( message, headers, Endpoint, sourceEndpointName, offset, additionalLogData); _statusInfo.RecordConsumedMessage(offset); var consumerPipelineContext = new ConsumerPipelineContext( envelope, this, GetSequenceStore(offset), ServiceProvider); await ExecutePipelineAsync(consumerPipelineContext).ConfigureAwait(false); }
public async Task Handle(ConsumerPipelineContext context, IServiceProvider serviceProvider, ConsumerBehaviorHandler next) { var envelope = context.Envelopes.Single(); var operationName = $"Consuming Message on topic {envelope.Endpoint.Name}"; ISpanBuilder spanBuilder; try { var headers = envelope.Headers.ToDictionary(pair => pair.Key, pair => pair.Value.ToString()); var parentSpanCtx = GlobalTracer.Instance.Extract(BuiltinFormats.TextMap, new TextMapExtractAdapter(headers)); spanBuilder = GlobalTracer.Instance.BuildSpan(operationName); if (parentSpanCtx != null) { spanBuilder = spanBuilder.AsChildOf(parentSpanCtx); } } catch { spanBuilder = GlobalTracer.Instance.BuildSpan(operationName); } spanBuilder .WithTag(Tags.SpanKind, Tags.SpanKindConsumer) .WithTag("endpoint", envelope.Endpoint.Name); using (spanBuilder.StartActive()) { await next(context, serviceProvider); } }
/// <summary> /// Initializes a new instance of the <see cref="ConsumerTransactionManager" /> class. /// </summary> /// <param name="context"> /// The current <see cref="ConsumerPipelineContext" />. /// </param> /// <param name="logger"> /// The <see cref="ISilverbackLogger" />. /// </param> public ConsumerTransactionManager( ConsumerPipelineContext context, ISilverbackIntegrationLogger <ConsumerTransactionManager> logger) { _context = context; _logger = logger; }
private async Task <bool> HandleExceptionAsync(ConsumerPipelineContext context, Exception exception) { _logger.LogProcessingError(context, exception); try { bool handled = await ErrorPoliciesHelper.ApplyErrorPoliciesAsync(context, exception) .ConfigureAwait(false); // TODO: Carefully test: exception handled once and always rolled back if (!handled) { if (context.Sequence != null && (context.Sequence.Context.ProcessingTask?.IsCompleted ?? true)) { await context.Sequence.Context.TransactionManager.RollbackAsync(exception) .ConfigureAwait(false); } else { await context.TransactionManager.RollbackAsync(exception).ConfigureAwait(false); } } return(handled); } finally { context.Dispose(); } }
private async Task AwaitProcessedIfNecessaryAsync(ConsumerPipelineContext context) { if (context.Sequence == null) { throw new InvalidOperationException("Sequence is null"); } // At the end of the sequence (or when the processing task exits prematurely), ensure that the // commit was performed or the error policies were applied before continuing if (context.IsSequenceEnd || context.Sequence.IsAborted) { if (context.Sequence is ISequenceImplementation sequenceImpl) { _logger.LogTraceWithMessageInfo( IntegrationEventIds.LowLevelTracing, "Sequence ended or aborted: awaiting processing task.", context); await sequenceImpl.ProcessingCompletedTask.ConfigureAwait(false); _logger.LogTraceWithMessageInfo( IntegrationEventIds.LowLevelTracing, "Sequence ended or aborted: processing task completed.", context); } context.Dispose(); } }
/// <inheritdoc cref="IErrorPolicyImplementation.HandleErrorAsync" /> public async Task <bool> HandleErrorAsync(ConsumerPipelineContext context, Exception exception) { Check.NotNull(context, nameof(context)); Check.NotNull(exception, nameof(exception)); var result = await ApplyPolicyAsync(context, exception).ConfigureAwait(false); if (_messageToPublishFactory == null) { return(result); } object?message = _messageToPublishFactory.Invoke(context.Envelope, exception); if (message == null) { return(result); } using var scope = _serviceProvider.CreateScope(); await scope.ServiceProvider.GetRequiredService <IPublisher>() .PublishAsync(message) .ConfigureAwait(false); return(result); }
private static async Task <IRawInboundEnvelope?> DeserializeAsync(ConsumerPipelineContext context) { var envelope = context.Envelope; if (envelope is IInboundEnvelope inboundEnvelope && inboundEnvelope.Message != null) { return(inboundEnvelope); } var(deserializedObject, deserializedType) = await envelope.Endpoint.Serializer.DeserializeAsync( envelope.RawMessage, envelope.Headers, new MessageSerializationContext(envelope.Endpoint, envelope.ActualEndpointName)) .ConfigureAwait(false); envelope.Headers.AddIfNotExists( DefaultMessageHeaders.MessageType, deserializedType.AssemblyQualifiedName); return(deserializedObject == null ? HandleNullMessage(context, envelope, deserializedType) : SerializationHelper.CreateTypedInboundEnvelope( envelope, deserializedObject, deserializedType)); }
public async Task HandleAsync(ConsumerPipelineContext context, ConsumerBehaviorHandler next) { Check.NotNull(context, nameof(context)); Check.NotNull(next, nameof(next)); try { // TODO: Ensure always disposed (TEST IT!) var scope = context.ServiceProvider.CreateScope(); context.ReplaceServiceScope(scope); context.TransactionManager = new ConsumerTransactionManager( context, context.ServiceProvider .GetRequiredService <ISilverbackIntegrationLogger <ConsumerTransactionManager> >()); await next(context).ConfigureAwait(false); if (context.Sequence == null) { await context.TransactionManager.CommitAsync().ConfigureAwait(false); context.Dispose(); } else { if (context.IsSequenceStart) { StartSequenceProcessingAwaiter(context); } await AwaitProcessedIfNecessaryAsync(context).ConfigureAwait(false); } } catch (Exception exception) { // Sequence errors are handled in AwaitSequenceProcessingAsync, just await the rollback and rethrow if (context.Sequence != null) { await context.Sequence.AbortAsync(SequenceAbortReason.Error, exception).ConfigureAwait(false); if (context.Sequence.Length > 0 && context.Sequence is ISequenceImplementation sequenceImpl) { _logger.LogTraceWithMessageInfo( IntegrationEventIds.LowLevelTracing, "Awaiting sequence processing completed before rethrowing.", context); await sequenceImpl.ProcessingCompletedTask.ConfigureAwait(false); } throw; } if (!await HandleExceptionAsync(context, exception).ConfigureAwait(false)) { throw; } } }
public async Task <bool> CheckIsAlreadyProcessedAsync(ConsumerPipelineContext context) { Check.NotNull(context, nameof(context)); var envelope = context.Envelope; if (!(envelope.Offset is IComparableOffset comparableOffset)) { throw new InvalidOperationException( "The message broker implementation doesn't seem to support comparable offsets. " + "The OffsetStoreExactlyOnceStrategy cannot be used, please resort to LogExactlyOnceStrategy " + "to ensure exactly-once processing."); } var latestOffset = await _offsetStore.GetLatestValueAsync(envelope.Offset.Key, envelope.Endpoint) .ConfigureAwait(false); if (latestOffset != null && latestOffset.CompareTo(comparableOffset) >= 0) { return(true); } context.TransactionManager.Enlist(_offsetStore); await _offsetStore.StoreAsync(comparableOffset, envelope.Endpoint).ConfigureAwait(false); return(false); }
/// <inheritdoc cref="IConsumerBehavior.HandleAsync" /> public async Task HandleAsync( ConsumerPipelineContext context, ConsumerBehaviorHandler next) { Check.NotNull(context, nameof(context)); Check.NotNull(next, nameof(next)); if (context.Envelope.Endpoint.Encryption != null && context.Envelope.RawMessage != null) { string?keyIdentifier = null; if (context.Envelope.Endpoint.Encryption is SymmetricDecryptionSettings settings && settings.KeyProvider != null) { keyIdentifier = context.Envelope.Headers.GetValue(DefaultMessageHeaders.EncryptionKeyId); } context.Envelope.RawMessage = _streamFactory.GetDecryptStream( context.Envelope.RawMessage, context.Envelope.Endpoint.Encryption, keyIdentifier); } await next(context).ConfigureAwait(false); }
/// <inheritdoc cref="SequencerConsumerBehaviorBase.PublishSequenceAsync" /> protected override Task PublishSequenceAsync( ConsumerPipelineContext context, ConsumerBehaviorHandler next) { Check.NotNull(next, nameof(next)); return(next(context)); }
/// <inheritdoc cref="IBrokerActivityEnricher.EnrichInboundActivity" /> public void EnrichInboundActivity(Activity activity, ConsumerPipelineContext consumerContext) { Check.NotNull(activity, nameof(activity)); Check.NotNull(consumerContext, nameof(consumerContext)); SetMessageId(activity, consumerContext.Envelope.BrokerMessageIdentifier); SetMessageKey(activity, consumerContext.Envelope.Headers); }
/// <summary> /// Gets the sequence identifier extracted from the current envelope. /// </summary> /// <param name="context"> /// The current <see cref="ConsumerPipelineContext" />. /// </param> /// <returns> /// A <see cref="Task{TResult}" /> representing the asynchronous operation. The task result contains /// the recognized sequence identifier, or <c>null</c>. /// </returns> protected virtual Task <string> GetSequenceId(ConsumerPipelineContext context) { Check.NotNull(context, nameof(context)); string messageId = context.Envelope.Headers.GetValue(DefaultMessageHeaders.MessageId) ?? "***default***"; return(Task.FromResult(messageId)); }
/// <summary> /// Retrieves the existing incomplete sequence from the store. /// </summary> /// <param name="context"> /// The current <see cref="ConsumerPipelineContext" />. /// </param> /// <param name="sequenceId"> /// The sequence identifier. /// </param> /// <returns> /// The <see cref="ISequence" /> or <c>null</c> if not found. /// </returns> protected virtual Task <ISequence?> GetExistingSequenceAsync( ConsumerPipelineContext context, string sequenceId) { Check.NotNull(context, nameof(context)); return(context.SequenceStore.GetAsync <ISequence>(sequenceId)); }
/// <inheritdoc cref="SequenceReaderBase.IsNewSequence" /> protected override async Task <bool> IsNewSequence(string sequenceId, ConsumerPipelineContext context) { Check.NotNull(context, nameof(context)); var currentSequence = await context.SequenceStore.GetAsync <BatchSequence>(sequenceId, true).ConfigureAwait(false); return(currentSequence == null || !currentSequence.IsPending || currentSequence.IsCompleting); }
/// <inheritdoc cref="SequenceReaderBase.CanHandleAsync" /> public override Task <bool> CanHandleAsync(ConsumerPipelineContext context) { Check.NotNull(context, nameof(context)); var canHandle = context.Envelope.Headers.Contains(DefaultMessageHeaders.ChunkIndex); return(Task.FromResult(canHandle)); }
/// <summary> /// Initializes a new instance of the <see cref="Sequence" /> class. /// </summary> /// <param name="sequenceId"> /// The identifier that is used to match the consumed messages with their belonging sequence. /// </param> /// <param name="context"> /// The current <see cref="ConsumerPipelineContext" />, assuming that it will be the one from which the /// sequence gets published to the internal bus. /// </param> /// <param name="enforceTimeout"> /// A value indicating whether the timeout has to be enforced. /// </param> /// <param name="timeout"> /// The timeout to be applied. If not specified the value of <c>Endpoint.Sequence.Timeout</c> will be /// used. /// </param> protected Sequence( string sequenceId, ConsumerPipelineContext context, bool enforceTimeout = true, TimeSpan?timeout = null) : base(sequenceId, context, enforceTimeout, timeout) { }
public int SortIndex => int.MaxValue; // Ignored if a proper sequence is detected /// <inheritdoc cref="SequenceReaderBase.CanHandleAsync" /> public override Task <bool> CanHandleAsync(ConsumerPipelineContext context) { Check.NotNull(context, nameof(context)); bool isBatchEnabled = context.Envelope.Endpoint.Batch != null && context.Envelope.Endpoint.Batch.Size > 1; return(Task.FromResult(isBatchEnabled)); }
private void StartSequenceProcessingAwaiter(ConsumerPipelineContext context) { #pragma warning disable 4014 // ReSharper disable AccessToDisposedClosure Task.Run(() => AwaitSequenceProcessingAsync(context)); // ReSharper restore AccessToDisposedClosure #pragma warning restore 4014 }
/// <inheritdoc cref="IConsumerBehavior.HandleAsync" /> public Task HandleAsync(ConsumerPipelineContext context, ConsumerBehaviorHandler next) { Check.NotNull(context, nameof(context)); Check.NotNull(next, nameof(next)); _integrationSpy.AddRawInboundEnvelope(context.Envelope); return(next(context)); }
public void LogProcessing(ConsumerPipelineContext context) { Check.NotNull(context, nameof(context)); LogInformationWithMessageInfo( IntegrationEventIds.ProcessingInboundMessage, "Processing inbound message.", context); }
/// <summary> /// Initializes a new instance of the <see cref="Sequence" /> class. /// </summary> /// <param name="sequenceId"> /// The identifier that is used to match the consumed messages with their belonging sequence. /// </param> /// <param name="context"> /// The current <see cref="ConsumerPipelineContext" />, assuming that it will be the one from which the /// sequence gets published to the internal bus. /// </param> /// <param name="enforceTimeout"> /// A value indicating whether the timeout has to be enforced. /// </param> /// <param name="timeout"> /// The timeout to be applied. If not specified the value of <c>Endpoint.Sequence.Timeout</c> will be /// used. /// </param> /// <param name="trackIdentifiers"> /// Specifies whether the message identifiers have to be collected, in order to be used for the commit /// later on. /// </param> protected Sequence( string sequenceId, ConsumerPipelineContext context, bool enforceTimeout = true, TimeSpan?timeout = null, bool trackIdentifiers = true) : base(sequenceId, context, enforceTimeout, timeout, trackIdentifiers: trackIdentifiers) { }
/// <summary> /// Initializes a new instance of the <see cref="RawSequence" /> class. /// </summary> /// <param name="sequenceId"> /// The identifier that is used to match the consumed messages with their belonging sequence. /// </param> /// <param name="context"> /// The current <see cref="ConsumerPipelineContext" />, assuming that it will be the one from which the /// sequence gets published to the internal bus. /// </param> /// <param name="enforceTimeout"> /// A value indicating whether the timeout has to be enforced. /// </param> /// <param name="timeout"> /// The timeout to be applied. If not specified the value of <c>Endpoint.Sequence.Timeout</c> will be /// used. /// </param> /// <param name="streamProvider"> /// The <see cref="IMessageStreamProvider" /> to be pushed. A new one will be created if not provided. /// </param> protected RawSequence( string sequenceId, ConsumerPipelineContext context, bool enforceTimeout = true, TimeSpan?timeout = null, IMessageStreamProvider?streamProvider = null) : base(sequenceId, context, enforceTimeout, timeout, streamProvider) { }
/// <inheritdoc cref="SequenceReaderBase.IsNewSequenceAsync" /> protected override Task <bool> IsNewSequenceAsync(string sequenceId, ConsumerPipelineContext context) { Check.NotNull(context, nameof(context)); var chunkIndex = context.Envelope.Headers.GetValue <int>(DefaultMessageHeaders.ChunkIndex) ?? throw new InvalidOperationException("Chunk index header not found."); return(Task.FromResult(chunkIndex == 0)); }
public void LogProcessingError(ConsumerPipelineContext context, Exception exception) { Check.NotNull(context, nameof(context)); LogWarningWithMessageInfo( IntegrationEventIds.ErrorProcessingInboundMessage, exception, "Error occurred processing the inbound message.", context); }
private async Task PublishSequenceAsync(ISequence sequence, ConsumerPipelineContext context) { var processingTask = await PublishStreamProviderAsync(sequence, context).ConfigureAwait(false); context.ProcessingTask = processingTask; _logger.LogTraceWithMessageInfo( IntegrationEventIds.LowLevelTracing, $"Published {sequence.GetType().Name} '{sequence.SequenceId}' (ProcessingTask.Id={processingTask.Id}).", context); }
public Task HandleAsync( ConsumerPipelineContext context, ConsumerBehaviorHandler next) { lock (_inboundEnvelopes) { _inboundEnvelopes.Add((IInboundEnvelope)context.Envelope); } return(next(context)); }
public UnboundedSequence(string sequenceId, ConsumerPipelineContext context) : base( sequenceId, context, false, streamProvider: new MessageStreamProvider <IInboundEnvelope> { AllowSubscribeAsEnumerable = false }) { }
/// <inheritdoc cref="IConsumerBehavior.HandleAsync" /> public async Task HandleAsync( ConsumerPipelineContext context, ConsumerBehaviorHandler next) { Check.NotNull(context, nameof(context)); Check.NotNull(next, nameof(next)); context.Envelope = await HandleAsync(context.Envelope).ConfigureAwait(false); await next(context).ConfigureAwait(false); }