public void Enrich( IRawInboundEnvelope inboundEnvelope, IOutboundEnvelope outboundEnvelope, Exception exception) { outboundEnvelope.Headers.AddOrReplace( KafkaMessageHeaders.SourceTopic, inboundEnvelope.ActualEndpointName); outboundEnvelope.Headers.AddOrReplace( KafkaMessageHeaders.SourceConsumerGroupId, ((KafkaConsumerEndpoint)inboundEnvelope.Endpoint).Configuration.GroupId); while (exception.InnerException != null) { exception = exception.InnerException; } outboundEnvelope.Headers.AddOrReplace( DefaultMessageHeaders.FailureReason, $"{exception.GetType().FullName} in {exception.Source}"); outboundEnvelope.Headers.AddOrReplace( KafkaMessageHeaders.SourcePartition, ((KafkaOffset)inboundEnvelope.BrokerMessageIdentifier).Partition); outboundEnvelope.Headers.AddOrReplace( KafkaMessageHeaders.SourceOffset, ((KafkaOffset)inboundEnvelope.BrokerMessageIdentifier).Offset); if (inboundEnvelope.Headers.Contains(KafkaMessageHeaders.TimestampKey)) { outboundEnvelope.Headers.AddOrReplace( KafkaMessageHeaders.SourceTimestamp, inboundEnvelope.Headers[KafkaMessageHeaders.TimestampKey]); } }
/// <inheritdoc cref="ISequenceWriter.CanHandle" /> public bool CanHandle(IOutboundEnvelope envelope) { Check.NotNull(envelope, nameof(envelope)); var chunkSettings = envelope.Endpoint.Chunk; if (chunkSettings == null || chunkSettings.Size == int.MaxValue) { return(false); } if (chunkSettings.AlwaysAddHeaders) { return(true); } if (envelope.RawMessage == null) { return(false); } if (!envelope.RawMessage.CanSeek) { return(true); } return(envelope.RawMessage != null && envelope.RawMessage.Length > chunkSettings.Size); }
public void OnOutbound(IOutboundEnvelope envelope) { lock (_outboundEnvelopes) { _outboundEnvelopes.Add(envelope); } }
/// <inheritdoc cref="Producer.ProduceCoreAsync" /> protected override async Task <IBrokerMessageIdentifier?> ProduceCoreAsync(IOutboundEnvelope envelope) { Check.NotNull(envelope, nameof(envelope)); await _queueWriter.WriteAsync(envelope).ConfigureAwait(false); return(null); }
/// <inheritdoc cref="Producer.ProduceCoreAsync" /> protected override Task <IOffset?> ProduceCoreAsync(IOutboundEnvelope envelope) { var queuedMessage = new QueuedMessage(envelope); _queue.Add(queuedMessage); return(queuedMessage.TaskCompletionSource.Task); }
public Task ProduceAsync(IOutboundEnvelope envelope) { Check.NotNull(envelope, nameof(envelope)); _logger.LogWrittenToOutbox(envelope); return(_outboundQueueBroker.GetProducer(envelope.Endpoint).ProduceAsync(envelope)); }
/// <inheritdoc cref="IProducer.ProduceAsync(IOutboundEnvelope)" /> public async Task ProduceAsync(IOutboundEnvelope envelope) => await ExecutePipelineIfNeededAsync( new ProducerPipelineContext(envelope, this, _serviceProvider), async finalContext => { ((RawOutboundEnvelope)finalContext.Envelope).Offset = await ProduceCoreAsync(finalContext.Envelope).ConfigureAwait(false); }).ConfigureAwait(false);
/// <inheritdoc cref="IOutboundMessageEnricher.Enrich" /> public void Enrich(IOutboundEnvelope envelope) { if (envelope == null || envelope is not IOutboundEnvelope <TMessage> typedEnvelope) { return; } envelope.Headers.AddOrReplace(_name, _valueProvider.Invoke(typedEnvelope)); }
public async IAsyncEnumerable <IOutboundEnvelope> ProcessMessage(IOutboundEnvelope envelope) { Check.NotNull(envelope, nameof(envelope)); if (envelope.RawMessage == null) { throw new InvalidOperationException("RawMessage is null"); } var settings = envelope.Endpoint.Chunk; var chunkSize = settings?.Size ?? int.MaxValue; var bufferArray = ArrayPool <byte> .Shared.Rent(chunkSize); var bufferMemory = bufferArray.AsMemory(0, chunkSize); var chunksCount = envelope.RawMessage.CanSeek ? (int?)Math.Ceiling(envelope.RawMessage.Length / (double)chunkSize) : null; IBrokerMessageOffset?firstChunkOffset = null; int chunkIndex = 0; var readBytesCount = await envelope.RawMessage.ReadAsync(bufferMemory).ConfigureAwait(false); while (readBytesCount > 0) { var chunkEnvelope = CreateChunkEnvelope( chunkIndex, chunksCount, bufferMemory.Slice(0, readBytesCount).ToArray(), envelope); if (chunkIndex > 0) { chunkEnvelope.Headers.AddOrReplace(DefaultMessageHeaders.FirstChunkOffset, firstChunkOffset?.Value); } // Read the next chunk and check if we were at the end of the stream (to not rely on Length property) readBytesCount = await envelope.RawMessage.ReadAsync(bufferMemory).ConfigureAwait(false); if (readBytesCount == 0) { chunkEnvelope.Headers.AddOrReplace(DefaultMessageHeaders.IsLastChunk, true.ToString()); } yield return(chunkEnvelope); // Read and store the offset of the first chunk, after it has been produced (after yield return) if (chunkIndex == 0) { firstChunkOffset = chunkEnvelope.BrokerMessageIdentifier as IBrokerMessageOffset; } chunkIndex++; } }
/// <inheritdoc cref="IOutboundMessageEnricher.Enrich" /> public void Enrich(IOutboundEnvelope envelope) { Check.NotNull(envelope, nameof(envelope)); if (envelope is not IOutboundEnvelope <TMessage> typedEnvelope) { return; } envelope.Headers.AddOrReplace(_name, _valueProvider.Invoke(typedEnvelope)); }
public Task ProduceAsync(IOutboundEnvelope envelope) { Check.NotNull(envelope, nameof(envelope)); _logger.LogDebugWithMessageInfo( IntegrationEventIds.OutboundMessageWrittenToOutbox, "Writing the outbound message to the transactional outbox.", envelope); return(_outboundQueueBroker.GetProducer(envelope.Endpoint).ProduceAsync(envelope)); }
private static async ValueTask <byte[]?> GetContentAsync(IOutboundEnvelope envelope) { var stream = envelope.RawMessage ?? await envelope.Endpoint.Serializer.SerializeAsync( envelope.Message, envelope.Headers, new MessageSerializationContext(envelope.Endpoint)) .ConfigureAwait(false); return(await stream.ReadAllAsync().ConfigureAwait(false)); }
/// <inheritdoc cref="IProducer.Produce(IOutboundEnvelope)" /> public void Produce(IOutboundEnvelope envelope) => AsyncHelper.RunSynchronously( () => ExecutePipelineIfNeededAsync( new ProducerPipelineContext(envelope, this, _serviceProvider), finalContext => { ((RawOutboundEnvelope)finalContext.Envelope).Offset = ProduceCore(finalContext.Envelope); return(Task.CompletedTask); }));
/// <inheritdoc cref="IOutboxWriter.WriteAsync" /> public async Task WriteAsync(IOutboundEnvelope envelope) { Check.NotNull(envelope, nameof(envelope)); var item = new OutboxStoredMessage( envelope.Message?.GetType(), await envelope.RawMessage.ReadAllAsync().ConfigureAwait(false), envelope.Headers, envelope.Endpoint.Name); await AddAsync(item).ConfigureAwait(false); }
/// <inheritdoc cref="IOutboxWriter.WriteAsync" /> public async Task WriteAsync(IOutboundEnvelope envelope) { Check.NotNull(envelope, nameof(envelope)); DbSet.Add( new OutboxMessage { MessageType = envelope.Message?.GetType().AssemblyQualifiedName, Content = await GetContentAsync(envelope).ConfigureAwait(false), SerializedHeaders = JsonSerializer.SerializeToUtf8Bytes((IEnumerable <MessageHeader>)envelope.Headers), EndpointName = envelope.Endpoint.Name, Created = DateTime.UtcNow }); }
protected override async Task <IOffset?> ProduceCoreAsync(IOutboundEnvelope envelope) { Check.NotNull(envelope, nameof(envelope)); try { var kafkaMessage = new Message <byte[]?, byte[]?> { Key = GetKafkaKeyAndRemoveHeader(envelope.Headers), Value = await envelope.RawMessage.ReadAllAsync().ConfigureAwait(false) }; if (envelope.Headers.Count >= 1) { kafkaMessage.Headers = envelope.Headers.ToConfluentHeaders(); } var deliveryResult = await GetConfluentProducer().ProduceAsync(Endpoint.Name, kafkaMessage) .ConfigureAwait(false); if (Endpoint.Configuration.ArePersistenceStatusReportsEnabled) { CheckPersistenceStatus(deliveryResult); } var offset = new KafkaOffset(deliveryResult.TopicPartitionOffset); envelope.AdditionalLogData["offset"] = $"{offset.Partition}@{offset.Offset}"; return(offset); } catch (KafkaException ex) { // Disposing and re-creating the producer will maybe fix the issue if (Endpoint.Configuration.DisposeOnException) { DisposeConfluentProducer(); } throw new ProduceException( "Error occurred producing the message. See inner exception for details.", ex); } }
public void LogProduceError(ISilverbackLogger logger, IOutboundEnvelope envelope, Exception exception) { if (!logger.IsEnabled(IntegrationLogEvents.MessageProduced)) { return; } (string?value1, string?value2) = _logEnricher.GetAdditionalValues( envelope.Endpoint, envelope.Headers, null); _errorProducingMessage.Invoke( logger.InnerLogger, envelope.ActualEndpointDisplayName, envelope.Headers.GetValue(DefaultMessageHeaders.MessageType), envelope.Headers.GetValue(DefaultMessageHeaders.MessageId), value1, value2, exception); }
public void LogWrittenToOutbox(ISilverbackLogger logger, IOutboundEnvelope envelope) { if (!logger.IsEnabled(IntegrationLogEvents.MessageWrittenToOutbox)) { return; } (string?value1, string?value2) = _logEnricher.GetAdditionalValues( envelope.Endpoint, envelope.Headers, envelope.BrokerMessageIdentifier); _messageWrittenToOutbox.Invoke( logger.InnerLogger, envelope.ActualEndpointDisplayName, envelope.Headers.GetValue(DefaultMessageHeaders.MessageType), envelope.Headers.GetValue(DefaultMessageHeaders.MessageId), value1, value2, null); }
public void LogErrorProducingOutboxStoredMessage( ISilverbackLogger logger, IOutboundEnvelope envelope, Exception exception) { if (!logger.IsEnabled(IntegrationLogEvents.ErrorProducingOutboxStoredMessage)) { return; } (string?value1, string?value2) = _logEnricher.GetAdditionalValues( envelope.Endpoint, envelope.Headers, envelope.BrokerMessageIdentifier); _errorProducingOutboxStoredMessage.Invoke( logger.InnerLogger, envelope.ActualEndpointName, envelope.Headers.GetValue(DefaultMessageHeaders.MessageType), envelope.Headers.GetValue(DefaultMessageHeaders.MessageId), value1, value2, exception); }
public void Enrich(IRawInboundEnvelope inboundEnvelope, IOutboundEnvelope outboundEnvelope, Exception exception) { // Do nothing }
/// <inheritdoc cref="Producer.ProduceCore" /> protected override IBrokerMessageIdentifier ProduceCore(IOutboundEnvelope envelope) { throw new InvalidOperationException("Only asynchronous operations are supported."); }
public void LogErrorProducingOutboxStoredMessage(IOutboundEnvelope envelope, Exception exception) => _loggerFactory.GetOutboundLogger(envelope.Endpoint) .LogErrorProducingOutboxStoredMessage(this, envelope, exception);
protected override Task <IOffset?> ProduceCoreAsync(IOutboundEnvelope envelope) { Produce(envelope.RawMessage, envelope.Headers); return(Task.FromResult <IOffset?>(null)); }
public string GetName(IOutboundEnvelope envelope) => "some-topic";
public int?GetPartition(IOutboundEnvelope envelope) => 42;
public void LogProduced(IOutboundEnvelope envelope) => _loggerFactory.GetOutboundLogger(envelope.Endpoint) .LogProduced(this, envelope);
protected override IOffset?ProduceCore(IOutboundEnvelope envelope) { ProducedMessages.Add(new ProducedMessage(envelope.RawMessage, envelope.Headers, Endpoint)); return(null); }
public void LogProduceError(IOutboundEnvelope envelope, Exception exception) => _loggerFactory.GetOutboundLogger(envelope.Endpoint) .LogProduceError(this, envelope, exception);
/// <summary> /// Initializes a new instance of the <see cref="ProducerPipelineContext" /> class. /// </summary> /// <param name="envelope"> /// The envelope containing the message to be produced. /// </param> /// <param name="producer"> /// The <see cref="IProducer" /> that triggered this pipeline. /// </param> /// <param name="serviceProvider"> /// The <see cref="IServiceProvider"/> to be used to resolve the required services. /// </param> public ProducerPipelineContext(IOutboundEnvelope envelope, IProducer producer, IServiceProvider serviceProvider) { Envelope = envelope; Producer = producer; ServiceProvider = serviceProvider; }
public void LogWrittenToOutbox(IOutboundEnvelope envelope) => _loggerFactory.GetOutboundLogger(envelope.Endpoint) .LogWrittenToOutbox(this, envelope);