public void Enrich(
IRawInboundEnvelope inboundEnvelope,
IOutboundEnvelope outboundEnvelope,
Exception exception)
{
outboundEnvelope.Headers.AddOrReplace(
KafkaMessageHeaders.SourceTopic,
inboundEnvelope.ActualEndpointName);
outboundEnvelope.Headers.AddOrReplace(
KafkaMessageHeaders.SourceConsumerGroupId,
((KafkaConsumerEndpoint)inboundEnvelope.Endpoint).Configuration.GroupId);
while (exception.InnerException != null)
{
exception = exception.InnerException;
}
outboundEnvelope.Headers.AddOrReplace(
DefaultMessageHeaders.FailureReason,
$"{exception.GetType().FullName} in {exception.Source}");
outboundEnvelope.Headers.AddOrReplace(
KafkaMessageHeaders.SourcePartition,
((KafkaOffset)inboundEnvelope.BrokerMessageIdentifier).Partition);
outboundEnvelope.Headers.AddOrReplace(
KafkaMessageHeaders.SourceOffset,
((KafkaOffset)inboundEnvelope.BrokerMessageIdentifier).Offset);
if (inboundEnvelope.Headers.Contains(KafkaMessageHeaders.TimestampKey))
{
outboundEnvelope.Headers.AddOrReplace(
KafkaMessageHeaders.SourceTimestamp,
inboundEnvelope.Headers[KafkaMessageHeaders.TimestampKey]);
}
}
/// <inheritdoc cref="ISequenceWriter.CanHandle" />
public bool CanHandle(IOutboundEnvelope envelope)
{
Check.NotNull(envelope, nameof(envelope));
var chunkSettings = envelope.Endpoint.Chunk;
if (chunkSettings == null || chunkSettings.Size == int.MaxValue)
{
return(false);
}
if (chunkSettings.AlwaysAddHeaders)
{
return(true);
}
if (envelope.RawMessage == null)
{
return(false);
}
if (!envelope.RawMessage.CanSeek)
{
return(true);
}
return(envelope.RawMessage != null && envelope.RawMessage.Length > chunkSettings.Size);
}
public void OnOutbound(IOutboundEnvelope envelope)
{
lock (_outboundEnvelopes)
{
_outboundEnvelopes.Add(envelope);
}
}
/// <inheritdoc cref="Producer.ProduceCoreAsync" />
protected override async Task <IBrokerMessageIdentifier?> ProduceCoreAsync(IOutboundEnvelope envelope)
{
Check.NotNull(envelope, nameof(envelope));
await _queueWriter.WriteAsync(envelope).ConfigureAwait(false);
return(null);
}
/// <inheritdoc cref="Producer.ProduceCoreAsync" />
protected override Task <IOffset?> ProduceCoreAsync(IOutboundEnvelope envelope)
{
var queuedMessage = new QueuedMessage(envelope);
_queue.Add(queuedMessage);
return(queuedMessage.TaskCompletionSource.Task);
}
public Task ProduceAsync(IOutboundEnvelope envelope)
{
Check.NotNull(envelope, nameof(envelope));
_logger.LogWrittenToOutbox(envelope);
return(_outboundQueueBroker.GetProducer(envelope.Endpoint).ProduceAsync(envelope));
}
/// <inheritdoc cref="IProducer.ProduceAsync(IOutboundEnvelope)" />
public async Task ProduceAsync(IOutboundEnvelope envelope) =>
await ExecutePipelineIfNeededAsync(
new ProducerPipelineContext(envelope, this, _serviceProvider),
async finalContext =>
{
((RawOutboundEnvelope)finalContext.Envelope).Offset =
await ProduceCoreAsync(finalContext.Envelope).ConfigureAwait(false);
}).ConfigureAwait(false);
/// <inheritdoc cref="IOutboundMessageEnricher.Enrich" />
public void Enrich(IOutboundEnvelope envelope)
{
if (envelope == null || envelope is not IOutboundEnvelope <TMessage> typedEnvelope)
{
return;
}
envelope.Headers.AddOrReplace(_name, _valueProvider.Invoke(typedEnvelope));
}
public async IAsyncEnumerable <IOutboundEnvelope> ProcessMessage(IOutboundEnvelope envelope)
{
Check.NotNull(envelope, nameof(envelope));
if (envelope.RawMessage == null)
{
throw new InvalidOperationException("RawMessage is null");
}
var settings = envelope.Endpoint.Chunk;
var chunkSize = settings?.Size ?? int.MaxValue;
var bufferArray = ArrayPool <byte> .Shared.Rent(chunkSize);
var bufferMemory = bufferArray.AsMemory(0, chunkSize);
var chunksCount = envelope.RawMessage.CanSeek
? (int?)Math.Ceiling(envelope.RawMessage.Length / (double)chunkSize)
: null;
IBrokerMessageOffset?firstChunkOffset = null;
int chunkIndex = 0;
var readBytesCount = await envelope.RawMessage.ReadAsync(bufferMemory).ConfigureAwait(false);
while (readBytesCount > 0)
{
var chunkEnvelope = CreateChunkEnvelope(
chunkIndex,
chunksCount,
bufferMemory.Slice(0, readBytesCount).ToArray(),
envelope);
if (chunkIndex > 0)
{
chunkEnvelope.Headers.AddOrReplace(DefaultMessageHeaders.FirstChunkOffset, firstChunkOffset?.Value);
}
// Read the next chunk and check if we were at the end of the stream (to not rely on Length property)
readBytesCount = await envelope.RawMessage.ReadAsync(bufferMemory).ConfigureAwait(false);
if (readBytesCount == 0)
{
chunkEnvelope.Headers.AddOrReplace(DefaultMessageHeaders.IsLastChunk, true.ToString());
}
yield return(chunkEnvelope);
// Read and store the offset of the first chunk, after it has been produced (after yield return)
if (chunkIndex == 0)
{
firstChunkOffset = chunkEnvelope.BrokerMessageIdentifier as IBrokerMessageOffset;
}
chunkIndex++;
}
}
/// <inheritdoc cref="IOutboundMessageEnricher.Enrich" />
public void Enrich(IOutboundEnvelope envelope)
{
Check.NotNull(envelope, nameof(envelope));
if (envelope is not IOutboundEnvelope <TMessage> typedEnvelope)
{
return;
}
envelope.Headers.AddOrReplace(_name, _valueProvider.Invoke(typedEnvelope));
}
public Task ProduceAsync(IOutboundEnvelope envelope)
{
Check.NotNull(envelope, nameof(envelope));
_logger.LogDebugWithMessageInfo(
IntegrationEventIds.OutboundMessageWrittenToOutbox,
"Writing the outbound message to the transactional outbox.",
envelope);
return(_outboundQueueBroker.GetProducer(envelope.Endpoint).ProduceAsync(envelope));
}
private static async ValueTask <byte[]?> GetContentAsync(IOutboundEnvelope envelope)
{
var stream =
envelope.RawMessage ??
await envelope.Endpoint.Serializer.SerializeAsync(
envelope.Message,
envelope.Headers,
new MessageSerializationContext(envelope.Endpoint))
.ConfigureAwait(false);
return(await stream.ReadAllAsync().ConfigureAwait(false));
}
/// <inheritdoc cref="IProducer.Produce(IOutboundEnvelope)" />
public void Produce(IOutboundEnvelope envelope) =>
AsyncHelper.RunSynchronously(
() =>
ExecutePipelineIfNeededAsync(
new ProducerPipelineContext(envelope, this, _serviceProvider),
finalContext =>
{
((RawOutboundEnvelope)finalContext.Envelope).Offset =
ProduceCore(finalContext.Envelope);
return(Task.CompletedTask);
}));
/// <inheritdoc cref="IOutboxWriter.WriteAsync" />
public async Task WriteAsync(IOutboundEnvelope envelope)
{
Check.NotNull(envelope, nameof(envelope));
var item = new OutboxStoredMessage(
envelope.Message?.GetType(),
await envelope.RawMessage.ReadAllAsync().ConfigureAwait(false),
envelope.Headers,
envelope.Endpoint.Name);
await AddAsync(item).ConfigureAwait(false);
}
/// <inheritdoc cref="IOutboxWriter.WriteAsync" />
public async Task WriteAsync(IOutboundEnvelope envelope)
{
Check.NotNull(envelope, nameof(envelope));
DbSet.Add(
new OutboxMessage
{
MessageType = envelope.Message?.GetType().AssemblyQualifiedName,
Content = await GetContentAsync(envelope).ConfigureAwait(false),
SerializedHeaders =
JsonSerializer.SerializeToUtf8Bytes((IEnumerable <MessageHeader>)envelope.Headers),
EndpointName = envelope.Endpoint.Name,
Created = DateTime.UtcNow
});
}
protected override async Task <IOffset?> ProduceCoreAsync(IOutboundEnvelope envelope)
{
Check.NotNull(envelope, nameof(envelope));
try
{
var kafkaMessage = new Message <byte[]?, byte[]?>
{
Key = GetKafkaKeyAndRemoveHeader(envelope.Headers),
Value = await envelope.RawMessage.ReadAllAsync().ConfigureAwait(false)
};
if (envelope.Headers.Count >= 1)
{
kafkaMessage.Headers = envelope.Headers.ToConfluentHeaders();
}
var deliveryResult = await GetConfluentProducer().ProduceAsync(Endpoint.Name, kafkaMessage)
.ConfigureAwait(false);
if (Endpoint.Configuration.ArePersistenceStatusReportsEnabled)
{
CheckPersistenceStatus(deliveryResult);
}
var offset = new KafkaOffset(deliveryResult.TopicPartitionOffset);
envelope.AdditionalLogData["offset"] = $"{offset.Partition}@{offset.Offset}";
return(offset);
}
catch (KafkaException ex)
{
// Disposing and re-creating the producer will maybe fix the issue
if (Endpoint.Configuration.DisposeOnException)
{
DisposeConfluentProducer();
}
throw new ProduceException(
"Error occurred producing the message. See inner exception for details.",
ex);
}
}
public void LogProduceError(ISilverbackLogger logger, IOutboundEnvelope envelope, Exception exception)
{
if (!logger.IsEnabled(IntegrationLogEvents.MessageProduced))
{
return;
}
(string?value1, string?value2) = _logEnricher.GetAdditionalValues(
envelope.Endpoint,
envelope.Headers,
null);
_errorProducingMessage.Invoke(
logger.InnerLogger,
envelope.ActualEndpointDisplayName,
envelope.Headers.GetValue(DefaultMessageHeaders.MessageType),
envelope.Headers.GetValue(DefaultMessageHeaders.MessageId),
value1,
value2,
exception);
}
public void LogWrittenToOutbox(ISilverbackLogger logger, IOutboundEnvelope envelope)
{
if (!logger.IsEnabled(IntegrationLogEvents.MessageWrittenToOutbox))
{
return;
}
(string?value1, string?value2) = _logEnricher.GetAdditionalValues(
envelope.Endpoint,
envelope.Headers,
envelope.BrokerMessageIdentifier);
_messageWrittenToOutbox.Invoke(
logger.InnerLogger,
envelope.ActualEndpointDisplayName,
envelope.Headers.GetValue(DefaultMessageHeaders.MessageType),
envelope.Headers.GetValue(DefaultMessageHeaders.MessageId),
value1,
value2,
null);
}
public void LogErrorProducingOutboxStoredMessage(
ISilverbackLogger logger,
IOutboundEnvelope envelope,
Exception exception)
{
if (!logger.IsEnabled(IntegrationLogEvents.ErrorProducingOutboxStoredMessage))
{
return;
}
(string?value1, string?value2) = _logEnricher.GetAdditionalValues(
envelope.Endpoint,
envelope.Headers,
envelope.BrokerMessageIdentifier);
_errorProducingOutboxStoredMessage.Invoke(
logger.InnerLogger,
envelope.ActualEndpointName,
envelope.Headers.GetValue(DefaultMessageHeaders.MessageType),
envelope.Headers.GetValue(DefaultMessageHeaders.MessageId),
value1,
value2,
exception);
}
public void Enrich(IRawInboundEnvelope inboundEnvelope, IOutboundEnvelope outboundEnvelope, Exception exception)
{
// Do nothing
}
/// <inheritdoc cref="Producer.ProduceCore" />
protected override IBrokerMessageIdentifier ProduceCore(IOutboundEnvelope envelope)
{
throw new InvalidOperationException("Only asynchronous operations are supported.");
}
public void LogErrorProducingOutboxStoredMessage(IOutboundEnvelope envelope, Exception exception) => _loggerFactory.GetOutboundLogger(envelope.Endpoint) .LogErrorProducingOutboxStoredMessage(this, envelope, exception);
protected override Task <IOffset?> ProduceCoreAsync(IOutboundEnvelope envelope)
{
Produce(envelope.RawMessage, envelope.Headers);
return(Task.FromResult <IOffset?>(null));
}
public string GetName(IOutboundEnvelope envelope) => "some-topic";
public int?GetPartition(IOutboundEnvelope envelope) => 42;
public void LogProduced(IOutboundEnvelope envelope) => _loggerFactory.GetOutboundLogger(envelope.Endpoint) .LogProduced(this, envelope);
protected override IOffset?ProduceCore(IOutboundEnvelope envelope)
{
ProducedMessages.Add(new ProducedMessage(envelope.RawMessage, envelope.Headers, Endpoint));
return(null);
}
public void LogProduceError(IOutboundEnvelope envelope, Exception exception) => _loggerFactory.GetOutboundLogger(envelope.Endpoint) .LogProduceError(this, envelope, exception);
/// <summary>
/// Initializes a new instance of the <see cref="ProducerPipelineContext" /> class.
/// </summary>
/// <param name="envelope">
/// The envelope containing the message to be produced.
/// </param>
/// <param name="producer">
/// The <see cref="IProducer" /> that triggered this pipeline.
/// </param>
/// <param name="serviceProvider">
/// The <see cref="IServiceProvider"/> to be used to resolve the required services.
/// </param>
public ProducerPipelineContext(IOutboundEnvelope envelope, IProducer producer, IServiceProvider serviceProvider)
{
Envelope = envelope;
Producer = producer;
ServiceProvider = serviceProvider;
}
public void LogWrittenToOutbox(IOutboundEnvelope envelope) => _loggerFactory.GetOutboundLogger(envelope.Endpoint) .LogWrittenToOutbox(this, envelope);
