public void ChunkIfNeeded_LargeMessage_Chunked()
{
var message = new BinaryMessage
{
MessageId = Guid.NewGuid(),
Content = GetByteArray(1400)
};
var headers = new MessageHeaderCollection
{
{ MessageHeader.MessageIdKey, "1234" }
};
var serializedMessage = _serializer.Serialize(message, headers);
var rawBrokerMessage =
new RawBrokerMessage(message, headers,
new TestProducerEndpoint("test")
{
Chunk = new ChunkSettings
{
Size = 500
}
})
{
RawContent = serializedMessage
};
var chunks = ChunkProducer.ChunkIfNeeded(rawBrokerMessage);
chunks.Should().HaveCount(4);
chunks.Should().Match(c => c.All(m => m.RawContent.Length < 1000));
}
public async Task Handle_WithTraceIdHeader_NewActivityStartedAndParentIdIsSet()
{
var rawMessage = new RawBrokerMessage(
"123",
new MessageHeaderCollection
{
{ DiagnosticsConstants.TraceIdHeaderKey, "00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01" }
},
TestEndpoint.GetDefault());
var entered = false;
await new ActivityConsumerBehavior().Handle(rawMessage, _ =>
{
Activity.Current.Should().NotBeNull();
Activity.Current.ParentId.Should().Be("00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01");
Activity.Current.Id.Should().StartWith("00-0af7651916cd43dd8448eb211c80319c");
entered = true;
return(Task.CompletedTask);
});
entered.Should().BeTrue();
}
public void ChunkIfNeeded_SmallMessage_ReturnedWithoutChanges()
{
var message = new BinaryMessage
{
MessageId = Guid.NewGuid(),
Content = GetByteArray(100)
};
var headers = new MessageHeaderCollection();
var serializedMessage = _serializer.Serialize(message, headers);
var rawBrokerMessage =
new RawBrokerMessage(message, headers,
new TestProducerEndpoint("test")
{
Chunk = new ChunkSettings
{
Size = 500
}
})
{
RawContent = serializedMessage
};
var chunks = ChunkProducer.ChunkIfNeeded(rawBrokerMessage);
chunks.Should().HaveCount(1);
chunks.First().Should().BeEquivalentTo(rawBrokerMessage);
}
private IEnumerable <RawBrokerMessage> GetRawMessages(object content, IEnumerable <MessageHeader> headers)
{
var headersCollection = new MessageHeaderCollection(headers);
_messageKeyProvider.EnsureKeyIsInitialized(content, headersCollection);
var rawMessage = new RawBrokerMessage(content, headersCollection, Endpoint);
return(ChunkProducer.ChunkIfNeeded(rawMessage));
}
public void Handle_NoStartedActivity_ActivityStartedAndTraceIdHeaderIsSet()
{
var rawMessage = new RawBrokerMessage("123", null, TestEndpoint.GetDefault());
new ActivityProducerBehavior().Handle(rawMessage, _ => Task.CompletedTask);
rawMessage.Headers.Should().Contain(
h => h.Key == DiagnosticsConstants.TraceIdHeaderKey && !string.IsNullOrEmpty(h.Value));
}
public void Handle_StartedActivity_TraceIdHeaderIsSet()
{
var activity = new Activity("test");
activity.SetParentId("00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01");
activity.Start();
var rawMessage = new RawBrokerMessage("123", null, TestEndpoint.GetDefault());
new ActivityProducerBehavior().Handle(rawMessage, _ => Task.CompletedTask);
rawMessage.Headers.Should().Contain(
h => h.Key == DiagnosticsConstants.TraceIdHeaderKey &&
h.Value.StartsWith("00-0af7651916cd43dd8448eb211c80319c"));
}
public async Task Handle(RawBrokerMessage message, RawBrokerMessageHandler next)
{
var activity = new Activity(DiagnosticsConstants.ActivityNameMessageConsuming);
try
{
activity.InitFromMessageHeaders(message.Headers);
activity.Start();
await next(message);
}
finally
{
activity.Stop();
}
}
public static IEnumerable <RawBrokerMessage> ChunkIfNeeded(RawBrokerMessage message)
{
var messageId = message.Headers.GetValue(MessageHeader.MessageIdKey);
var settings = (message.Endpoint as IProducerEndpoint)?.Chunk;
var chunkSize = settings?.Size ?? int.MaxValue;
if (chunkSize >= message.RawContent.Length)
{
yield return(message);
yield break;
}
if (string.IsNullOrEmpty(messageId))
{
throw new InvalidOperationException(
"Dividing into chunks is pointless if no unique MessageId can be retrieved. " +
"Please add an Id or MessageId property to the message model or " +
"use a custom IMessageKeyProvider.");
}
var span = message.RawContent.AsMemory();
var chunksCount = (int)Math.Ceiling(message.RawContent.Length / (double)chunkSize);
var offset = 0;
for (var i = 0; i < chunksCount; i++)
{
var slice = span.Slice(offset, Math.Min(chunkSize, message.RawContent.Length - offset)).ToArray();
var messageChunk = new RawBrokerMessage(slice, message.Headers, message.Endpoint);
messageChunk.Headers.AddOrReplace(MessageHeader.ChunkIdKey, i);
messageChunk.Headers.AddOrReplace(MessageHeader.ChunksCountKey, chunksCount);
yield return(messageChunk);
offset += chunkSize;
}
}
public async Task Handle_FromConsume_NewActivityStartedAndParentIdIsSet()
{
var services = new ServiceCollection();
services
.AddSingleton <ILoggerFactory, NullLoggerFactory>()
.AddSingleton(typeof(ILogger <>), typeof(NullLogger <>))
.AddSilverback().WithConnectionTo <TestBroker>();
var serviceProvider = services.BuildServiceProvider();
var broker = (TestBroker)serviceProvider.GetRequiredService <IBroker>();
var rawMessage = new RawBrokerMessage(
"123",
new MessageHeaderCollection
{
{ DiagnosticsConstants.TraceIdHeaderKey, "00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01" }
},
TestEndpoint.GetDefault());
var consumer = (TestConsumer)broker.GetConsumer(TestEndpoint.GetDefault());
var entered = false;
consumer.Received += (sender, args) =>
{
Activity.Current.Should().NotBeNull();
Activity.Current.ParentId.Should().Be("00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01");
Activity.Current.Id.Should().StartWith("00-0af7651916cd43dd8448eb211c80319c");
entered = true;
return(Task.CompletedTask);
};
broker.Connect();
await consumer.TestPush(rawMessage.RawContent, rawMessage.Headers);
entered.Should().BeTrue();
}
protected override async Task <IOffset> ProduceAsync(RawBrokerMessage message)
{
try
{
var kafkaMessage = new Confluent.Kafka.Message <byte[], byte[]>
{
Key = GetPartitioningKey(message.Headers),
Value = message.RawContent
};
if (message.Headers != null && message.Headers.Any())
{
kafkaMessage.Headers = new Confluent.Kafka.Headers();
message.Headers.ForEach(h => kafkaMessage.Headers.Add(h.ToConfluentHeader()));
}
var deliveryReport = await GetInnerProducer().ProduceAsync(Endpoint.Name, kafkaMessage);
if (Endpoint.Configuration.ArePersistenceStatusReportsEnabled)
{
CheckPersistenceStatus(deliveryReport);
}
return(new KafkaOffset(deliveryReport.TopicPartitionOffset));
}
catch (Confluent.Kafka.KafkaException ex)
{
// Disposing and re-creating the producer will maybe fix the issue
if (Endpoint.Configuration.DisposeOnException)
{
DisposeInnerProducer();
}
throw new ProduceException("Error occurred producing the message. See inner exception for details.", ex);
}
}
protected override IOffset Produce(RawBrokerMessage message)
{
ProducedMessages.Add(new TestBroker.ProducedMessage(message.RawContent, message.Headers, Endpoint));
return(null);
}
public MessageReceivedEventArgs(RawBrokerMessage message)
{
Message = message;
}
protected abstract Task <IOffset> ProduceAsync(RawBrokerMessage message);
protected override Task <IOffset> ProduceAsync(RawBrokerMessage message) => Broker.GetTopic(Endpoint.Name).PublishAsync(message.RawContent, message.Headers);
private async Task ExecutePipeline(IEnumerable <IConsumerBehavior> behaviors, RawBrokerMessage message, RawBrokerMessageHandler finalAction)
{
if (behaviors != null && behaviors.Any())
{
await behaviors.First().Handle(message, m => ExecutePipeline(behaviors.Skip(1), m, finalAction));
}
else
{
await finalAction(message);
}
}
protected override Task <IOffset> ProduceAsync(RawBrokerMessage message)
{
Produce(message.RawContent, message.Headers);
return(Task.FromResult <IOffset>(null));
}
protected override IOffset Produce(RawBrokerMessage message) => AsyncHelper.RunSynchronously(() => ProduceAsync(message));
private async Task ExecutePipeline(IEnumerable <IProducerBehavior> behaviors, RawBrokerMessage message, RawBrokerMessageHandler finalAction)
{
if (behaviors != null && behaviors.Any())
{
await behaviors.First().Handle(message, m => ExecutePipeline(behaviors.Skip(1), m, finalAction));
}
else
{
await finalAction(message);
_messageLogger.LogInformation(_logger, "Message produced.", message);
}
}
protected abstract IOffset Produce(RawBrokerMessage message);
protected override IOffset Produce(RawBrokerMessage message) => Broker.GetTopic(Endpoint.Name).Publish(message.RawContent, message.Headers);
