public async Task DeserializeAsync_NullMessageWithHardcodedType_CustomBinaryFileReturned()
{
var headers = new MessageHeaderCollection();
var serializer = new BinaryFileMessageSerializer <InheritedBinaryFileMessage>();
var(deserializedObject, type) = await serializer
.DeserializeAsync(null, headers, MessageSerializationContext.Empty);
deserializedObject.Should().BeOfType <InheritedBinaryFileMessage>();
deserializedObject.Should().BeEquivalentTo(
new InheritedBinaryFileMessage
{
Content = null
});
type.Should().Be(typeof(InheritedBinaryFileMessage));
}
public async Task RawProducer_Stream_ProducedAndConsumed()
{
var message = new TestEventOne
{
Content = "Hello E2E!"
};
var headers = new MessageHeaderCollection();
Stream rawMessageStream = await Endpoint.DefaultSerializer.SerializeAsync(
message,
headers,
MessageSerializationContext.Empty) ??
throw new InvalidOperationException("Serializer returned null");
Host.ConfigureServices(
services => services
.AddLogging()
.AddSilverback()
.UseModel()
.WithConnectionToMessageBroker(options => options.AddMockedKafka())
.AddEndpoints(
endpoints => endpoints
.AddOutbound <IIntegrationEvent>(new KafkaProducerEndpoint(DefaultTopicName))
.AddInbound(
new KafkaConsumerEndpoint(DefaultTopicName)
{
Configuration = new KafkaConsumerConfig
{
GroupId = "consumer1",
AutoCommitIntervalMs = 100
}
}))
.AddSingletonBrokerBehavior <SpyBrokerBehavior>()
.AddSingletonSubscriber <OutboundInboundSubscriber>())
.Run();
var producer = Broker.GetProducer(new KafkaProducerEndpoint(DefaultTopicName));
await producer.ProduceAsync(rawMessageStream, headers);
await KafkaTestingHelper.WaitUntilAllMessagesAreConsumedAsync();
Subscriber.InboundEnvelopes.Should().HaveCount(1);
SpyBehavior.InboundEnvelopes.Should().HaveCount(1);
SpyBehavior.InboundEnvelopes[0].Message.Should().BeOfType <TestEventOne>();
SpyBehavior.InboundEnvelopes[0].Message.As <TestEventOne>().Content.Should().BeEquivalentTo("Hello E2E!");
}
public void DeserializeAsync_BadTypeHeader_ExceptionThrown()
{
var rawMessage = new MemoryStream(Encoding.UTF8.GetBytes("{\"Content\":\"the message\"}"));
var headers = new MessageHeaderCollection
{
{
"x-message-type",
"Bad.TestEventOne, Silverback.Integration.Tests"
}
};
var serializer = new JsonMessageSerializer();
Func <Task> act = async() => await serializer
.DeserializeAsync(rawMessage, headers, MessageSerializationContext.Empty);
act.Should().Throw <TypeLoadException>();
}
public async Task DeserializeAsync_BadTypeHeader_ExceptionThrown()
{
var rawContent = new MemoryStream(new byte[] { 0x01, 0x02, 0x03, 0x04, 0x05 });
var headers = new MessageHeaderCollection
{
{
"x-message-type",
"Bad.TestEventOne, Silverback.Integration.Tests"
}
};
var serializer = new JsonMessageSerializer();
Func <Task> act = async() => await serializer
.DeserializeAsync(rawContent, headers, MessageSerializationContext.Empty);
await act.Should().ThrowAsync <TypeLoadException>();
}
public void Apply_NoMappings_HeadersUnchanged()
{
var mappings = new CustomHeadersMappings();
var headers = new MessageHeaderCollection
{
{ DefaultMessageHeaders.ChunkIndex, 1 },
{ DefaultMessageHeaders.ChunksCount, 2 },
{ DefaultMessageHeaders.TraceId, "abc" }
};
mappings.Apply(headers);
headers.Should().BeEquivalentTo(
new MessageHeader("x-chunk-index", "1"),
new MessageHeader("x-chunk-count", "2"),
new MessageHeader("traceparent", "abc"));
}
public void ToConfluentHeaders_PartitionIndexHeaderIgnored()
{
var headers = new MessageHeaderCollection
{
{ "one", "1" },
{ KafkaMessageHeaders.KafkaPartitionIndex, "42" },
{ "two", "2" }
};
var confluentHeaders = headers.ToConfluentHeaders();
confluentHeaders.Should().BeEquivalentTo(
new[]
{
new Header("one", Encoding.UTF8.GetBytes("1")),
new Header("two", Encoding.UTF8.GetBytes("2"))
});
}
public async Task DeserializeAsync_WithHardcodedType_CustomBinaryFileReturned()
{
var rawContent = new MemoryStream(new byte[] { 0x01, 0x02, 0x03, 0x04, 0x05 });
var headers = new MessageHeaderCollection();
var serializer = new BinaryFileMessageSerializer <InheritedBinaryFileMessage>();
var(deserializedObject, type) = await serializer
.DeserializeAsync(rawContent, headers, MessageSerializationContext.Empty);
deserializedObject.Should().BeOfType <InheritedBinaryFileMessage>();
deserializedObject.Should().BeEquivalentTo(
new InheritedBinaryFileMessage
{
Content = rawContent
});
type.Should().Be(typeof(InheritedBinaryFileMessage));
}
public async Task Run()
{
var producer = _broker.GetProducer(
new KafkaProducerEndpoint("silverback-examples-events")
{
Configuration = new KafkaProducerConfig
{
BootstrapServers = "PLAINTEXT://localhost:9092"
}
});
var headers = new MessageHeaderCollection
{
["use-case"] = "empty-message"
};
await producer.ProduceAsync(null, headers);
}
public async Task SerializeAsync_Message_TypeHeaderAdded()
{
var message = new BinaryFileMessage
{
Content = new MemoryStream(new byte[] { 0x01, 0x02, 0x03, 0x04, 0x05 })
};
var headers = new MessageHeaderCollection();
var serializer = new BinaryFileMessageSerializer();
await serializer.SerializeAsync(message, headers, MessageSerializationContext.Empty);
var typeHeaderValue = headers["x-message-type"];
typeHeaderValue.Should().NotBeNullOrEmpty();
typeHeaderValue.Should()
.StartWith("Silverback.Messaging.Messages.BinaryFileMessage, Silverback.Integration,");
}
/// <inheritdoc cref="IMessageSerializer.DeserializeAsync" />
public override async ValueTask <(object?, Type)> DeserializeAsync(
Stream?messageStream,
MessageHeaderCollection messageHeaders,
MessageSerializationContext context)
{
if (messageStream == null)
{
return(null, _type);
}
var buffer = await messageStream.ReadAllAsync().ConfigureAwait(false);
var jsonString = GetSystemEncoding().GetString(buffer !);
var deserializedObject = JsonConvert.DeserializeObject(jsonString, _type, Settings);
return(deserializedObject, _type);
}
public async Task TestHandleMessage(byte[]?rawMessage, MessageHeaderCollection headers, IOffset?offset = null)
{
if (!Broker.IsConnected)
{
throw new InvalidOperationException("The broker is not connected.");
}
if (!IsConnected)
{
throw new InvalidOperationException("The consumer is not ready.");
}
await HandleMessageAsync(
rawMessage,
headers,
"test-topic",
offset ?? new TestOffset(),
null);
}
public async Task SerializeDeserializeAsync_HardcodedType_CorrectlyDeserialized()
{
var message = new InheritedBinaryFileMessage
{
Content = new MemoryStream(new byte[] { 0x01, 0x02, 0x03, 0x04, 0x05 })
};
var headers = new MessageHeaderCollection();
var serializer = new BinaryFileMessageSerializer <InheritedBinaryFileMessage>();
var serialized = await serializer.SerializeAsync(message, headers, MessageSerializationContext.Empty);
var(deserialized, _) = await serializer
.DeserializeAsync(serialized, headers, MessageSerializationContext.Empty);
var message2 = deserialized as BinaryFileMessage;
message2.Should().NotBeNull();
message2.Should().BeEquivalentTo(message);
}
public async Task SerializeDeserializeAsync_Message_CorrectlyDeserialized()
{
var message = new TestEventOne {
Content = "the message"
};
var headers = new MessageHeaderCollection();
var serializer = new NewtonsoftJsonMessageSerializer();
var serialized = await serializer.SerializeAsync(message, headers, MessageSerializationContext.Empty);
var(deserialized, _) = await serializer
.DeserializeAsync(serialized, headers, MessageSerializationContext.Empty);
var message2 = deserialized as TestEventOne;
message2.Should().NotBeNull();
message2.Should().BeEquivalentTo(message);
}
public void Apply_SomeMappings_HeadersMapped()
{
var mappings = new CustomHeadersMappings();
mappings.Add(DefaultMessageHeaders.ChunkIndex, "mapped1");
mappings.Add(DefaultMessageHeaders.ChunksCount, "mapped2");
var headers = new MessageHeaderCollection
{
{ DefaultMessageHeaders.ChunkIndex, 1 },
{ DefaultMessageHeaders.ChunksCount, 2 },
{ DefaultMessageHeaders.TraceId, "abc" }
};
mappings.Apply(headers);
headers.Should().BeEquivalentTo(
new MessageHeader("mapped1", "1"),
new MessageHeader("mapped2", "2"),
new MessageHeader("traceparent", "abc"));
}
public async Task SerializeDeserializeAsync_Message_CorrectlyDeserialized()
{
var message = new TestEventOne {
Content = "the message"
};
var headers = new MessageHeaderCollection();
var serializer = new NewtonsoftJsonMessageSerializer <TestEventOne>();
var serialized = (await serializer.SerializeAsync(message, headers, MessageSerializationContext.Empty)) !;
Encoding.UTF8.GetString(serialized.ReadAll() !).Should().NotContain("TestEventOne");
var(deserialized, _) = await serializer
.DeserializeAsync(serialized, headers, MessageSerializationContext.Empty);
var message2 = deserialized as TestEventOne;
message2.Should().NotBeNull();
message2.Should().BeEquivalentTo(message);
}
/// <inheritdoc cref="IMessageSerializer.DeserializeAsync" />
public override async ValueTask <(object?, Type)> DeserializeAsync(
Stream?messageStream,
MessageHeaderCollection messageHeaders,
MessageSerializationContext context)
{
if (messageStream == null)
{
return(null, _type);
}
if (messageStream.CanSeek && messageStream.Length == 0)
{
return(null, _type);
}
var deserializedObject = await JsonSerializer.DeserializeAsync(messageStream, _type, Options)
.ConfigureAwait(false) ??
throw new MessageSerializerException("The deserialization returned null.");
return(deserializedObject, _type);
}
internal async Task HandleMessageAsync(
Message <byte[]?, byte[]?> message,
TopicPartitionOffset topicPartitionOffset)
{
if (_serializer == null)
{
throw new InvalidOperationException("The consumer is not connected.");
}
Dictionary <string, string> logData = new Dictionary <string, string>();
var offset = new KafkaOffset(topicPartitionOffset);
logData["offset"] = $"{offset.Partition}@{offset.Offset}";
var headers = new MessageHeaderCollection(message.Headers.ToSilverbackHeaders());
if (message.Key != null)
{
string deserializedKafkaKey = _serializer.DeserializeKey(
message.Key,
headers,
new MessageSerializationContext(Endpoint, topicPartitionOffset.Topic));
headers.AddOrReplace(KafkaMessageHeaders.KafkaMessageKey, deserializedKafkaKey);
headers.AddIfNotExists(DefaultMessageHeaders.MessageId, deserializedKafkaKey);
logData["kafkaKey"] = deserializedKafkaKey;
}
headers.AddOrReplace(KafkaMessageHeaders.TimestampKey, message.Timestamp.UtcDateTime.ToString("O"));
await HandleMessageAsync(
message.Value,
headers,
topicPartitionOffset.Topic,
offset,
logData)
.ConfigureAwait(false);
}
public async Task HandleErrorAsync_InboundMessage_MessagePreserved()
{
var policy = new MoveMessageErrorPolicy(TestProducerEndpoint.GetDefault()).Build(_serviceProvider);
var message = new TestEventOne {
Content = "hey oh!"
};
var headers = new MessageHeaderCollection
{
{ "key1", "value1" },
{ "key2", "value2" }
};
var rawContent = await TestConsumerEndpoint.GetDefault().Serializer
.SerializeAsync(message, headers, MessageSerializationContext.Empty);
var envelope = new InboundEnvelope(
message,
rawContent,
headers,
new TestOffset(),
TestConsumerEndpoint.GetDefault(),
TestConsumerEndpoint.GetDefault().Name);
await policy.HandleErrorAsync(
ConsumerPipelineContextHelper.CreateSubstitute(envelope, _serviceProvider),
new InvalidOperationException("test"));
var producer = (TestProducer)_broker.GetProducer(TestProducerEndpoint.GetDefault());
var producedMessage = producer.ProducedMessages.Last();
var(deserializedMessage, _) =
await producedMessage.Endpoint.Serializer.DeserializeAsync(
new MemoryStream(producedMessage.Message !),
producedMessage.Headers,
MessageSerializationContext.Empty);
deserializedMessage.Should().BeEquivalentTo(envelope.Message);
}
public void SplitHeadersAndBody()
{
string headers, body;
// No headers, just a body.
MimeMessage.SeparateBodyPartHeadersAndBody(MtaParameters.NewLine + "Just a body.", out headers, out body);
Assert.AreEqual(string.Empty, headers);
Assert.AreEqual("Just a body.", body);
// Couple of headers and a body.
MimeMessage.SeparateBodyPartHeadersAndBody("From: <*****@*****.**>" + MtaParameters.NewLine + "To: <*****@*****.**>" + MtaParameters.NewLine + MtaParameters.NewLine + "Here's the body.", out headers, out body);
Assert.AreEqual("From: <*****@*****.**>" + MtaParameters.NewLine + "To: <*****@*****.**>", headers);
Assert.AreEqual("Here's the body.", body);
// Check headers are separated correctly.
MessageHeaderCollection headersColl = MimeMessage.GetMessageHeaders("From: <*****@*****.**>" + MtaParameters.NewLine + "To: <*****@*****.**>");
Assert.AreEqual("From", headersColl[0].Name);
Assert.AreEqual("<*****@*****.**>", headersColl[0].Value);
Assert.AreEqual("To", headersColl[1].Name);
Assert.AreEqual("<*****@*****.**>", headersColl[1].Value);
// Check headers are unfolded correctly, ensuring that only "\r\n" is used as a line ending ("\r" and "\n" may appear, but should be considered
// as part of a header's value.
headersColl = MimeMessage.GetMessageHeaders("FoldedHeader1: beginning" + MtaParameters.NewLine + " end" + MtaParameters.NewLine + "FoldedHeader2: beginning" + MtaParameters.NewLine + "\t\tend" + MtaParameters.NewLine + "From: <*****@*****.**>" + MtaParameters.NewLine + "To: <*****@*****.**>" + MtaParameters.NewLine + "FoldedHeader3: line1\nstill line 1" + MtaParameters.NewLine + "\t\tline2\rstill line 2");
Assert.AreEqual("FoldedHeader1", headersColl[0].Name);
Assert.AreEqual("beginning end", headersColl[0].Value);
Assert.AreEqual("FoldedHeader2", headersColl[1].Name);
Assert.AreEqual("beginning\t\tend", headersColl[1].Value);
Assert.AreEqual("From", headersColl[2].Name);
Assert.AreEqual("<*****@*****.**>", headersColl[2].Value);
Assert.AreEqual("To", headersColl[3].Name);
Assert.AreEqual("<*****@*****.**>", headersColl[3].Value);
Assert.AreEqual("FoldedHeader3", headersColl[4].Name);
Assert.AreEqual("line1\nstill line 1\t\tline2\rstill line 2", headersColl[4].Value);
}
public void SetFromHeaders_DecoratedMessage_PropertiesSet()
{
var headers = new MessageHeaderCollection
{
{ "x-string", "string1" },
{ "x-string-default", "string2" },
{ "x-readonly-string", "ignored" },
{ "x-int", "1" },
{ "x-int-default", "2" },
{ "x-readonly-int", "3" }
};
var message = new TestEventWithHeaders();
HeaderAttributeHelper.SetFromHeaders(message, headers);
message.StringHeader.Should().Be("string1");
message.StringHeaderWithDefault.Should().Be("string2");
message.ReadOnlyStringHeader.Should().Be("readonly");
message.IntHeader.Should().Be(1);
message.IntHeaderWithDefault.Should().Be(2);
message.ReadOnlyIntHeader.Should().Be(42);
}
/// <inheritdoc cref="IMessageSerializer.DeserializeAsync" />
public override async ValueTask <(object?, Type)> DeserializeAsync(
Stream?messageStream,
MessageHeaderCollection messageHeaders,
MessageSerializationContext context)
{
Check.NotNull(messageHeaders, nameof(messageHeaders));
var type = SerializationHelper.GetTypeFromHeaders(messageHeaders) ??
throw new InvalidOperationException("Message type is null.");
if (messageStream == null)
{
return(null, type);
}
var buffer = await messageStream.ReadAllAsync().ConfigureAwait(false);
var jsonString = GetSystemEncoding().GetString(buffer !);
var deserializedObject = JsonConvert.DeserializeObject(jsonString, type, Settings);
return(deserializedObject, type);
}
public static void SetMessageHeaders(this Activity activity, MessageHeaderCollection headers)
{
if (activity?.Id == null)
{
throw new InvalidOperationException(
"Activity.Id is null. Consider to start a new activity, before calling this method.");
}
headers.Add(new MessageHeader(DiagnosticsConstants.TraceIdHeaderKey, activity.Id));
var traceState = activity.TraceStateString;
if (traceState != null)
{
headers.Add(DiagnosticsConstants.TraceStateHeaderKey, traceState);
}
if (activity.Baggage.Any())
{
headers.Add(new MessageHeader(DiagnosticsConstants.TraceBaggageHeaderKey,
BaggageConverter.Serialize(activity.Baggage)));
}
}
private byte[]? GetKafkaKeyAndRemoveHeader(MessageHeaderCollection headers)
{
var kafkaKeyHeader = headers.FirstOrDefault(header => header.Name == KafkaMessageHeaders.KafkaMessageKey);
if (kafkaKeyHeader == null)
{
return(null);
}
headers.Remove(kafkaKeyHeader);
if (kafkaKeyHeader.Value == null)
{
return(null);
}
return(Endpoint.Serializer is IKafkaMessageSerializer kafkaSerializer
? kafkaSerializer.SerializeKey(
kafkaKeyHeader.Value,
headers,
new MessageSerializationContext(Endpoint, Endpoint.Name))
: Encoding.UTF8.GetBytes(kafkaKeyHeader.Value));
}
public void LogProduced_NoEnvelope_Logged()
{
var endpoint = new TestProducerEndpoint("[dynamic]");
var actualEndpointName = "test1";
var headers = new MessageHeaderCollection
{
{ DefaultMessageHeaders.MessageType, "Message.Type" },
{ DefaultMessageHeaders.MessageId, "1234" }
};
var brokerMessageIdentifier = new TestOffset("a", "42");
var expectedMessage =
"Message produced. | " +
"endpointName: test1, " +
"messageType: Message.Type, " +
"messageId: 1234, " +
"unused1: (null), " +
"unused2: (null)";
_outboundLogger.LogProduced(endpoint, actualEndpointName, headers, brokerMessageIdentifier);
_loggerSubstitute.Received(LogLevel.Information, null, expectedMessage, 1031);
}
public ValueTask <Stream?> SerializeAsync(
object?message,
MessageHeaderCollection messageHeaders,
MessageSerializationContext context)
{
Check.NotNull(messageHeaders, nameof(messageHeaders));
if (message == null)
{
return(ValueTaskFactory.FromResult <Stream?>(null));
}
if (message is Stream inputStream)
{
return(ValueTaskFactory.FromResult <Stream?>(inputStream));
}
if (message is byte[] inputBytes)
{
return(ValueTaskFactory.FromResult <Stream?>(new MemoryStream(inputBytes)));
}
var binaryFileMessage = message as IBinaryFileMessage;
if (binaryFileMessage == null)
{
throw new ArgumentException(
"The message is not implementing the IBinaryFileMessage interface.",
nameof(message));
}
messageHeaders.AddOrReplace(
DefaultMessageHeaders.MessageType,
message.GetType().AssemblyQualifiedName);
return(ValueTaskFactory.FromResult(binaryFileMessage.Content));
}
public override IEnumerable <IProducerEndpoint> GetDestinationEndpoints(
TestPrioritizedCommand message,
MessageHeaderCollection headers)
{
yield return(AllMessagesEndpoint);
switch (message.Priority)
{
case Priority.Low:
yield return(LowPriorityEndpoint);
break;
case Priority.High:
yield return(HighPriorityEndpoint);
break;
default:
yield return(NormalPriorityEndpoint);
break;
}
}
public async Task HandleErrorAsync_InboundMessage_HeadersPreserved()
{
var policy = new MoveMessageErrorPolicy(TestProducerEndpoint.GetDefault()).Build(_serviceProvider);
var headers = new MessageHeaderCollection
{
{ "key1", "value1" },
{ "key2", "value2" }
};
var envelope = new InboundEnvelope(
"hey oh!",
new MemoryStream(Encoding.UTF8.GetBytes("hey oh!")),
headers,
new TestOffset(),
TestConsumerEndpoint.GetDefault(),
TestConsumerEndpoint.GetDefault().Name);
await policy.HandleErrorAsync(
ConsumerPipelineContextHelper.CreateSubstitute(envelope, _serviceProvider),
new InvalidOperationException("test"));
var producer = (TestProducer)_broker.GetProducer(TestProducerEndpoint.GetDefault());
producer.ProducedMessages.Last().Headers.Should().Contain(envelope.Headers);
}
public void LogProduced_NoEnvelope_Logged()
{
var endpoint = new KafkaProducerEndpoint("[dynamic]");
var actualEndpointName = "test1";
var headers = new MessageHeaderCollection
{
{ DefaultMessageHeaders.MessageType, "Message.Type" },
{ DefaultMessageHeaders.MessageId, "1234" },
{ KafkaMessageHeaders.KafkaMessageKey, "key1234" }
};
var brokerMessageIdentifier = new KafkaOffset("topic2", 2, 42);
var expectedMessage =
"Message produced. | " +
"endpointName: test1, " +
"messageType: Message.Type, " +
"messageId: 1234, " +
"offset: [2]@42, " +
"kafkaKey: key1234";
_outboundLogger.LogProduced(endpoint, actualEndpointName, headers, brokerMessageIdentifier);
_loggerSubstitute.Received(LogLevel.Information, null, expectedMessage, 1031);
}
public override ValueTask <Stream?> SerializeAsync(
object?message,
MessageHeaderCollection messageHeaders,
MessageSerializationContext context)
{
if (message == null)
{
return(ValueTaskFactory.FromResult <Stream?>(null));
}
if (message is Stream inputStream)
{
return(ValueTaskFactory.FromResult <Stream?>(inputStream));
}
if (message is byte[] inputBytes)
{
return(ValueTaskFactory.FromResult <Stream?>(new MemoryStream(inputBytes)));
}
var jsonString = JsonConvert.SerializeObject(message, _type, Settings);
return(ValueTaskFactory.FromResult <Stream?>(new MemoryStream(GetSystemEncoding().GetBytes(jsonString))));
}
public override ValueTask <Stream?> SerializeAsync(
object?message,
MessageHeaderCollection messageHeaders,
MessageSerializationContext context)
{
if (message == null)
{
return(ValueTaskFactory.FromResult <Stream?>(null));
}
if (message is Stream inputStream)
{
return(ValueTaskFactory.FromResult <Stream?>(inputStream));
}
if (message is byte[] inputBytes)
{
return(ValueTaskFactory.FromResult <Stream?>(new MemoryStream(inputBytes)));
}
var bytes = JsonSerializer.SerializeToUtf8Bytes(message, _type, Options);
return(ValueTaskFactory.FromResult <Stream?>(new MemoryStream(bytes)));
}
