public void ChunkIfNeeded_LargeMessage_Chunked() { var message = new BinaryMessage { MessageId = Guid.NewGuid(), Content = GetByteArray(1400) }; var headers = new MessageHeaderCollection { { MessageHeader.MessageIdKey, "1234" } }; var serializedMessage = _serializer.Serialize(message, headers); var rawBrokerMessage = new RawBrokerMessage(message, headers, new TestProducerEndpoint("test") { Chunk = new ChunkSettings { Size = 500 } }) { RawContent = serializedMessage }; var chunks = ChunkProducer.ChunkIfNeeded(rawBrokerMessage); chunks.Should().HaveCount(4); chunks.Should().Match(c => c.All(m => m.RawContent.Length < 1000)); }
public async Task Handle_WithTraceIdHeader_NewActivityStartedAndParentIdIsSet() { var rawMessage = new RawBrokerMessage( "123", new MessageHeaderCollection { { DiagnosticsConstants.TraceIdHeaderKey, "00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01" } }, TestEndpoint.GetDefault()); var entered = false; await new ActivityConsumerBehavior().Handle(rawMessage, _ => { Activity.Current.Should().NotBeNull(); Activity.Current.ParentId.Should().Be("00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01"); Activity.Current.Id.Should().StartWith("00-0af7651916cd43dd8448eb211c80319c"); entered = true; return(Task.CompletedTask); }); entered.Should().BeTrue(); }
public void ChunkIfNeeded_SmallMessage_ReturnedWithoutChanges() { var message = new BinaryMessage { MessageId = Guid.NewGuid(), Content = GetByteArray(100) }; var headers = new MessageHeaderCollection(); var serializedMessage = _serializer.Serialize(message, headers); var rawBrokerMessage = new RawBrokerMessage(message, headers, new TestProducerEndpoint("test") { Chunk = new ChunkSettings { Size = 500 } }) { RawContent = serializedMessage }; var chunks = ChunkProducer.ChunkIfNeeded(rawBrokerMessage); chunks.Should().HaveCount(1); chunks.First().Should().BeEquivalentTo(rawBrokerMessage); }
private IEnumerable <RawBrokerMessage> GetRawMessages(object content, IEnumerable <MessageHeader> headers) { var headersCollection = new MessageHeaderCollection(headers); _messageKeyProvider.EnsureKeyIsInitialized(content, headersCollection); var rawMessage = new RawBrokerMessage(content, headersCollection, Endpoint); return(ChunkProducer.ChunkIfNeeded(rawMessage)); }
public void Handle_NoStartedActivity_ActivityStartedAndTraceIdHeaderIsSet() { var rawMessage = new RawBrokerMessage("123", null, TestEndpoint.GetDefault()); new ActivityProducerBehavior().Handle(rawMessage, _ => Task.CompletedTask); rawMessage.Headers.Should().Contain( h => h.Key == DiagnosticsConstants.TraceIdHeaderKey && !string.IsNullOrEmpty(h.Value)); }
public void Handle_StartedActivity_TraceIdHeaderIsSet() { var activity = new Activity("test"); activity.SetParentId("00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01"); activity.Start(); var rawMessage = new RawBrokerMessage("123", null, TestEndpoint.GetDefault()); new ActivityProducerBehavior().Handle(rawMessage, _ => Task.CompletedTask); rawMessage.Headers.Should().Contain( h => h.Key == DiagnosticsConstants.TraceIdHeaderKey && h.Value.StartsWith("00-0af7651916cd43dd8448eb211c80319c")); }
public async Task Handle(RawBrokerMessage message, RawBrokerMessageHandler next) { var activity = new Activity(DiagnosticsConstants.ActivityNameMessageConsuming); try { activity.InitFromMessageHeaders(message.Headers); activity.Start(); await next(message); } finally { activity.Stop(); } }
public static IEnumerable <RawBrokerMessage> ChunkIfNeeded(RawBrokerMessage message) { var messageId = message.Headers.GetValue(MessageHeader.MessageIdKey); var settings = (message.Endpoint as IProducerEndpoint)?.Chunk; var chunkSize = settings?.Size ?? int.MaxValue; if (chunkSize >= message.RawContent.Length) { yield return(message); yield break; } if (string.IsNullOrEmpty(messageId)) { throw new InvalidOperationException( "Dividing into chunks is pointless if no unique MessageId can be retrieved. " + "Please add an Id or MessageId property to the message model or " + "use a custom IMessageKeyProvider."); } var span = message.RawContent.AsMemory(); var chunksCount = (int)Math.Ceiling(message.RawContent.Length / (double)chunkSize); var offset = 0; for (var i = 0; i < chunksCount; i++) { var slice = span.Slice(offset, Math.Min(chunkSize, message.RawContent.Length - offset)).ToArray(); var messageChunk = new RawBrokerMessage(slice, message.Headers, message.Endpoint); messageChunk.Headers.AddOrReplace(MessageHeader.ChunkIdKey, i); messageChunk.Headers.AddOrReplace(MessageHeader.ChunksCountKey, chunksCount); yield return(messageChunk); offset += chunkSize; } }
public async Task Handle_FromConsume_NewActivityStartedAndParentIdIsSet() { var services = new ServiceCollection(); services .AddSingleton <ILoggerFactory, NullLoggerFactory>() .AddSingleton(typeof(ILogger <>), typeof(NullLogger <>)) .AddSilverback().WithConnectionTo <TestBroker>(); var serviceProvider = services.BuildServiceProvider(); var broker = (TestBroker)serviceProvider.GetRequiredService <IBroker>(); var rawMessage = new RawBrokerMessage( "123", new MessageHeaderCollection { { DiagnosticsConstants.TraceIdHeaderKey, "00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01" } }, TestEndpoint.GetDefault()); var consumer = (TestConsumer)broker.GetConsumer(TestEndpoint.GetDefault()); var entered = false; consumer.Received += (sender, args) => { Activity.Current.Should().NotBeNull(); Activity.Current.ParentId.Should().Be("00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01"); Activity.Current.Id.Should().StartWith("00-0af7651916cd43dd8448eb211c80319c"); entered = true; return(Task.CompletedTask); }; broker.Connect(); await consumer.TestPush(rawMessage.RawContent, rawMessage.Headers); entered.Should().BeTrue(); }
protected override async Task <IOffset> ProduceAsync(RawBrokerMessage message) { try { var kafkaMessage = new Confluent.Kafka.Message <byte[], byte[]> { Key = GetPartitioningKey(message.Headers), Value = message.RawContent }; if (message.Headers != null && message.Headers.Any()) { kafkaMessage.Headers = new Confluent.Kafka.Headers(); message.Headers.ForEach(h => kafkaMessage.Headers.Add(h.ToConfluentHeader())); } var deliveryReport = await GetInnerProducer().ProduceAsync(Endpoint.Name, kafkaMessage); if (Endpoint.Configuration.ArePersistenceStatusReportsEnabled) { CheckPersistenceStatus(deliveryReport); } return(new KafkaOffset(deliveryReport.TopicPartitionOffset)); } catch (Confluent.Kafka.KafkaException ex) { // Disposing and re-creating the producer will maybe fix the issue if (Endpoint.Configuration.DisposeOnException) { DisposeInnerProducer(); } throw new ProduceException("Error occurred producing the message. See inner exception for details.", ex); } }
protected override IOffset Produce(RawBrokerMessage message) { ProducedMessages.Add(new TestBroker.ProducedMessage(message.RawContent, message.Headers, Endpoint)); return(null); }
public MessageReceivedEventArgs(RawBrokerMessage message) { Message = message; }
protected abstract Task <IOffset> ProduceAsync(RawBrokerMessage message);
protected override Task <IOffset> ProduceAsync(RawBrokerMessage message) => Broker.GetTopic(Endpoint.Name).PublishAsync(message.RawContent, message.Headers);
private async Task ExecutePipeline(IEnumerable <IConsumerBehavior> behaviors, RawBrokerMessage message, RawBrokerMessageHandler finalAction) { if (behaviors != null && behaviors.Any()) { await behaviors.First().Handle(message, m => ExecutePipeline(behaviors.Skip(1), m, finalAction)); } else { await finalAction(message); } }
protected override Task <IOffset> ProduceAsync(RawBrokerMessage message) { Produce(message.RawContent, message.Headers); return(Task.FromResult <IOffset>(null)); }
protected override IOffset Produce(RawBrokerMessage message) => AsyncHelper.RunSynchronously(() => ProduceAsync(message));
private async Task ExecutePipeline(IEnumerable <IProducerBehavior> behaviors, RawBrokerMessage message, RawBrokerMessageHandler finalAction) { if (behaviors != null && behaviors.Any()) { await behaviors.First().Handle(message, m => ExecutePipeline(behaviors.Skip(1), m, finalAction)); } else { await finalAction(message); _messageLogger.LogInformation(_logger, "Message produced.", message); } }
protected abstract IOffset Produce(RawBrokerMessage message);
protected override IOffset Produce(RawBrokerMessage message) => Broker.GetTopic(Endpoint.Name).Publish(message.RawContent, message.Headers);