public async Task PlainPartitionedSource_should_be_signalled_about_serialization_errors() { var topic = CreateTopic(1); var group = CreateGroup(1); await ProduceStrings(topic, new int[] { 0 }, ProducerSettings); // Produce "0" string var settings = CreateConsumerSettings <int>(group).WithValueDeserializer(Deserializers.Int32); var(control1, partitionedProbe) = KafkaConsumer.PlainPartitionedSource(settings, Subscriptions.Topics(topic)) .WithAttributes(ActorAttributes.CreateSupervisionStrategy(Deciders.StoppingDecider)) .ToMaterialized(this.SinkProbe <(TopicPartition, Source <ConsumeResult <Null, int>, NotUsed>)>(), Keep.Both) .Run(Materializer); partitionedProbe.Request(3); var subsources = partitionedProbe.Within(TimeSpan.FromSeconds(10), () => partitionedProbe.ExpectNextN(3).Select(t => t.Item2).ToList()); var substream = subsources.Aggregate((s1, s2) => s1.Merge(s2)).RunWith(this.SinkProbe <ConsumeResult <Null, int> >(), Materializer); substream.Request(1); await ProduceStrings(topic, new int[] { 0 }, ProducerSettings); // Produce "0" string Within(TimeSpan.FromSeconds(10), () => substream.ExpectError().Should().BeOfType <SerializationException>()); var shutdown = control1.Shutdown(); AwaitCondition(() => shutdown.IsCompleted, TimeSpan.FromSeconds(10)); }
public async Task PlainPartitionedSource_Should_split_messages_by_partitions() { var topic = CreateTopic(1); var group = CreateGroup(1); var totalMessages = 100; var consumerSettings = CreateConsumerSettings <string>(group); var control = KafkaConsumer.PlainPartitionedSource(consumerSettings, Subscriptions.Topics(topic)) .SelectAsync(6, async tuple => { var(topicPartition, source) = tuple; Log.Info($"Sub-source for {topicPartition}"); var consumedPartitions = await source .Select(m => m.TopicPartition.Partition) .RunWith(Sink.Seq <Partition>(), Materializer); // Return flag that all messages in child source are from the same, expected partition return(consumedPartitions.All(partition => partition == topicPartition.Partition)); }) .As <Source <bool, IControl> >() .ToMaterialized(Sink.Aggregate <bool, bool>(true, (result, childSourceIsValid) => result && childSourceIsValid), Keep.Both) .MapMaterializedValue(tuple => DrainingControl <bool> .Create(tuple.Item1, tuple.Item2)) .Run(Materializer); await ProduceStrings(topic, Enumerable.Range(1, totalMessages), ProducerSettings); // Give it some time to consume all messages await Task.Delay(5000); var shutdown = control.DrainAndShutdown(); AwaitCondition(() => shutdown.IsCompleted, TimeSpan.FromSeconds(10)); shutdown.Result.Should().BeTrue(); }
static void StartConsume() { KafkaConsumer consumer = new KafkaConsumer(); try { Console.WriteLine("Started consuming, press CTRL + C to break."); CancellationTokenSource cancelSource = new CancellationTokenSource(); Console.CancelKeyPress += (_, e) => { e.Cancel = true; // prevent the process from terminating. cancelSource.Cancel(); }; Guid groupId = Guid.NewGuid(); consumer.ConsumeAsync(KafkaBootstrapper, TopicName, groupId.ToString(), cancelSource, (result) => { Console.WriteLine(result.Topic); Console.WriteLine(result.Broker); Console.WriteLine(result.Partition); Console.WriteLine(result.Message); }); Console.WriteLine("Done!"); Console.ReadLine(); } catch (Exception ex) { Console.WriteLine(ex.Message); } }
public async Task ProduceToMultiplePartitions(int numberOfPartitions, int numberOfKeys, int numberOfMessages) { var keySerializer = new Int32Serializer(); var valueSerializer = new StringSerializer(); var messagePartitioner = new Int32Partitioner(); using (var temporaryTopic = testCluster.CreateTemporaryTopic(partitions: 2)) using (var brokers = new KafkaBrokers(testCluster.CreateBrokerUris())) { var topic = temporaryTopic.Name; { var producer = KafkaProducer.Create(brokers, keySerializer, valueSerializer, messagePartitioner); var messages = Enumerable .Range(0, numberOfMessages) .Select(i => KeyedMessage.Create(topic, i % numberOfKeys, i % numberOfPartitions, "Message " + i)); await producer.SendAsync(messages, CancellationToken.None); } { var selectors = Enumerable .Range(0, numberOfPartitions) .Select(partition => new TopicSelector { Partition = partition, Topic = topic }) .ToArray(); var consumer = KafkaConsumer.Create(defaultConsumerGroup, brokers, keySerializer, valueSerializer, selectors); var responses = await consumer.ReceiveAsync(CancellationToken.None); Assert.That(responses, Has.Count.EqualTo(numberOfMessages)); var received = new bool[numberOfMessages]; var offsets = new long[numberOfPartitions]; foreach (var response in responses) { var split = response.Value.Split(' '); Assert.That(split, Has.Length.EqualTo(2)); Assert.That(split[0], Is.EqualTo("Message")); int messageNumber; var parsed = Int32.TryParse(split[1], out messageNumber); Assert.That(parsed, Is.True); Assert.That(messageNumber, Is.InRange(0, numberOfMessages - 1)); var key = messageNumber % numberOfKeys; Assert.That(response.Key, Is.EqualTo(key)); var partition = messageNumber % numberOfPartitions; Assert.That(response.Partition, Is.EqualTo(partition)); Assert.That(received[messageNumber], Is.False); received[messageNumber] = true; Assert.That(response.Offset, Is.EqualTo(offsets[response.Partition])); offsets[response.Partition] += 1; Assert.That(response.Topic, Is.EqualTo(topic)); } } } }
public async Task PlainSource_should_stop_on_errors() { int elementsCount = 10; var topic1 = CreateTopic(1); var group1 = CreateGroup(1); await ProduceStrings(new TopicPartition(topic1, 0), Enumerable.Range(1, elementsCount), ProducerSettings); var settings = CreateConsumerSettings <int>(group1).WithValueDeserializer(Deserializers.Int32); var probe = KafkaConsumer .PlainSource(settings, Subscriptions.Assignment(new TopicPartition(topic1, 0))) .WithAttributes(ActorAttributes.CreateSupervisionStrategy(Deciders.StoppingDecider)) .Select(c => c.Value) .RunWith(this.SinkProbe <int>(), Materializer); var error = probe.Request(elementsCount).ExpectEvent(TimeSpan.FromSeconds(5)); error.Should().BeOfType <TestSubscriber.OnError>(); var exception = ((TestSubscriber.OnError)error).Cause; exception.Should().BeOfType <ConsumeException>(); ((ConsumeException)exception).Error.IsSerializationError().Should().BeTrue(); probe.ExpectNoMsg(TimeSpan.FromSeconds(5)); probe.Cancel(); }
public async Task PlainPartitionedManualOffsetSource_Should_begin_consuming_with_offset() { var topic = CreateTopic(1); var group = CreateGroup(1); var consumerSettings = CreateConsumerSettings <string>(group); await ProduceStrings(topic, Enumerable.Range(1, 100), ProducerSettings); var probe = KafkaConsumer.PlainPartitionedManualOffsetSource( consumerSettings, Subscriptions.Topics(topic), getOffsetsOnAssign: topicPartitions => { // Skip first message from first partition var firstPartition = topicPartitions.OrderBy(tp => tp.Partition.Value).First(); var offset = ImmutableHashSet <TopicPartitionOffset> .Empty.Add(new TopicPartitionOffset(firstPartition, 1)); return(Task.FromResult <IImmutableSet <TopicPartitionOffset> >(offset)); }, onRevoke: _ => { } ).MergeMany(3, tuple => tuple.Item2.MapMaterializedValue(notUsed => new NoopControl())) .Select(m => m.Value) .RunWith(this.SinkProbe <string>(), Materializer); probe.Request(99); var messages = probe.Within(TimeSpan.FromSeconds(10), () => probe.ExpectNextN(99)); messages.ToHashSet().Count.Should().Be(99); // All consumed messages should be different (only one value is missing) probe.Cancel(); }
public void KafkaConsumerTest_CanCreateConsumer() { var consumerMock = new Mock <IConsumer <string, string> >(); var objectUnderTest = new KafkaConsumer(_loggerMock.Object, consumerMock.Object); Assert.NotNull(objectUnderTest); }
public async Task TestSimpleConsumerWorksOk() { var keySerializer = new NullSerializer <object>(); var valueSerializer = new StringSerializer(); var messagePartitioner = new LoadBalancedPartitioner <object>(); using (var temporaryTopic = testCluster.CreateTemporaryTopic()) using (var brokers = new KafkaBrokers(testCluster.CreateBrokerUris())) { var topic = temporaryTopic.Name; var producer = KafkaProducer.Create(brokers, keySerializer, valueSerializer, messagePartitioner); var consumer = KafkaConsumer.Create(defaultConsumerGroup, brokers, keySerializer, valueSerializer, new TopicSelector { Partition = 0, Topic = topic }); await producer.SendAsync(KeyedMessage.Create(topic, "Message"), CancellationToken.None); var responses = await consumer.ReceiveAsync(CancellationToken.None); Assert.That(responses, Is.Not.Null); Assert.That(responses, Has.Count.EqualTo(1)); var first = responses.First(); Assert.That(first.Key, Is.Null); Assert.That(first.Offset, Is.EqualTo(0)); Assert.That(first.Partition, Is.EqualTo(0)); Assert.That(first.Topic, Is.EqualTo(topic)); Assert.That(first.Value, Is.EqualTo("Message")); } }
private static void OnLog( LogMessage logMessage, KafkaConsumer consumer, ISilverbackLogger logger) { switch (logMessage.Level) { case SyslogLevel.Emergency: case SyslogLevel.Alert: case SyslogLevel.Critical: logger.LogConfluentConsumerLogCritical(logMessage, consumer); break; case SyslogLevel.Error: logger.LogConfluentConsumerLogError(logMessage, consumer); break; case SyslogLevel.Warning: logger.LogConfluentConsumerLogWarning(logMessage, consumer); break; case SyslogLevel.Notice: case SyslogLevel.Info: logger.LogConfluentConsumerLogInformation(logMessage, consumer); break; default: logger.LogConfluentConsumerLogDebug(logMessage, consumer); break; } }
public void KafkaConsumerTest_ProcessMessages_Many() { var consumerMock = new Mock <IConsumer <string, string> >(); consumerMock.Setup(p => p.Subscribe(It.IsAny <string>())); var result1 = new ConsumeResult <string, string>() { Message = new Message <string, string> { Key = "key1", Value = "value1" } , TopicPartitionOffset = new TopicPartitionOffset(new TopicPartition("t", new Partition()), new Offset(0)) }; var result2 = new ConsumeResult <string, string>() { Message = new Message <string, string> { Key = "key2", Value = "value2" } , TopicPartitionOffset = new TopicPartitionOffset(new TopicPartition("t", new Partition()), new Offset(1)) }; consumerMock.Setup(p => p.Consume(It.IsAny <TimeSpan>())).Returns(result1); consumerMock.Setup(p => p.Consume(It.IsAny <TimeSpan>())).Returns(result2); var objectUnderTest = new KafkaConsumer(_loggerMock.Object, consumerMock.Object); var ctn = new CancellationTokenSource(TimeSpan.FromMilliseconds(17)); objectUnderTest.ProcessMessages(ctn.Token, It.IsAny <string>(), It.IsAny <IMessageHandler>()); consumerMock.Verify(x => x.Subscribe(It.IsAny <string>()), Times.Once()); //consumerMock.Verify(x => x.Consume(It.IsAny<TimeSpan>()),Times.Exactly(2)); }
private static void OnOffsetsCommitted( CommittedOffsets offsets, KafkaConsumer consumer, IBrokerCallbacksInvoker callbacksInvoker, ISilverbackLogger logger) { foreach (var topicPartitionOffsetError in offsets.Offsets) { if (topicPartitionOffsetError.Offset == Offset.Unset) { continue; } if (topicPartitionOffsetError.Error != null && topicPartitionOffsetError.Error.Code != ErrorCode.NoError) { logger.LogOffsetCommitError(topicPartitionOffsetError, consumer); } else { logger.LogOffsetCommitted(topicPartitionOffsetError.TopicPartitionOffset, consumer); } } callbacksInvoker.Invoke <IKafkaOffsetCommittedCallback>( handler => handler.OnOffsetsCommitted(offsets, consumer)); }
public async Task CommitWithMetadataSource_Commit_metadata_in_message_Should_work() { var topic = CreateTopic(1); var group = CreateGroup(1); string MetadataFromMessage <K, V>(ConsumeResult <K, V> message) => message.Offset.ToString(); await ProduceStrings(topic, Enumerable.Range(1, 10), ProducerSettings); var(control, probe) = KafkaConsumer.CommitWithMetadataSource(CreateConsumerSettings <string>(group), Subscriptions.Topics(topic), MetadataFromMessage) .ToMaterialized(this.SinkProbe <CommittableMessage <Null, string> >(), Keep.Both) .Run(Materializer); probe.Request(10); probe.Within(TimeSpan.FromSeconds(10), () => probe.ExpectNextN(10)).ForEach(message => { var offsetWithMeta = message.CommitableOffset as ICommittableOffsetMetadata; offsetWithMeta.Should().NotBeNull(); offsetWithMeta.Metadata.Should().Be(message.CommitableOffset.Offset.Offset.ToString()); }); probe.Cancel(); AwaitCondition(() => control.IsShutdown.IsCompletedSuccessfully, TimeSpan.FromSeconds(10)); }
public async Task CanConsumeEvent_Lots() { SetLogLevelTo(LogEventLevel.Information); var receivedEvents = new ConcurrentQueue <string>(); const string groupName = "default6"; var consumer1 = new KafkaConsumer( address: "localhost:9092", topics: new[] { "lots" }, group: groupName, eventHandler: async evt => receivedEvents.Enqueue(evt.Body) ); Using(consumer1); var consumer2 = new KafkaConsumer( address: "localhost:9092", topics: new[] { "lots" }, group: groupName, eventHandler: async evt => receivedEvents.Enqueue(evt.Body) ); Using(consumer2); consumer1.Start(); consumer2.Start(); await receivedEvents.WaitOrDie(q => q.Count == 11000, timeoutSeconds : 70); await Task.Delay(TimeSpan.FromSeconds(5)); }
public async Task PlainSource_should_resume_on_deserialization_errors() { Directive Decider(Exception cause) => cause is SerializationException ? Directive.Resume : Directive.Stop; int elementsCount = 10; var topic1 = CreateTopic(1); var group1 = CreateGroup(1); await Produce(topic1, Enumerable.Range(1, elementsCount), ProducerSettings); var settings = ConsumerSettings <Null, int> .Create(Sys, null, new IntDeserializer()) .WithBootstrapServers(KafkaUrl) .WithProperty("auto.offset.reset", "earliest") .WithGroupId(group1); var probe = KafkaConsumer .PlainSource(settings, Subscriptions.Assignment(new TopicPartition(topic1, 0))) .WithAttributes(ActorAttributes.CreateSupervisionStrategy(Decider)) .Select(c => c.Value) .RunWith(this.SinkProbe <int>(), _materializer); probe.Request(elementsCount); probe.ExpectNoMsg(TimeSpan.FromSeconds(10)); probe.Cancel(); }
public void KafkaConsumerTest_When_Consumer_ConsumeException() { var mockLogger = new Mock <ILogger <KafkaConsumer> >(); mockLogger.Setup(x => x.Log(It.IsAny <LogLevel>(), It.IsAny <EventId>(), It.IsAny <FormattedLogValues>(), It.IsAny <Exception>(), It.IsAny <Func <object, Exception, string> >())); var consumerMock = new Mock <IConsumer <string, string> >(); consumerMock.Setup(p => p.Subscribe(It.IsAny <string>())); var messageHandlerMock = new Mock <IMessageHandler>(); var result1 = new ConsumeResult <byte[], byte[]>() { Message = new Message <byte[], byte[]>() }; consumerMock.Setup(p => p.Consume(It.IsAny <TimeSpan>())).Throws(new ConsumeException(result1, new Error(ErrorCode.BrokerNotAvailable, "failure"))); var objectUnderTest = new KafkaConsumer(mockLogger.Object, consumerMock.Object); var ctn = new CancellationTokenSource(TimeSpan.FromMilliseconds(10)); objectUnderTest.ProcessMessages(ctn.Token, It.IsAny <string>(), messageHandlerMock.Object); consumerMock.Verify(x => x.Subscribe(It.IsAny <string>()), Times.Once()); consumerMock.Verify(x => x.Consume(It.IsAny <TimeSpan>()), Times.Once()); mockLogger.Verify(x => x.Log(LogLevel.Error, It.IsAny <EventId>(), It.IsAny <FormattedLogValues>(), It.IsAny <Exception>(), It.IsAny <Func <object, Exception, string> >()), Times.Once()); messageHandlerMock.Verify(x => x.HandleMessage(It.IsAny <string>(), It.IsAny <string>()), Times.Never()); }
public async Task Transactional_source_with_sink_Should_work() { var settings = CreateConsumerSettings <string>(CreateGroup(1)); var sourceTopic = CreateTopic(1); var targetTopic = CreateTopic(2); var transactionalId = Guid.NewGuid().ToString(); const int totalMessages = 10; var control = KafkaConsumer.TransactionalSource(settings, Subscriptions.Topics(sourceTopic)) .Via(Business <TransactionalMessage <Null, string> >()) .Select(message => { return(ProducerMessage.Single( new ProducerRecord <Null, string>(targetTopic, message.Record.Key, message.Record.Value), passThrough: message.PartitionOffset)); }) .ToMaterialized(KafkaProducer.TransactionalSink(ProducerSettings, transactionalId), Keep.Both) .MapMaterializedValue(DrainingControl <NotUsed> .Create) .Run(Materializer); var consumer = ConsumeStrings(targetTopic, totalMessages); await ProduceStrings(sourceTopic, Enumerable.Range(1, totalMessages), ProducerSettings); AssertTaskCompletesWithin(TimeSpan.FromSeconds(totalMessages), consumer.IsShutdown); AssertTaskCompletesWithin(TimeSpan.FromSeconds(totalMessages), control.DrainAndShutdown()); consumer.DrainAndShutdown().Result.Should().HaveCount(totalMessages); }
public void KafkaConsumerTest_ProcessMessages_Success() { var consumerMock = new Mock <IConsumer <string, string> >(); var ctn = new CancellationTokenSource(); var messageHandlerMock = new Mock <IMessageHandler>(); var objectUnderTest = new KafkaConsumer(_loggerMock.Object, consumerMock.Object); var consumeRes = new ConsumeResult <string, string>() { Message = new Message <string, string> { Key = "key1", Value = "value1" }, TopicPartitionOffset = new TopicPartitionOffset(new TopicPartition("t", new Partition()), new Offset(0)) }; consumerMock.Setup(x => x.Consume(TimeSpan.FromMilliseconds(100))).Returns(consumeRes).Callback(() => ctn.Cancel()); consumerMock.Setup(x => x.Subscribe(It.IsAny <string>())); //ACTION objectUnderTest.ProcessMessages(ctn.Token, "TestTopic", messageHandlerMock.Object); //ASSERT //1. Verify consumer.subscribe has been called //2. verify on success next method is called consumerMock.Verify(x => x.Subscribe(It.IsAny <string>()), Times.Once); consumerMock.Verify(x => x.Consume(TimeSpan.FromMilliseconds(100)), Times.AtLeastOnce); messageHandlerMock.Verify(x => x.HandleMessage(It.IsAny <string>(), It.IsAny <string>()), Times.Once); }
public async Task Overriden_default_decider_on_PlainSource_should_work() { int elementsCount = 10; var topic1 = CreateTopic(1); var group1 = CreateGroup(1); var sourceTask = ProduceStrings(new TopicPartition(topic1, 0), Enumerable.Range(1, elementsCount), ProducerSettings); await GuardWithTimeoutAsync(sourceTask, TimeSpan.FromSeconds(3)); var settings = CreateConsumerSettings <int>(group1).WithValueDeserializer(Deserializers.Int32); var decider = new OverridenConsumerDecider(settings.AutoCreateTopicsEnabled); var probe = KafkaConsumer .PlainSource(settings, Subscriptions.Assignment(new TopicPartition(topic1, 0))) .WithAttributes(ActorAttributes.CreateSupervisionStrategy(decider.Decide)) .Select(c => c.Value) .RunWith(this.SinkProbe <int>(), Materializer); probe.Request(elementsCount); probe.ExpectNoMsg(TimeSpan.FromSeconds(10)); // this is twice elementCount because Decider is called twice on each exceptions decider.CallCount.Should().Be(elementsCount * 2); probe.Cancel(); }
private static async Task Main() { //todo temp - for manual tests // var message = new KafkaMessage // { // UtcStartDateTime = DateTime.Now.AddMinutes(-1).ToUniversalTime(), // FilePath = @"<some path>" // }; Log.Logger = new LoggerConfiguration() .MinimumLevel.Information() .Enrich.With(new UtcTimestampEnricher()) .WriteTo.ColoredConsole( outputTemplate: "{UtcTimestamp:yyyy-MM-dd HH:mm:ss.fff} [{Level}] {CallerMemberName} {Message:lj}{NewLine}{Exception}", formatProvider: CultureInfo.InvariantCulture) .CreateLogger(); var cfg = JsonConvert.DeserializeObject <MainConfig>(await File.ReadAllTextAsync("config.json")); cfg.ValidateConfig(); await KafkaConsumer.ConsumeMessage(cfg, new CancellationToken(), Log.Logger); //// <comment upper line and uncomment this to use without Kafka in testing> //await SortWorker.ProcessSorting(cfg, message, Log.Logger); //Console.ReadKey(); }
public async Task AtMostOnceSource_Should_work() { var topic = CreateTopic(1); var settings = CreateConsumerSettings <string>(CreateGroup(1)); var totalMessages = 10; var lastMessage = new TaskCompletionSource <Done>(); await ProduceStrings(topic, Enumerable.Range(1, 10), ProducerSettings); var(task, probe) = KafkaConsumer.AtMostOnceSource(settings, Subscriptions.Topics(topic)) .SelectAsync(1, m => { if (m.Value == totalMessages.ToString()) { lastMessage.SetResult(Done.Instance); } return(Task.FromResult(Done.Instance)); }) .ToMaterialized(this.SinkProbe <Done>(), Keep.Both) .Run(Materializer); probe.Request(10); await lastMessage.Task; probe.Cancel(); probe.ExpectNextN(10); }
public async Task IntegrationTest() { int counter = 0; //消费 var consumer = new KafkaConsumer($"integration.{group}", hosts) { EnableAutoCommit = false }; await consumer.ListenAsync(new string[] { $"integration.{topic}" }, result => { Output.WriteLine($"integration.{topic}({result.Key}):" + result.Message); counter++; result.Commit(); }); //发布 var producer = new KafkaProducer(hosts) { DefaultTopic = $"integration.{topic}", DefaultKey = $"integration.key" }; await producer.PublishAsync("hello kafka"); BlockUntil(() => counter >= 1, 3000); producer.Dispose(); consumer.Dispose(); Assert.Equal(1, counter); }
public async Task CommitableSource_consumes_messages_from_Producer_without_commits() { int elementsCount = 100; var topic1 = CreateTopic(1); var group1 = CreateGroup(1); var topicPartition1 = new TopicPartition(topic1, 0); await GivenInitializedTopic(topicPartition1); await Source .From(Enumerable.Range(1, elementsCount)) .Select(elem => new ProducerRecord <Null, string>(topicPartition1, elem.ToString())) .RunWith(KafkaProducer.PlainSink(ProducerSettings), Materializer); var consumerSettings = CreateConsumerSettings <string>(group1); var probe = KafkaConsumer .CommittableSource(consumerSettings, Subscriptions.Assignment(topicPartition1)) .Where(c => !c.Record.Value.Equals(InitialMsg)) .Select(c => c.Record.Value) .RunWith(this.SinkProbe <string>(), Materializer); probe.Request(elementsCount); foreach (var i in Enumerable.Range(1, elementsCount).Select(c => c.ToString())) { probe.ExpectNext(i, TimeSpan.FromSeconds(10)); } probe.Cancel(); }
public async Task VerifyABrokerStoppingAndRestartingCanBeHandledByTheConsumer() { using (var cluster = new KafkaTestCluster("server.home", 1)) { var topic = "test"; cluster.CreateTopic(topic); using (var brokers = new KafkaBrokers(cluster.CreateBrokerUris())) { var producer = KafkaProducer.Create(brokers, new StringSerializer()); await producer.SendAsync(KeyedMessage.Create(topic, "Test"), CancellationToken.None); await Task.Delay(1000); cluster.StopKafkaBroker(0); cluster.RestartKafkaBroker(0); var consumer = KafkaConsumer.Create(topic, brokers, new StringSerializer(), new TopicSelector { Topic = topic, Partition = 0, Offset = 0 }); var result = await consumer.ReceiveAsync(CancellationToken.None); Assert.That(result, Has.Count.EqualTo(1)); var first = result[0]; Assert.That(first.Value, Is.EqualTo("Test")); Assert.That(first.Offset, Is.EqualTo(0)); } cluster.DeleteTopic(topic); } }
public async Task StartAsync(CancellationToken stopCancellationToken = default) { foreach (var consumerConfiguration in this.configuration.Clusters.SelectMany(cl => cl.Consumers)) { var dependencyScope = this.dependencyResolver.CreateScope(); var middlewares = consumerConfiguration.MiddlewareConfiguration.Factories .Select(factory => factory(dependencyScope.Resolver)) .ToList(); var consumerWorkerPool = new ConsumerWorkerPool( dependencyScope.Resolver, this.logHandler, new MiddlewareExecutor(middlewares), consumerConfiguration.DistributionStrategyFactory); var consumer = new KafkaConsumer( consumerConfiguration, this.consumerManager, this.logHandler, consumerWorkerPool, stopCancellationToken); this.consumers.Add(consumer); await consumer.StartAsync().ConfigureAwait(false); } }
public async Task PlainSource_with_directive_override_should_resume_on_deserialization_errors() { var callCount = 0; Directive Decider(Exception cause) { if (cause is ConsumeException ex && ex.Error.IsSerializationError()) { callCount++; return(Directive.Resume); } return(Directive.Stop); } int elementsCount = 10; var topic1 = CreateTopic(1); var group1 = CreateGroup(1); await ProduceStrings(new TopicPartition(topic1, 0), Enumerable.Range(1, elementsCount), ProducerSettings); var settings = CreateConsumerSettings <int>(group1).WithValueDeserializer(Deserializers.Int32); var probe = KafkaConsumer .PlainSource(settings, Subscriptions.Assignment(new TopicPartition(topic1, 0))) .WithAttributes(ActorAttributes.CreateSupervisionStrategy(Decider)) .Select(c => c.Value) .RunWith(this.SinkProbe <int>(), Materializer); probe.Request(elementsCount); probe.ExpectNoMsg(TimeSpan.FromSeconds(10)); // this is twice elementCount because Decider is called twice on each exceptions callCount.Should().Be(elementsCount * 2); probe.Cancel(); }
public async Task PlainPartitionedSource_should_stop_partition_sources_when_stopped() { var topic = CreateTopic(1); var group = CreateGroup(1); var totalMessages = 100; await ProduceStrings(topic, Enumerable.Range(1, totalMessages), ProducerSettings); var consumerSettings = CreateConsumerSettings <string>(group).WithStopTimeout(TimeSpan.FromMilliseconds(10)); var(control, probe) = KafkaConsumer.PlainPartitionedSource(consumerSettings, Subscriptions.Topics(topic)) .MergeMany(3, tuple => tuple.Item2.MapMaterializedValue(notUsed => new NoopControl())) .Select(message => { Log.Debug($"Consumed partition {message.Partition.Value}"); return(message.Value); }) .ToMaterialized(this.SinkProbe <string>(), Keep.Both) .Run(Materializer); probe.Request(totalMessages).Within(TimeSpan.FromSeconds(10), () => probe.ExpectNextN(totalMessages)); var stopped = control.Stop(); probe.ExpectComplete(); AwaitCondition(() => stopped.IsCompleted, TimeSpan.FromSeconds(10)); await control.Shutdown(); probe.Cancel(); }
private static void OnError(Error error, KafkaConsumer consumer, ISilverbackIntegrationLogger logger) { // Ignore errors if not consuming anymore // (lidrdkafka randomly throws some "brokers are down" while disconnecting) if (!consumer.IsConnected) { return; } try { if (consumer.Endpoint.Events.ErrorHandler != null && consumer.Endpoint.Events.ErrorHandler.Invoke(error, consumer)) { return; } } catch (Exception ex) { logger.LogError( KafkaEventIds.KafkaErrorHandlerError, ex, "Error in Kafka consumer error handler."); } logger.Log( error.IsFatal ? LogLevel.Critical : LogLevel.Error, KafkaEventIds.ConsumerError, "Error in Kafka consumer ({consumerId}): {error} (topic(s): {topics})", consumer.Id, error, consumer.Endpoint.Names); }
public async Task PlainPartitionedSource_should_not_leave_gaps_when_subsource_is_cancelled() { var topic = CreateTopic(1); var group = CreateGroup(1); var totalMessages = 100; await ProduceStrings(topic, Enumerable.Range(1, totalMessages), ProducerSettings); var consumedMessagesTask = KafkaConsumer.PlainPartitionedSource(CreateConsumerSettings <string>(group), Subscriptions.Topics(topic)) .Log(topic, m => $"Consuming topic partition {m.Item1}") .MergeMany(3, tuple => { var(topicPartition, source) = tuple; return(source .MapMaterializedValue(notUsed => new NoopControl()) .Log(topicPartition.ToString(), m => $"Consumed offset {m.Offset} (value: {m.Value})") .Take(10)); }) .Select(m => int.Parse(m.Value)) .Log("Merged stream", m => m) .Scan(0, (c, _) => c + 1) .TakeWhile(m => m < totalMessages, inclusive: true) .RunWith(Sink.Last <int>(), Materializer); AwaitCondition(() => consumedMessagesTask.IsCompleted, TimeSpan.FromSeconds(10)); consumedMessagesTask.Result.Should().Be(totalMessages); }
public async Task SubscribeAsync <T>(Func <T, SubscribeContext, Task> handler, SubscribeOptions subscribeOptions = null, CancellationToken cancellationToken = default(CancellationToken)) where T : class { string topic = GetTopic(typeof(T)); var groupId = subscribeOptions?.GroupId; groupId = !string.IsNullOrEmpty(groupId) ? groupId : _kafkaOptions.ConsumerConfig.GroupId; var threadCount = subscribeOptions?.ConsumerThreadCount ?? 0; threadCount = threadCount > 0 ? threadCount : _kafkaOptions.DefaultConsumerThreadCount; AssertUtils.IsTrue(threadCount > 0, "消费者线程数必须大于0"); ValidateSubscribe(topic, groupId); _logger.LogInformation($"订阅Topic:{topic},GroupId:{groupId},ConsumerThreadCount:{threadCount}"); for (int i = 0; i < threadCount; i++) { var consumer = new KafkaConsumer <string, KafkaMessageBusData>(_serviceProvider); consumer.OnMessage += consumeResult => { return(With.NoException(_logger, async() => { var obj = _kafkaOptions.Serializer.Deserialize <T>(consumeResult.Message.Value.Data); var subscribeContext = new SubscribeContext { Topic = topic, GroupId = groupId }; await handler(obj, subscribeContext); }, $"消费数据{consumeResult.Message.Value.Topic}")); }; _consumerList.Add(consumer); await consumer.Subscribe(topic, groupId, cancellationToken); } }
public static void Main(string[] args) { Config fallbackConfig = ConfigurationFactory.ParseString(@" akka.suppress-json-serializer-warning=true akka.loglevel = DEBUG ").WithFallback(ConfigurationFactory.FromResource <ConsumerSettings <object, object> >("Akka.Streams.Kafka.reference.conf")); var system = ActorSystem.Create("TestKafka", fallbackConfig); var materializer = system.Materializer(); var consumerSettings = ConsumerSettings <Null, string> .Create(system, null, null) .WithBootstrapServers("localhost:29092") .WithGroupId("group1"); var subscription = Subscriptions.Topics("akka100"); KafkaConsumer.PlainSource(consumerSettings, subscription) .RunForeach(result => { Console.WriteLine($"Consumer: {result.Topic}/{result.Partition} {result.Offset}: {result.Value}"); }, materializer); Console.ReadLine(); }
static void Main(string[] args) { var host = "192.168.33.12:9092"; var topic = "test2"; var count = 50000000; var prop = new Properties(); prop.put("bootstrap.servers", host); prop.put("group.id", "test3"); prop.put("auto.offset.reset", "earliest"); prop.put("enable.auto.commit", "true"); prop.put("auto.commit.interval.ms", "1000"); prop.put("socket.receive.buffer.bytes", (2*1024*1024).ToString()); prop.put("fetch.message.max.bytes", (1024*1024).ToString()); var c = new KafkaConsumer(prop, new ByteArrayDeserializer(), new ByteArrayDeserializer()); var topics = new ArrayList(1); topics.add(topic); var time = DateTime.UtcNow; c.subscribe(topics); var bytes = 0; var i = count; var recordCount = 0; while (i > 0) { var r = c.poll(1000); var records = r.records(topic); for (var it = records.iterator(); it.hasNext() && i > 0; i--, recordCount++) { var rec = (ConsumerRecord)it.next(); var b = (byte[]) rec.value(); bytes += b.Length; } Console.WriteLine(recordCount); } var mb = bytes / 1024.0 / 1024.0; var seconds = (DateTime.UtcNow - time).TotalSeconds; Console.WriteLine($"{mb / seconds} MB/sec"); Console.WriteLine($"{count / seconds} Msg/sec"); }