public async Task CommitableSource_consumes_messages_from_Producer_without_commits() { int elementsCount = 100; var topic1 = CreateTopic(1); var group1 = CreateGroup(1); var topicPartition1 = new TopicPartition(topic1, 0); await GivenInitializedTopic(topicPartition1); await Source .From(Enumerable.Range(1, elementsCount)) .Select(elem => new ProducerRecord <Null, string>(topicPartition1, elem.ToString())) .RunWith(KafkaProducer.PlainSink(ProducerSettings), Materializer); var consumerSettings = CreateConsumerSettings <string>(group1); var probe = KafkaConsumer .CommittableSource(consumerSettings, Subscriptions.Assignment(topicPartition1)) .Where(c => !c.Record.Value.Equals(InitialMsg)) .Select(c => c.Record.Value) .RunWith(this.SinkProbe <string>(), Materializer); probe.Request(elementsCount); foreach (var i in Enumerable.Range(1, elementsCount).Select(c => c.ToString())) { probe.ExpectNext(i, TimeSpan.FromSeconds(10)); } probe.Cancel(); }
public static async Task Main(string[] args) { Config fallbackConfig = ConfigurationFactory.ParseString(@" akka.suppress-json-serializer-warning=true akka.loglevel = DEBUG ").WithFallback(ConfigurationFactory.FromResource <ConsumerSettings <object, object> >("Akka.Streams.Kafka.reference.conf")); var system = ActorSystem.Create("TestKafka", fallbackConfig); var materializer = system.Materializer(); var consumerSettings = ConsumerSettings <Null, string> .Create(system, null, null) .WithBootstrapServers($"{EventHubNamespace}.servicebus.windows.net:9093") .WithGroupId(EventHubConsumerGroup) .WithProperties(new Dictionary <string, string> { { "security.protocol", "SASL_SSL" }, { "sasl.mechanism", "PLAIN" }, { "sasl.username", "$ConnectionString" }, { "sasl.password", EventHubConnectionString }, }); var subscription = Subscriptions.Topics(EventHubName); var committerDefaults = CommitterSettings.Create(system); // Comment for simple no-commit consumer DrainingControl <NotUsed> control = KafkaConsumer.CommittableSource(consumerSettings, subscription) .SelectAsync(1, msg => Business(msg.Record).ContinueWith(done => (ICommittable)msg.CommitableOffset)) .ToMaterialized( Committer.Sink(committerDefaults.WithMaxBatch(1)), DrainingControl <NotUsed> .Create) .Run(materializer); // Uncomment for simple no-commit consumer /* * await KafkaConsumer.PlainSource(consumerSettings, subscription) * .RunForeach(result => * { * Console.WriteLine($"Consumer: {result.Topic}/{result.Partition} {result.Offset}: {result.Value}"); * }, materializer); */ Console.WriteLine("Press any key to stop consumer."); Console.ReadKey(); // Comment for simple no-commit consumer await control.Stop(); await system.Terminate(); }
public async Task CommitterFlow_commits_offsets_from_CommittableSource(int batchSize) { var topic1 = CreateTopic(1); var topicPartition1 = new TopicPartition(topic1, 0); var group1 = CreateGroup(1); await GivenInitializedTopic(topicPartition1); await Source .From(Enumerable.Range(1, 100)) .Select(elem => new ProducerRecord <Null, string>(topicPartition1, elem.ToString())) .RunWith(KafkaProducer.PlainSink(ProducerSettings), Materializer); var consumerSettings = CreateConsumerSettings <string>(group1); var committedElements = new ConcurrentQueue <string>(); var committerSettings = CommitterSettings.WithMaxBatch(batchSize); var(task, probe1) = KafkaConsumer.CommittableSource(consumerSettings, Subscriptions.Assignment(topicPartition1)) .WhereNot(c => c.Record.Value == InitialMsg) .SelectAsync(10, elem => { committedElements.Enqueue(elem.Record.Value); return(Task.FromResult(elem.CommitableOffset as ICommittable)); }) .Via(Committer.Flow(committerSettings)) .ToMaterialized(this.SinkProbe <Done>(), Keep.Both) .Run(Materializer); probe1.Request(25 / batchSize); foreach (var _ in Enumerable.Range(1, 25 / batchSize)) { probe1.ExpectNext(Done.Instance, TimeSpan.FromSeconds(10)); } probe1.Cancel(); AwaitCondition(() => task.IsShutdown.IsCompletedSuccessfully); var probe2 = KafkaConsumer.PlainSource(consumerSettings, Subscriptions.Assignment(new TopicPartition(topic1, 0))) .Select(_ => _.Value) .RunWith(this.SinkProbe <string>(), Materializer); probe2.Request(75); foreach (var i in Enumerable.Range(committedElements.Count + 1, 75).Select(c => c.ToString())) { probe2.ExpectNext(i, TimeSpan.FromSeconds(10)); } probe2.Cancel(); }
[InlineData(20, 10)] //20 개의 메시지를 생산하고,소비한다,테스트는 10초이내에 완료되어야함(완료시 종료됨) public void Test1(int limit, int cutoff) { string lastSignal = Guid.NewGuid().ToString(); int readyTimeForConsume = 3; int recCnt = 0; KafkaConsumer.CommittableSource(consumerSettings, subscription) .RunForeach(result => { Console.WriteLine($"Consumer: {result.Record.Topic}/{result.Record.Partition} {result.Record.Offset}: {result.Record.Value}"); if (lastSignal == result.Record.Value) { probe.Tell("처리모두완료"); } result.CommitableOffset.Commit(); }, materializer_consumer); Source <int, NotUsed> source = Source.From(Enumerable.Range(1, limit)); source .Throttle(2, TimeSpan.FromSeconds(1), 1, ThrottleMode.Shaping) //출력 조절 : 초당 2개처리 .Select(c => { var result = $"No:{c.ToString()}"; if (c == limit) { result = lastSignal; } return(result); }) .Select(elem => ProducerMessage.Single(new ProducerRecord <Null, string>(testTopic, elem))) .Via(KafkaProducer.FlexiFlow <Null, string, NotUsed>(producerSettings)) .Select(result => { var response = result as Result <Null, string, NotUsed>; Console.WriteLine($"Producer: {response.Metadata.Topic}/{response.Metadata.Partition} {response.Metadata.Offset}: {response.Metadata.Value}"); return(result); }) .RunWith(Sink.Ignore <IResults <Null, string, NotUsed> >(), materializer_producer); Within(TimeSpan.FromSeconds(cutoff), () => { probe.ExpectMsg("처리모두완료", TimeSpan.FromSeconds(cutoff)); }); }
private Source <CommittableMessage <K, V>, IControl> CreateCommitableSource( MockConsumer <K, V> mock, string groupId = "group1", string[] topics = null) { topics ??= new[] { "topic" }; var settings = ConsumerSettings <K, V> .Create(Sys, Deserializers.Utf8, Deserializers.Utf8) .WithGroupId(groupId) .WithCloseTimeout(MockConsumer.CloseTimeout) .WithStopTimeout(MockConsumer.CloseTimeout) .WithCommitTimeout(TimeSpan.FromMilliseconds(500)) .WithConsumerFactory(_ => mock.Mock); mock.Settings = settings; return(KafkaConsumer.CommittableSource( settings, Subscriptions.Topics(topics))); }
public void Start(ConsumerAkkaOption consumerActorOption) { IAutoSubscription makeshop_neworder = Subscriptions.Topics(consumerActorOption.Topics); var consumerSettings = ConsumerSettings <Null, string> .Create(consumerSystem, null, null) .WithBootstrapServers(consumerActorOption.BootstrapServers) .WithGroupId(consumerActorOption.KafkaGroupId); var materializer_consumer = consumerSystem.Materializer(); KafkaConsumer.CommittableSource(consumerSettings, makeshop_neworder) .RunForeach(result => { result.CommitableOffset.Commit(); Console.WriteLine($"Consumer: {result.Record.Partition}/{result.Record.Topic} {result.Record.Offset}: {result.Record.Value}"); if (consumerActorOption.RelayActor != null) { consumerActorOption.RelayActor.Tell(result.Record.Value); //ForgetAndFire 발송 } }, materializer_consumer); }
public async Task CommitableSource_resume_from_commited_offset() { var topic1 = CreateTopic(1); var topicPartition1 = new TopicPartition(topic1, 0); var group1 = CreateGroup(1); var group2 = CreateGroup(2); await GivenInitializedTopic(topicPartition1); await Source .From(Enumerable.Range(1, 100)) .Select(elem => new ProducerRecord <Null, string>(topicPartition1, elem.ToString())) .RunWith(KafkaProducer.PlainSink(ProducerSettings), Materializer); var consumerSettings = CreateConsumerSettings <string>(group1); var committedElements = new ConcurrentQueue <string>(); var(task, probe1) = KafkaConsumer.CommittableSource(consumerSettings, Subscriptions.Assignment(topicPartition1)) .WhereNot(c => c.Record.Value == InitialMsg) .SelectAsync(10, async elem => { await elem.CommitableOffset.Commit(); committedElements.Enqueue(elem.Record.Value); return(Done.Instance); }) .ToMaterialized(this.SinkProbe <Done>(), Keep.Both) .Run(Materializer); probe1.Request(25); foreach (var _ in Enumerable.Range(1, 25)) { probe1.ExpectNext(Done.Instance, TimeSpan.FromSeconds(10)); } probe1.Cancel(); AwaitCondition(() => task.IsShutdown.IsCompletedSuccessfully); var probe2 = KafkaConsumer.CommittableSource(consumerSettings, Subscriptions.Assignment(new TopicPartition(topic1, 0))) .Select(_ => _.Record.Value) .RunWith(this.SinkProbe <string>(), Materializer); // Note that due to buffers and SelectAsync(10) the committed offset is more // than 26, and that is not wrong // some concurrent publish await Source .From(Enumerable.Range(101, 100)) .Select(elem => new ProducerRecord <Null, string>(topicPartition1, elem.ToString())) .RunWith(KafkaProducer.PlainSink(ProducerSettings), Materializer); probe2.Request(100); foreach (var i in Enumerable.Range(committedElements.Count + 1, 100).Select(c => c.ToString())) { probe2.ExpectNext(i, TimeSpan.FromSeconds(10)); } probe2.Cancel(); // another consumer should see all var probe3 = KafkaConsumer.CommittableSource(consumerSettings.WithGroupId(group2), Subscriptions.Assignment(new TopicPartition(topic1, 0))) .WhereNot(c => c.Record.Value == InitialMsg) .Select(_ => _.Record.Value) .RunWith(this.SinkProbe <string>(), Materializer); probe3.Request(100); foreach (var i in Enumerable.Range(1, 100).Select(c => c.ToString())) { probe3.ExpectNext(i, TimeSpan.FromSeconds(10)); } probe3.Cancel(); }
public async Task SupervisionStrategy_Decider_on_complex_stream_should_work() { var topic = CreateTopic(1); var group = CreateGroup(1); var topicPartition = new TopicPartition(topic, 0); var committedTopicPartition = new TopicPartition($"{topic}-done", 0); var callCount = 0; Directive Decider(Exception cause) { callCount++; return(Directive.Resume); } var committerSettings = CommitterSettings.Create(Sys); var consumerSettings = CreateConsumerSettings <string>(group); var counter = 0; // arrange await Source.From(Enumerable.Range(1, 10)) .Select(elem => new ProducerRecord <Null, string>(topicPartition, elem.ToString())) .RunWith(KafkaProducer.PlainSink(ProducerSettings), Materializer); // act var drainingControl = KafkaConsumer.CommittableSource(consumerSettings, Subscriptions.Assignment(topicPartition)) .Via(Flow.Create <CommittableMessage <Null, string> >().Select(x => { counter++; // Exception happened here, fail once, when counter is 5 if (counter == 5) { throw new Exception("BOOM!"); } return(x); })) .WithAttributes(Attributes.CreateName("CommitableSource").And(ActorAttributes.CreateSupervisionStrategy(Decider))) .Select(c => (c.Record.Topic, c.Record.Message.Value, c.CommitableOffset)) .SelectAsync(1, async t => { Log.Info($"[{t.Topic}]: {t.Value}"); // simulate a request-response call that takes 10ms to complete here await Task.Delay(10); return(t); }) .Select(t => ProducerMessage.Single(new ProducerRecord <Null, string>(committedTopicPartition, t.Value), t.CommitableOffset)) .Via(KafkaProducer.FlexiFlow <Null, string, ICommittableOffset>(ProducerSettings)).WithAttributes(Attributes.CreateName("FlexiFlow")) .Select(m => (ICommittable)m.PassThrough) .AlsoToMaterialized(Committer.Sink(committerSettings), DrainingControl <NotUsed> .Create) .To(Flow.Create <ICommittable>() .Async() .GroupedWithin(1000, TimeSpan.FromSeconds(1)) .Select(c => c.Count()) .Log("MsgCount").AddAttributes(Attributes.CreateLogLevels(LogLevel.InfoLevel)) .To(Sink.Ignore <int>())) .Run(Sys.Materializer()); await Task.Delay(TimeSpan.FromSeconds(5)); await GuardWithTimeoutAsync(drainingControl.DrainAndShutdown(), TimeSpan.FromSeconds(10)); // There should be only 1 decider call callCount.Should().Be(1); // Assert that all of the messages, except for those that failed in the stage, got committed var settings = CreateConsumerSettings <Null, string>(group); var probe = KafkaConsumer .PlainSource(settings, Subscriptions.Assignment(committedTopicPartition)) .Select(c => c.Message.Value) .RunWith(this.SinkProbe <string>(), Materializer); probe.Request(9); var messages = new List <string>(); for (var i = 0; i < 9; ++i) { var message = probe.RequestNext(); messages.Add(message); } // Message "5" is missing because the exception happened downstream of the source and we chose to // ignore it in the decider messages.Should().BeEquivalentTo(new[] { "1", "2", "3", "4", "6", "7", "8", "9", "10" }); probe.Cancel(); }
public async Task Committable_consumer_with_failed_downstream_stage_result_should_be_gapless() { var topic = CreateTopic(1); var group = CreateGroup(1); var topicPartition = new TopicPartition(topic, 0); var consumerSettings = ConsumerSettings <Null, string> .Create(Sys, null, null) .WithBootstrapServers(Fixture.KafkaServer) .WithStopTimeout(TimeSpan.FromSeconds(1)) .WithProperty("auto.offset.reset", "earliest") .WithGroupId(group); var counter = 0; await Source.From(Enumerable.Range(1, 11)) .Select(elem => new ProducerRecord <Null, string>(topicPartition, elem.ToString())) .RunWith(KafkaProducer.PlainSink(ProducerSettings), Materializer); var probe = KafkaConsumer.CommittableSource(consumerSettings, Subscriptions.AssignmentWithOffset(new TopicPartitionOffset(topicPartition, Offset.Unset))) .Select(t => { counter++; // fail once, on the 7th message if (counter == 7) { throw new Exception("BOOM!"); } return(t); }) .SelectAsync(1, async elem => { await elem.CommitableOffset.Commit(); return(elem.Record.Value); }) .ToMaterialized(this.SinkProbe <string>(), Keep.Right) .Run(Materializer); var messages = new List <string>(); probe.Request(11); for (var i = 0; i < 6; i++) { messages.Add(probe.ExpectNext(TimeSpan.FromSeconds(5))); } // stream fails at index 7 var err = probe.ExpectEvent(); err.Should().BeOfType <TestSubscriber.OnError>(); var exception = ((TestSubscriber.OnError)err).Cause; exception.Message.Should().Be("BOOM!"); // stream should be dead here probe.ExpectNoMsg(TimeSpan.FromSeconds(5)); probe.Cancel(); // restart dead stream probe = KafkaConsumer.CommittableSource(consumerSettings, Subscriptions.AssignmentWithOffset(new TopicPartitionOffset(topicPartition, Offset.Unset))) .SelectAsync(1, async elem => { await elem.CommitableOffset.Commit(); return(elem.Record.Value); }) .ToMaterialized(this.SinkProbe <string>(), Keep.Right) .Run(Materializer); probe.Request(11); for (var i = 0; i < 5; i++) { messages.Add(probe.ExpectNext(TimeSpan.FromSeconds(5))); } probe.Cancel(); // end result should be gapless messages.Select(s => int.Parse(s)).Should().BeEquivalentTo(Enumerable.Range(1, 11)); }