public async Task CommitableSource_consumes_messages_from_Producer_without_commits()
{
int elementsCount = 100;
var topic1 = CreateTopic(1);
var group1 = CreateGroup(1);
var topicPartition1 = new TopicPartition(topic1, 0);
await GivenInitializedTopic(topicPartition1);
await Source
.From(Enumerable.Range(1, elementsCount))
.Select(elem => new ProducerRecord <Null, string>(topicPartition1, elem.ToString()))
.RunWith(KafkaProducer.PlainSink(ProducerSettings), Materializer);
var consumerSettings = CreateConsumerSettings <string>(group1);
var probe = KafkaConsumer
.CommittableSource(consumerSettings, Subscriptions.Assignment(topicPartition1))
.Where(c => !c.Record.Value.Equals(InitialMsg))
.Select(c => c.Record.Value)
.RunWith(this.SinkProbe <string>(), Materializer);
probe.Request(elementsCount);
foreach (var i in Enumerable.Range(1, elementsCount).Select(c => c.ToString()))
{
probe.ExpectNext(i, TimeSpan.FromSeconds(10));
}
probe.Cancel();
}
public static async Task Main(string[] args)
{
Config fallbackConfig = ConfigurationFactory.ParseString(@"
akka.suppress-json-serializer-warning=true
akka.loglevel = DEBUG
").WithFallback(ConfigurationFactory.FromResource <ConsumerSettings <object, object> >("Akka.Streams.Kafka.reference.conf"));
var system = ActorSystem.Create("TestKafka", fallbackConfig);
var materializer = system.Materializer();
var consumerSettings = ConsumerSettings <Null, string> .Create(system, null, null)
.WithBootstrapServers($"{EventHubNamespace}.servicebus.windows.net:9093")
.WithGroupId(EventHubConsumerGroup)
.WithProperties(new Dictionary <string, string>
{
{ "security.protocol", "SASL_SSL" },
{ "sasl.mechanism", "PLAIN" },
{ "sasl.username", "$ConnectionString" },
{ "sasl.password", EventHubConnectionString },
});
var subscription = Subscriptions.Topics(EventHubName);
var committerDefaults = CommitterSettings.Create(system);
// Comment for simple no-commit consumer
DrainingControl <NotUsed> control = KafkaConsumer.CommittableSource(consumerSettings, subscription)
.SelectAsync(1, msg =>
Business(msg.Record).ContinueWith(done => (ICommittable)msg.CommitableOffset))
.ToMaterialized(
Committer.Sink(committerDefaults.WithMaxBatch(1)),
DrainingControl <NotUsed> .Create)
.Run(materializer);
// Uncomment for simple no-commit consumer
/*
* await KafkaConsumer.PlainSource(consumerSettings, subscription)
* .RunForeach(result =>
* {
* Console.WriteLine($"Consumer: {result.Topic}/{result.Partition} {result.Offset}: {result.Value}");
* }, materializer);
*/
Console.WriteLine("Press any key to stop consumer.");
Console.ReadKey();
// Comment for simple no-commit consumer
await control.Stop();
await system.Terminate();
}
public async Task CommitterFlow_commits_offsets_from_CommittableSource(int batchSize)
{
var topic1 = CreateTopic(1);
var topicPartition1 = new TopicPartition(topic1, 0);
var group1 = CreateGroup(1);
await GivenInitializedTopic(topicPartition1);
await Source
.From(Enumerable.Range(1, 100))
.Select(elem => new ProducerRecord <Null, string>(topicPartition1, elem.ToString()))
.RunWith(KafkaProducer.PlainSink(ProducerSettings), Materializer);
var consumerSettings = CreateConsumerSettings <string>(group1);
var committedElements = new ConcurrentQueue <string>();
var committerSettings = CommitterSettings.WithMaxBatch(batchSize);
var(task, probe1) = KafkaConsumer.CommittableSource(consumerSettings, Subscriptions.Assignment(topicPartition1))
.WhereNot(c => c.Record.Value == InitialMsg)
.SelectAsync(10, elem =>
{
committedElements.Enqueue(elem.Record.Value);
return(Task.FromResult(elem.CommitableOffset as ICommittable));
})
.Via(Committer.Flow(committerSettings))
.ToMaterialized(this.SinkProbe <Done>(), Keep.Both)
.Run(Materializer);
probe1.Request(25 / batchSize);
foreach (var _ in Enumerable.Range(1, 25 / batchSize))
{
probe1.ExpectNext(Done.Instance, TimeSpan.FromSeconds(10));
}
probe1.Cancel();
AwaitCondition(() => task.IsShutdown.IsCompletedSuccessfully);
var probe2 = KafkaConsumer.PlainSource(consumerSettings, Subscriptions.Assignment(new TopicPartition(topic1, 0)))
.Select(_ => _.Value)
.RunWith(this.SinkProbe <string>(), Materializer);
probe2.Request(75);
foreach (var i in Enumerable.Range(committedElements.Count + 1, 75).Select(c => c.ToString()))
{
probe2.ExpectNext(i, TimeSpan.FromSeconds(10));
}
probe2.Cancel();
}
[InlineData(20, 10)] //20 개의 메시지를 생산하고,소비한다,테스트는 10초이내에 완료되어야함(완료시 종료됨)
public void Test1(int limit, int cutoff)
{
string lastSignal = Guid.NewGuid().ToString();
int readyTimeForConsume = 3;
int recCnt = 0;
KafkaConsumer.CommittableSource(consumerSettings, subscription)
.RunForeach(result =>
{
Console.WriteLine($"Consumer: {result.Record.Topic}/{result.Record.Partition} {result.Record.Offset}: {result.Record.Value}");
if (lastSignal == result.Record.Value)
{
probe.Tell("처리모두완료");
}
result.CommitableOffset.Commit();
}, materializer_consumer);
Source <int, NotUsed> source = Source.From(Enumerable.Range(1, limit));
source
.Throttle(2, TimeSpan.FromSeconds(1), 1, ThrottleMode.Shaping) //출력 조절 : 초당 2개처리
.Select(c =>
{
var result = $"No:{c.ToString()}";
if (c == limit)
{
result = lastSignal;
}
return(result);
})
.Select(elem => ProducerMessage.Single(new ProducerRecord <Null, string>(testTopic, elem)))
.Via(KafkaProducer.FlexiFlow <Null, string, NotUsed>(producerSettings))
.Select(result =>
{
var response = result as Result <Null, string, NotUsed>;
Console.WriteLine($"Producer: {response.Metadata.Topic}/{response.Metadata.Partition} {response.Metadata.Offset}: {response.Metadata.Value}");
return(result);
})
.RunWith(Sink.Ignore <IResults <Null, string, NotUsed> >(), materializer_producer);
Within(TimeSpan.FromSeconds(cutoff), () =>
{
probe.ExpectMsg("처리모두완료", TimeSpan.FromSeconds(cutoff));
});
}
private Source <CommittableMessage <K, V>, IControl> CreateCommitableSource(
MockConsumer <K, V> mock, string groupId = "group1", string[] topics = null)
{
topics ??= new[] { "topic" };
var settings = ConsumerSettings <K, V> .Create(Sys, Deserializers.Utf8, Deserializers.Utf8)
.WithGroupId(groupId)
.WithCloseTimeout(MockConsumer.CloseTimeout)
.WithStopTimeout(MockConsumer.CloseTimeout)
.WithCommitTimeout(TimeSpan.FromMilliseconds(500))
.WithConsumerFactory(_ => mock.Mock);
mock.Settings = settings;
return(KafkaConsumer.CommittableSource(
settings,
Subscriptions.Topics(topics)));
}
public void Start(ConsumerAkkaOption consumerActorOption)
{
IAutoSubscription makeshop_neworder = Subscriptions.Topics(consumerActorOption.Topics);
var consumerSettings = ConsumerSettings <Null, string> .Create(consumerSystem, null, null)
.WithBootstrapServers(consumerActorOption.BootstrapServers)
.WithGroupId(consumerActorOption.KafkaGroupId);
var materializer_consumer = consumerSystem.Materializer();
KafkaConsumer.CommittableSource(consumerSettings, makeshop_neworder)
.RunForeach(result =>
{
result.CommitableOffset.Commit();
Console.WriteLine($"Consumer: {result.Record.Partition}/{result.Record.Topic} {result.Record.Offset}: {result.Record.Value}");
if (consumerActorOption.RelayActor != null)
{
consumerActorOption.RelayActor.Tell(result.Record.Value); //ForgetAndFire 발송
}
}, materializer_consumer);
}
public async Task CommitableSource_resume_from_commited_offset()
{
var topic1 = CreateTopic(1);
var topicPartition1 = new TopicPartition(topic1, 0);
var group1 = CreateGroup(1);
var group2 = CreateGroup(2);
await GivenInitializedTopic(topicPartition1);
await Source
.From(Enumerable.Range(1, 100))
.Select(elem => new ProducerRecord <Null, string>(topicPartition1, elem.ToString()))
.RunWith(KafkaProducer.PlainSink(ProducerSettings), Materializer);
var consumerSettings = CreateConsumerSettings <string>(group1);
var committedElements = new ConcurrentQueue <string>();
var(task, probe1) = KafkaConsumer.CommittableSource(consumerSettings, Subscriptions.Assignment(topicPartition1))
.WhereNot(c => c.Record.Value == InitialMsg)
.SelectAsync(10, async elem =>
{
await elem.CommitableOffset.Commit();
committedElements.Enqueue(elem.Record.Value);
return(Done.Instance);
})
.ToMaterialized(this.SinkProbe <Done>(), Keep.Both)
.Run(Materializer);
probe1.Request(25);
foreach (var _ in Enumerable.Range(1, 25))
{
probe1.ExpectNext(Done.Instance, TimeSpan.FromSeconds(10));
}
probe1.Cancel();
AwaitCondition(() => task.IsShutdown.IsCompletedSuccessfully);
var probe2 = KafkaConsumer.CommittableSource(consumerSettings, Subscriptions.Assignment(new TopicPartition(topic1, 0)))
.Select(_ => _.Record.Value)
.RunWith(this.SinkProbe <string>(), Materializer);
// Note that due to buffers and SelectAsync(10) the committed offset is more
// than 26, and that is not wrong
// some concurrent publish
await Source
.From(Enumerable.Range(101, 100))
.Select(elem => new ProducerRecord <Null, string>(topicPartition1, elem.ToString()))
.RunWith(KafkaProducer.PlainSink(ProducerSettings), Materializer);
probe2.Request(100);
foreach (var i in Enumerable.Range(committedElements.Count + 1, 100).Select(c => c.ToString()))
{
probe2.ExpectNext(i, TimeSpan.FromSeconds(10));
}
probe2.Cancel();
// another consumer should see all
var probe3 = KafkaConsumer.CommittableSource(consumerSettings.WithGroupId(group2), Subscriptions.Assignment(new TopicPartition(topic1, 0)))
.WhereNot(c => c.Record.Value == InitialMsg)
.Select(_ => _.Record.Value)
.RunWith(this.SinkProbe <string>(), Materializer);
probe3.Request(100);
foreach (var i in Enumerable.Range(1, 100).Select(c => c.ToString()))
{
probe3.ExpectNext(i, TimeSpan.FromSeconds(10));
}
probe3.Cancel();
}
public async Task SupervisionStrategy_Decider_on_complex_stream_should_work()
{
var topic = CreateTopic(1);
var group = CreateGroup(1);
var topicPartition = new TopicPartition(topic, 0);
var committedTopicPartition = new TopicPartition($"{topic}-done", 0);
var callCount = 0;
Directive Decider(Exception cause)
{
callCount++;
return(Directive.Resume);
}
var committerSettings = CommitterSettings.Create(Sys);
var consumerSettings = CreateConsumerSettings <string>(group);
var counter = 0;
// arrange
await Source.From(Enumerable.Range(1, 10))
.Select(elem => new ProducerRecord <Null, string>(topicPartition, elem.ToString()))
.RunWith(KafkaProducer.PlainSink(ProducerSettings), Materializer);
// act
var drainingControl = KafkaConsumer.CommittableSource(consumerSettings, Subscriptions.Assignment(topicPartition))
.Via(Flow.Create <CommittableMessage <Null, string> >().Select(x =>
{
counter++;
// Exception happened here, fail once, when counter is 5
if (counter == 5)
{
throw new Exception("BOOM!");
}
return(x);
}))
.WithAttributes(Attributes.CreateName("CommitableSource").And(ActorAttributes.CreateSupervisionStrategy(Decider)))
.Select(c => (c.Record.Topic, c.Record.Message.Value, c.CommitableOffset))
.SelectAsync(1, async t =>
{
Log.Info($"[{t.Topic}]: {t.Value}");
// simulate a request-response call that takes 10ms to complete here
await Task.Delay(10);
return(t);
})
.Select(t => ProducerMessage.Single(new ProducerRecord <Null, string>(committedTopicPartition, t.Value),
t.CommitableOffset))
.Via(KafkaProducer.FlexiFlow <Null, string, ICommittableOffset>(ProducerSettings)).WithAttributes(Attributes.CreateName("FlexiFlow"))
.Select(m => (ICommittable)m.PassThrough)
.AlsoToMaterialized(Committer.Sink(committerSettings), DrainingControl <NotUsed> .Create)
.To(Flow.Create <ICommittable>()
.Async()
.GroupedWithin(1000, TimeSpan.FromSeconds(1))
.Select(c => c.Count())
.Log("MsgCount").AddAttributes(Attributes.CreateLogLevels(LogLevel.InfoLevel))
.To(Sink.Ignore <int>()))
.Run(Sys.Materializer());
await Task.Delay(TimeSpan.FromSeconds(5));
await GuardWithTimeoutAsync(drainingControl.DrainAndShutdown(), TimeSpan.FromSeconds(10));
// There should be only 1 decider call
callCount.Should().Be(1);
// Assert that all of the messages, except for those that failed in the stage, got committed
var settings = CreateConsumerSettings <Null, string>(group);
var probe = KafkaConsumer
.PlainSource(settings, Subscriptions.Assignment(committedTopicPartition))
.Select(c => c.Message.Value)
.RunWith(this.SinkProbe <string>(), Materializer);
probe.Request(9);
var messages = new List <string>();
for (var i = 0; i < 9; ++i)
{
var message = probe.RequestNext();
messages.Add(message);
}
// Message "5" is missing because the exception happened downstream of the source and we chose to
// ignore it in the decider
messages.Should().BeEquivalentTo(new[] { "1", "2", "3", "4", "6", "7", "8", "9", "10" });
probe.Cancel();
}
public async Task Committable_consumer_with_failed_downstream_stage_result_should_be_gapless()
{
var topic = CreateTopic(1);
var group = CreateGroup(1);
var topicPartition = new TopicPartition(topic, 0);
var consumerSettings = ConsumerSettings <Null, string> .Create(Sys, null, null)
.WithBootstrapServers(Fixture.KafkaServer)
.WithStopTimeout(TimeSpan.FromSeconds(1))
.WithProperty("auto.offset.reset", "earliest")
.WithGroupId(group);
var counter = 0;
await Source.From(Enumerable.Range(1, 11))
.Select(elem => new ProducerRecord <Null, string>(topicPartition, elem.ToString()))
.RunWith(KafkaProducer.PlainSink(ProducerSettings), Materializer);
var probe = KafkaConsumer.CommittableSource(consumerSettings, Subscriptions.AssignmentWithOffset(new TopicPartitionOffset(topicPartition, Offset.Unset)))
.Select(t =>
{
counter++;
// fail once, on the 7th message
if (counter == 7)
{
throw new Exception("BOOM!");
}
return(t);
})
.SelectAsync(1, async elem =>
{
await elem.CommitableOffset.Commit();
return(elem.Record.Value);
})
.ToMaterialized(this.SinkProbe <string>(), Keep.Right)
.Run(Materializer);
var messages = new List <string>();
probe.Request(11);
for (var i = 0; i < 6; i++)
{
messages.Add(probe.ExpectNext(TimeSpan.FromSeconds(5)));
}
// stream fails at index 7
var err = probe.ExpectEvent();
err.Should().BeOfType <TestSubscriber.OnError>();
var exception = ((TestSubscriber.OnError)err).Cause;
exception.Message.Should().Be("BOOM!");
// stream should be dead here
probe.ExpectNoMsg(TimeSpan.FromSeconds(5));
probe.Cancel();
// restart dead stream
probe = KafkaConsumer.CommittableSource(consumerSettings, Subscriptions.AssignmentWithOffset(new TopicPartitionOffset(topicPartition, Offset.Unset)))
.SelectAsync(1, async elem =>
{
await elem.CommitableOffset.Commit();
return(elem.Record.Value);
})
.ToMaterialized(this.SinkProbe <string>(), Keep.Right)
.Run(Materializer);
probe.Request(11);
for (var i = 0; i < 5; i++)
{
messages.Add(probe.ExpectNext(TimeSpan.FromSeconds(5)));
}
probe.Cancel();
// end result should be gapless
messages.Select(s => int.Parse(s)).Should().BeEquivalentTo(Enumerable.Range(1, 11));
}