public async Task When_Options_Are_Set_Should_Be_Set_In_Consumer_Config() { var executor = new Mock <ITriggeredFunctionExecutor>(); var consumer = new Mock <IConsumer <Ignore, string> >(); var listenerConfig = new KafkaListenerConfiguration() { BrokerList = "testBroker", Topic = "topic", ConsumerGroup = "group1", SslKeyPassword = "******", SslCertificateLocation = "path/to/cert", SslKeyLocation = "path/to/key", SslCaLocation = "path/to/cacert" }; var kafkaOptions = new KafkaOptions(); var target = new KafkaListenerForTest <Ignore, string>( executor.Object, true, kafkaOptions, listenerConfig, requiresKey: true, valueDeserializer: null, NullLogger.Instance, functionId: "testId" ); target.SetConsumer(consumer.Object); await target.StartAsync(default);
public async Task When_Options_Are_Set_Should_Be_Set_In_Consumer_Config() { var executor = new Mock <ITriggeredFunctionExecutor>(); var consumer = new Mock <IConsumer <Ignore, string> >(); var listenerConfig = new KafkaListenerConfiguration() { BrokerList = "testBroker", Topic = "topic", ConsumerGroup = "group1", }; var kafkaOptions = new KafkaOptions(); var target = new KafkaListenerForTest <Ignore, string>( executor.Object, true, kafkaOptions, listenerConfig, valueDeserializer: null, NullLogger.Instance ); target.SetConsumer(consumer.Object); await target.StartAsync(default);
public async Task When_Using_SingleItem_Binding_10_Events_Should_Execute_Function_Ten_Times() { const int ExpectedEventCount = 10; var executor = new Mock <ITriggeredFunctionExecutor>(); var consumer = new Mock <IConsumer <Ignore, string> >(); var offset = 0L; consumer.Setup(x => x.Consume(It.IsNotNull <TimeSpan>())) .Returns(() => { if (offset < ExpectedEventCount) { offset++; return(CreateConsumeResult <Ignore, string>(offset.ToString(), 0, offset)); } return(null); }); var executorFinished = new SemaphoreSlim(0); var executorCalls = 0; executor.Setup(x => x.TryExecuteAsync(It.IsNotNull <TriggeredFunctionData>(), It.IsAny <CancellationToken>())) .Callback(() => { Interlocked.Increment(ref executorCalls); executorFinished.Release(); }) .ReturnsAsync(new FunctionResult(true)); var listenerConfig = new KafkaListenerConfiguration() { BrokerList = "testBroker", Topic = "topic", ConsumerGroup = "group1", }; var target = new KafkaListenerForTest <Ignore, string>( executor.Object, singleDispatch: true, options: new KafkaOptions(), listenerConfig, requiresKey: true, valueDeserializer: null, logger: NullLogger.Instance, functionId: "testId" ); target.SetConsumer(consumer.Object); await target.StartAsync(default(CancellationToken)); Assert.True(await executorFinished.WaitAsync(TimeSpan.FromSeconds(5))); await target.StopAsync(default(CancellationToken)); }
public async Task When_Using_MultiItem_Binding_10_Events_Should_Execute_Function_Once() { const int ExpectedEventCount = 10; var executor = new Mock <ITriggeredFunctionExecutor>(); var consumer = new Mock <IConsumer <Ignore, string> >(); var offset = 0L; consumer.Setup(x => x.Consume(It.IsNotNull <TimeSpan>())) .Returns(() => { if (offset < ExpectedEventCount) { offset++; return(CreateConsumeResult <Ignore, string>(offset.ToString(), 0, offset)); } return(null); }); var executorFinished = new SemaphoreSlim(0); var processedItemCount = 0; executor.Setup(x => x.TryExecuteAsync(It.IsNotNull <TriggeredFunctionData>(), It.IsAny <CancellationToken>())) .Callback <TriggeredFunctionData, CancellationToken>((td, _) => { var triggerData = (KafkaTriggerInput)td.TriggerValue; var alreadyProcessed = Interlocked.Add(ref processedItemCount, triggerData.Events.Length); if (alreadyProcessed == ExpectedEventCount) { executorFinished.Release(); } }) .ReturnsAsync(new FunctionResult(true)); var target = new KafkaListenerForTest <Ignore, string>( executor.Object, singleDispatch: false, options: new KafkaOptions(), brokerList: "testBroker", topic: "topic", consumerGroup: "group1", eventHubConnectionString: null, logger: NullLogger.Instance ); target.SetConsumer(consumer.Object); await target.StartAsync(default(CancellationToken)); Assert.True(await executorFinished.WaitAsync(TimeSpan.FromSeconds(5))); await target.StopAsync(default(CancellationToken)); executor.Verify(x => x.TryExecuteAsync(It.IsNotNull <TriggeredFunctionData>(), It.IsAny <CancellationToken>()), Times.Once); }
public async Task When_Topic_Has_Multiple_Partitions_Should_Execute_And_Commit_In_Order(bool singleDispatch) { const int MessagesPerPartition = 5; const int PartitionCount = 2; const int BatchCount = 2; const long Offset_A = 0; const long Offset_B = 1; const long Offset_C = 2; const long Offset_D = 3; const long Offset_E = 4; const long Offset_1 = 0; const long Offset_2 = 1; const long Offset_3 = 2; const long Offset_4 = 3; const long Offset_5 = 4; var executor = new Mock <ITriggeredFunctionExecutor>(); var consumer = new Mock <IConsumer <Null, string> >(); var committed = new ConcurrentQueue <TopicPartitionOffset>(); consumer.Setup(x => x.StoreOffset(It.IsNotNull <TopicPartitionOffset>())) .Callback <TopicPartitionOffset>((topicPartitionOffset) => { committed.Enqueue(topicPartitionOffset); }); // Batch 1: AB12C // Batch 2: 34DE5 consumer.SetupSequence(x => x.Consume(It.IsNotNull <TimeSpan>())) .Returns(CreateConsumeResult <Null, string>("A", 0, Offset_A)) .Returns(CreateConsumeResult <Null, string>("B", 0, Offset_B)) .Returns(CreateConsumeResult <Null, string>("1", 1, Offset_1)) .Returns(CreateConsumeResult <Null, string>("2", 1, Offset_2)) .Returns(CreateConsumeResult <Null, string>("C", 0, Offset_C)) .Returns((ConsumeResult <Null, string>)null) .Returns(CreateConsumeResult <Null, string>("3", 1, Offset_3)) .Returns(CreateConsumeResult <Null, string>("4", 1, Offset_4)) .Returns(CreateConsumeResult <Null, string>("D", 0, Offset_D)) .Returns(CreateConsumeResult <Null, string>("E", 0, Offset_E)) .Returns(() => { // from now on return null consumer.Setup(x => x.Consume(It.IsNotNull <TimeSpan>())) .Returns((ConsumeResult <Null, string>)null); return(CreateConsumeResult <Null, string>("5", 1, Offset_5)); }); var partition0 = new ConcurrentQueue <string>(); var partition1 = new ConcurrentQueue <string>(); var executorFinished = new SemaphoreSlim(0); executor.Setup(x => x.TryExecuteAsync(It.IsNotNull <TriggeredFunctionData>(), It.IsAny <CancellationToken>())) .Callback <TriggeredFunctionData, CancellationToken>((t, _) => { var triggerData = (KafkaTriggerInput)t.TriggerValue; if (singleDispatch) { Assert.Single(triggerData.Events); } foreach (var ev in triggerData.Events) { switch (ev.Partition) { case 0: partition0.Enqueue(ev.Value.ToString()); break; case 1: partition1.Enqueue(ev.Value.ToString()); break; default: Assert.True(false, "Unknown partition"); break; } } if (partition0.Count == 5 && partition1.Count == 5) { executorFinished.Release(); } }) .ReturnsAsync(new FunctionResult(true)); var listenerConfig = new KafkaListenerConfiguration() { BrokerList = "testBroker", Topic = "topic", ConsumerGroup = "group1", }; var target = new KafkaListenerForTest <Null, string>( executor.Object, singleDispatch, new KafkaOptions(), listenerConfig, requiresKey: true, valueDeserializer: null, NullLogger.Instance, functionId: "testId" ); target.SetConsumer(consumer.Object); await target.StartAsync(default(CancellationToken)); Assert.True(await executorFinished.WaitAsync(TimeSpan.FromSeconds(5))); // Give time for the commit to be saved await Task.Delay(1500); Assert.Equal(new[] { "A", "B", "C", "D", "E" }, partition0.ToArray()); Assert.Equal(new[] { "1", "2", "3", "4", "5" }, partition1.ToArray()); // Committing will be the one we read + 1 // Batch 1: AB12C // Batch 2: 34DE5 var committedArray = committed.ToArray(); if (singleDispatch) { // In single dispatch we expected to commit once per message / per partition Assert.Equal(MessagesPerPartition * PartitionCount, committedArray.Length); // In single dispatch each item will be committed individually Assert.Equal(new[] { Offset_A + 1, Offset_B + 1, Offset_C + 1, Offset_D + 1, Offset_E + 1 }, committedArray.Where(x => x.Partition == 0).Select(x => (long)x.Offset).ToArray()); Assert.Equal(new[] { Offset_1 + 1, Offset_2 + 1, Offset_3 + 1, Offset_4 + 1, Offset_5 + 1 }, committedArray.Where(x => x.Partition == 1).Select(x => (long)x.Offset).ToArray()); } else { // In multi dispatch we expected to commit once per batch / per partition Assert.Equal(BatchCount * PartitionCount, committedArray.Length); // In multi dispatch we batch/partition pair will be committed Assert.Equal(new[] { Offset_C + 1, Offset_E + 1 }, committedArray.Where(x => x.Partition == 0).Select(x => (long)x.Offset).ToArray()); Assert.Equal(new[] { Offset_2 + 1, Offset_5 + 1 }, committedArray.Where(x => x.Partition == 1).Select(x => (long)x.Offset).ToArray()); } await target.StopAsync(default(CancellationToken)); }
public async Task When_Topic_Has_Multiple_Partitions_Should_Execute_And_Commit_In_Order(bool singleDispatch) { const long Offset_A = 0; const long Offset_B = 1; const long Offset_C = 2; const long Offset_D = 3; const long Offset_E = 4; const long Offset_1 = 0; const long Offset_2 = 1; const long Offset_3 = 2; const long Offset_4 = 3; const long Offset_5 = 4; var executor = new Mock <ITriggeredFunctionExecutor>(); var consumer = new Mock <IConsumer <Ignore, string> >(); var committed = new ConcurrentQueue <TopicPartitionOffset>(); consumer.Setup(x => x.Commit(It.IsAny <IEnumerable <TopicPartitionOffset> >())) .Callback <IEnumerable <TopicPartitionOffset> >((topicPartitionOffsetCollection) => { foreach (var item in topicPartitionOffsetCollection) { committed.Enqueue(item); } }); // Batch 1: AB12C // Batch 2: 34DE5 consumer.SetupSequence(x => x.Consume(It.IsNotNull <TimeSpan>())) .Returns(CreateConsumeResult <Ignore, string>("A", 0, Offset_A)) .Returns(CreateConsumeResult <Ignore, string>("B", 0, Offset_B)) .Returns(CreateConsumeResult <Ignore, string>("1", 1, Offset_1)) .Returns(CreateConsumeResult <Ignore, string>("2", 1, Offset_2)) .Returns(CreateConsumeResult <Ignore, string>("C", 0, Offset_C)) .Returns((ConsumeResult <Ignore, string>)null) .Returns(CreateConsumeResult <Ignore, string>("3", 1, Offset_3)) .Returns(CreateConsumeResult <Ignore, string>("4", 1, Offset_4)) .Returns(CreateConsumeResult <Ignore, string>("D", 0, Offset_D)) .Returns(CreateConsumeResult <Ignore, string>("E", 0, Offset_E)) .Returns(() => { // from now on return null consumer.Setup(x => x.Consume(It.IsNotNull <TimeSpan>())) .Returns((ConsumeResult <Ignore, string>)null); return(CreateConsumeResult <Ignore, string>("5", 1, Offset_5)); }); var partition0 = new ConcurrentQueue <string>(); var partition1 = new ConcurrentQueue <string>(); var executorFinished = new SemaphoreSlim(0); executor.Setup(x => x.TryExecuteAsync(It.IsNotNull <TriggeredFunctionData>(), It.IsAny <CancellationToken>())) .Callback <TriggeredFunctionData, CancellationToken>((t, _) => { var triggerData = (KafkaTriggerInput)t.TriggerValue; if (singleDispatch) { Assert.Single(triggerData.Events); } foreach (var ev in triggerData.Events) { switch (ev.Partition) { case 0: partition0.Enqueue(ev.Value.ToString()); break; case 1: partition1.Enqueue(ev.Value.ToString()); break; default: Assert.True(false, "Unknown partition"); break; } } if (partition0.Count == 5 && partition1.Count == 5) { executorFinished.Release(); } }) .ReturnsAsync(new FunctionResult(true)); var target = new KafkaListenerForTest <Ignore, string>( executor.Object, singleDispatch, new KafkaOptions(), "testBroker", "topic", "group1", null, NullLogger.Instance ); target.SetConsumer(consumer.Object); await target.StartAsync(default(CancellationToken)); Assert.True(await executorFinished.WaitAsync(TimeSpan.FromSeconds(5))); // Give time for the commit to be saved await Task.Delay(1500); Assert.Equal(new[] { "A", "B", "C", "D", "E" }, partition0.ToArray()); Assert.Equal(new[] { "1", "2", "3", "4", "5" }, partition1.ToArray()); // Committing will be the one we read + 1 // Batch 1: AB12C // Batch 2: 34DE5 var committedArray = committed.ToArray(); Assert.Equal(4, committedArray.Length); Assert.Equal(new[] { Offset_C + 1, Offset_E + 1 }, committedArray.Where(x => x.Partition == 0).Select(x => (long)x.Offset).ToArray()); Assert.Equal(new[] { Offset_2 + 1, Offset_5 + 1 }, committedArray.Where(x => x.Partition == 1).Select(x => (long)x.Offset).ToArray()); await target.StopAsync(default(CancellationToken)); }