public void TestConsumer_OnlyOneConsumerGroup() { var mocks = InitCluster(); var consumer = new ConsumeRouter(mocks.Cluster.Object, new Configuration { TaskScheduler = new CurrentThreadTaskScheduler(), ConsumeBatchSize = 1 }, 1); consumer.StartConsumeSubscription(mocks.Group.Object, new[] { "the topic" }); Assert.That(() => consumer.StartConsumeSubscription(mocks.Group.Object, new[] { "the topic" }), Throws.ArgumentException); }
public async Task TestConsumer_RaisesPartitionsRevokedOnRebalance() { var mocks = InitCluster(); mocks.Group.Setup(g => g.Heartbeat()).ReturnsAsync(ErrorCode.RebalanceInProgress); mocks.Group.SetupGet(g => g.Configuration).Returns( new ConsumerGroupConfiguration { AutoCommitEveryMs = -1, SessionTimeoutMs = 10 }); var consumer = new ConsumeRouter( cluster: mocks.Cluster.Object, configuration: new Configuration { TaskScheduler = new CurrentThreadTaskScheduler(), ConsumeBatchSize = 1 }, resolution: 1); var partitionsRevokedEventIsCalled = false; consumer.PartitionsRevoked += () => partitionsRevokedEventIsCalled = true; consumer.StartConsumeSubscription(mocks.Group.Object, new[] { "the topic" }); Thread.Sleep(20); Assert.That(partitionsRevokedEventIsCalled, Is.True); await consumer.Stop(); }
public async Task TestConsumer_ConsumerGroupLeaveWhenStop() { var mocks = InitCluster(); mocks.Group.SetupGet(g => g.Configuration) .Returns(new ConsumerGroupConfiguration { AutoCommitEveryMs = -1, SessionTimeoutMs = 10 }); var consumer = new ConsumeRouter(mocks.Cluster.Object, new Configuration { TaskScheduler = new CurrentThreadTaskScheduler(), ConsumeBatchSize = 1 }, 1); mocks.Group.Setup(g => g.Heartbeat()).ReturnsAsync(ErrorCode.RebalanceInProgress); consumer.StartConsumeSubscription(mocks.Group.Object, new[] { "the topic" }); var partitionsRevokedEventIsCalled = false; consumer.PartitionsRevoked += () => partitionsRevokedEventIsCalled = true; await consumer.Stop(); mocks.Group.Verify(g => g.LeaveGroup(), Times.Once); mocks.Group.Verify(g => g.Commit(It.IsAny <IEnumerable <TopicData <OffsetCommitPartitionData> > >())); Assert.That(partitionsRevokedEventIsCalled, Is.True); }
public async Task TestConsumer_ConsumerGroupCommitAsync() { var mocks = InitCluster(); mocks.Group.SetupGet(g => g.Configuration) .Returns(new ConsumerGroupConfiguration { AutoCommitEveryMs = -1, SessionTimeoutMs = 10 }); var consumer = new ConsumeRouter(mocks.Cluster.Object, new Configuration { TaskScheduler = new CurrentThreadTaskScheduler(), ConsumeBatchSize = 1 }, 1); consumer.StartConsumeSubscription(mocks.Group.Object, new[] { "the topic" }); await consumer.CommitAsync("the topic", 1, 42); mocks.Group.Verify(g => g.Commit(It.Is <IEnumerable <TopicData <OffsetCommitPartitionData> > >(l => l.Count() == 1 && l.First().TopicName == "the topic" && l.First().PartitionsData.Count() == 1 && l.First().PartitionsData.First().Partition == 1 && l.First().PartitionsData.First().Metadata == "" && l.First().PartitionsData.First().Offset == 42)), // Offset saved should be next expected offset Times.Once); mocks.Group.Setup(g => g.Commit(It.IsAny <IEnumerable <TopicData <OffsetCommitPartitionData> > >())) .ThrowsAsync(new InvalidOperationException()); // NUnit 3 (used for .Net Core build) requires to use ThrowsAsync which doesn't exist in 2.6 Assert.Throws <InvalidOperationException>(consumer.CommitAsync("the topic", 1, 42).GetAwaiter().GetResult); }
public void TestConsumer_ConsumerGroupStartConsume() { var mocks = InitCluster(); var consumer = new ConsumeRouter(mocks.Cluster.Object, new Configuration { TaskScheduler = new CurrentThreadTaskScheduler(), ConsumeBatchSize = 1 }, 1); consumer.StartConsumeSubscription(mocks.Group.Object, new[] { "the topic" }); mocks.Group.Verify(g => g.Join(It.IsAny <IEnumerable <string> >()), Times.Once); mocks.Node.Verify(n => n.Fetch(It.IsAny <FetchMessage>()), Times.Once); // 1 partition with specific offset mocks.Node.Verify(n => n.Offset(It.IsAny <OffsetMessage>()), Times.Once); // 1 partition with offset -1 Thread.Sleep(20); // wait for at least one heartbeat to be sent mocks.Group.Verify(g => g.Heartbeat()); consumer.Acknowledge(new CommonAcknowledgement <FetchResponse> { ReceivedDate = DateTime.UtcNow, Response = new FetchResponse { FetchPartitionResponse = new CommonResponse <FetchPartitionResponse> { TopicsResponse = new[] { new TopicData <FetchPartitionResponse> { TopicName = "the topic", PartitionsData = new[] { new FetchPartitionResponse { Partition = 1, Messages = new List <ResponseMessage> { new ResponseMessage { Offset = 28, Message = new Message() } } } } } } } } }); mocks.Node.Verify(n => n.Fetch(It.IsAny <FetchMessage>()), Times.Exactly(2)); // response should have triggered one more fetch mocks.Group.Verify(g => g.Commit(It.IsAny <IEnumerable <TopicData <OffsetCommitPartitionData> > >())); // should have auto commited consumer.Stop().Wait(); }
public async Task TestConsumer_ConsumerGroupHeartbeatErrors() { var mocks = InitCluster(); mocks.Group.SetupGet(g => g.Configuration) .Returns(new ConsumerGroupConfiguration { AutoCommitEveryMs = -1, SessionTimeoutMs = 10 }); var consumer = new ConsumeRouter(mocks.Cluster.Object, new Configuration { TaskScheduler = new CurrentThreadTaskScheduler(), ConsumeBatchSize = 1 }, 1); mocks.Group.Setup(g => g.Heartbeat()).ReturnsAsync(ErrorCode.RebalanceInProgress) .Callback(() => mocks.HeartbeatCalled.Set()); consumer.StartConsumeSubscription(mocks.Group.Object, new[] { "the topic" }); await HeartbeatFinishedProcessing(mocks, consumer); // At least 2 Join (one on start, one on next heartbeat) mocks.Group.Verify(g => g.Join(It.IsAny <IEnumerable <string> >()), Times.AtLeast(2)); // Commit should have been called due to RebalanceInProgressError mocks.Group.Verify(g => g.Commit(It.IsAny <IEnumerable <TopicData <OffsetCommitPartitionData> > >())); consumer.Stop().Wait(); mocks = InitCluster(); mocks.Group.SetupGet(g => g.Configuration) .Returns(new ConsumerGroupConfiguration { AutoCommitEveryMs = -1, SessionTimeoutMs = 10 }); mocks.Group.Setup(g => g.Heartbeat()).ThrowsAsync(new Exception()) .Callback(() => mocks.HeartbeatCalled.Set()); consumer = new ConsumeRouter(mocks.Cluster.Object, new Configuration { TaskScheduler = new CurrentThreadTaskScheduler(), ConsumeBatchSize = 1 }, 1); consumer.StartConsumeSubscription(mocks.Group.Object, new[] { "the topic" }); await HeartbeatFinishedProcessing(mocks, consumer); mocks.Group.Verify(g => g.Join(It.IsAny <IEnumerable <string> >()), Times.AtLeast(2)); // No Commit tried in case of ""hard" heartbeat errors mocks.Group.Verify(g => g.Commit(It.IsAny <IEnumerable <TopicData <OffsetCommitPartitionData> > >()), Times.Never); }
public async Task TestConsumer_RaisesPartitionsAssignedEventOnJoin() { var mocks = InitCluster(); var consumer = new ConsumeRouter( cluster: mocks.Cluster.Object, configuration: new Configuration { TaskScheduler = new CurrentThreadTaskScheduler(), ConsumeBatchSize = 1 }, resolution: 1); var partitionsAssignedEventIsCalled = false; consumer.PartitionsAssigned += _ => partitionsAssignedEventIsCalled = true; consumer.StartConsumeSubscription(mocks.Group.Object, new[] { "the topic" }); await consumer.Stop(); Assert.That(partitionsAssignedEventIsCalled, Is.True); }
public void TestConsumer_ConsumerGroupCommit() { var mocks = InitCluster(); mocks.Group.SetupGet(g => g.Configuration) .Returns(new ConsumerGroupConfiguration { AutoCommitEveryMs = -1, SessionTimeoutMs = 10 }); var consumer = new ConsumeRouter(mocks.Cluster.Object, new Configuration { TaskScheduler = new CurrentThreadTaskScheduler(), ConsumeBatchSize = 1 }, 1); consumer.StartConsumeSubscription(mocks.Group.Object, new[] { "the topic" }); Thread.Sleep(10); // wait for at least one heartbeat to be sent consumer.Acknowledge(new CommonAcknowledgement <FetchResponse> { ReceivedDate = DateTime.UtcNow, Response = new FetchResponse { FetchPartitionResponse = new CommonResponse <FetchPartitionResponse> { TopicsResponse = new[] { new TopicData <FetchPartitionResponse> { TopicName = "the topic", PartitionsData = new[] { new FetchPartitionResponse { Partition = 1, Messages = new List <ResponseMessage> { new ResponseMessage { Offset = 28, Message = new Message() } } } } } } } } }); mocks.Group.Verify(g => g.Commit(It.IsAny <IEnumerable <TopicData <OffsetCommitPartitionData> > >()), Times.Never); // no auto commit consumer.RequireCommit(); mocks.Group.Verify(g => g.Commit(It.Is <IEnumerable <TopicData <OffsetCommitPartitionData> > >(l => l.Count() == 1 && l.First().TopicName == "the topic" && l.First().PartitionsData.Count() == 2 && l.First().PartitionsData.First().Partition == 1 && l.First().PartitionsData.First().Metadata == "" && l.First().PartitionsData.First().Offset == 29)), // Offset saved should be next expected offset Times.Once); }
public async Task TestConsumer_ConsumerGroupRestartConsume() { var mocks = InitCluster(); var consumer = new ConsumeRouter(mocks.Cluster.Object, new Configuration { TaskScheduler = new CurrentThreadTaskScheduler(), ConsumeBatchSize = 1 }, 1); var consumerStartEvent = new AutoResetEvent(false); var consumerStopEvent = new AutoResetEvent(false); consumer.ConsumerStopped += () => { consumerStopEvent.Set(); }; consumer.StartConsumeSubscription(mocks.Group.Object, new[] { "the topic" }); mocks.Group.Verify(g => g.Join(It.IsAny <IEnumerable <string> >()), Times.Once); mocks.Node.Verify(n => n.Fetch(It.IsAny <FetchMessage>()), Times.Once); // 1 partition with specific offset mocks.Node.Verify(n => n.Offset(It.IsAny <OffsetMessage>()), Times.Once); // 1 partition with offset -1 WaitOneSecondMaxForEvent("heatbeat", mocks.HeartbeatCalled); mocks.Group.Verify(g => g.Heartbeat()); consumer.Acknowledge(new CommonAcknowledgement <FetchResponse> { ReceivedDate = DateTime.UtcNow, Response = new FetchResponse { FetchPartitionResponse = new CommonResponse <FetchPartitionResponse> { TopicsResponse = new[] { new TopicData <FetchPartitionResponse> { TopicName = "the topic", PartitionsData = new[] { new FetchPartitionResponse { Partition = 1, Messages = new List <ResponseMessage> { new ResponseMessage { Offset = 28, Message = new Message() } } } } } } } } }); consumer.StopConsume("the topic", Partitions.All, Offsets.Now); WaitOneSecondMaxForEvent("stop", consumerStopEvent); mocks.Node.Verify(n => n.Fetch(It.IsAny <FetchMessage>()), Times.Exactly(2)); // response should have triggered one more fetch consumer.Acknowledge(new CommonAcknowledgement <FetchResponse> { ReceivedDate = DateTime.UtcNow, Response = new FetchResponse { FetchPartitionResponse = new CommonResponse <FetchPartitionResponse> { TopicsResponse = new[] { new TopicData <FetchPartitionResponse> { TopicName = "the topic", PartitionsData = new[] { new FetchPartitionResponse { Partition = 1, Messages = new List <ResponseMessage> { new ResponseMessage { Offset = 29, Message = new Message() } } } } } } } } }); consumer.ConsumerStarted += () => { consumerStartEvent.Set(); }; consumer.StartConsume("the topic", Partitions.All, Offsets.Now); WaitOneSecondMaxForEvent("start", consumerStartEvent); mocks.Node.Verify(n => n.Fetch(It.IsAny <FetchMessage>()), Times.Exactly(3)); consumer.Stop().Wait(); }