public async Task TestConsumer_ConsumerGroupLeaveWhenStop()
{
var mocks = InitCluster();
mocks.Group.SetupGet(g => g.Configuration)
.Returns(new ConsumerGroupConfiguration {
AutoCommitEveryMs = -1, SessionTimeoutMs = 10
});
var consumer = new ConsumeRouter(mocks.Cluster.Object,
new Configuration {
TaskScheduler = new CurrentThreadTaskScheduler(), ConsumeBatchSize = 1
}, 1);
mocks.Group.Setup(g => g.Heartbeat()).ReturnsAsync(ErrorCode.RebalanceInProgress);
consumer.StartConsumeSubscription(mocks.Group.Object, new[] { "the topic" });
var partitionsRevokedEventIsCalled = false;
consumer.PartitionsRevoked += () => partitionsRevokedEventIsCalled = true;
await consumer.Stop();
mocks.Group.Verify(g => g.LeaveGroup(), Times.Once);
mocks.Group.Verify(g => g.Commit(It.IsAny <IEnumerable <TopicData <OffsetCommitPartitionData> > >()));
Assert.That(partitionsRevokedEventIsCalled, Is.True);
}
public async Task TestConsumer_RaisesPartitionsRevokedOnRebalance()
{
var mocks = InitCluster();
mocks.Group.Setup(g => g.Heartbeat()).ReturnsAsync(ErrorCode.RebalanceInProgress);
mocks.Group.SetupGet(g => g.Configuration).Returns(
new ConsumerGroupConfiguration {
AutoCommitEveryMs = -1, SessionTimeoutMs = 10
});
var consumer = new ConsumeRouter(
cluster: mocks.Cluster.Object,
configuration: new Configuration
{
TaskScheduler = new CurrentThreadTaskScheduler(),
ConsumeBatchSize = 1
},
resolution: 1);
var partitionsRevokedEventIsCalled = false;
consumer.PartitionsRevoked += () => partitionsRevokedEventIsCalled = true;
consumer.StartConsumeSubscription(mocks.Group.Object, new[] { "the topic" });
Thread.Sleep(20);
Assert.That(partitionsRevokedEventIsCalled, Is.True);
await consumer.Stop();
}
public void TestConsumer_ConsumerGroupStartConsume()
{
var mocks = InitCluster();
var consumer = new ConsumeRouter(mocks.Cluster.Object,
new Configuration {
TaskScheduler = new CurrentThreadTaskScheduler(), ConsumeBatchSize = 1
}, 1);
consumer.StartConsumeSubscription(mocks.Group.Object, new[] { "the topic" });
mocks.Group.Verify(g => g.Join(It.IsAny <IEnumerable <string> >()), Times.Once);
mocks.Node.Verify(n => n.Fetch(It.IsAny <FetchMessage>()), Times.Once); // 1 partition with specific offset
mocks.Node.Verify(n => n.Offset(It.IsAny <OffsetMessage>()), Times.Once); // 1 partition with offset -1
Thread.Sleep(20); // wait for at least one heartbeat to be sent
mocks.Group.Verify(g => g.Heartbeat());
consumer.Acknowledge(new CommonAcknowledgement <FetchResponse>
{
ReceivedDate = DateTime.UtcNow,
Response = new FetchResponse {
FetchPartitionResponse =
new CommonResponse <FetchPartitionResponse>
{
TopicsResponse =
new[]
{
new TopicData <FetchPartitionResponse>
{
TopicName = "the topic",
PartitionsData =
new[]
{
new FetchPartitionResponse
{
Partition = 1,
Messages =
new List <ResponseMessage>
{
new ResponseMessage {
Offset = 28, Message = new Message()
}
}
}
}
}
}
}
}
});
mocks.Node.Verify(n => n.Fetch(It.IsAny <FetchMessage>()), Times.Exactly(2)); // response should have triggered one more fetch
mocks.Group.Verify(g => g.Commit(It.IsAny <IEnumerable <TopicData <OffsetCommitPartitionData> > >())); // should have auto commited
consumer.Stop().Wait();
}
public async Task TestConsumer_ConsumerGroupHeartbeatErrors()
{
var mocks = InitCluster();
mocks.Group.SetupGet(g => g.Configuration)
.Returns(new ConsumerGroupConfiguration {
AutoCommitEveryMs = -1, SessionTimeoutMs = 10
});
var consumer = new ConsumeRouter(mocks.Cluster.Object,
new Configuration {
TaskScheduler = new CurrentThreadTaskScheduler(), ConsumeBatchSize = 1
}, 1);
mocks.Group.Setup(g => g.Heartbeat()).ReturnsAsync(ErrorCode.RebalanceInProgress)
.Callback(() => mocks.HeartbeatCalled.Set());
consumer.StartConsumeSubscription(mocks.Group.Object, new[] { "the topic" });
await HeartbeatFinishedProcessing(mocks, consumer);
// At least 2 Join (one on start, one on next heartbeat)
mocks.Group.Verify(g => g.Join(It.IsAny <IEnumerable <string> >()), Times.AtLeast(2));
// Commit should have been called due to RebalanceInProgressError
mocks.Group.Verify(g => g.Commit(It.IsAny <IEnumerable <TopicData <OffsetCommitPartitionData> > >()));
consumer.Stop().Wait();
mocks = InitCluster();
mocks.Group.SetupGet(g => g.Configuration)
.Returns(new ConsumerGroupConfiguration {
AutoCommitEveryMs = -1, SessionTimeoutMs = 10
});
mocks.Group.Setup(g => g.Heartbeat()).ThrowsAsync(new Exception())
.Callback(() => mocks.HeartbeatCalled.Set());
consumer = new ConsumeRouter(mocks.Cluster.Object,
new Configuration {
TaskScheduler = new CurrentThreadTaskScheduler(), ConsumeBatchSize = 1
}, 1);
consumer.StartConsumeSubscription(mocks.Group.Object, new[] { "the topic" });
await HeartbeatFinishedProcessing(mocks, consumer);
mocks.Group.Verify(g => g.Join(It.IsAny <IEnumerable <string> >()), Times.AtLeast(2));
// No Commit tried in case of ""hard" heartbeat errors
mocks.Group.Verify(g => g.Commit(It.IsAny <IEnumerable <TopicData <OffsetCommitPartitionData> > >()), Times.Never);
}
public async Task Stop()
{
if (!_started)
{
return;
}
_timeoutScheduler.Dispose();
_refreshMetadataTimer.Dispose();
await ConsumeRouter.Stop();
await ProduceRouter.Stop();
_agent.Complete();
await _agent.Completion;
await Task.WhenAll(_nodes.Keys.Select(n => n.Stop()));
_started = false;
}
public async Task TestConsumer_RaisesPartitionsAssignedEventOnJoin()
{
var mocks = InitCluster();
var consumer = new ConsumeRouter(
cluster: mocks.Cluster.Object,
configuration: new Configuration
{
TaskScheduler = new CurrentThreadTaskScheduler(),
ConsumeBatchSize = 1
},
resolution: 1);
var partitionsAssignedEventIsCalled = false;
consumer.PartitionsAssigned += _ => partitionsAssignedEventIsCalled = true;
consumer.StartConsumeSubscription(mocks.Group.Object, new[] { "the topic" });
await consumer.Stop();
Assert.That(partitionsAssignedEventIsCalled, Is.True);
}
public async Task TestConsumer_ConsumerGroupRestartConsume()
{
var mocks = InitCluster();
var consumer = new ConsumeRouter(mocks.Cluster.Object,
new Configuration {
TaskScheduler = new CurrentThreadTaskScheduler(), ConsumeBatchSize = 1
}, 1);
var consumerStartEvent = new AutoResetEvent(false);
var consumerStopEvent = new AutoResetEvent(false);
consumer.ConsumerStopped += () => { consumerStopEvent.Set(); };
consumer.StartConsumeSubscription(mocks.Group.Object, new[] { "the topic" });
mocks.Group.Verify(g => g.Join(It.IsAny <IEnumerable <string> >()), Times.Once);
mocks.Node.Verify(n => n.Fetch(It.IsAny <FetchMessage>()), Times.Once); // 1 partition with specific offset
mocks.Node.Verify(n => n.Offset(It.IsAny <OffsetMessage>()), Times.Once); // 1 partition with offset -1
WaitOneSecondMaxForEvent("heatbeat", mocks.HeartbeatCalled);
mocks.Group.Verify(g => g.Heartbeat());
consumer.Acknowledge(new CommonAcknowledgement <FetchResponse>
{
ReceivedDate = DateTime.UtcNow,
Response = new FetchResponse {
FetchPartitionResponse =
new CommonResponse <FetchPartitionResponse>
{
TopicsResponse =
new[]
{
new TopicData <FetchPartitionResponse>
{
TopicName = "the topic",
PartitionsData =
new[]
{
new FetchPartitionResponse
{
Partition = 1,
Messages =
new List <ResponseMessage>
{
new ResponseMessage {
Offset = 28, Message = new Message()
}
}
}
}
}
}
}
}
});
consumer.StopConsume("the topic", Partitions.All, Offsets.Now);
WaitOneSecondMaxForEvent("stop", consumerStopEvent);
mocks.Node.Verify(n => n.Fetch(It.IsAny <FetchMessage>()),
Times.Exactly(2)); // response should have triggered one more fetch
consumer.Acknowledge(new CommonAcknowledgement <FetchResponse>
{
ReceivedDate = DateTime.UtcNow,
Response = new FetchResponse {
FetchPartitionResponse =
new CommonResponse <FetchPartitionResponse>
{
TopicsResponse =
new[]
{
new TopicData <FetchPartitionResponse>
{
TopicName = "the topic",
PartitionsData =
new[]
{
new FetchPartitionResponse
{
Partition = 1,
Messages =
new List <ResponseMessage>
{
new ResponseMessage {
Offset = 29, Message = new Message()
}
}
}
}
}
}
}
}
});
consumer.ConsumerStarted += () => { consumerStartEvent.Set(); };
consumer.StartConsume("the topic", Partitions.All, Offsets.Now);
WaitOneSecondMaxForEvent("start", consumerStartEvent);
mocks.Node.Verify(n => n.Fetch(It.IsAny <FetchMessage>()), Times.Exactly(3));
consumer.Stop().Wait();
}