/// <summary>
/// Called from the consumer when joining/rejoining..
///
/// See https://cwiki.apache.org/confluence/display/KAFKA/Kafka+Client-side+Assignment+Proposal
/// </summary>
internal void OnJoinGroup(JoinGroupResponse response)
{
if (response.member_id != MemberId)
{
throw new ArgumentOutOfRangeException(nameof(response), $"Member is not valid ({MemberId} != {response.member_id})");
}
if (_disposeCount > 0)
{
throw new ObjectDisposedException($"Consumer {{GroupId:{GroupId},MemberId:{MemberId}}} is no longer valid");
}
_joinSemaphore.Lock(
() => {
IsLeader = response.leader_id == MemberId;
GenerationId = response.generation_id;
_groupProtocol = response.group_protocol;
_memberMetadata = response.members.ToImmutableDictionary(member => member.member_id, member => member.member_metadata);
Router.Log.Info(() => LogEvent.Create(GenerationId > 1
? $"Consumer {MemberId} Rejoined {GroupId} Generation{GenerationId}"
: $"Consumer {MemberId} Joined {GroupId}"));
}, _disposeToken.Token);
}
public void JoinGroupResponse(
[Values(
ErrorResponseCode.None,
ErrorResponseCode.OffsetMetadataTooLarge
)] ErrorResponseCode errorCode,
[Values(0, 1, 20000)] int generationId,
[Values("consumer", "other")] string protocol,
[Values("test", "a groupId")] string leaderId,
[Values("", "an existing member")] string memberId,
[Values(1, 10)] int memberCount)
{
var members = new List <JoinGroupResponse.Member>();
for (var m = 0; m < memberCount; m++)
{
var bytes = new byte[memberCount * 100];
_randomizer.NextBytes(bytes);
members.Add(new JoinGroupResponse.Member(memberId + m, new ByteMember(bytes)));
}
var response = new JoinGroupResponse(errorCode, generationId, protocol, leaderId, memberId, members);
response.AssertCanEncodeDecodeResponse(0);
}
public void JoinGroupResponse(
[Values(
ErrorCode.NONE,
ErrorCode.OFFSET_METADATA_TOO_LARGE
)] ErrorCode errorCode,
[Values(0, 1, 20000)] int generationId,
[Values("consumer", "other")] string protocol,
[Values("test", "a groupId")] string leaderId,
[Values("", "an existing member")] string memberId,
[Values(1, 10)] int memberCount)
{
var members = new List <JoinGroupResponse.Member>();
for (var m = 0; m < memberCount; m++)
{
var bytes = new byte[memberCount * 100];
_randomizer.NextBytes(bytes);
members.Add(new JoinGroupResponse.Member(memberId + m, new ByteTypeMetadata("known", new ArraySegment <byte>(bytes))));
}
var response = new JoinGroupResponse(errorCode, generationId, "known", leaderId, memberId, members);
response.AssertCanEncodeDecodeResponse(0, new ByteMembershipEncoder(protocol));
}
public async Task ConsumerHeartbeatsUntilDisposed(int heartbeatMilliseconds)
{
var protocol = new JoinGroupRequest.GroupProtocol(new ConsumerProtocolMetadata("mine"));
var router = Substitute.For <IRouter>();
var conn = Substitute.For <IConnection>();
router.GetConnectionAsync(Arg.Any <string>(), Arg.Any <string>(), Arg.Any <CancellationToken>())
.Returns(_ => Task.FromResult(conn));
router.SyncGroupAsync(Arg.Any <SyncGroupRequest>(), Arg.Any <IRequestContext>(), Arg.Any <IRetry>(), Arg.Any <CancellationToken>())
.Returns(_ => Task.FromResult(new SyncGroupResponse(ErrorCode.NONE, new ConsumerMemberAssignment(new [] { new TopicPartition("name", 0) }))));
conn.SendAsync(Arg.Any <HeartbeatRequest>(), Arg.Any <CancellationToken>(), Arg.Any <IRequestContext>())
.Returns(_ => Task.FromResult(new HeartbeatResponse(ErrorCode.NETWORK_EXCEPTION)));
var heartbeat = TimeSpan.FromMilliseconds(heartbeatMilliseconds);
var config = new ConsumerConfiguration(heartbeatTimeout: heartbeat, coordinationRetry: Retry.Until(heartbeat, maximumDelay: TimeSpan.FromMilliseconds(50)));
var request = new JoinGroupRequest(TestConfig.GroupId(), config.GroupHeartbeat, "", ConsumerEncoder.Protocol, new [] { protocol });
var memberId = Guid.NewGuid().ToString("N");
var response = new JoinGroupResponse(ErrorCode.NONE, 1, protocol.protocol_name, memberId, memberId, new [] { new JoinGroupResponse.Member(memberId, new ConsumerProtocolMetadata("mine")) });
using (new GroupConsumer(router, request.group_id, request.protocol_type, response, config)) {
await Task.Delay(heartbeatMilliseconds * 3);
#pragma warning disable CS4014 // Because this call is not awaited, execution of the current method continues before the call is completed
conn.DidNotReceive().SendAsync(
Arg.Is((LeaveGroupRequest s) => s.group_id == request.group_id && s.member_id == memberId),
Arg.Any <CancellationToken>(),
Arg.Any <IRequestContext>());
}
conn.Received().SendAsync(
Arg.Is((LeaveGroupRequest s) => s.group_id == request.group_id && s.member_id == memberId),
Arg.Any <CancellationToken>(),
Arg.Any <IRequestContext>());
#pragma warning restore CS4014 // Because this call is not awaited, execution of the current method continues before the call is completed
Assert.That(conn.ReceivedCalls().Count(c => c.GetMethodInfo().Name == nameof(Connection.SendAsync) && (c.GetArguments()[0] as HeartbeatRequest) != null), Is.AtLeast(2));
}
public async Task ConsumerHeartbeatsAtDesiredIntervals(int expectedHeartbeats, int heartbeatMilliseconds, int totalMilliseconds)
{
var protocol = new JoinGroupRequest.GroupProtocol(new ConsumerProtocolMetadata("mine"));
var router = Substitute.For <IRouter>();
var conn = Substitute.For <IConnection>();
router.GetConnectionAsync(Arg.Any <string>(), Arg.Any <string>(), Arg.Any <CancellationToken>())
.Returns(_ => Task.FromResult(conn));
router.SyncGroupAsync(Arg.Any <SyncGroupRequest>(), Arg.Any <IRequestContext>(), Arg.Any <IRetry>(), Arg.Any <CancellationToken>())
.Returns(_ => Task.FromResult(new SyncGroupResponse(ErrorCode.NONE, new ConsumerMemberAssignment(new [] { new TopicPartition("name", 0) }))));
conn.SendAsync(Arg.Any <HeartbeatRequest>(), Arg.Any <CancellationToken>(), Arg.Any <IRequestContext>())
.Returns(_ => Task.FromResult(new HeartbeatResponse(ErrorCode.NONE)));
var config = new ConsumerConfiguration(heartbeatTimeout: TimeSpan.FromMilliseconds(heartbeatMilliseconds * 2));
var request = new JoinGroupRequest(TestConfig.GroupId(), config.GroupHeartbeat, "", ConsumerEncoder.Protocol, new [] { protocol });
var memberId = Guid.NewGuid().ToString("N");
var response = new JoinGroupResponse(ErrorCode.NONE, 1, protocol.protocol_name, memberId, memberId, new [] { new JoinGroupResponse.Member(memberId, new ConsumerProtocolMetadata("mine")) });
using (new GroupConsumer(router, request.group_id, request.protocol_type, response, config)) {
await Task.Delay(totalMilliseconds);
}
Assert.That(conn.ReceivedCalls()
.Count(c => {
if (c.GetMethodInfo().Name != nameof(Connection.SendAsync))
{
return(false);
}
var s = c.GetArguments()[0] as HeartbeatRequest;
if (s == null)
{
return(false);
}
return(s.group_id == request.group_id && s.member_id == memberId && s.generation_id == response.generation_id);
}), Is.InRange(expectedHeartbeats - 1, expectedHeartbeats + 1));
}
public void JoinGroupResponse()
{
var response1 = new JoinGroupResponse {
GenerationId = _random.Next(),
GroupProtocol = Guid.NewGuid().ToString(),
LeaderId = Guid.NewGuid().ToString(),
MemberId = Guid.NewGuid().ToString(),
Members = new[] {
new JoinGroupResponseMember {
MemberId = Guid.NewGuid().ToString(),
MemberMetadata = new JoinGroupMemberMetadata {
Topics = new [] {
Guid.NewGuid().ToString(),
},
UserData = new Byte[] { 1, 2, 3 },
Version = (Int16)Guid.NewGuid().GetHashCode()
}
}
}
};
Stream binary1 = new MemoryStream();
response1.Serialize(binary1);
binary1.Seek(0L, SeekOrigin.Begin);
var response2 = new JoinGroupResponse();
response2.Deserialize(binary1);
var compareLogic = new CompareLogic();
var result = compareLogic.Compare(response1, response2);
Assert.True(result.AreEqual);
Stream binary2 = new MemoryStream();
response2.Serialize(binary2);
Assert.Equal(binary1.Length, binary2.Length);
using (var stream1 = new MemoryStream())
using (var stream2 = new MemoryStream()) {
binary1.Seek(0L, SeekOrigin.Begin);
binary1.CopyTo(stream1);
binary2.Seek(0L, SeekOrigin.Begin);
binary2.CopyTo(stream2);
Assert.Equal(stream1.Length, stream2.Length);
stream1.Seek(0L, SeekOrigin.Begin);
var bytes1 = stream1.ToArray();
stream2.Seek(0L, SeekOrigin.Begin);
var bytes2 = stream2.ToArray();
Assert.Equal(bytes1.Length, bytes2.Length);
for (int i = 0; i < bytes1.Length; i++)
{
Assert.Equal(bytes1[i], bytes2[i]);
}
}
}
private static bool TryEncodeResponse(IKafkaWriter writer, IRequestContext context, JoinGroupResponse response)
{
if (response == null)
{
return(false);
}
writer.Write(response.error_code)
.Write(response.generation_id)
.Write(response.group_protocol)
.Write(response.leader_id)
.Write(response.member_id)
.Write(response.members.Count);
var encoder = context.GetEncoder();
foreach (var member in response.members)
{
writer.Write(member.member_id)
.Write(member.member_metadata, encoder);
}
return(true);
}
private static bool TryEncodeResponse(IKafkaWriter writer, IRequestContext context, JoinGroupResponse response)
{
if (response == null)
{
return(false);
}
writer.Write(response.ErrorCode)
.Write(response.GenerationId)
.Write(response.GroupProtocol)
.Write(response.LeaderId)
.Write(response.MemberId)
.Write(response.Members.Count);
var encoder = context.GetEncoder();
foreach (var member in response.Members)
{
writer.Write(member.MemberId)
.Write(member.Metadata, encoder);
}
return(true);
}
