/// <summary> /// Called from the consumer when joining/rejoining.. /// /// See https://cwiki.apache.org/confluence/display/KAFKA/Kafka+Client-side+Assignment+Proposal /// </summary> internal void OnJoinGroup(JoinGroupResponse response) { if (response.member_id != MemberId) { throw new ArgumentOutOfRangeException(nameof(response), $"Member is not valid ({MemberId} != {response.member_id})"); } if (_disposeCount > 0) { throw new ObjectDisposedException($"Consumer {{GroupId:{GroupId},MemberId:{MemberId}}} is no longer valid"); } _joinSemaphore.Lock( () => { IsLeader = response.leader_id == MemberId; GenerationId = response.generation_id; _groupProtocol = response.group_protocol; _memberMetadata = response.members.ToImmutableDictionary(member => member.member_id, member => member.member_metadata); Router.Log.Info(() => LogEvent.Create(GenerationId > 1 ? $"Consumer {MemberId} Rejoined {GroupId} Generation{GenerationId}" : $"Consumer {MemberId} Joined {GroupId}")); }, _disposeToken.Token); }
public void JoinGroupResponse( [Values( ErrorResponseCode.None, ErrorResponseCode.OffsetMetadataTooLarge )] ErrorResponseCode errorCode, [Values(0, 1, 20000)] int generationId, [Values("consumer", "other")] string protocol, [Values("test", "a groupId")] string leaderId, [Values("", "an existing member")] string memberId, [Values(1, 10)] int memberCount) { var members = new List <JoinGroupResponse.Member>(); for (var m = 0; m < memberCount; m++) { var bytes = new byte[memberCount * 100]; _randomizer.NextBytes(bytes); members.Add(new JoinGroupResponse.Member(memberId + m, new ByteMember(bytes))); } var response = new JoinGroupResponse(errorCode, generationId, protocol, leaderId, memberId, members); response.AssertCanEncodeDecodeResponse(0); }
public void JoinGroupResponse( [Values( ErrorCode.NONE, ErrorCode.OFFSET_METADATA_TOO_LARGE )] ErrorCode errorCode, [Values(0, 1, 20000)] int generationId, [Values("consumer", "other")] string protocol, [Values("test", "a groupId")] string leaderId, [Values("", "an existing member")] string memberId, [Values(1, 10)] int memberCount) { var members = new List <JoinGroupResponse.Member>(); for (var m = 0; m < memberCount; m++) { var bytes = new byte[memberCount * 100]; _randomizer.NextBytes(bytes); members.Add(new JoinGroupResponse.Member(memberId + m, new ByteTypeMetadata("known", new ArraySegment <byte>(bytes)))); } var response = new JoinGroupResponse(errorCode, generationId, "known", leaderId, memberId, members); response.AssertCanEncodeDecodeResponse(0, new ByteMembershipEncoder(protocol)); }
public async Task ConsumerHeartbeatsUntilDisposed(int heartbeatMilliseconds) { var protocol = new JoinGroupRequest.GroupProtocol(new ConsumerProtocolMetadata("mine")); var router = Substitute.For <IRouter>(); var conn = Substitute.For <IConnection>(); router.GetConnectionAsync(Arg.Any <string>(), Arg.Any <string>(), Arg.Any <CancellationToken>()) .Returns(_ => Task.FromResult(conn)); router.SyncGroupAsync(Arg.Any <SyncGroupRequest>(), Arg.Any <IRequestContext>(), Arg.Any <IRetry>(), Arg.Any <CancellationToken>()) .Returns(_ => Task.FromResult(new SyncGroupResponse(ErrorCode.NONE, new ConsumerMemberAssignment(new [] { new TopicPartition("name", 0) })))); conn.SendAsync(Arg.Any <HeartbeatRequest>(), Arg.Any <CancellationToken>(), Arg.Any <IRequestContext>()) .Returns(_ => Task.FromResult(new HeartbeatResponse(ErrorCode.NETWORK_EXCEPTION))); var heartbeat = TimeSpan.FromMilliseconds(heartbeatMilliseconds); var config = new ConsumerConfiguration(heartbeatTimeout: heartbeat, coordinationRetry: Retry.Until(heartbeat, maximumDelay: TimeSpan.FromMilliseconds(50))); var request = new JoinGroupRequest(TestConfig.GroupId(), config.GroupHeartbeat, "", ConsumerEncoder.Protocol, new [] { protocol }); var memberId = Guid.NewGuid().ToString("N"); var response = new JoinGroupResponse(ErrorCode.NONE, 1, protocol.protocol_name, memberId, memberId, new [] { new JoinGroupResponse.Member(memberId, new ConsumerProtocolMetadata("mine")) }); using (new GroupConsumer(router, request.group_id, request.protocol_type, response, config)) { await Task.Delay(heartbeatMilliseconds * 3); #pragma warning disable CS4014 // Because this call is not awaited, execution of the current method continues before the call is completed conn.DidNotReceive().SendAsync( Arg.Is((LeaveGroupRequest s) => s.group_id == request.group_id && s.member_id == memberId), Arg.Any <CancellationToken>(), Arg.Any <IRequestContext>()); } conn.Received().SendAsync( Arg.Is((LeaveGroupRequest s) => s.group_id == request.group_id && s.member_id == memberId), Arg.Any <CancellationToken>(), Arg.Any <IRequestContext>()); #pragma warning restore CS4014 // Because this call is not awaited, execution of the current method continues before the call is completed Assert.That(conn.ReceivedCalls().Count(c => c.GetMethodInfo().Name == nameof(Connection.SendAsync) && (c.GetArguments()[0] as HeartbeatRequest) != null), Is.AtLeast(2)); }
public async Task ConsumerHeartbeatsAtDesiredIntervals(int expectedHeartbeats, int heartbeatMilliseconds, int totalMilliseconds) { var protocol = new JoinGroupRequest.GroupProtocol(new ConsumerProtocolMetadata("mine")); var router = Substitute.For <IRouter>(); var conn = Substitute.For <IConnection>(); router.GetConnectionAsync(Arg.Any <string>(), Arg.Any <string>(), Arg.Any <CancellationToken>()) .Returns(_ => Task.FromResult(conn)); router.SyncGroupAsync(Arg.Any <SyncGroupRequest>(), Arg.Any <IRequestContext>(), Arg.Any <IRetry>(), Arg.Any <CancellationToken>()) .Returns(_ => Task.FromResult(new SyncGroupResponse(ErrorCode.NONE, new ConsumerMemberAssignment(new [] { new TopicPartition("name", 0) })))); conn.SendAsync(Arg.Any <HeartbeatRequest>(), Arg.Any <CancellationToken>(), Arg.Any <IRequestContext>()) .Returns(_ => Task.FromResult(new HeartbeatResponse(ErrorCode.NONE))); var config = new ConsumerConfiguration(heartbeatTimeout: TimeSpan.FromMilliseconds(heartbeatMilliseconds * 2)); var request = new JoinGroupRequest(TestConfig.GroupId(), config.GroupHeartbeat, "", ConsumerEncoder.Protocol, new [] { protocol }); var memberId = Guid.NewGuid().ToString("N"); var response = new JoinGroupResponse(ErrorCode.NONE, 1, protocol.protocol_name, memberId, memberId, new [] { new JoinGroupResponse.Member(memberId, new ConsumerProtocolMetadata("mine")) }); using (new GroupConsumer(router, request.group_id, request.protocol_type, response, config)) { await Task.Delay(totalMilliseconds); } Assert.That(conn.ReceivedCalls() .Count(c => { if (c.GetMethodInfo().Name != nameof(Connection.SendAsync)) { return(false); } var s = c.GetArguments()[0] as HeartbeatRequest; if (s == null) { return(false); } return(s.group_id == request.group_id && s.member_id == memberId && s.generation_id == response.generation_id); }), Is.InRange(expectedHeartbeats - 1, expectedHeartbeats + 1)); }
public void JoinGroupResponse() { var response1 = new JoinGroupResponse { GenerationId = _random.Next(), GroupProtocol = Guid.NewGuid().ToString(), LeaderId = Guid.NewGuid().ToString(), MemberId = Guid.NewGuid().ToString(), Members = new[] { new JoinGroupResponseMember { MemberId = Guid.NewGuid().ToString(), MemberMetadata = new JoinGroupMemberMetadata { Topics = new [] { Guid.NewGuid().ToString(), }, UserData = new Byte[] { 1, 2, 3 }, Version = (Int16)Guid.NewGuid().GetHashCode() } } } }; Stream binary1 = new MemoryStream(); response1.Serialize(binary1); binary1.Seek(0L, SeekOrigin.Begin); var response2 = new JoinGroupResponse(); response2.Deserialize(binary1); var compareLogic = new CompareLogic(); var result = compareLogic.Compare(response1, response2); Assert.True(result.AreEqual); Stream binary2 = new MemoryStream(); response2.Serialize(binary2); Assert.Equal(binary1.Length, binary2.Length); using (var stream1 = new MemoryStream()) using (var stream2 = new MemoryStream()) { binary1.Seek(0L, SeekOrigin.Begin); binary1.CopyTo(stream1); binary2.Seek(0L, SeekOrigin.Begin); binary2.CopyTo(stream2); Assert.Equal(stream1.Length, stream2.Length); stream1.Seek(0L, SeekOrigin.Begin); var bytes1 = stream1.ToArray(); stream2.Seek(0L, SeekOrigin.Begin); var bytes2 = stream2.ToArray(); Assert.Equal(bytes1.Length, bytes2.Length); for (int i = 0; i < bytes1.Length; i++) { Assert.Equal(bytes1[i], bytes2[i]); } } }
private static bool TryEncodeResponse(IKafkaWriter writer, IRequestContext context, JoinGroupResponse response) { if (response == null) { return(false); } writer.Write(response.error_code) .Write(response.generation_id) .Write(response.group_protocol) .Write(response.leader_id) .Write(response.member_id) .Write(response.members.Count); var encoder = context.GetEncoder(); foreach (var member in response.members) { writer.Write(member.member_id) .Write(member.member_metadata, encoder); } return(true); }
private static bool TryEncodeResponse(IKafkaWriter writer, IRequestContext context, JoinGroupResponse response) { if (response == null) { return(false); } writer.Write(response.ErrorCode) .Write(response.GenerationId) .Write(response.GroupProtocol) .Write(response.LeaderId) .Write(response.MemberId) .Write(response.Members.Count); var encoder = context.GetEncoder(); foreach (var member in response.Members) { writer.Write(member.MemberId) .Write(member.Metadata, encoder); } return(true); }