public void SyncConsumerGroupRequest( [Values("test", "a groupId")] string groupId, [Values(0, 1, 20000)] int generationId, [Values("", "an existing member")] string memberId, [Values("consumer")] string protocolType, [Values(1, 10)] int assignmentsPerRequest) { var encoder = new ConsumerEncoder(); var assignments = new List <SyncGroupRequest.GroupAssignment>(); for (var a = 0; a < assignmentsPerRequest; a++) { var topics = new List <TopicPartition>(); for (var t = 0; t < assignmentsPerRequest; t++) { topics.Add(new TopicPartition(groupId + t, t)); } var userData = new byte[assignmentsPerRequest * 100]; _randomizer.NextBytes(userData); var assignment = new ConsumerMemberAssignment(topics, new ArraySegment <byte>(userData), 0); assignments.Add(new SyncGroupRequest.GroupAssignment(protocolType + a, assignment)); } var request = new SyncGroupRequest(groupId, generationId, memberId, assignments); request.AssertCanEncodeDecodeRequest(0, encoder); }
public async Task SyncGroupAsync(CancellationToken cancellationToken) { if (_disposeCount > 0) { throw new ObjectDisposedException($"Consumer {{GroupId:{GroupId},MemberId:{MemberId}}} is no longer valid"); } var groupAssignments = await _joinSemaphore.LockAsync( async() => { if (IsLeader) { var encoder = Encoders[ProtocolType]; var assigner = encoder.GetAssignor(_groupProtocol); var assignments = await assigner.AssignMembersAsync(Router, GroupId, GenerationId, _memberMetadata, cancellationToken).ConfigureAwait(false); return(assignments.Select(pair => new SyncGroupRequest.GroupAssignment(pair.Key, pair.Value))); } return(null); }, _disposeToken.Token).ConfigureAwait(false); await Task.WhenAll(_batches.Values.Select(b => b.CommitMarkedIgnoringDisposedAsync(cancellationToken))).ConfigureAwait(false); SyncGroupResponse response; try { var request = new SyncGroupRequest(GroupId, GenerationId, MemberId, groupAssignments); response = await Router.SyncGroupAsync(request, new RequestContext(protocolType : ProtocolType), Configuration.GroupCoordinationRetry, cancellationToken).ConfigureAwait(false); } catch (RequestException ex) { switch (ex.ErrorCode) { case ErrorCode.REBALANCE_IN_PROGRESS: Router.Log.Info(() => LogEvent.Create(ex.Message)); TriggerRejoin(); return; case ErrorCode.GROUP_AUTHORIZATION_FAILED: case ErrorCode.UNKNOWN_MEMBER_ID: Router.Log.Warn(() => LogEvent.Create(ex)); _leaveOnDispose = false; // no point in attempting to leave the group since it will fail _disposeToken.Cancel(); break; } throw; } _syncSemaphore.Lock(() => { _assignment = response.member_assignment; var validPartitions = response.member_assignment.PartitionAssignments.ToImmutableHashSet(); var invalidPartitions = _batches.Where(pair => !validPartitions.Contains(pair.Key)).ToList(); foreach (var invalidPartition in invalidPartitions) { invalidPartition.Value.Dispose(); } _batches = _batches.RemoveRange(invalidPartitions.Select(pair => pair.Key)); }, _disposeToken.Token); if (Interlocked.Increment(ref _syncCount) == 1) { _fetchSemaphore.Release(); } }
public void InterfacesAreFormattedWithinProtocol() { var request = new SyncGroupRequest("group", 5, "member", new[] { new SyncGroupRequest.GroupAssignment("member", new ConsumerMemberAssignment(new[] { new TopicPartition("topic-foo", 0), new TopicPartition("topic", 1) })) }); var formatted = request.ToString(); Assert.That(formatted.Contains("topic:topic-foo")); Assert.That(formatted.Contains("partition_id:1")); }
public Task <SyncGroupResponse> SyncGroupAsync(SyncGroupRequest request, IRequestContext context, IRetry retryPolicy, CancellationToken cancellationToken) { if (request.group_assignments.Count > 0) { var value = new Tuple <IImmutableList <SyncGroupRequest.GroupAssignment>, int>(request.group_assignments, request.generation_id); _memberAssignmentCache.AddOrUpdate(request.group_id, value, (key, old) => value); } return(this.SendAsync(request, request.group_id, cancellationToken, context, retryPolicy)); }
public SyncGroupResponse SyncGroup(String groupId, String memberId, Int32 generationId, IList <SyncGroupGroupAssignment> assignments) { var request = new SyncGroupRequest(groupId, generationId, memberId); request.GroupAssignments = assignments; var response = (SyncGroupResponse)_client.SubmitRequest(_coordinateBroker, request); //response.TryThrowFirstErrorOccured(); return(response); }
public void SyncGroupRequest( [Values("test", "a groupId")] string groupId, [Values(0, 1, 20000)] int generationId, [Values("", "an existing member")] string memberId, [Values("consumer", "other")] string protocolType, [Values(1, 10)] int assignmentsPerRequest) { var assignments = new List <SyncGroupRequest.GroupAssignment>(); for (var a = 0; a < assignmentsPerRequest; a++) { var bytes = new byte[assignmentsPerRequest * 100]; _randomizer.NextBytes(bytes); assignments.Add(new SyncGroupRequest.GroupAssignment(protocolType + a, new ByteMember(bytes))); } var request = new SyncGroupRequest(groupId, generationId, memberId, assignments); request.AssertCanEncodeDecodeRequest(0); }
public ValueTask <SyncGroupResponse> SyncGroup(SyncGroupRequest request) => SendRequest(ApiKey.SyncGroup, 3, request, new SyncGroupRequestWriter(), new SyncGroupResponseReader());
public void SyncGroupRequest() { var request = new SyncGroupRequest(); request.GroupId = Guid.NewGuid().ToString(); request.GenerationId = _random.Next(); request.MemberId = Guid.NewGuid().ToString(); request.GroupAssignments = new[] { new SyncGroupGroupAssignment { MemberId = Guid.NewGuid().ToString(), MemberAssignment = new SyncGroupMemberAssignment { PartitionAssignments = new [] { new SyncGroupPartitionAssignment { Topic = Guid.NewGuid().ToString(), Partitions = new [] { Guid.NewGuid().GetHashCode(), Guid.NewGuid().GetHashCode() } } }, Version = (Int16)Guid.NewGuid().GetHashCode(), UserData = new Byte[] { 1, 2, 3 } } } }; Stream binary1 = new MemoryStream(); request.Serialize(binary1); binary1.Seek(0L, SeekOrigin.Begin); var request2 = new SyncGroupRequest(); request2.Deserialize(binary1); var compareLogic = new CompareLogic(); var result = compareLogic.Compare(request, request2); Assert.True(result.AreEqual); Stream binary2 = new MemoryStream(); request.Serialize(binary2); Assert.Equal(binary1.Length, binary2.Length); using (var stream1 = new MemoryStream()) using (var stream2 = new MemoryStream()) { binary1.Seek(0L, SeekOrigin.Begin); binary1.CopyTo(stream1); binary2.Seek(0L, SeekOrigin.Begin); binary2.CopyTo(stream2); Assert.Equal(stream1.Length, stream2.Length); stream1.Seek(0L, SeekOrigin.Begin); var bytes1 = stream1.ToArray(); stream2.Seek(0L, SeekOrigin.Begin); var bytes2 = stream2.ToArray(); Assert.Equal(bytes1.Length, bytes2.Length); for (int i = 0; i < bytes1.Length; i++) { Assert.Equal(bytes1[i], bytes2[i]); } } }