public void SyncConsumerGroupRequest(
[Values("test", "a groupId")] string groupId,
[Values(0, 1, 20000)] int generationId,
[Values("", "an existing member")] string memberId,
[Values("consumer")] string protocolType,
[Values(1, 10)] int assignmentsPerRequest)
{
var encoder = new ConsumerEncoder();
var assignments = new List <SyncGroupRequest.GroupAssignment>();
for (var a = 0; a < assignmentsPerRequest; a++)
{
var topics = new List <TopicPartition>();
for (var t = 0; t < assignmentsPerRequest; t++)
{
topics.Add(new TopicPartition(groupId + t, t));
}
var userData = new byte[assignmentsPerRequest * 100];
_randomizer.NextBytes(userData);
var assignment = new ConsumerMemberAssignment(topics, new ArraySegment <byte>(userData), 0);
assignments.Add(new SyncGroupRequest.GroupAssignment(protocolType + a, assignment));
}
var request = new SyncGroupRequest(groupId, generationId, memberId, assignments);
request.AssertCanEncodeDecodeRequest(0, encoder);
}
public async Task SyncGroupAsync(CancellationToken cancellationToken)
{
if (_disposeCount > 0)
{
throw new ObjectDisposedException($"Consumer {{GroupId:{GroupId},MemberId:{MemberId}}} is no longer valid");
}
var groupAssignments = await _joinSemaphore.LockAsync(
async() => {
if (IsLeader)
{
var encoder = Encoders[ProtocolType];
var assigner = encoder.GetAssignor(_groupProtocol);
var assignments = await assigner.AssignMembersAsync(Router, GroupId, GenerationId, _memberMetadata, cancellationToken).ConfigureAwait(false);
return(assignments.Select(pair => new SyncGroupRequest.GroupAssignment(pair.Key, pair.Value)));
}
return(null);
}, _disposeToken.Token).ConfigureAwait(false);
await Task.WhenAll(_batches.Values.Select(b => b.CommitMarkedIgnoringDisposedAsync(cancellationToken))).ConfigureAwait(false);
SyncGroupResponse response;
try {
var request = new SyncGroupRequest(GroupId, GenerationId, MemberId, groupAssignments);
response = await Router.SyncGroupAsync(request, new RequestContext(protocolType : ProtocolType), Configuration.GroupCoordinationRetry, cancellationToken).ConfigureAwait(false);
} catch (RequestException ex) {
switch (ex.ErrorCode)
{
case ErrorCode.REBALANCE_IN_PROGRESS:
Router.Log.Info(() => LogEvent.Create(ex.Message));
TriggerRejoin();
return;
case ErrorCode.GROUP_AUTHORIZATION_FAILED:
case ErrorCode.UNKNOWN_MEMBER_ID:
Router.Log.Warn(() => LogEvent.Create(ex));
_leaveOnDispose = false; // no point in attempting to leave the group since it will fail
_disposeToken.Cancel();
break;
}
throw;
}
_syncSemaphore.Lock(() => {
_assignment = response.member_assignment;
var validPartitions = response.member_assignment.PartitionAssignments.ToImmutableHashSet();
var invalidPartitions = _batches.Where(pair => !validPartitions.Contains(pair.Key)).ToList();
foreach (var invalidPartition in invalidPartitions)
{
invalidPartition.Value.Dispose();
}
_batches = _batches.RemoveRange(invalidPartitions.Select(pair => pair.Key));
}, _disposeToken.Token);
if (Interlocked.Increment(ref _syncCount) == 1)
{
_fetchSemaphore.Release();
}
}
public void InterfacesAreFormattedWithinProtocol()
{
var request = new SyncGroupRequest("group", 5, "member", new[] { new SyncGroupRequest.GroupAssignment("member", new ConsumerMemberAssignment(new[] { new TopicPartition("topic-foo", 0), new TopicPartition("topic", 1) })) });
var formatted = request.ToString();
Assert.That(formatted.Contains("topic:topic-foo"));
Assert.That(formatted.Contains("partition_id:1"));
}
public Task <SyncGroupResponse> SyncGroupAsync(SyncGroupRequest request, IRequestContext context, IRetry retryPolicy, CancellationToken cancellationToken)
{
if (request.group_assignments.Count > 0)
{
var value = new Tuple <IImmutableList <SyncGroupRequest.GroupAssignment>, int>(request.group_assignments, request.generation_id);
_memberAssignmentCache.AddOrUpdate(request.group_id, value, (key, old) => value);
}
return(this.SendAsync(request, request.group_id, cancellationToken, context, retryPolicy));
}
public SyncGroupResponse SyncGroup(String groupId, String memberId, Int32 generationId, IList <SyncGroupGroupAssignment> assignments)
{
var request = new SyncGroupRequest(groupId, generationId, memberId);
request.GroupAssignments = assignments;
var response = (SyncGroupResponse)_client.SubmitRequest(_coordinateBroker, request);
//response.TryThrowFirstErrorOccured();
return(response);
}
public void SyncGroupRequest(
[Values("test", "a groupId")] string groupId,
[Values(0, 1, 20000)] int generationId,
[Values("", "an existing member")] string memberId,
[Values("consumer", "other")] string protocolType,
[Values(1, 10)] int assignmentsPerRequest)
{
var assignments = new List <SyncGroupRequest.GroupAssignment>();
for (var a = 0; a < assignmentsPerRequest; a++)
{
var bytes = new byte[assignmentsPerRequest * 100];
_randomizer.NextBytes(bytes);
assignments.Add(new SyncGroupRequest.GroupAssignment(protocolType + a, new ByteMember(bytes)));
}
var request = new SyncGroupRequest(groupId, generationId, memberId, assignments);
request.AssertCanEncodeDecodeRequest(0);
}
public ValueTask <SyncGroupResponse> SyncGroup(SyncGroupRequest request) => SendRequest(ApiKey.SyncGroup, 3, request, new SyncGroupRequestWriter(), new SyncGroupResponseReader());
public void SyncGroupRequest()
{
var request = new SyncGroupRequest();
request.GroupId = Guid.NewGuid().ToString();
request.GenerationId = _random.Next();
request.MemberId = Guid.NewGuid().ToString();
request.GroupAssignments = new[] {
new SyncGroupGroupAssignment {
MemberId = Guid.NewGuid().ToString(),
MemberAssignment = new SyncGroupMemberAssignment {
PartitionAssignments = new [] {
new SyncGroupPartitionAssignment {
Topic = Guid.NewGuid().ToString(),
Partitions = new [] {
Guid.NewGuid().GetHashCode(),
Guid.NewGuid().GetHashCode()
}
}
},
Version = (Int16)Guid.NewGuid().GetHashCode(),
UserData = new Byte[] { 1, 2, 3 }
}
}
};
Stream binary1 = new MemoryStream();
request.Serialize(binary1);
binary1.Seek(0L, SeekOrigin.Begin);
var request2 = new SyncGroupRequest();
request2.Deserialize(binary1);
var compareLogic = new CompareLogic();
var result = compareLogic.Compare(request, request2);
Assert.True(result.AreEqual);
Stream binary2 = new MemoryStream();
request.Serialize(binary2);
Assert.Equal(binary1.Length, binary2.Length);
using (var stream1 = new MemoryStream())
using (var stream2 = new MemoryStream()) {
binary1.Seek(0L, SeekOrigin.Begin);
binary1.CopyTo(stream1);
binary2.Seek(0L, SeekOrigin.Begin);
binary2.CopyTo(stream2);
Assert.Equal(stream1.Length, stream2.Length);
stream1.Seek(0L, SeekOrigin.Begin);
var bytes1 = stream1.ToArray();
stream2.Seek(0L, SeekOrigin.Begin);
var bytes2 = stream2.ToArray();
Assert.Equal(bytes1.Length, bytes2.Length);
for (int i = 0; i < bytes1.Length; i++)
{
Assert.Equal(bytes1[i], bytes2[i]);
}
}
}
