private async Task <ImmutableList <ProduceTask> > GetNextBatchAsync()
{
var batch = ImmutableList <ProduceTask> .Empty;
try {
if (await _produceMessageQueue.OutputAvailableAsync(_disposeToken.Token).ConfigureAwait(false))
{
using (var cancellation = new TimedCancellation(_disposeToken.Token, Configuration.BatchMaxDelay)) {
var messageCount = 0;
while (messageCount < Configuration.BatchSize && !cancellation.Token.IsCancellationRequested)
{
// Try rather than simply Take (in case the collection has been closed and is not empty)
var result = await _produceMessageQueue.DequeueAsync(cancellation.Token).ConfigureAwait(false);
if (result.CancellationToken.IsCancellationRequested)
{
result.Tcs.SetCanceled();
}
else
{
batch = batch.Add(result);
messageCount += result.Messages.Count;
}
}
}
}
} catch (InvalidOperationException) { // From DequeueAsync
} catch (OperationCanceledException) { // cancellation token fired while attempting to get tasks: normal behavior
}
return(batch);
}
public async Task CanConsumeFromMultipleGroups(int groups, int members)
{
using (var timed = new TimedCancellation(CancellationToken.None, TimeSpan.FromMinutes(1))) {
var batchSize = 50;
var totalMessages = 200;
using (var router = await TestConfig.IntegrationOptions.CreateRouterAsync()) {
await router.TemporaryTopicAsync(async topicName =>
{
var allFetched = 0;
var tasks = new List <Task>();
var groupIdPrefix = TestConfig.GroupId();
for (var group = 0; group < groups; group++)
{
var groupId = $"{groupIdPrefix}.{group}";
var groupFetched = 0;
await ProduceMessages(router, topicName, groupId, totalMessages, Enumerable.Range(0, members));
var cancellation = new CancellationTokenSource();
for (var index = 0; index < members; index++)
{
tasks.Add(Task.Run(async() => {
using (var merged = CancellationTokenSource.CreateLinkedTokenSource(timed.Token, cancellation.Token)) {
var consumer = await router.CreateGroupConsumerAsync(groupId, new ConsumerProtocolMetadata(topicName), TestConfig.IntegrationOptions.ConsumerConfiguration, TestConfig.IntegrationOptions.Encoders, merged.Token);
await consumer.UsingAsync(async() => {
try {
await consumer.FetchAsync(async(batch, token) => {
router.Log.Info(() => LogEvent.Create($"Member {consumer.MemberId} starting batch of {batch.Messages.Count}"));
foreach (var message in batch.Messages)
{
await Task.Delay(1); // do the work...
batch.MarkSuccessful(message);
Interlocked.Increment(ref allFetched);
if (Interlocked.Increment(ref groupFetched) >= totalMessages)
{
cancellation.Cancel();
break;
}
}
router.Log.Info(() => LogEvent.Create($"Member {consumer.MemberId} finished batch size {batch.Messages.Count} ({groupFetched} of {totalMessages})"));
}, merged.Token, batchSize);
} catch (ObjectDisposedException) {
// ignore
}
});
}
}, timed.Token));
}
}
await Task.WhenAll(tasks);
Assert.That(allFetched, Is.AtLeast(totalMessages *groups));
// await Task.WhenAny(Task.WhenAll(tasks), Task.Delay(TimeSpan.FromMinutes(1)));
}, 10);
}
}
}
public async Task ShouldDisposeEvenWhilePollingToReconnect()
{
var connectionAttempt = -1;
var config = new ConnectionConfiguration(Retry.Until(TimeSpan.FromSeconds(10)), onConnecting: (e, a, _) => connectionAttempt = a);
var endpoint = TestConfig.ServerEndpoint();
using (var transport = CreateTransport(endpoint, config, TestConfig.Log)) {
var taskResult = transport.ConnectAsync(CancellationToken.None);
await AssertAsync.ThatEventually(() => connectionAttempt >= 0, () => $"attempt {connectionAttempt}");
transport.Dispose();
using (var cancellation = new TimedCancellation(CancellationToken.None, TimeSpan.FromSeconds(3))) {
await AssertAsync.Throws <ObjectDisposedException>(() => taskResult.ThrowIfCancellationRequested(cancellation.Token));
}
}
}
