private async Task ProduceAndSendBatchAsync(List<TopicMessage> messages, CancellationToken cancellationToken)
{
Interlocked.Add(ref _inFlightMessageCount, messages.Count);
var topics = messages.GroupBy(batch => batch.Topic).Select(batch => batch.Key).ToArray();
await BrokerRouter.RefreshMissingTopicMetadata(topics);
//we must send a different produce request for each ack level and timeout combination.
foreach (var ackLevelBatch in messages.GroupBy(batch => new { batch.Acks, batch.Timeout }))
{
var messageByRouter = ackLevelBatch.Select(batch => new
{
TopicMessage = batch,
Route = batch.Partition.HasValue ? BrokerRouter.SelectBrokerRouteFromLocalCache(batch.Topic, batch.Partition.Value) : BrokerRouter.SelectBrokerRouteFromLocalCache(batch.Topic, batch.Message.Key)
})
.GroupBy(x => new { x.Route, x.TopicMessage.Topic, x.TopicMessage.Codec });
var sendTasks = new List<BrokerRouteSendBatch>();
foreach (var group in messageByRouter)
{
var payload = new Payload
{
Codec = group.Key.Codec,
Topic = group.Key.Topic,
Partition = group.Key.Route.PartitionId,
Messages = group.Select(x => x.TopicMessage.Message).ToList()
};
var request = new ProduceRequest
{
Acks = ackLevelBatch.Key.Acks,
TimeoutMS = (int)ackLevelBatch.Key.Timeout.TotalMilliseconds,
Payload = new List<Payload> { payload }
};
await _semaphoreMaximumAsync.WaitAsync(cancellationToken).ConfigureAwait(false);
var sendGroupTask = _protocolGateway.SendProtocolRequest(request, group.Key.Topic, group.Key.Route.PartitionId);// group.Key.Route.Connection.SendAsync(request);
var brokerSendTask = new BrokerRouteSendBatch
{
Route = group.Key.Route,
Task = sendGroupTask,
MessagesSent = group.Select(x => x.TopicMessage).ToList()
};
//ensure the async is released as soon as each task is completed
brokerSendTask.Task.ContinueWith(t => { _semaphoreMaximumAsync.Release(); }, cancellationToken);
sendTasks.Add(brokerSendTask);
}
try
{
await Task.WhenAll(sendTasks.Select(x => x.Task)).ConfigureAwait(false);
foreach (var task in sendTasks)
{
//TODO when we dont ask for an ACK, result is an empty list. Which FirstOrDefault returns null. Dont like this...
task.MessagesSent.ForEach(async x => x.Tcs.TrySetResult(await task.Task));
}
}
catch
{
//if an error occurs here, all we know is some or all of the messages in this ackBatch failed.
var failedTask = sendTasks.FirstOrDefault(t => t.Task.IsFaulted);
if (failedTask != null)
{
foreach (var topicMessageBatch in ackLevelBatch)
{
topicMessageBatch.Tcs.TrySetException(
new KafkaApplicationException(
"An exception occured while executing a send operation against {0}. Exception:{1}",
failedTask.Route, failedTask.Task.Exception));
}
}
}
finally
{
Interlocked.Add(ref _inFlightMessageCount, messages.Count * -1);
}
}
}
private async Task ProduceAndSendBatchAsync(List<TopicMessage> messages, CancellationToken cancellationToken)
{
Interlocked.Add(ref _inFlightMessageCount, messages.Count);
var topics = messages.GroupBy(batch => batch.Topic).Select(batch => batch.Key).ToArray();
await BrokerRouter.RefreshMissingTopicMetadata(topics).ConfigureAwait(false);
//we must send a different produce request for each ack level and timeout combination.
foreach (var ackLevelBatch in messages.GroupBy(batch => new { batch.Acks, batch.Timeout }))
{
var messageByRouter = ackLevelBatch.Select(batch => new
{
TopicMessage = batch,
AckLevel = ackLevelBatch.Key.Acks,
Route = batch.Partition.HasValue ? BrokerRouter.SelectBrokerRouteFromLocalCache(batch.Topic, batch.Partition.Value) : BrokerRouter.SelectBrokerRouteFromLocalCache(batch.Topic, batch.Message.Key)
}).GroupBy(x => new { x.Route, x.TopicMessage.Topic, x.TopicMessage.Codec, x.AckLevel });
var sendTasks = new List<BrokerRouteSendBatch>();
foreach (var group in messageByRouter)
{
var payload = new Payload
{
Codec = group.Key.Codec,
Topic = group.Key.Topic,
Partition = group.Key.Route.PartitionId,
Messages = group.Select(x => x.TopicMessage.Message).ToList()
};
var request = new ProduceRequest
{
Acks = ackLevelBatch.Key.Acks,
TimeoutMS = (int)ackLevelBatch.Key.Timeout.TotalMilliseconds,
Payload = new List<Payload> { payload }
};
await _semaphoreMaximumAsync.WaitAsync(cancellationToken).ConfigureAwait(false);
var sendGroupTask = _protocolGateway.SendProtocolRequest(request, group.Key.Topic, group.Key.Route.PartitionId);
var brokerSendTask = new BrokerRouteSendBatch
{
Route = group.Key.Route,
Task = sendGroupTask,
MessagesSent = group.Select(x => x.TopicMessage).ToList(),
AckLevel = group.Key.AckLevel
};
//ensure the async is released as soon as each task is completed //TODO: remove it from ack level 0 , don't like it
brokerSendTask.Task.ContinueWith(t => { _semaphoreMaximumAsync.Release(); }, cancellationToken);
sendTasks.Add(brokerSendTask);
}
try
{
await Task.WhenAll(sendTasks.Select(x => x.Task)).ConfigureAwait(false);
}
catch (Exception ex)
{
BrokerRouter.Log.ErrorFormat("Exception[{0}] stacktrace[{1}]", ex.Message, ex.StackTrace);
}
await SetResult(sendTasks);
Interlocked.Add(ref _inFlightMessageCount, messages.Count * -1);
}
}
private async Task ProduceAndSendBatchAsync(List<TopicMessage> batchs, CancellationToken cancellationToken)
{
//we must send a different produce request for each ack level and timeout combination.
foreach (var ackLevelBatch in batchs.GroupBy(batch => new { batch.Acks, batch.Timeout }))
{
var messageByRouter = ackLevelBatch.Select(batch => new
{
TopicMessage = batch,
Route = BrokerRouter.SelectBrokerRoute(batch.Topic, batch.Message.Key),
})
.GroupBy(x => new { x.Route, x.TopicMessage.Topic, x.TopicMessage.Codec });
var sendTasks = new List<BrokerRouteSendBatch>();
foreach (var group in messageByRouter)
{
var payload = new Payload
{
Codec = group.Key.Codec,
Topic = group.Key.Topic,
Partition = group.Key.Route.PartitionId,
Messages = group.Select(x => x.TopicMessage.Message).ToList()
};
var request = new ProduceRequest
{
Acks = ackLevelBatch.Key.Acks,
TimeoutMS = (int)ackLevelBatch.Key.Timeout.TotalMilliseconds,
Payload = new List<Payload> { payload }
};
await _semaphoreMaximumAsync.WaitAsync(cancellationToken).ConfigureAwait(false);
var brokerSendTask = new BrokerRouteSendBatch
{
Route = group.Key.Route,
Task = group.Key.Route.Connection.SendAsync(request),
MessagesSent = group.Select(x => x.TopicMessage).ToList()
};
//ensure the async is released as soon as each task is completed
brokerSendTask.Task.ContinueWith(t => { _semaphoreMaximumAsync.Release(); }, cancellationToken,
TaskContinuationOptions.ExecuteSynchronously, TaskScheduler.Default);
sendTasks.Add(brokerSendTask);
}
try
{
await Task.WhenAll(sendTasks.Select(x => x.Task)).ConfigureAwait(false);
foreach (var task in sendTasks)
{
task.MessagesSent.ForEach(x => x.Tcs.TrySetResult(task.Task.Result.FirstOrDefault()));
}
}
catch
{
//if an error occurs here, all we know is some or all of the messages in this ackBatch failed.
var failedTask = sendTasks.FirstOrDefault(t => t.Task.IsFaulted);
if (failedTask != null)
{
foreach (var topicMessageBatch in ackLevelBatch)
{
topicMessageBatch.Tcs.TrySetException(new KafkaApplicationException("An exception occured while executing a send operation against {0}. Exception:{1}",
failedTask.Route, failedTask.Task.Exception));
}
}
}
}
}