private async Task ProduceAndSendBatchAsync(List <TopicMessage> messages, CancellationToken cancellationToken) { Interlocked.Add(ref _inFlightMessageCount, messages.Count); //we must send a different produce request for each ack level and timeout combination. foreach (var ackLevelBatch in messages.GroupBy(batch => new { batch.Acks, batch.Timeout })) { var messageByRouter = ackLevelBatch.Select(batch => new { TopicMessage = batch, Route = BrokerRouter.SelectBrokerRoute(batch.Topic, batch.Message.Key), }) .GroupBy(x => new { x.Route, x.TopicMessage.Topic, x.TopicMessage.Codec }); var sendTasks = new List <BrokerRouteSendBatch>(); foreach (var group in messageByRouter) { var payload = new Payload { Codec = group.Key.Codec, Topic = group.Key.Topic, Partition = group.Key.Route.PartitionId, Messages = group.Select(x => x.TopicMessage.Message).ToList() }; var request = new ProduceRequest { Acks = ackLevelBatch.Key.Acks, TimeoutMS = (int)ackLevelBatch.Key.Timeout.TotalMilliseconds, Payload = new List <Payload> { payload } }; await _semaphoreMaximumAsync.WaitAsync(cancellationToken).ConfigureAwait(false); var brokerSendTask = new BrokerRouteSendBatch { Route = group.Key.Route, Task = group.Key.Route.Connection.SendAsync(request), MessagesSent = group.Select(x => x.TopicMessage).ToList() }; //ensure the async is released as soon as each task is completed brokerSendTask.Task.ContinueWith(t => { _semaphoreMaximumAsync.Release(); }, cancellationToken); sendTasks.Add(brokerSendTask); } try { await Task.WhenAll(sendTasks.Select(x => x.Task)).ConfigureAwait(false); foreach (var task in sendTasks) { //TODO when we dont ask for an ACK, result is an empty list. Which FirstOrDefault returns null. Dont like this... task.MessagesSent.ForEach(x => x.Tcs.TrySetResult(task.Task.Result.FirstOrDefault())); } } catch { //if an error occurs here, all we know is some or all of the messages in this ackBatch failed. var failedTask = sendTasks.FirstOrDefault(t => t.Task.IsFaulted); if (failedTask != null) { foreach (var topicMessageBatch in ackLevelBatch) { topicMessageBatch.Tcs.TrySetException( new KafkaApplicationException( "An exception occured while executing a send operation against {0}. Exception:{1}", failedTask.Route, failedTask.Task.Exception)); } } } finally { Interlocked.Add(ref _inFlightMessageCount, messages.Count * -1); } } }
private async Task ProduceAndSendBatchAsync(List<TopicMessage> messages, CancellationToken cancellationToken) { Interlocked.Add(ref _inFlightMessageCount, messages.Count); var topics = messages.GroupBy(batch => batch.Topic).Select(batch => batch.Key).ToArray(); await BrokerRouter.RefreshMissingTopicMetadata(topics).ConfigureAwait(false); //we must send a different produce request for each ack level and timeout combination. foreach (var ackLevelBatch in messages.GroupBy(batch => new { batch.Acks, batch.Timeout })) { var messageByRouter = ackLevelBatch.Select(batch => new { TopicMessage = batch, AckLevel = ackLevelBatch.Key.Acks, Route = batch.Partition.HasValue ? BrokerRouter.SelectBrokerRouteFromLocalCache(batch.Topic, batch.Partition.Value) : BrokerRouter.SelectBrokerRouteFromLocalCache(batch.Topic, batch.Message.Key) }).GroupBy(x => new { x.Route, x.TopicMessage.Topic, x.TopicMessage.Codec, x.AckLevel }); var sendTasks = new List<BrokerRouteSendBatch>(); foreach (var group in messageByRouter) { var payload = new Payload { Codec = group.Key.Codec, Topic = group.Key.Topic, Partition = group.Key.Route.PartitionId, Messages = group.Select(x => x.TopicMessage.Message).ToList() }; var request = new ProduceRequest { Acks = ackLevelBatch.Key.Acks, TimeoutMS = (int)ackLevelBatch.Key.Timeout.TotalMilliseconds, Payload = new List<Payload> { payload } }; await _semaphoreMaximumAsync.WaitAsync(cancellationToken).ConfigureAwait(false); var sendGroupTask = _protocolGateway.SendProtocolRequest(request, group.Key.Topic, group.Key.Route.PartitionId); var brokerSendTask = new BrokerRouteSendBatch { Route = group.Key.Route, Task = sendGroupTask, MessagesSent = group.Select(x => x.TopicMessage).ToList(), AckLevel = group.Key.AckLevel }; //ensure the async is released as soon as each task is completed //TODO: remove it from ack level 0 , don't like it brokerSendTask.Task.ContinueWith(t => { _semaphoreMaximumAsync.Release(); }, cancellationToken); sendTasks.Add(brokerSendTask); } try { await Task.WhenAll(sendTasks.Select(x => x.Task)).ConfigureAwait(false); } catch (Exception ex) { BrokerRouter.Log.ErrorFormat("Exception[{0}] stacktrace[{1}]", ex.Message, ex.StackTrace); } await SetResult(sendTasks); Interlocked.Add(ref _inFlightMessageCount, messages.Count * -1); } }
private async Task ProduceAndSendBatchAsync(List<TopicMessage> messages, CancellationToken cancellationToken) { Interlocked.Add(ref _inFlightMessageCount, messages.Count); var topics = messages.GroupBy(batch => batch.Topic).Select(batch => batch.Key).ToArray(); await BrokerRouter.RefreshMissingTopicMetadata(topics); //we must send a different produce request for each ack level and timeout combination. foreach (var ackLevelBatch in messages.GroupBy(batch => new { batch.Acks, batch.Timeout })) { var messageByRouter = ackLevelBatch.Select(batch => new { TopicMessage = batch, Route = batch.Partition.HasValue ? BrokerRouter.SelectBrokerRouteFromLocalCache(batch.Topic, batch.Partition.Value) : BrokerRouter.SelectBrokerRouteFromLocalCache(batch.Topic, batch.Message.Key) }) .GroupBy(x => new { x.Route, x.TopicMessage.Topic, x.TopicMessage.Codec }); var sendTasks = new List<BrokerRouteSendBatch>(); foreach (var group in messageByRouter) { var payload = new Payload { Codec = group.Key.Codec, Topic = group.Key.Topic, Partition = group.Key.Route.PartitionId, Messages = group.Select(x => x.TopicMessage.Message).ToList() }; var request = new ProduceRequest { Acks = ackLevelBatch.Key.Acks, TimeoutMS = (int)ackLevelBatch.Key.Timeout.TotalMilliseconds, Payload = new List<Payload> { payload } }; await _semaphoreMaximumAsync.WaitAsync(cancellationToken).ConfigureAwait(false); var sendGroupTask = _protocolGateway.SendProtocolRequest(request, group.Key.Topic, group.Key.Route.PartitionId);// group.Key.Route.Connection.SendAsync(request); var brokerSendTask = new BrokerRouteSendBatch { Route = group.Key.Route, Task = sendGroupTask, MessagesSent = group.Select(x => x.TopicMessage).ToList() }; //ensure the async is released as soon as each task is completed brokerSendTask.Task.ContinueWith(t => { _semaphoreMaximumAsync.Release(); }, cancellationToken); sendTasks.Add(brokerSendTask); } try { await Task.WhenAll(sendTasks.Select(x => x.Task)).ConfigureAwait(false); foreach (var task in sendTasks) { //TODO when we dont ask for an ACK, result is an empty list. Which FirstOrDefault returns null. Dont like this... task.MessagesSent.ForEach(async x => x.Tcs.TrySetResult(await task.Task)); } } catch { //if an error occurs here, all we know is some or all of the messages in this ackBatch failed. var failedTask = sendTasks.FirstOrDefault(t => t.Task.IsFaulted); if (failedTask != null) { foreach (var topicMessageBatch in ackLevelBatch) { topicMessageBatch.Tcs.TrySetException( new KafkaApplicationException( "An exception occured while executing a send operation against {0}. Exception:{1}", failedTask.Route, failedTask.Task.Exception)); } } } finally { Interlocked.Add(ref _inFlightMessageCount, messages.Count * -1); } } }
private async Task ProduceAndSendBatchAsync(List <TopicMessage> messages, CancellationToken cancellationToken) { Interlocked.Add(ref _inFlightMessageCount, messages.Count); var topics = messages.GroupBy(batch => batch.Topic).Select(batch => batch.Key).ToArray(); await BrokerRouter.RefreshMissingTopicMetadata(topics).ConfigureAwait(false); //we must send a different produce request for each ack level and timeout combination. foreach (var ackLevelBatch in messages.GroupBy(batch => new { batch.Acks, batch.Timeout })) { var messageByRouter = ackLevelBatch.Select(batch => new { TopicMessage = batch, AckLevel = ackLevelBatch.Key.Acks, Route = batch.Partition.HasValue ? BrokerRouter.SelectBrokerRouteFromLocalCache(batch.Topic, batch.Partition.Value) : BrokerRouter.SelectBrokerRouteFromLocalCache(batch.Topic, batch.Message.Key) }).GroupBy(x => new { x.Route, x.TopicMessage.Topic, x.TopicMessage.Codec, x.AckLevel }); var sendTasks = new List <BrokerRouteSendBatch>(); foreach (var group in messageByRouter) { var payload = new Payload { Codec = group.Key.Codec, Topic = group.Key.Topic, Partition = group.Key.Route.PartitionId, Messages = group.Select(x => x.TopicMessage.Message).ToList() }; var request = new ProduceRequest { Acks = ackLevelBatch.Key.Acks, TimeoutMS = (int)ackLevelBatch.Key.Timeout.TotalMilliseconds, Payload = new List <Payload> { payload } }; await _semaphoreMaximumAsync.WaitAsync(cancellationToken).ConfigureAwait(false); var sendGroupTask = _protocolGateway.SendProtocolRequest(request, group.Key.Topic, group.Key.Route.PartitionId); var brokerSendTask = new BrokerRouteSendBatch { Route = group.Key.Route, Task = sendGroupTask, MessagesSent = group.Select(x => x.TopicMessage).ToList(), AckLevel = group.Key.AckLevel }; //ensure the async is released as soon as each task is completed //TODO: remove it from ack level 0 , don't like it brokerSendTask.Task.ContinueWith(t => { _semaphoreMaximumAsync.Release(); }, cancellationToken); sendTasks.Add(brokerSendTask); } try { await Task.WhenAll(sendTasks.Select(x => x.Task)).ConfigureAwait(false); } catch (Exception ex) { BrokerRouter.Log.ErrorFormat("Exception[{0}] stacktrace[{1}]", ex.Message, ex.StackTrace); } await SetResult(sendTasks).ConfigureAwait(false); Interlocked.Add(ref _inFlightMessageCount, messages.Count * -1); } }
private async Task ProduceAndSendBatchAsync(List<TopicMessage> batchs, CancellationToken cancellationToken) { //we must send a different produce request for each ack level and timeout combination. foreach (var ackLevelBatch in batchs.GroupBy(batch => new { batch.Acks, batch.Timeout })) { var messageByRouter = ackLevelBatch.Select(batch => new { TopicMessage = batch, Route = BrokerRouter.SelectBrokerRoute(batch.Topic, batch.Message.Key), }) .GroupBy(x => new { x.Route, x.TopicMessage.Topic, x.TopicMessage.Codec }); var sendTasks = new List<BrokerRouteSendBatch>(); foreach (var group in messageByRouter) { var payload = new Payload { Codec = group.Key.Codec, Topic = group.Key.Topic, Partition = group.Key.Route.PartitionId, Messages = group.Select(x => x.TopicMessage.Message).ToList() }; var request = new ProduceRequest { Acks = ackLevelBatch.Key.Acks, TimeoutMS = (int)ackLevelBatch.Key.Timeout.TotalMilliseconds, Payload = new List<Payload> { payload } }; await _semaphoreMaximumAsync.WaitAsync(cancellationToken).ConfigureAwait(false); var brokerSendTask = new BrokerRouteSendBatch { Route = group.Key.Route, Task = group.Key.Route.Connection.SendAsync(request), MessagesSent = group.Select(x => x.TopicMessage).ToList() }; //ensure the async is released as soon as each task is completed brokerSendTask.Task.ContinueWith(t => { _semaphoreMaximumAsync.Release(); }, cancellationToken, TaskContinuationOptions.ExecuteSynchronously, TaskScheduler.Default); sendTasks.Add(brokerSendTask); } try { await Task.WhenAll(sendTasks.Select(x => x.Task)).ConfigureAwait(false); foreach (var task in sendTasks) { task.MessagesSent.ForEach(x => x.Tcs.TrySetResult(task.Task.Result.FirstOrDefault())); } } catch { //if an error occurs here, all we know is some or all of the messages in this ackBatch failed. var failedTask = sendTasks.FirstOrDefault(t => t.Task.IsFaulted); if (failedTask != null) { foreach (var topicMessageBatch in ackLevelBatch) { topicMessageBatch.Tcs.TrySetException(new KafkaApplicationException("An exception occured while executing a send operation against {0}. Exception:{1}", failedTask.Route, failedTask.Task.Exception)); } } } } }