private async Task CommitOffset(long offsetToCommit) { Task commitTask; using (TimerTimeToCommitOffset.NewContext()) { commitTask = _consumer.UpdateOrCreateOffset(_options.ConsumerGroupName, offsetToCommit); await Task.WhenAny(commitTask, Task.Delay(_options.ReceiveWaitTimeInMs)); } if (!commitTask.IsCompleted) { var innerException = commitTask.IsFaulted ? (Exception)commitTask.Exception : new TimeoutException("Commit operation timed out"); var newException = new KafkaStreamProviderException("Commit offset operation has failed", innerException); _logger.Error((int)KafkaErrorCodes.KafkaApplicationError, String.Format( "KafkaQueueAdapterReceiver - Commit offset operation has failed. ConsumerGroup is {0}, offset is {1}", _options.ConsumerGroupName, offsetToCommit), newException); throw new KafkaStreamProviderException(); } _logger.Info( "KafkaQueueAdapterReceiver - Commited an offset to the ConsumerGroup. ConsumerGroup is {0}, offset is {1}", _options.ConsumerGroupName, offsetToCommit); }
public async Task QueueMessageBatchAsync <T>(Guid streamGuid, string streamNamespace, IEnumerable <T> events, StreamSequenceToken token, Dictionary <string, object> requestContext) { var queueId = _streamQueueMapper.GetQueueForStream(streamGuid, streamNamespace); var partitionId = (int)queueId.GetNumericId(); _logger.Info("KafkaQueueAdapter - For StreamId: {0}, StreamNamespace:{1} using partition {2}", streamGuid, streamNamespace, partitionId); var enumeratedEvents = events.ToList(); var payload = _batchFactory.ToKafkaMessage(streamGuid, streamNamespace, enumeratedEvents.AsEnumerable(), requestContext); if (payload == null) { _logger.Info("The batch factory returned a faulty message, the message was not sent"); return; } var messageToSend = new List <Message> { payload }; IEnumerable <ProduceResponse> response; using (TimerTimeToProduceMessage.NewContext()) { response = await _producer.SendMessageAsync(_options.TopicName, messageToSend, (short)_options.AckLevel, partition : partitionId); } // This is ackLevel != 0 check if (response != null && !response.Contains(null)) { var responsesWithError = response.Where(messageResponse => messageResponse.Error != (int)ErrorResponseCode.NoError).ToList(); if (responsesWithError.Any()) { List <Exception> allResponsesExceptions = new List <Exception>(); // Checking all the responses foreach (var messageResponse in responsesWithError) { _logger.Warn((int)KafkaErrorCodes.KafkaStreamProviderBase, "KafkaQueueAdapter - Error sending message through kafka client, the error code is {0}, message offset is {1}", messageResponse.Error, messageResponse.Offset); var newException = new KafkaApplicationException( "Failed at producing the message with offset {0} for queue {1} in topic {2}", messageResponse.Offset, queueId, _options.TopicName) { ErrorCode = messageResponse.Error, Source = "KafkaStreamProvider" }; allResponsesExceptions.Add(newException); } // Aggregating the exceptions, and putting them inside a KafkaStreamProviderException AggregateException exceptions = new AggregateException(allResponsesExceptions); KafkaStreamProviderException exceptionToThrow = new KafkaStreamProviderException("Producing message failed for one or more requests", exceptions); throw exceptionToThrow; } } HistogramProducedMessageBatchSize.Update(enumeratedEvents.Count(), queueId.ToString()); MeterProducedMessagesPerSecond.Mark(queueId.ToString(), 1); }
private async Task CommitOffset(long offsetToCommit) { Task commitTask; using (TimerTimeToCommitOffset.NewContext()) { commitTask = _consumer.UpdateOrCreateOffset(_options.ConsumerGroupName, offsetToCommit); await Task.WhenAny(commitTask, Task.Delay(_options.ReceiveWaitTimeInMs)); } if (!commitTask.IsCompleted) { var innerException = commitTask.IsFaulted ? (Exception)commitTask.Exception : new TimeoutException("Commit operation timed out"); var newException = new KafkaStreamProviderException("Commit offset operation has failed", innerException); _logger.Error((int)KafkaErrorCodes.KafkaApplicationError, String.Format( "KafkaQueueAdapterReceiver - Commit offset operation has failed. ConsumerGroup is {0}, offset is {1}", _options.ConsumerGroupName, offsetToCommit), newException); throw new KafkaStreamProviderException(); } _logger.Verbose( "KafkaQueueAdapterReceiver - Commited an offset to the ConsumerGroup. ConsumerGroup is {0}, offset is {1}", _options.ConsumerGroupName, offsetToCommit); }