private async Task CommitOffset(long offsetToCommit)
{
Task commitTask;
using (TimerTimeToCommitOffset.NewContext())
{
commitTask = _consumer.UpdateOrCreateOffset(_options.ConsumerGroupName, offsetToCommit);
await Task.WhenAny(commitTask, Task.Delay(_options.ReceiveWaitTimeInMs));
}
if (!commitTask.IsCompleted)
{
var innerException = commitTask.IsFaulted
? (Exception)commitTask.Exception
: new TimeoutException("Commit operation timed out");
var newException = new KafkaStreamProviderException("Commit offset operation has failed", innerException);
_logger.Error((int)KafkaErrorCodes.KafkaApplicationError, String.Format(
"KafkaQueueAdapterReceiver - Commit offset operation has failed. ConsumerGroup is {0}, offset is {1}",
_options.ConsumerGroupName, offsetToCommit), newException);
throw new KafkaStreamProviderException();
}
_logger.Info(
"KafkaQueueAdapterReceiver - Commited an offset to the ConsumerGroup. ConsumerGroup is {0}, offset is {1}",
_options.ConsumerGroupName, offsetToCommit);
}
public async Task QueueMessageBatchAsync <T>(Guid streamGuid, string streamNamespace, IEnumerable <T> events, StreamSequenceToken token, Dictionary <string, object> requestContext)
{
var queueId = _streamQueueMapper.GetQueueForStream(streamGuid, streamNamespace);
var partitionId = (int)queueId.GetNumericId();
_logger.Info("KafkaQueueAdapter - For StreamId: {0}, StreamNamespace:{1} using partition {2}", streamGuid, streamNamespace, partitionId);
var enumeratedEvents = events.ToList();
var payload = _batchFactory.ToKafkaMessage(streamGuid, streamNamespace, enumeratedEvents.AsEnumerable(), requestContext);
if (payload == null)
{
_logger.Info("The batch factory returned a faulty message, the message was not sent");
return;
}
var messageToSend = new List <Message> {
payload
};
IEnumerable <ProduceResponse> response;
using (TimerTimeToProduceMessage.NewContext())
{
response = await _producer.SendMessageAsync(_options.TopicName, messageToSend, (short)_options.AckLevel, partition : partitionId);
}
// This is ackLevel != 0 check
if (response != null && !response.Contains(null))
{
var responsesWithError =
response.Where(messageResponse => messageResponse.Error != (int)ErrorResponseCode.NoError).ToList();
if (responsesWithError.Any())
{
List <Exception> allResponsesExceptions = new List <Exception>();
// Checking all the responses
foreach (var messageResponse in responsesWithError)
{
_logger.Warn((int)KafkaErrorCodes.KafkaStreamProviderBase,
"KafkaQueueAdapter - Error sending message through kafka client, the error code is {0}, message offset is {1}",
messageResponse.Error, messageResponse.Offset);
var newException = new KafkaApplicationException(
"Failed at producing the message with offset {0} for queue {1} in topic {2}",
messageResponse.Offset,
queueId, _options.TopicName)
{
ErrorCode = messageResponse.Error,
Source = "KafkaStreamProvider"
};
allResponsesExceptions.Add(newException);
}
// Aggregating the exceptions, and putting them inside a KafkaStreamProviderException
AggregateException exceptions = new AggregateException(allResponsesExceptions);
KafkaStreamProviderException exceptionToThrow =
new KafkaStreamProviderException("Producing message failed for one or more requests", exceptions);
throw exceptionToThrow;
}
}
HistogramProducedMessageBatchSize.Update(enumeratedEvents.Count(), queueId.ToString());
MeterProducedMessagesPerSecond.Mark(queueId.ToString(), 1);
}
private async Task CommitOffset(long offsetToCommit)
{
Task commitTask;
using (TimerTimeToCommitOffset.NewContext())
{
commitTask = _consumer.UpdateOrCreateOffset(_options.ConsumerGroupName, offsetToCommit);
await Task.WhenAny(commitTask, Task.Delay(_options.ReceiveWaitTimeInMs));
}
if (!commitTask.IsCompleted)
{
var innerException = commitTask.IsFaulted
? (Exception)commitTask.Exception
: new TimeoutException("Commit operation timed out");
var newException = new KafkaStreamProviderException("Commit offset operation has failed", innerException);
_logger.Error((int)KafkaErrorCodes.KafkaApplicationError, String.Format(
"KafkaQueueAdapterReceiver - Commit offset operation has failed. ConsumerGroup is {0}, offset is {1}",
_options.ConsumerGroupName, offsetToCommit), newException);
throw new KafkaStreamProviderException();
}
_logger.Verbose(
"KafkaQueueAdapterReceiver - Commited an offset to the ConsumerGroup. ConsumerGroup is {0}, offset is {1}",
_options.ConsumerGroupName, offsetToCommit);
}
