/// <summary>
/// Updating the cosumerGroup's offset for the partition in topic
/// </summary>
/// <param name="consumerGroup">The consumer group</param>
/// <param name="offset">The new offset. must be larger than or equal to zero</param>
/// <returns></returns>
public async Task UpdateOrCreateOffset(string consumerGroup, long offset)
{
if (string.IsNullOrEmpty(consumerGroup))
{
throw new ArgumentNullException("consumerGroup");
}
if (offset < 0)
{
throw new ArgumentOutOfRangeException("offset", "offset must be positive or zero");
}
OffsetCommitRequest request = CreateOffsetCommitRequest(offset, consumerGroup);
await _gateway.SendProtocolRequest(request, _topic, _partitionId).ConfigureAwait(false);
}
private async Task ProduceAndSendBatchAsync(List <TopicMessage> messages, CancellationToken cancellationToken)
{
Interlocked.Add(ref _inFlightMessageCount, messages.Count);
var topics = messages.GroupBy(batch => batch.Topic).Select(batch => batch.Key).ToArray();
await BrokerRouter.RefreshMissingTopicMetadata(topics).ConfigureAwait(false);
//we must send a different produce request for each ack level and timeout combination.
foreach (var ackLevelBatch in messages.GroupBy(batch => new { batch.Acks, batch.Timeout }))
{
var messageByRouter = ackLevelBatch.Select(batch => new
{
TopicMessage = batch,
AckLevel = ackLevelBatch.Key.Acks,
Route = batch.Partition.HasValue ? BrokerRouter.SelectBrokerRouteFromLocalCache(batch.Topic, batch.Partition.Value) : BrokerRouter.SelectBrokerRouteFromLocalCache(batch.Topic, batch.Message.Key)
}).GroupBy(x => new { x.Route, x.TopicMessage.Topic, x.TopicMessage.Codec, x.AckLevel });
var sendTasks = new List <BrokerRouteSendBatch>();
foreach (var group in messageByRouter)
{
var payload = new Payload
{
Codec = group.Key.Codec,
Topic = group.Key.Topic,
Partition = group.Key.Route.PartitionId,
Messages = group.Select(x => x.TopicMessage.Message).ToList()
};
var request = new ProduceRequest
{
Acks = ackLevelBatch.Key.Acks,
TimeoutMS = (int)ackLevelBatch.Key.Timeout.TotalMilliseconds,
Payload = new List <Payload> {
payload
}
};
await _semaphoreMaximumAsync.WaitAsync(cancellationToken).ConfigureAwait(false);
var sendGroupTask = _protocolGateway.SendProtocolRequest(request, group.Key.Topic, group.Key.Route.PartitionId);
var brokerSendTask = new BrokerRouteSendBatch
{
Route = group.Key.Route,
Task = sendGroupTask,
MessagesSent = group.Select(x => x.TopicMessage).ToList(),
AckLevel = group.Key.AckLevel
};
//ensure the async is released as soon as each task is completed //TODO: remove it from ack level 0 , don't like it
brokerSendTask.Task.ContinueWith(t => { _semaphoreMaximumAsync.Release(); }, cancellationToken);
sendTasks.Add(brokerSendTask);
}
try
{
await Task.WhenAll(sendTasks.Select(x => x.Task)).ConfigureAwait(false);
}
catch (Exception ex)
{
BrokerRouter.Log.ErrorFormat("Exception[{0}] stacktrace[{1}]", ex.Message, ex.StackTrace);
}
await SetResult(sendTasks).ConfigureAwait(false);
Interlocked.Add(ref _inFlightMessageCount, messages.Count * -1);
}
}