/// <summary> /// Updating the cosumerGroup's offset for the partition in topic /// </summary> /// <param name="consumerGroup">The consumer group</param> /// <param name="offset">The new offset. must be larger than or equal to zero</param> /// <returns></returns> public async Task UpdateOrCreateOffset(string consumerGroup, long offset) { if (string.IsNullOrEmpty(consumerGroup)) { throw new ArgumentNullException("consumerGroup"); } if (offset < 0) { throw new ArgumentOutOfRangeException("offset", "offset must be positive or zero"); } OffsetCommitRequest request = CreateOffsetCommitRequest(offset, consumerGroup); await _gateway.SendProtocolRequest(request, _topic, _partitionId).ConfigureAwait(false); }
private async Task ProduceAndSendBatchAsync(List <TopicMessage> messages, CancellationToken cancellationToken) { Interlocked.Add(ref _inFlightMessageCount, messages.Count); var topics = messages.GroupBy(batch => batch.Topic).Select(batch => batch.Key).ToArray(); await BrokerRouter.RefreshMissingTopicMetadata(topics).ConfigureAwait(false); //we must send a different produce request for each ack level and timeout combination. foreach (var ackLevelBatch in messages.GroupBy(batch => new { batch.Acks, batch.Timeout })) { var messageByRouter = ackLevelBatch.Select(batch => new { TopicMessage = batch, AckLevel = ackLevelBatch.Key.Acks, Route = batch.Partition.HasValue ? BrokerRouter.SelectBrokerRouteFromLocalCache(batch.Topic, batch.Partition.Value) : BrokerRouter.SelectBrokerRouteFromLocalCache(batch.Topic, batch.Message.Key) }).GroupBy(x => new { x.Route, x.TopicMessage.Topic, x.TopicMessage.Codec, x.AckLevel }); var sendTasks = new List <BrokerRouteSendBatch>(); foreach (var group in messageByRouter) { var payload = new Payload { Codec = group.Key.Codec, Topic = group.Key.Topic, Partition = group.Key.Route.PartitionId, Messages = group.Select(x => x.TopicMessage.Message).ToList() }; var request = new ProduceRequest { Acks = ackLevelBatch.Key.Acks, TimeoutMS = (int)ackLevelBatch.Key.Timeout.TotalMilliseconds, Payload = new List <Payload> { payload } }; await _semaphoreMaximumAsync.WaitAsync(cancellationToken).ConfigureAwait(false); var sendGroupTask = _protocolGateway.SendProtocolRequest(request, group.Key.Topic, group.Key.Route.PartitionId); var brokerSendTask = new BrokerRouteSendBatch { Route = group.Key.Route, Task = sendGroupTask, MessagesSent = group.Select(x => x.TopicMessage).ToList(), AckLevel = group.Key.AckLevel }; //ensure the async is released as soon as each task is completed //TODO: remove it from ack level 0 , don't like it brokerSendTask.Task.ContinueWith(t => { _semaphoreMaximumAsync.Release(); }, cancellationToken); sendTasks.Add(brokerSendTask); } try { await Task.WhenAll(sendTasks.Select(x => x.Task)).ConfigureAwait(false); } catch (Exception ex) { BrokerRouter.Log.ErrorFormat("Exception[{0}] stacktrace[{1}]", ex.Message, ex.StackTrace); } await SetResult(sendTasks).ConfigureAwait(false); Interlocked.Add(ref _inFlightMessageCount, messages.Count * -1); } }