Ejemplo n.º 1
0
        /// <summary>
        /// Updating the cosumerGroup's offset for the partition in topic
        /// </summary>
        /// <param name="consumerGroup">The consumer group</param>
        /// <param name="offset">The new offset. must be larger than or equal to zero</param>
        /// <returns></returns>
        public async Task UpdateOrCreateOffset(string consumerGroup, long offset)
        {
            if (string.IsNullOrEmpty(consumerGroup))
            {
                throw new ArgumentNullException("consumerGroup");
            }
            if (offset < 0)
            {
                throw new ArgumentOutOfRangeException("offset", "offset must be positive or zero");
            }

            OffsetCommitRequest request = CreateOffsetCommitRequest(offset, consumerGroup);
            await _gateway.SendProtocolRequest(request, _topic, _partitionId).ConfigureAwait(false);
        }
Ejemplo n.º 2
0
        private async Task ProduceAndSendBatchAsync(List <TopicMessage> messages, CancellationToken cancellationToken)
        {
            Interlocked.Add(ref _inFlightMessageCount, messages.Count);

            var topics = messages.GroupBy(batch => batch.Topic).Select(batch => batch.Key).ToArray();
            await BrokerRouter.RefreshMissingTopicMetadata(topics).ConfigureAwait(false);

            //we must send a different produce request for each ack level and timeout combination.
            foreach (var ackLevelBatch in messages.GroupBy(batch => new { batch.Acks, batch.Timeout }))
            {
                var messageByRouter = ackLevelBatch.Select(batch => new
                {
                    TopicMessage = batch,
                    AckLevel     = ackLevelBatch.Key.Acks,
                    Route        = batch.Partition.HasValue ? BrokerRouter.SelectBrokerRouteFromLocalCache(batch.Topic, batch.Partition.Value) : BrokerRouter.SelectBrokerRouteFromLocalCache(batch.Topic, batch.Message.Key)
                }).GroupBy(x => new { x.Route, x.TopicMessage.Topic, x.TopicMessage.Codec, x.AckLevel });

                var sendTasks = new List <BrokerRouteSendBatch>();
                foreach (var group in messageByRouter)
                {
                    var payload = new Payload
                    {
                        Codec     = group.Key.Codec,
                        Topic     = group.Key.Topic,
                        Partition = group.Key.Route.PartitionId,
                        Messages  = group.Select(x => x.TopicMessage.Message).ToList()
                    };

                    var request = new ProduceRequest
                    {
                        Acks      = ackLevelBatch.Key.Acks,
                        TimeoutMS = (int)ackLevelBatch.Key.Timeout.TotalMilliseconds,
                        Payload   = new List <Payload> {
                            payload
                        }
                    };

                    await _semaphoreMaximumAsync.WaitAsync(cancellationToken).ConfigureAwait(false);

                    var sendGroupTask  = _protocolGateway.SendProtocolRequest(request, group.Key.Topic, group.Key.Route.PartitionId);
                    var brokerSendTask = new BrokerRouteSendBatch
                    {
                        Route        = group.Key.Route,
                        Task         = sendGroupTask,
                        MessagesSent = group.Select(x => x.TopicMessage).ToList(),
                        AckLevel     = group.Key.AckLevel
                    };

                    //ensure the async is released as soon as each task is completed //TODO: remove it from ack level 0 , don't like it
                    brokerSendTask.Task.ContinueWith(t => { _semaphoreMaximumAsync.Release(); }, cancellationToken);

                    sendTasks.Add(brokerSendTask);
                }

                try
                {
                    await Task.WhenAll(sendTasks.Select(x => x.Task)).ConfigureAwait(false);
                }
                catch (Exception ex)
                {
                    BrokerRouter.Log.ErrorFormat("Exception[{0}] stacktrace[{1}]", ex.Message, ex.StackTrace);
                }

                await SetResult(sendTasks).ConfigureAwait(false);

                Interlocked.Add(ref _inFlightMessageCount, messages.Count * -1);
            }
        }