Load() public method

Return Partition -> Message
public Load ( String topic, Int32>.Dictionary offsetByPartition, Int32 blockSize ) : Message>>.IEnumerable
topic String
offsetByPartition Int32>.Dictionary
blockSize System.Int32
return Message>>.IEnumerable
PartitionConsumer Class Documentation
Exemplo n.º 1
0
        private void Start()
        {
            _started      = true;
            _stateStorage = new ConsumerStateStorage(_stateStorageDirectory);
            _streamState  = _stateStorage.ReadStreamState(Topic, "default-group", Partitions);

            var task = Task.Factory.StartNew(() =>
            {
                var consumer = new PartitionConsumer(_brokerAddress, _context);

                var offsetByPartition = new Dictionary <Int32, Int32>();
                foreach (var pair in _streamState.OffsetByPartition)
                {
                    offsetByPartition.Add(pair.Key, pair.Value);
                }

                while (true)
                {
                    // If we already have enough messages in queue - wait
                    if (Messages.Count > 100)
                    {
                        continue;
                    }

                    var result = consumer.Load(Topic, offsetByPartition, 300);

                    var messageCount = 0;
                    foreach (var tuple in result)
                    {
                        Messages.Enqueue(tuple);
                        offsetByPartition[tuple.Item1] += Message.CalculateOnDiskMessageLength(tuple.Item2.Payload.Length);
                        messageCount++;
                    }

                    // Wait for 500 msecond, if there is no new messages
                    // This value should be configurable
                    if (messageCount == 0)
                    {
                        Thread.Sleep(500);
                    }
                }
            });
        }
Exemplo n.º 2
0
        private void Start()
        {
            _started = true;
            _stateStorage = new ConsumerStateStorage(_stateStorageDirectory);
            _streamState = _stateStorage.ReadStreamState(Topic, "default-group", Partitions);

            var task = Task.Factory.StartNew(() =>
            {
                var consumer = new PartitionConsumer(_brokerAddress, _context);

                var offsetByPartition = new Dictionary<Int32, Int32>();
                foreach (var pair in _streamState.OffsetByPartition)
                    offsetByPartition.Add(pair.Key, pair.Value);

                while(true)
                {
                    // If we already have enough messages in queue - wait
                    if (Messages.Count > 100)
                        continue;

                    var result = consumer.Load(Topic, offsetByPartition, 300);

                    var messageCount = 0;
                    foreach (var tuple in result)
                    {
                        Messages.Enqueue(tuple);
                        offsetByPartition[tuple.Item1] += Message.CalculateOnDiskMessageLength(tuple.Item2.Payload.Length);
                        messageCount++;
                    }

                    // Wait for 500 msecond, if there is no new messages
                    // This value should be configurable
                    if (messageCount == 0)
                        Thread.Sleep(500);
                }
            });
        }