SetOffsetPosition() public method

Force reset the offset position for a specific partition to a specific offset value.
public SetOffsetPosition ( ) : void
return void
Example #1
2
        static void Main(string[] args)
        {
            try
            {
                _topic = ConfigurationManager.AppSettings["topic"];
                var brokers = from x in ConfigurationManager.AppSettings["kafkaBrokers"].Split(',')
                              select new Uri(x);
                Console.WriteLine("Connecting to kafka queue brokers {0} with topic {1}", string.Join(",", brokers), _topic);
                var options = new KafkaOptions(brokers.ToArray());
                var router = new BrokerRouter(options);
                var coption = new ConsumerOptions(_topic, router);
                _consumer = new KafkaNet.Consumer(coption);
                var offset = _consumer.GetTopicOffsetAsync(_topic, 1000000).Result;
                var t = from x in offset select new OffsetPosition(x.PartitionId, x.Offsets.Max());
                _consumer.SetOffsetPosition(t.ToArray());

                foreach (var message in _consumer.Consume())
                {
                    Console.WriteLine("Response: P{0},O{1} : {2}",
                    message.Meta.PartitionId, 
                    message.Meta.Offset,
                    System.Text.Encoding.Default.GetString(message.Value));
                }
            }
            catch(Exception ex)
            {
                Console.WriteLine(ex.Message);
            }
            Console.ReadLine();
        }
        public void ConsumerShouldBeAbleToSeekBackToEarlierOffset()
        {
            var producer = new Producer(_router);

            var offsets = producer.GetTopicOffsetAsync("LoadTest").Result
                .Select(x => new OffsetPosition(x.PartitionId, x.Offsets.Max())).ToArray();

            var consumer = new Consumer(new ConsumerOptions("LoadTest", _router), offsets);

            var tasks = new List<Task<List<ProduceResponse>>>();
            for (int i = 0; i < 20; i++)
            {
                tasks.Add(producer.SendMessageAsync("LoadTest", new[] { new Message { Value = i.ToString(), Key = "1" } }));
            }
            Task.WaitAll(tasks.ToArray());

            var results = consumer.Consume().Take(20).ToList();

            //ensure the produced messages arrived
            for (int i = 0; i < 20; i++)
            {
                Assert.That(results[i].Value == i.ToString());
            }

            //seek back to initial offset
            consumer.SetOffsetPosition(offsets);

            //ensure all produced messages arrive again
            for (int i = 0; i < 20; i++)
            {
                Assert.That(results[i].Value == i.ToString());
            }
        }
Example #3
0
        static void Main(string[] args)
        {
            try
            {
                _topic = ConfigurationManager.AppSettings["topic"];
                var brokers = from x in ConfigurationManager.AppSettings["kafkaBrokers"].Split(',')
                              select new Uri(x);
                Console.WriteLine("Connecting to kafka queue brokers {0} with topic {1}", string.Join(",", brokers), _topic);
                var options = new KafkaOptions(brokers.ToArray());
                var router  = new BrokerRouter(options);
                var coption = new ConsumerOptions(_topic, router);
                _consumer = new KafkaNet.Consumer(coption);
                var offset = _consumer.GetTopicOffsetAsync(_topic, 1000000).Result;
                var t      = from x in offset select new OffsetPosition(x.PartitionId, x.Offsets.Max());
                _consumer.SetOffsetPosition(t.ToArray());

                foreach (var message in _consumer.Consume())
                {
                    Console.WriteLine("Response: P{0},O{1} : {2}",
                                      message.Meta.PartitionId,
                                      message.Meta.Offset,
                                      System.Text.Encoding.Default.GetString(message.Value));
                }
            }
            catch (Exception ex)
            {
                Console.WriteLine(ex.Message);
            }
            Console.ReadLine();
        }
        public void ConsumerShouldBeAbleToSeekBackToEarlierOffset()
        {
            var expected = new List<string> { "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19" };
            var testId = Guid.NewGuid().ToString();

            using (var router = new BrokerRouter(new KafkaOptions(IntegrationConfig.IntegrationUri)))
            using (var producer = new Producer(router))
            {
                var offsets = producer.GetTopicOffsetAsync(IntegrationConfig.IntegrationTopic).Result
                    .Select(x => new OffsetPosition(x.PartitionId, x.Offsets.Max())).ToArray();

                using (var consumer = new Consumer(new ConsumerOptions(IntegrationConfig.IntegrationTopic, router), offsets))
                {
                    for (int i = 0; i < 20; i++)
                    {
                        producer.SendMessageAsync(IntegrationConfig.IntegrationTopic, new[] { new Message(i.ToString(), testId) }).Wait();
                    }

                    var sentMessages = consumer.Consume().Take(20).ToList();

                    //ensure the produced messages arrived
                    Console.WriteLine("Message order:  {0}", string.Join(", ", sentMessages.Select(x => x.Value.ToUtf8String()).ToList()));

                    Assert.That(sentMessages.Count, Is.EqualTo(20));
                    Assert.That(sentMessages.Select(x => x.Value.ToUtf8String()).ToList(), Is.EqualTo(expected));
                    Assert.That(sentMessages.Any(x => x.Key.ToUtf8String() != testId), Is.False);

                    //seek back to initial offset
                    consumer.SetOffsetPosition(offsets);

                    var resetPositionMessages = consumer.Consume().Take(20).ToList();

                    //ensure all produced messages arrive again
                    Console.WriteLine("Message order:  {0}", string.Join(", ", resetPositionMessages.Select(x => x.Value).ToList()));

                    Assert.That(resetPositionMessages.Count, Is.EqualTo(20));
                    Assert.That(resetPositionMessages.Select(x => x.Value.ToUtf8String()).ToList(), Is.EqualTo(expected));
                    Assert.That(resetPositionMessages.Any(x => x.Key.ToUtf8String() != testId), Is.False);
                }
            }
        }