static void Main(string[] args) { try { _topic = ConfigurationManager.AppSettings["topic"]; var brokers = from x in ConfigurationManager.AppSettings["kafkaBrokers"].Split(',') select new Uri(x); Console.WriteLine("Connecting to kafka queue brokers {0} with topic {1}", string.Join(",", brokers), _topic); var options = new KafkaOptions(brokers.ToArray()); var router = new BrokerRouter(options); var coption = new ConsumerOptions(_topic, router); _consumer = new KafkaNet.Consumer(coption); var offset = _consumer.GetTopicOffsetAsync(_topic, 1000000).Result; var t = from x in offset select new OffsetPosition(x.PartitionId, x.Offsets.Max()); _consumer.SetOffsetPosition(t.ToArray()); foreach (var message in _consumer.Consume()) { Console.WriteLine("Response: P{0},O{1} : {2}", message.Meta.PartitionId, message.Meta.Offset, System.Text.Encoding.Default.GetString(message.Value)); } } catch(Exception ex) { Console.WriteLine(ex.Message); } Console.ReadLine(); }
public void ConsumerShouldBeAbleToSeekBackToEarlierOffset() { var producer = new Producer(_router); var offsets = producer.GetTopicOffsetAsync("LoadTest").Result .Select(x => new OffsetPosition(x.PartitionId, x.Offsets.Max())).ToArray(); var consumer = new Consumer(new ConsumerOptions("LoadTest", _router), offsets); var tasks = new List<Task<List<ProduceResponse>>>(); for (int i = 0; i < 20; i++) { tasks.Add(producer.SendMessageAsync("LoadTest", new[] { new Message { Value = i.ToString(), Key = "1" } })); } Task.WaitAll(tasks.ToArray()); var results = consumer.Consume().Take(20).ToList(); //ensure the produced messages arrived for (int i = 0; i < 20; i++) { Assert.That(results[i].Value == i.ToString()); } //seek back to initial offset consumer.SetOffsetPosition(offsets); //ensure all produced messages arrive again for (int i = 0; i < 20; i++) { Assert.That(results[i].Value == i.ToString()); } }
static void Main(string[] args) { try { _topic = ConfigurationManager.AppSettings["topic"]; var brokers = from x in ConfigurationManager.AppSettings["kafkaBrokers"].Split(',') select new Uri(x); Console.WriteLine("Connecting to kafka queue brokers {0} with topic {1}", string.Join(",", brokers), _topic); var options = new KafkaOptions(brokers.ToArray()); var router = new BrokerRouter(options); var coption = new ConsumerOptions(_topic, router); _consumer = new KafkaNet.Consumer(coption); var offset = _consumer.GetTopicOffsetAsync(_topic, 1000000).Result; var t = from x in offset select new OffsetPosition(x.PartitionId, x.Offsets.Max()); _consumer.SetOffsetPosition(t.ToArray()); foreach (var message in _consumer.Consume()) { Console.WriteLine("Response: P{0},O{1} : {2}", message.Meta.PartitionId, message.Meta.Offset, System.Text.Encoding.Default.GetString(message.Value)); } } catch (Exception ex) { Console.WriteLine(ex.Message); } Console.ReadLine(); }
public void ConsumerShouldBeAbleToSeekBackToEarlierOffset() { var expected = new List<string> { "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19" }; var testId = Guid.NewGuid().ToString(); using (var router = new BrokerRouter(new KafkaOptions(IntegrationConfig.IntegrationUri))) using (var producer = new Producer(router)) { var offsets = producer.GetTopicOffsetAsync(IntegrationConfig.IntegrationTopic).Result .Select(x => new OffsetPosition(x.PartitionId, x.Offsets.Max())).ToArray(); using (var consumer = new Consumer(new ConsumerOptions(IntegrationConfig.IntegrationTopic, router), offsets)) { for (int i = 0; i < 20; i++) { producer.SendMessageAsync(IntegrationConfig.IntegrationTopic, new[] { new Message(i.ToString(), testId) }).Wait(); } var sentMessages = consumer.Consume().Take(20).ToList(); //ensure the produced messages arrived Console.WriteLine("Message order: {0}", string.Join(", ", sentMessages.Select(x => x.Value.ToUtf8String()).ToList())); Assert.That(sentMessages.Count, Is.EqualTo(20)); Assert.That(sentMessages.Select(x => x.Value.ToUtf8String()).ToList(), Is.EqualTo(expected)); Assert.That(sentMessages.Any(x => x.Key.ToUtf8String() != testId), Is.False); //seek back to initial offset consumer.SetOffsetPosition(offsets); var resetPositionMessages = consumer.Consume().Take(20).ToList(); //ensure all produced messages arrive again Console.WriteLine("Message order: {0}", string.Join(", ", resetPositionMessages.Select(x => x.Value).ToList())); Assert.That(resetPositionMessages.Count, Is.EqualTo(20)); Assert.That(resetPositionMessages.Select(x => x.Value.ToUtf8String()).ToList(), Is.EqualTo(expected)); Assert.That(resetPositionMessages.Any(x => x.Key.ToUtf8String() != testId), Is.False); } } }