static void Main(string[] args)
{
try
{
_topic = ConfigurationManager.AppSettings["topic"];
var brokers = from x in ConfigurationManager.AppSettings["kafkaBrokers"].Split(',')
select new Uri(x);
Console.WriteLine("Connecting to kafka queue brokers {0} with topic {1}", string.Join(",", brokers), _topic);
var options = new KafkaOptions(brokers.ToArray());
var router = new BrokerRouter(options);
var coption = new ConsumerOptions(_topic, router);
_consumer = new KafkaNet.Consumer(coption);
var offset = _consumer.GetTopicOffsetAsync(_topic, 1000000).Result;
var t = from x in offset select new OffsetPosition(x.PartitionId, x.Offsets.Max());
_consumer.SetOffsetPosition(t.ToArray());
foreach (var message in _consumer.Consume())
{
Console.WriteLine("Response: P{0},O{1} : {2}",
message.Meta.PartitionId,
message.Meta.Offset,
System.Text.Encoding.Default.GetString(message.Value));
}
}
catch(Exception ex)
{
Console.WriteLine(ex.Message);
}
Console.ReadLine();
}
public void ConsumerShouldBeAbleToSeekBackToEarlierOffset()
{
var producer = new Producer(_router);
var offsets = producer.GetTopicOffsetAsync("LoadTest").Result
.Select(x => new OffsetPosition(x.PartitionId, x.Offsets.Max())).ToArray();
var consumer = new Consumer(new ConsumerOptions("LoadTest", _router), offsets);
var tasks = new List<Task<List<ProduceResponse>>>();
for (int i = 0; i < 20; i++)
{
tasks.Add(producer.SendMessageAsync("LoadTest", new[] { new Message { Value = i.ToString(), Key = "1" } }));
}
Task.WaitAll(tasks.ToArray());
var results = consumer.Consume().Take(20).ToList();
//ensure the produced messages arrived
for (int i = 0; i < 20; i++)
{
Assert.That(results[i].Value == i.ToString());
}
//seek back to initial offset
consumer.SetOffsetPosition(offsets);
//ensure all produced messages arrive again
for (int i = 0; i < 20; i++)
{
Assert.That(results[i].Value == i.ToString());
}
}
static void Main(string[] args)
{
try
{
_topic = ConfigurationManager.AppSettings["topic"];
var brokers = from x in ConfigurationManager.AppSettings["kafkaBrokers"].Split(',')
select new Uri(x);
Console.WriteLine("Connecting to kafka queue brokers {0} with topic {1}", string.Join(",", brokers), _topic);
var options = new KafkaOptions(brokers.ToArray());
var router = new BrokerRouter(options);
var coption = new ConsumerOptions(_topic, router);
_consumer = new KafkaNet.Consumer(coption);
var offset = _consumer.GetTopicOffsetAsync(_topic, 1000000).Result;
var t = from x in offset select new OffsetPosition(x.PartitionId, x.Offsets.Max());
_consumer.SetOffsetPosition(t.ToArray());
foreach (var message in _consumer.Consume())
{
Console.WriteLine("Response: P{0},O{1} : {2}",
message.Meta.PartitionId,
message.Meta.Offset,
System.Text.Encoding.Default.GetString(message.Value));
}
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
Console.ReadLine();
}
public void ConsumerShouldBeAbleToSeekBackToEarlierOffset()
{
var expected = new List<string> { "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19" };
var testId = Guid.NewGuid().ToString();
using (var router = new BrokerRouter(new KafkaOptions(IntegrationConfig.IntegrationUri)))
using (var producer = new Producer(router))
{
var offsets = producer.GetTopicOffsetAsync(IntegrationConfig.IntegrationTopic).Result
.Select(x => new OffsetPosition(x.PartitionId, x.Offsets.Max())).ToArray();
using (var consumer = new Consumer(new ConsumerOptions(IntegrationConfig.IntegrationTopic, router), offsets))
{
for (int i = 0; i < 20; i++)
{
producer.SendMessageAsync(IntegrationConfig.IntegrationTopic, new[] { new Message(i.ToString(), testId) }).Wait();
}
var sentMessages = consumer.Consume().Take(20).ToList();
//ensure the produced messages arrived
Console.WriteLine("Message order: {0}", string.Join(", ", sentMessages.Select(x => x.Value.ToUtf8String()).ToList()));
Assert.That(sentMessages.Count, Is.EqualTo(20));
Assert.That(sentMessages.Select(x => x.Value.ToUtf8String()).ToList(), Is.EqualTo(expected));
Assert.That(sentMessages.Any(x => x.Key.ToUtf8String() != testId), Is.False);
//seek back to initial offset
consumer.SetOffsetPosition(offsets);
var resetPositionMessages = consumer.Consume().Take(20).ToList();
//ensure all produced messages arrive again
Console.WriteLine("Message order: {0}", string.Join(", ", resetPositionMessages.Select(x => x.Value).ToList()));
Assert.That(resetPositionMessages.Count, Is.EqualTo(20));
Assert.That(resetPositionMessages.Select(x => x.Value.ToUtf8String()).ToList(), Is.EqualTo(expected));
Assert.That(resetPositionMessages.Any(x => x.Key.ToUtf8String() != testId), Is.False);
}
}
}