public static long GetCurrentKafkaOffset(string topic, string address, int port)
{
OffsetRequest request = new OffsetRequest(topic, 0, DateTime.Now.AddDays(-5).Ticks, 10);
ConsumerConfig consumerConfig = new ConsumerConfig();
consumerConfig.Host = address;
consumerConfig.Port = port;
IConsumer consumer = new Consumers.Consumer(consumerConfig);
IList<long> list = consumer.GetOffsetsBefore(request);
if (list.Count > 0)
{
return list[0];
}
else
{
return 0;
}
}
public void AsyncProducerSendsAndConsumerReceivesSingleSimpleMessage()
{
Message sourceMessage = new Message(Encoding.UTF8.GetBytes("test message"));
var config = new AsyncProducerConfig(clientConfig);
var producer = new AsyncProducer(config);
var producerRequest = new ProducerRequest(CurrentTestTopic, 0, new List<Message>() { sourceMessage });
long currentOffset = TestHelper.GetCurrentKafkaOffset(CurrentTestTopic, clientConfig);
producer.Send(producerRequest);
ConsumerConfig consumerConfig = new ConsumerConfig(clientConfig);
IConsumer consumer = new Consumers.Consumer(consumerConfig);
FetchRequest request = new FetchRequest(CurrentTestTopic, 0, currentOffset);
BufferedMessageSet response;
int totalWaitTimeInMiliseconds = 0;
int waitSingle = 100;
while (true)
{
Thread.Sleep(waitSingle);
response = consumer.Fetch(request);
if (response != null && response.Messages.Count() > 0)
{
break;
}
else
{
totalWaitTimeInMiliseconds += waitSingle;
if (totalWaitTimeInMiliseconds >= MaxTestWaitTimeInMiliseconds)
{
break;
}
}
}
Assert.NotNull(response);
Assert.AreEqual(1, response.Messages.Count());
Message resultMessage = response.Messages.First();
Assert.AreEqual(sourceMessage.ToString(), resultMessage.ToString());
}
public void ProducerSends1Message()
{
int totalWaitTimeInMiliseconds = 0;
int waitSingle = 100;
var originalMessage = new Message(Encoding.UTF8.GetBytes("TestData"));
var multipleBrokersHelper = new TestMultipleBrokersHelper(CurrentTestTopic);
multipleBrokersHelper.GetCurrentOffsets();
var producerConfig = new ProducerConfig(clientConfig);
var mockPartitioner = new MockAlwaysZeroPartitioner();
using (var producer = new Producer<string, Message>(producerConfig, mockPartitioner, new DefaultEncoder(), null))
{
var producerData = new ProducerData<string, Message>(
CurrentTestTopic, "somekey", new List<Message> { originalMessage });
producer.Send(producerData);
Thread.Sleep(waitSingle);
while (!multipleBrokersHelper.CheckIfAnyBrokerHasChanged())
{
totalWaitTimeInMiliseconds += waitSingle;
Thread.Sleep(waitSingle);
if (totalWaitTimeInMiliseconds > this.maxTestWaitTimeInMiliseconds)
{
Assert.Fail("None of the brokers changed their offset after sending a message");
}
}
totalWaitTimeInMiliseconds = 0;
var consumerConfig = new ConsumerConfig(clientConfig)
{
Host = multipleBrokersHelper.BrokerThatHasChanged.Address,
Port = multipleBrokersHelper.BrokerThatHasChanged.Port
};
IConsumer consumer = new Consumers.Consumer(consumerConfig);
var request = new FetchRequest(CurrentTestTopic, 0, multipleBrokersHelper.OffsetFromBeforeTheChange);
BufferedMessageSet response;
while (true)
{
Thread.Sleep(waitSingle);
response = consumer.Fetch(request);
if (response != null && response.Messages.Count() > 0)
{
break;
}
totalWaitTimeInMiliseconds += waitSingle;
if (totalWaitTimeInMiliseconds >= this.maxTestWaitTimeInMiliseconds)
{
break;
}
}
Assert.NotNull(response);
Assert.AreEqual(1, response.Messages.Count());
Assert.AreEqual(originalMessage.ToString(), response.Messages.First().ToString());
}
}
public void ProducerSendsAndConsumerReceivesMultiRequest()
{
string testTopic1 = CurrentTestTopic + "1";
string testTopic2 = CurrentTestTopic + "2";
string testTopic3 = CurrentTestTopic + "3";
Message sourceMessage1 = new Message(Encoding.UTF8.GetBytes("1: TestMessage"));
Message sourceMessage2 = new Message(Encoding.UTF8.GetBytes("2: TestMessage"));
Message sourceMessage3 = new Message(Encoding.UTF8.GetBytes("3: TestMessage"));
Message sourceMessage4 = new Message(Encoding.UTF8.GetBytes("4: TestMessage"));
List<ProducerRequest> requests = new List<ProducerRequest>
{
new ProducerRequest(testTopic1, 0, new List<Message> { sourceMessage1 }),
new ProducerRequest(testTopic1, 0, new List<Message> { sourceMessage2 }),
new ProducerRequest(testTopic2, 0, new List<Message> { sourceMessage3 }),
new ProducerRequest(testTopic3, 0, new List<Message> { sourceMessage4 })
};
var config = new SyncProducerConfig(clientConfig);
var producer = new SyncProducer(config);
long currentOffset1 = TestHelper.GetCurrentKafkaOffset(testTopic1, clientConfig);
long currentOffset2 = TestHelper.GetCurrentKafkaOffset(testTopic2, clientConfig);
long currentOffset3 = TestHelper.GetCurrentKafkaOffset(testTopic3, clientConfig);
producer.MultiSend(requests);
ConsumerConfig consumerConfig = new ConsumerConfig(clientConfig);
IConsumer consumer = new Consumers.Consumer(consumerConfig);
MultiFetchRequest request = new MultiFetchRequest(new List<FetchRequest>
{
new FetchRequest(testTopic1, 0, currentOffset1),
new FetchRequest(testTopic2, 0, currentOffset2),
new FetchRequest(testTopic3, 0, currentOffset3)
});
IList<BufferedMessageSet> messageSets;
int totalWaitTimeInMiliseconds = 0;
int waitSingle = 100;
while (true)
{
Thread.Sleep(waitSingle);
messageSets = consumer.MultiFetch(request);
if (messageSets.Count > 2 && messageSets[0].Messages.Count() > 0 && messageSets[1].Messages.Count() > 0 && messageSets[2].Messages.Count() > 0)
{
break;
}
else
{
totalWaitTimeInMiliseconds += waitSingle;
if (totalWaitTimeInMiliseconds >= MaxTestWaitTimeInMiliseconds)
{
break;
}
}
}
Assert.AreEqual(3, messageSets.Count);
Assert.AreEqual(2, messageSets[0].Messages.Count());
Assert.AreEqual(1, messageSets[1].Messages.Count());
Assert.AreEqual(1, messageSets[2].Messages.Count());
Assert.AreEqual(sourceMessage1.ToString(), messageSets[0].Messages.First().ToString());
Assert.AreEqual(sourceMessage2.ToString(), messageSets[0].Messages.Skip(1).First().ToString());
Assert.AreEqual(sourceMessage3.ToString(), messageSets[1].Messages.First().ToString());
Assert.AreEqual(sourceMessage4.ToString(), messageSets[2].Messages.First().ToString());
}
public void ConsumerMultiFetchGetsMessage()
{
ProducerSendMultiRequest();
ConsumerConfig config = new ConsumerConfig(clientConfig);
IConsumer cons = new Consumers.Consumer(config);
MultiFetchRequest request = new MultiFetchRequest(new List<FetchRequest>
{
new FetchRequest(CurrentTestTopic, 0, 0),
new FetchRequest(CurrentTestTopic, 0, 0),
new FetchRequest(CurrentTestTopic + "2", 0, 0)
});
IList<BufferedMessageSet> response = cons.MultiFetch(request);
for (int ix = 0; ix < response.Count; ix++)
{
IEnumerable<Message> messageSet = response[ix].Messages;
Console.WriteLine(string.Format("Request #{0}-->", ix));
foreach (Message msg in messageSet)
{
Console.WriteLine(msg);
}
}
}
public void ConsumerGetsOffsets()
{
OffsetRequest request = new OffsetRequest(CurrentTestTopic, 0, DateTime.Now.AddHours(-24).Ticks, 10);
ConsumerConfig config = new ConsumerConfig(clientConfig);
IConsumer consumer = new Consumers.Consumer(config);
IList<long> list = consumer.GetOffsetsBefore(request);
foreach (long l in list)
{
Console.Out.WriteLine(l);
}
}