/// <summary>
/// SyncProducer makes a connection to the broker inside constructor
/// </summary>
/// <param name="brokerId"></param>
/// <param name="host"></param>
/// <param name="port"></param>
/// <returns></returns>
public static SyncProducer TryCreateSyncProducer(int brokerId, string host, int port)
{
SyncProducer syncProducer = null;
if (string.IsNullOrEmpty(host) || port <= 0)
{
Logger.ErrorFormat("TryCreateSyncProducer invalid arguments,host=[{0}] must not be empty or port=[{1}] must be valid", host, port);
return null;
}
try
{
syncProducer = new SyncProducer(new SyncProducerConfiguration() { BrokerId = brokerId, Host = host, Port = port });
}
catch (Exception e)
{
Logger.Error(string.Format("TryCreateSyncProducer exception,brokerId={0},host={1},port={2}", brokerId, host, port), e);
syncProducer = null;
}
return syncProducer;
}
public void ProducerSendsMessage()
{
string payload1 = "kafka 1.";
byte[] payloadData1 = Encoding.UTF8.GetBytes(payload1);
Message msg1 = new Message(payloadData1);
string payload2 = "kafka 2.";
byte[] payloadData2 = Encoding.UTF8.GetBytes(payload2);
Message msg2 = new Message(payloadData2);
var config = new SyncProducerConfig(clientConfig);
var producer = new SyncProducer(config);
var producerRequest = new ProducerRequest(CurrentTestTopic, 0, new List<Message>() { msg1, msg2 });
producer.Send(producerRequest);
}
public void ProducerSendsMessageWithLongTopic()
{
Message msg = new Message(Encoding.UTF8.GetBytes("test message"));
string topic = "ThisIsAVeryLongTopicThisIsAVeryLongTopicThisIsAVeryLongTopicThisIsAVeryLongTopicThisIsAVeryLongTopicThisIsAVeryLongTopic";
var config = new SyncProducerConfig(clientConfig);
var producer = new SyncProducer(config);
var producerRequest = new ProducerRequest(topic, 0, new List<Message>() { msg });
producer.Send(producerRequest);
}
public void ProducerSendsAndConsumerReceivesMultiRequest()
{
string testTopic1 = CurrentTestTopic + "1";
string testTopic2 = CurrentTestTopic + "2";
string testTopic3 = CurrentTestTopic + "3";
Message sourceMessage1 = new Message(Encoding.UTF8.GetBytes("1: TestMessage"));
Message sourceMessage2 = new Message(Encoding.UTF8.GetBytes("2: TestMessage"));
Message sourceMessage3 = new Message(Encoding.UTF8.GetBytes("3: TestMessage"));
Message sourceMessage4 = new Message(Encoding.UTF8.GetBytes("4: TestMessage"));
List<ProducerRequest> requests = new List<ProducerRequest>
{
new ProducerRequest(testTopic1, 0, new List<Message> { sourceMessage1 }),
new ProducerRequest(testTopic1, 0, new List<Message> { sourceMessage2 }),
new ProducerRequest(testTopic2, 0, new List<Message> { sourceMessage3 }),
new ProducerRequest(testTopic3, 0, new List<Message> { sourceMessage4 })
};
var config = new SyncProducerConfig(clientConfig);
var producer = new SyncProducer(config);
long currentOffset1 = TestHelper.GetCurrentKafkaOffset(testTopic1, clientConfig);
long currentOffset2 = TestHelper.GetCurrentKafkaOffset(testTopic2, clientConfig);
long currentOffset3 = TestHelper.GetCurrentKafkaOffset(testTopic3, clientConfig);
producer.MultiSend(requests);
ConsumerConfig consumerConfig = new ConsumerConfig(clientConfig);
IConsumer consumer = new Consumers.Consumer(consumerConfig);
MultiFetchRequest request = new MultiFetchRequest(new List<FetchRequest>
{
new FetchRequest(testTopic1, 0, currentOffset1),
new FetchRequest(testTopic2, 0, currentOffset2),
new FetchRequest(testTopic3, 0, currentOffset3)
});
IList<BufferedMessageSet> messageSets;
int totalWaitTimeInMiliseconds = 0;
int waitSingle = 100;
while (true)
{
Thread.Sleep(waitSingle);
messageSets = consumer.MultiFetch(request);
if (messageSets.Count > 2 && messageSets[0].Messages.Count() > 0 && messageSets[1].Messages.Count() > 0 && messageSets[2].Messages.Count() > 0)
{
break;
}
else
{
totalWaitTimeInMiliseconds += waitSingle;
if (totalWaitTimeInMiliseconds >= MaxTestWaitTimeInMiliseconds)
{
break;
}
}
}
Assert.AreEqual(3, messageSets.Count);
Assert.AreEqual(2, messageSets[0].Messages.Count());
Assert.AreEqual(1, messageSets[1].Messages.Count());
Assert.AreEqual(1, messageSets[2].Messages.Count());
Assert.AreEqual(sourceMessage1.ToString(), messageSets[0].Messages.First().ToString());
Assert.AreEqual(sourceMessage2.ToString(), messageSets[0].Messages.Skip(1).First().ToString());
Assert.AreEqual(sourceMessage3.ToString(), messageSets[1].Messages.First().ToString());
Assert.AreEqual(sourceMessage4.ToString(), messageSets[2].Messages.First().ToString());
}
public void ProducerSendsAndConsumerReceivesSingleSimpleMessage()
{
Message sourceMessage = new Message(Encoding.UTF8.GetBytes("test message"));
var config = new SyncProducerConfig(clientConfig);
var producer = new SyncProducer(config);
var producerRequest = new ProducerRequest(CurrentTestTopic, 0, new List<Message>() { sourceMessage });
long currentOffset = TestHelper.GetCurrentKafkaOffset(CurrentTestTopic, clientConfig);
producer.Send(producerRequest);
ConsumerConfig consumerConfig = new ConsumerConfig(clientConfig);
IConsumer consumer = new Consumers.Consumer(consumerConfig);
FetchRequest request = new FetchRequest(CurrentTestTopic, 0, currentOffset);
BufferedMessageSet response;
int totalWaitTimeInMiliseconds = 0;
int waitSingle = 100;
while (true)
{
Thread.Sleep(waitSingle);
response = consumer.Fetch(request);
if (response != null && response.Messages.Count() > 0)
{
break;
}
else
{
totalWaitTimeInMiliseconds += waitSingle;
if (totalWaitTimeInMiliseconds >= MaxTestWaitTimeInMiliseconds)
{
break;
}
}
}
Assert.NotNull(response);
Assert.AreEqual(1, response.Messages.Count());
Message resultMessage = response.Messages.First();
Assert.AreEqual(sourceMessage.ToString(), resultMessage.ToString());
}
public void ProducerSendsMessageWithLongTopic()
{
var prodConfig = this.SyncProducerConfig1;
var msg = new Message(Encoding.UTF8.GetBytes("test message"));
string topic = "ThisIsAVeryLongTopicThisIsAVeryLongTopicThisIsAVeryLongTopicThisIsAVeryLongTopicThisIsAVeryLongTopicThisIsAVeryLongTopic";
using (var producer = new SyncProducer(prodConfig))
{
var producerRequest = new ProducerRequest(topic, 0, new List<Message> { msg });
producer.Send(producerRequest);
}
}
public void ProducerSendMultiRequest()
{
List<ProducerRequest> requests = new List<ProducerRequest>
{
new ProducerRequest(CurrentTestTopic, 0, new List<Message> { new Message(Encoding.UTF8.GetBytes("1: " + DateTime.UtcNow)) }),
new ProducerRequest(CurrentTestTopic, 0, new List<Message> { new Message(Encoding.UTF8.GetBytes("2: " + DateTime.UtcNow)) }),
new ProducerRequest(CurrentTestTopic, 0, new List<Message> { new Message(Encoding.UTF8.GetBytes("3: " + DateTime.UtcNow)) }),
new ProducerRequest(CurrentTestTopic, 0, new List<Message> { new Message(Encoding.UTF8.GetBytes("4: " + DateTime.UtcNow)) })
};
var config = new SyncProducerConfig(clientConfig);
var producer = new SyncProducer(config);
producer.MultiSend(requests);
}
public void ProducerSendsMessage()
{
var prodConfig = this.SyncProducerConfig1;
string payload1 = "kafka 1.";
byte[] payloadData1 = Encoding.UTF8.GetBytes(payload1);
var msg1 = new Message(payloadData1);
string payload2 = "kafka 2.";
byte[] payloadData2 = Encoding.UTF8.GetBytes(payload2);
var msg2 = new Message(payloadData2);
using (var producer = new SyncProducer(prodConfig))
{
var producerRequest = new ProducerRequest(CurrentTestTopic, 0, new List<Message> { msg1, msg2 });
producer.Send(producerRequest);
}
}
public void ProducerSendMultiRequest()
{
var prodConfig = this.SyncProducerConfig1;
var requests = new List<ProducerRequest>
{
new ProducerRequest(CurrentTestTopic, 0, new List<Message> { new Message(Encoding.UTF8.GetBytes("1: " + DateTime.UtcNow)) }),
new ProducerRequest(CurrentTestTopic, 0, new List<Message> { new Message(Encoding.UTF8.GetBytes("2: " + DateTime.UtcNow)) }),
new ProducerRequest(CurrentTestTopic, 0, new List<Message> { new Message(Encoding.UTF8.GetBytes("3: " + DateTime.UtcNow)) }),
new ProducerRequest(CurrentTestTopic, 0, new List<Message> { new Message(Encoding.UTF8.GetBytes("4: " + DateTime.UtcNow)) })
};
using (var producer = new SyncProducer(prodConfig))
{
producer.MultiSend(requests);
}
}
public void ConsumerConnectorConsumesTwoDifferentTopics()
{
string topic1 = CurrentTestTopic + "1";
string topic2 = CurrentTestTopic + "2";
// first producing
string payload1 = "kafka 1.";
byte[] payloadData1 = Encoding.UTF8.GetBytes(payload1);
var msg1 = new Message(payloadData1);
string payload2 = "kafka 2.";
byte[] payloadData2 = Encoding.UTF8.GetBytes(payload2);
var msg2 = new Message(payloadData2);
var producerConfig = new SyncProducerConfig(clientConfig);
var producer = new SyncProducer(producerConfig);
var producerRequest1 = new ProducerRequest(topic1, 0, new List<Message> { msg1 });
producer.Send(producerRequest1);
var producerRequest2 = new ProducerRequest(topic2, 0, new List<Message> { msg2 });
producer.Send(producerRequest2);
// now consuming
var config = new ConsumerConfig(clientConfig) { AutoCommit = false };
var resultMessages1 = new List<Message>();
var resultMessages2 = new List<Message>();
using (IConsumerConnector consumerConnector = new ZookeeperConsumerConnector(config, true))
{
var topicCount = new Dictionary<string, int> { { topic1, 1 }, { topic2, 1 } };
var messages = consumerConnector.CreateMessageStreams(topicCount);
Assert.IsTrue(messages.ContainsKey(topic1));
Assert.IsTrue(messages.ContainsKey(topic2));
var sets1 = messages[topic1];
try
{
foreach (var set in sets1)
{
foreach (var message in set)
{
resultMessages1.Add(message);
}
}
}
catch (ConsumerTimeoutException)
{
// do nothing, this is expected
}
var sets2 = messages[topic2];
try
{
foreach (var set in sets2)
{
foreach (var message in set)
{
resultMessages2.Add(message);
}
}
}
catch (ConsumerTimeoutException)
{
// do nothing, this is expected
}
}
Assert.AreEqual(1, resultMessages1.Count);
Assert.AreEqual(msg1.ToString(), resultMessages1[0].ToString());
Assert.AreEqual(1, resultMessages2.Count);
Assert.AreEqual(msg2.ToString(), resultMessages2[0].ToString());
}
public void SimpleSyncProducerSends2MessagesAndConsumerConnectorGetsThemBack()
{
// first producing
string payload1 = "kafka 1.";
byte[] payloadData1 = Encoding.UTF8.GetBytes(payload1);
var msg1 = new Message(payloadData1);
string payload2 = "kafka 2.";
byte[] payloadData2 = Encoding.UTF8.GetBytes(payload2);
var msg2 = new Message(payloadData2);
var producerConfig = new SyncProducerConfig(clientConfig);
var producer = new SyncProducer(producerConfig);
var producerRequest = new ProducerRequest(CurrentTestTopic, 0, new List<Message> { msg1, msg2 });
producer.Send(producerRequest);
// now consuming
var config = new ConsumerConfig(clientConfig) { AutoCommit = false };
var resultMessages = new List<Message>();
using (IConsumerConnector consumerConnector = new ZookeeperConsumerConnector(config, true))
{
var topicCount = new Dictionary<string, int> { { CurrentTestTopic, 1 } };
var messages = consumerConnector.CreateMessageStreams(topicCount);
var sets = messages[CurrentTestTopic];
try
{
foreach (var set in sets)
{
foreach (var message in set)
{
resultMessages.Add(message);
}
}
}
catch (ConsumerTimeoutException)
{
// do nothing, this is expected
}
}
Assert.AreEqual(2, resultMessages.Count);
Assert.AreEqual(msg1.ToString(), resultMessages[0].ToString());
Assert.AreEqual(msg2.ToString(), resultMessages[1].ToString());
}
public void OneMessageIsSentAndReceivedThenExceptionsWhenNoMessageThenAnotherMessageIsSentAndReceived()
{
// first producing
string payload1 = "kafka 1.";
byte[] payloadData1 = Encoding.UTF8.GetBytes(payload1);
var msg1 = new Message(payloadData1);
var producerConfig = new SyncProducerConfig(clientConfig);
var producer = new SyncProducer(producerConfig);
var producerRequest = new ProducerRequest(CurrentTestTopic, 0, new List<Message> { msg1 });
producer.Send(producerRequest);
// now consuming
var config = new ConsumerConfig(clientConfig) { AutoCommit = false, Timeout = 5000 };
using (IConsumerConnector consumerConnector = new ZookeeperConsumerConnector(config, true))
{
var topicCount = new Dictionary<string, int> { { CurrentTestTopic, 1 } };
var messages = consumerConnector.CreateMessageStreams(topicCount);
var sets = messages[CurrentTestTopic];
KafkaMessageStream myStream = sets[0];
var enumerator = myStream.GetEnumerator();
Assert.IsTrue(enumerator.MoveNext());
Assert.AreEqual(msg1.ToString(), enumerator.Current.ToString());
Assert.Throws<ConsumerTimeoutException>(() => enumerator.MoveNext());
Assert.Throws<Exception>(() => enumerator.MoveNext()); // iterator is in failed state
enumerator.Reset();
// producing again
string payload2 = "kafka 2.";
byte[] payloadData2 = Encoding.UTF8.GetBytes(payload2);
var msg2 = new Message(payloadData2);
var producerRequest2 = new ProducerRequest(CurrentTestTopic, 0, new List<Message> { msg2 });
producer.Send(producerRequest2);
Thread.Sleep(3000);
Assert.IsTrue(enumerator.MoveNext());
Assert.AreEqual(msg2.ToString(), enumerator.Current.ToString());
}
}
