public void AsyncProducerSendsAndConsumerReceivesSingleSimpleMessage()
{
Message sourceMessage = new Message(Encoding.UTF8.GetBytes("test message"));
var config = new AsyncProducerConfig(clientConfig);
var producer = new AsyncProducer(config);
var producerRequest = new ProducerRequest(CurrentTestTopic, 0, new List<Message>() { sourceMessage });
long currentOffset = TestHelper.GetCurrentKafkaOffset(CurrentTestTopic, clientConfig);
producer.Send(producerRequest);
ConsumerConfig consumerConfig = new ConsumerConfig(clientConfig);
IConsumer consumer = new Consumers.Consumer(consumerConfig);
FetchRequest request = new FetchRequest(CurrentTestTopic, 0, currentOffset);
BufferedMessageSet response;
int totalWaitTimeInMiliseconds = 0;
int waitSingle = 100;
while (true)
{
Thread.Sleep(waitSingle);
response = consumer.Fetch(request);
if (response != null && response.Messages.Count() > 0)
{
break;
}
else
{
totalWaitTimeInMiliseconds += waitSingle;
if (totalWaitTimeInMiliseconds >= MaxTestWaitTimeInMiliseconds)
{
break;
}
}
}
Assert.NotNull(response);
Assert.AreEqual(1, response.Messages.Count());
Message resultMessage = response.Messages.First();
Assert.AreEqual(sourceMessage.ToString(), resultMessage.ToString());
}
public void WriteToValidFormat()
{
string topicName = "topic";
ProducerRequest request = new ProducerRequest(
topicName, 0, new List<Message> { new Message(new byte[10]) });
// format = len(request) + requesttype + len(topic) + topic + partition + len(messagepack) + message
// total byte count = (4 + 2 + 2 + 5 + 4 + 4 + 19)
System.IO.MemoryStream ms = new MemoryStream();
request.WriteTo(ms);
byte[] bytes = ms.ToArray();
Assert.IsNotNull(bytes);
Assert.AreEqual(41, bytes.Length);
// next 4 bytes = the length of the request
Assert.AreEqual(37, BitConverter.ToInt32(BitWorks.ReverseBytes(bytes.Take(4).ToArray<byte>()), 0));
// next 2 bytes = the RequestType which in this case should be Produce
Assert.AreEqual((short)RequestTypes.Produce, BitConverter.ToInt16(BitWorks.ReverseBytes(bytes.Skip(4).Take(2).ToArray<byte>()), 0));
// next 2 bytes = the length of the topic
Assert.AreEqual((short)5, BitConverter.ToInt16(BitWorks.ReverseBytes(bytes.Skip(6).Take(2).ToArray<byte>()), 0));
// next 5 bytes = the topic
Assert.AreEqual(topicName, Encoding.ASCII.GetString(bytes.Skip(8).Take(5).ToArray<byte>()));
// next 4 bytes = the partition
Assert.AreEqual(0, BitConverter.ToInt32(BitWorks.ReverseBytes(bytes.Skip(13).Take(4).ToArray<byte>()), 0));
// next 4 bytes = the length of the individual messages in the pack
Assert.AreEqual(20, BitConverter.ToInt32(BitWorks.ReverseBytes(bytes.Skip(17).Take(4).ToArray<byte>()), 0));
// fianl bytes = the individual messages in the pack
Assert.AreEqual(20, bytes.Skip(21).ToArray<byte>().Length);
}
/// <summary>
/// Sends request to Kafka server synchronously
/// </summary>
/// <param name="request">
/// The request.
/// </param>
public void Send(ProducerRequest request)
{
this.EnsuresNotDisposed();
this.connection.Write(request);
}
/// <summary>
/// Sends request to Kafka server asynchronously
/// </summary>
/// <param name="request">
/// The request.
/// </param>
public void Send(ProducerRequest request)
{
Guard.Assert<ArgumentNullException>(() => request != null);
Guard.Assert<ArgumentException>(() => request.MessageSet.Messages.All(x => x.PayloadSize <= this.Config.MaxMessageSize));
if (this.callbackHandler != null)
{
this.Send(request, this.callbackHandler.Handle);
}
else
{
using (var conn = new KafkaConnection(this.config.Host, this.config.Port))
{
conn.BeginWrite(request);
}
}
}
/// <summary>
/// Sends request to Kafka server asynchronously
/// </summary>
/// <param name="request">
/// The request.
/// </param>
/// <param name="callback">
/// The callback invoked when a request is finished being sent.
/// </param>
public void Send(ProducerRequest request, MessageSent<ProducerRequest> callback)
{
Guard.Assert<ArgumentNullException>(() => request != null);
Guard.Assert<ArgumentNullException>(() => request.MessageSet != null);
Guard.Assert<ArgumentNullException>(() => request.MessageSet.Messages != null);
Guard.Assert<ArgumentException>(
() => request.MessageSet.Messages.All(x => x.PayloadSize <= this.Config.MaxMessageSize));
using (var conn = new KafkaConnection(this.config.Host, this.config.Port))
{
conn.BeginWrite(request, callback);
}
}
public void ProducerSendsMessageWithLongTopic()
{
Message msg = new Message(Encoding.UTF8.GetBytes("test message"));
string topic = "ThisIsAVeryLongTopicThisIsAVeryLongTopicThisIsAVeryLongTopicThisIsAVeryLongTopicThisIsAVeryLongTopicThisIsAVeryLongTopic";
var config = new SyncProducerConfig(clientConfig);
var producer = new SyncProducer(config);
var producerRequest = new ProducerRequest(topic, 0, new List<Message>() { msg });
producer.Send(producerRequest);
}
public void ProducerSendsMessage()
{
string payload1 = "kafka 1.";
byte[] payloadData1 = Encoding.UTF8.GetBytes(payload1);
Message msg1 = new Message(payloadData1);
string payload2 = "kafka 2.";
byte[] payloadData2 = Encoding.UTF8.GetBytes(payload2);
Message msg2 = new Message(payloadData2);
var config = new SyncProducerConfig(clientConfig);
var producer = new SyncProducer(config);
var producerRequest = new ProducerRequest(CurrentTestTopic, 0, new List<Message>() { msg1, msg2 });
producer.Send(producerRequest);
}
public void ProducerSendsMessageWithLongTopic()
{
var prodConfig = this.SyncProducerConfig1;
var msg = new Message(Encoding.UTF8.GetBytes("test message"));
string topic = "ThisIsAVeryLongTopicThisIsAVeryLongTopicThisIsAVeryLongTopicThisIsAVeryLongTopicThisIsAVeryLongTopicThisIsAVeryLongTopic";
using (var producer = new SyncProducer(prodConfig))
{
var producerRequest = new ProducerRequest(topic, 0, new List<Message> { msg });
producer.Send(producerRequest);
}
}
public void ProducerSendsMessage()
{
var prodConfig = this.SyncProducerConfig1;
string payload1 = "kafka 1.";
byte[] payloadData1 = Encoding.UTF8.GetBytes(payload1);
var msg1 = new Message(payloadData1);
string payload2 = "kafka 2.";
byte[] payloadData2 = Encoding.UTF8.GetBytes(payload2);
var msg2 = new Message(payloadData2);
using (var producer = new SyncProducer(prodConfig))
{
var producerRequest = new ProducerRequest(CurrentTestTopic, 0, new List<Message> { msg1, msg2 });
producer.Send(producerRequest);
}
}
public void AsyncProducerSendsFewShortFixedMessagesInSeparateSendActions()
{
var prodConfig = this.AsyncProducerConfig1;
using (var producer = new AsyncProducer(prodConfig))
{
var req1 = new ProducerRequest(
CurrentTestTopic,
0,
new List<Message> { new Message(Encoding.UTF8.GetBytes("Async Test Message 1")) });
producer.Send(req1);
var req2 = new ProducerRequest(
CurrentTestTopic,
0,
new List<Message> { new Message(Encoding.UTF8.GetBytes("Async Test Message 2")) });
producer.Send(req2);
var req3 = new ProducerRequest(
CurrentTestTopic,
0,
new List<Message> { new Message(Encoding.UTF8.GetBytes("Async Test Message 3")) });
producer.Send(req3);
}
}
/// <summary>
/// Sends request to Kafka server asynchronously
/// </summary>
/// <param name="request">
/// The request.
/// </param>
/// <param name="callback">
/// The callback invoked when a request is finished being sent.
/// </param>
public void Send(ProducerRequest request, MessageSent<ProducerRequest> callback)
{
this.EnsuresNotDisposed();
Guard.NotNull(request, "request");
Guard.NotNull(request.MessageSet, "request.MessageSet");
Guard.NotNull(request.MessageSet.Messages, "request.MessageSet.Messages");
Guard.Assert<ArgumentException>(
() => request.MessageSet.Messages.All(x => x.PayloadSize <= this.Config.MaxMessageSize));
connection.BeginWrite(request, callback);
}
/// <summary>
/// Sends request to Kafka server asynchronously
/// </summary>
/// <param name="request">
/// The request.
/// </param>
public void Send(ProducerRequest request)
{
this.EnsuresNotDisposed();
Guard.NotNull(request, "request");
Guard.Assert<ArgumentException>(() => request.MessageSet.Messages.All(x => x.PayloadSize <= this.Config.MaxMessageSize));
if (this.callbackHandler != null)
{
this.Send(request, this.callbackHandler.Handle);
}
else
{
this.connection.BeginWrite(request);
}
}
public void ConsumerConnectorConsumesTwoDifferentTopics()
{
string topic1 = CurrentTestTopic + "1";
string topic2 = CurrentTestTopic + "2";
// first producing
string payload1 = "kafka 1.";
byte[] payloadData1 = Encoding.UTF8.GetBytes(payload1);
var msg1 = new Message(payloadData1);
string payload2 = "kafka 2.";
byte[] payloadData2 = Encoding.UTF8.GetBytes(payload2);
var msg2 = new Message(payloadData2);
var producerConfig = new SyncProducerConfig(clientConfig);
var producer = new SyncProducer(producerConfig);
var producerRequest1 = new ProducerRequest(topic1, 0, new List<Message> { msg1 });
producer.Send(producerRequest1);
var producerRequest2 = new ProducerRequest(topic2, 0, new List<Message> { msg2 });
producer.Send(producerRequest2);
// now consuming
var config = new ConsumerConfig(clientConfig) { AutoCommit = false };
var resultMessages1 = new List<Message>();
var resultMessages2 = new List<Message>();
using (IConsumerConnector consumerConnector = new ZookeeperConsumerConnector(config, true))
{
var topicCount = new Dictionary<string, int> { { topic1, 1 }, { topic2, 1 } };
var messages = consumerConnector.CreateMessageStreams(topicCount);
Assert.IsTrue(messages.ContainsKey(topic1));
Assert.IsTrue(messages.ContainsKey(topic2));
var sets1 = messages[topic1];
try
{
foreach (var set in sets1)
{
foreach (var message in set)
{
resultMessages1.Add(message);
}
}
}
catch (ConsumerTimeoutException)
{
// do nothing, this is expected
}
var sets2 = messages[topic2];
try
{
foreach (var set in sets2)
{
foreach (var message in set)
{
resultMessages2.Add(message);
}
}
}
catch (ConsumerTimeoutException)
{
// do nothing, this is expected
}
}
Assert.AreEqual(1, resultMessages1.Count);
Assert.AreEqual(msg1.ToString(), resultMessages1[0].ToString());
Assert.AreEqual(1, resultMessages2.Count);
Assert.AreEqual(msg2.ToString(), resultMessages2[0].ToString());
}
public void SimpleSyncProducerSends2MessagesAndConsumerConnectorGetsThemBack()
{
// first producing
string payload1 = "kafka 1.";
byte[] payloadData1 = Encoding.UTF8.GetBytes(payload1);
var msg1 = new Message(payloadData1);
string payload2 = "kafka 2.";
byte[] payloadData2 = Encoding.UTF8.GetBytes(payload2);
var msg2 = new Message(payloadData2);
var producerConfig = new SyncProducerConfig(clientConfig);
var producer = new SyncProducer(producerConfig);
var producerRequest = new ProducerRequest(CurrentTestTopic, 0, new List<Message> { msg1, msg2 });
producer.Send(producerRequest);
// now consuming
var config = new ConsumerConfig(clientConfig) { AutoCommit = false };
var resultMessages = new List<Message>();
using (IConsumerConnector consumerConnector = new ZookeeperConsumerConnector(config, true))
{
var topicCount = new Dictionary<string, int> { { CurrentTestTopic, 1 } };
var messages = consumerConnector.CreateMessageStreams(topicCount);
var sets = messages[CurrentTestTopic];
try
{
foreach (var set in sets)
{
foreach (var message in set)
{
resultMessages.Add(message);
}
}
}
catch (ConsumerTimeoutException)
{
// do nothing, this is expected
}
}
Assert.AreEqual(2, resultMessages.Count);
Assert.AreEqual(msg1.ToString(), resultMessages[0].ToString());
Assert.AreEqual(msg2.ToString(), resultMessages[1].ToString());
}
public void OneMessageIsSentAndReceivedThenExceptionsWhenNoMessageThenAnotherMessageIsSentAndReceived()
{
// first producing
string payload1 = "kafka 1.";
byte[] payloadData1 = Encoding.UTF8.GetBytes(payload1);
var msg1 = new Message(payloadData1);
var producerConfig = new SyncProducerConfig(clientConfig);
var producer = new SyncProducer(producerConfig);
var producerRequest = new ProducerRequest(CurrentTestTopic, 0, new List<Message> { msg1 });
producer.Send(producerRequest);
// now consuming
var config = new ConsumerConfig(clientConfig) { AutoCommit = false, Timeout = 5000 };
using (IConsumerConnector consumerConnector = new ZookeeperConsumerConnector(config, true))
{
var topicCount = new Dictionary<string, int> { { CurrentTestTopic, 1 } };
var messages = consumerConnector.CreateMessageStreams(topicCount);
var sets = messages[CurrentTestTopic];
KafkaMessageStream myStream = sets[0];
var enumerator = myStream.GetEnumerator();
Assert.IsTrue(enumerator.MoveNext());
Assert.AreEqual(msg1.ToString(), enumerator.Current.ToString());
Assert.Throws<ConsumerTimeoutException>(() => enumerator.MoveNext());
Assert.Throws<Exception>(() => enumerator.MoveNext()); // iterator is in failed state
enumerator.Reset();
// producing again
string payload2 = "kafka 2.";
byte[] payloadData2 = Encoding.UTF8.GetBytes(payload2);
var msg2 = new Message(payloadData2);
var producerRequest2 = new ProducerRequest(CurrentTestTopic, 0, new List<Message> { msg2 });
producer.Send(producerRequest2);
Thread.Sleep(3000);
Assert.IsTrue(enumerator.MoveNext());
Assert.AreEqual(msg2.ToString(), enumerator.Current.ToString());
}
}
/// <summary>
/// Sends request to Kafka server synchronously
/// </summary>
/// <param name="request">
/// The request.
/// </param>
public void Send(ProducerRequest request)
{
Guard.Assert<ArgumentNullException>(() => request != null);
using (var conn = new KafkaConnection(this.config.Host, this.config.Port))
{
conn.Write(request);
}
}