public void ZKAwareProducerSends1Message() { int totalWaitTimeInMiliseconds = 0; int waitSingle = 100; var originalMessage = new Message(Encoding.UTF8.GetBytes("TestData")); var multipleBrokersHelper = new TestMultipleBrokersHelper(CurrentTestTopic); multipleBrokersHelper.GetCurrentOffsets(); var producerConfig = new ProducerConfig(clientConfig); var mockPartitioner = new MockAlwaysZeroPartitioner(); using (var producer = new Producer<string, Message>(producerConfig, mockPartitioner, new DefaultEncoder())) { var producerData = new ProducerData<string, Message>( CurrentTestTopic, "somekey", new List<Message>() { originalMessage }); producer.Send(producerData); while (!multipleBrokersHelper.CheckIfAnyBrokerHasChanged()) { totalWaitTimeInMiliseconds += waitSingle; Thread.Sleep(waitSingle); if (totalWaitTimeInMiliseconds > MaxTestWaitTimeInMiliseconds) { Assert.Fail("None of the brokers changed their offset after sending a message"); } } totalWaitTimeInMiliseconds = 0; var consumerConfig = new ConsumerConfig(clientConfig) { Host = multipleBrokersHelper.BrokerThatHasChanged.Address, Port = multipleBrokersHelper.BrokerThatHasChanged.Port }; IConsumer consumer = new Consumers.Consumer(consumerConfig); var request = new FetchRequest(CurrentTestTopic, 0, multipleBrokersHelper.OffsetFromBeforeTheChange); BufferedMessageSet response; while (true) { Thread.Sleep(waitSingle); response = consumer.Fetch(request); if (response != null & response.Messages.Count() > 0) { break; } totalWaitTimeInMiliseconds += waitSingle; if (totalWaitTimeInMiliseconds >= MaxTestWaitTimeInMiliseconds) { break; } } Assert.NotNull(response); Assert.AreEqual(1, response.Messages.Count()); Assert.AreEqual(originalMessage.ToString(), response.Messages.First().ToString()); } }
public void ProducerSends1Message() { var prodConfig = this.ConfigBasedSyncProdConfig; int totalWaitTimeInMiliseconds = 0; int waitSingle = 100; var originalMessage = new Message(Encoding.UTF8.GetBytes("TestData")); var multipleBrokersHelper = new TestMultipleBrokersHelper(CurrentTestTopic); multipleBrokersHelper.GetCurrentOffsets( new[] { this.SyncProducerConfig1, this.SyncProducerConfig2, this.SyncProducerConfig3 }); using (var producer = new Producer(prodConfig)) { var producerData = new ProducerData<string, Message>( CurrentTestTopic, new List<Message> { originalMessage }); producer.Send(producerData); Thread.Sleep(waitSingle); } while ( !multipleBrokersHelper.CheckIfAnyBrokerHasChanged( new[] { this.SyncProducerConfig1, this.SyncProducerConfig2, this.SyncProducerConfig3 })) { totalWaitTimeInMiliseconds += waitSingle; Thread.Sleep(waitSingle); if (totalWaitTimeInMiliseconds > this.maxTestWaitTimeInMiliseconds) { Assert.Fail("None of the brokers changed their offset after sending a message"); } } totalWaitTimeInMiliseconds = 0; var consumerConfig = new ConsumerConfiguration( multipleBrokersHelper.BrokerThatHasChanged.Host, multipleBrokersHelper.BrokerThatHasChanged.Port); IConsumer consumer = new Consumer(consumerConfig); var request1 = new FetchRequest(CurrentTestTopic, multipleBrokersHelper.PartitionThatHasChanged, multipleBrokersHelper.OffsetFromBeforeTheChange); BufferedMessageSet response; while (true) { Thread.Sleep(waitSingle); response = consumer.Fetch(request1); if (response != null && response.Messages.Count() > 0) { break; } totalWaitTimeInMiliseconds += waitSingle; if (totalWaitTimeInMiliseconds >= this.maxTestWaitTimeInMiliseconds) { break; } } Assert.NotNull(response); Assert.AreEqual(1, response.Messages.Count()); Assert.AreEqual(originalMessage.ToString(), response.Messages.First().ToString()); }
internal void AddMessage(KafkaMessages.Message message) { var slidingDoor = SlidingDoors.GetOrAdd(message.PartitionId.Value, partition => { return(new SlidingDoor(CommitOffset, partition, Configuration.Instance.GetCommitPerMessage())); }); slidingDoor.AddOffset(message.Offset); }
public static ConsumedMessage AsConsumedMessage(this KafkaMessage message) { return(new ConsumedMessage { Partition = message.PartitionId ?? 0, Offset = message.Offset, Key = message.Key.Decode(), Value = message.Payload.Decode(), Codec = (Compression)message.CompressionCodec }); }
public void TestChecksum() { foreach (var v in this.messages) { Assert.False(v.Message.Equals(null)); Assert.False(v.Message.Equals("asdf")); Assert.True(v.Message.Equals(v.Message)); var copy = new Message(v.Payload, v.Key, v.Codec); Assert.True(v.Message.Equals(copy)); } }
public void BufferedMessageSetWriteToValidSequence() { byte[] messageBytes = new byte[] { 1, 2, 3, 4, 5 }; Message msg1 = new Message(messageBytes); Message msg2 = new Message(messageBytes); MessageSet messageSet = new BufferedMessageSet(new List<Message>() { msg1, msg2 }); MemoryStream ms = new MemoryStream(); messageSet.WriteTo(ms); ////first message byte[] messageLength = new byte[MessageLengthPartLength]; Array.Copy(ms.ToArray(), MessageLengthPartOffset, messageLength, 0, MessageLengthPartLength); if (BitConverter.IsLittleEndian) { Array.Reverse(messageLength); } Assert.AreEqual(MagicNumberPartLength + AttributesPartLength + ChecksumPartLength + messageBytes.Length, BitConverter.ToInt32(messageLength, 0)); Assert.AreEqual(1, ms.ToArray()[MagicNumberPartOffset]); // default magic number should be 1 byte[] checksumPart = new byte[ChecksumPartLength]; Array.Copy(ms.ToArray(), ChecksumPartOffset, checksumPart, 0, ChecksumPartLength); Assert.AreEqual(Crc32Hasher.Compute(messageBytes), checksumPart); byte[] dataPart = new byte[messageBytes.Length]; Array.Copy(ms.ToArray(), DataPartOffset, dataPart, 0, messageBytes.Length); Assert.AreEqual(messageBytes, dataPart); ////second message int secondMessageOffset = MessageLengthPartLength + MagicNumberPartLength + AttributesPartLength + ChecksumPartLength + messageBytes.Length; messageLength = new byte[MessageLengthPartLength]; Array.Copy(ms.ToArray(), secondMessageOffset + MessageLengthPartOffset, messageLength, 0, MessageLengthPartLength); if (BitConverter.IsLittleEndian) { Array.Reverse(messageLength); } Assert.AreEqual(MagicNumberPartLength + AttributesPartLength + ChecksumPartLength + messageBytes.Length, BitConverter.ToInt32(messageLength, 0)); Assert.AreEqual(1, ms.ToArray()[secondMessageOffset + MagicNumberPartOffset]); // default magic number should be 1 checksumPart = new byte[ChecksumPartLength]; Array.Copy(ms.ToArray(), secondMessageOffset + ChecksumPartOffset, checksumPart, 0, ChecksumPartLength); Assert.AreEqual(Crc32Hasher.Compute(messageBytes), checksumPart); dataPart = new byte[messageBytes.Length]; Array.Copy(ms.ToArray(), secondMessageOffset + DataPartOffset, dataPart, 0, messageBytes.Length); Assert.AreEqual(messageBytes, dataPart); }
public void ShouldAbleToEnumerateMessages() { var msg1 = new Message(new byte[101]) {Offset = 0}; var msg2 = new Message(new byte[102]) {Offset = 1}; var set = new BufferedMessageSet(new List<Message>() {msg1, msg2}, 0); set.MoveNext().Should().BeTrue(); set.Current.Message.Payload.Length.Should().Be(101); set.Current.Message.Offset.Should().Be(0); set.MoveNext().Should().BeTrue(); set.Current.Message.Payload.Length.Should().Be(102); set.Current.Message.Offset.Should().Be(1); set.MoveNext().Should().BeFalse(); }
private static void Initializer(ProducePerfTestHelperOption producewrapperOption) { Logger.InfoFormat("prepare perf test, {0} ", DateTime.Now); listOfDataNeedSendInOneBatch = new List<ProducerData<byte[], Message>>(); for (int i = 0; i < producewrapperOption.MessageCountPerBatch; i++) { String vKey = KafkaClientHelperUtils.GetRandomString(32); byte[] bKey = System.Text.Encoding.UTF8.GetBytes(vKey); String val = KafkaClientHelperUtils.GetRandomString(producewrapperOption.MessageSize); byte[] bVal = System.Text.Encoding.UTF8.GetBytes(val); Message message = new Message(bVal, bKey, producewrapperOption.CompressionCodec); listOfDataNeedSendInOneBatch.Add(new ProducerData<byte[], Message>(producewrapperOption.Topic, message)); } }
public void ShouldAbleToWriteMessageSetWithExtraBytes() { var stream = new MemoryStream(); var writer = new KafkaBinaryWriter(stream); var msg1 = new Message(new byte[101]) {Offset = 0}; var msg2 = new Message(new byte[102]) {Offset = 1}; var set = new BufferedMessageSet(new List<Message>() {msg1, msg2}, 0); set.WriteTo(writer); writer.Write(new byte[10]); // less than offset and size var size = (int) stream.Position; stream.Seek(0, SeekOrigin.Begin); var reader = new KafkaBinaryReader(stream); var newSet = BufferedMessageSet.ParseFrom(reader, size, 0); var messages = newSet.Messages.ToList(); messages.Count().Should().Be(2); messages[0].Payload.Count().Should().Be(101); messages[1].Payload.Count().Should().Be(102); }
public void GetBytesValidSequence() { Message message = new Message(new byte[10], (byte)245); MemoryStream ms = new MemoryStream(); message.WriteTo(ms); // len(payload) + 1 + 4 Assert.AreEqual(15, ms.Length); // first 4 bytes = the magic number Assert.AreEqual((byte)245, ms.ToArray()[0]); // next 4 bytes = the checksum Assert.IsTrue(message.Checksum.SequenceEqual(ms.ToArray().Skip(1).Take(4).ToArray<byte>())); // remaining bytes = the payload Assert.AreEqual(10, ms.ToArray().Skip(5).ToArray<byte>().Length); }
public void AsyncProducerSendsAndConsumerReceivesSingleSimpleMessage() { Message sourceMessage = new Message(Encoding.UTF8.GetBytes("test message")); var config = new AsyncProducerConfig(clientConfig); var producer = new AsyncProducer(config); var producerRequest = new ProducerRequest(CurrentTestTopic, 0, new List<Message>() { sourceMessage }); long currentOffset = TestHelper.GetCurrentKafkaOffset(CurrentTestTopic, clientConfig); producer.Send(producerRequest); ConsumerConfig consumerConfig = new ConsumerConfig(clientConfig); IConsumer consumer = new Consumers.Consumer(consumerConfig); FetchRequest request = new FetchRequest(CurrentTestTopic, 0, currentOffset); BufferedMessageSet response; int totalWaitTimeInMiliseconds = 0; int waitSingle = 100; while (true) { Thread.Sleep(waitSingle); response = consumer.Fetch(request); if (response != null && response.Messages.Count() > 0) { break; } else { totalWaitTimeInMiliseconds += waitSingle; if (totalWaitTimeInMiliseconds >= MaxTestWaitTimeInMiliseconds) { break; } } } Assert.NotNull(response); Assert.AreEqual(1, response.Messages.Count()); Message resultMessage = response.Messages.First(); Assert.AreEqual(sourceMessage.ToString(), resultMessage.ToString()); }
public void GetBytesValidSequence() { Message message = new Message(new byte[10], CompressionCodecs.NoCompressionCodec); MemoryStream ms = new MemoryStream(); message.WriteTo(ms); // len(payload) + 1 + 4 Assert.AreEqual(16, ms.Length); // first 4 bytes = the magic number Assert.AreEqual((byte)1, ms.ToArray()[0]); // attributes Assert.AreEqual((byte)0, ms.ToArray()[1]); // next 4 bytes = the checksum Assert.IsTrue(message.Checksum.SequenceEqual(ms.ToArray().Skip(2).Take(4).ToArray<byte>())); // remaining bytes = the payload Assert.AreEqual(10, ms.ToArray().Skip(6).ToArray<byte>().Length); }
public void then_properties_should_be_set() { var message = new KafkaMessage( "value1".Encode(), "key1".Encode(), CompressionCodecs.GZIPCompressionCodec ) { PartitionId = 1, Offset = 1234 }; var actual = message.AsConsumedMessage(); var expected = new ConsumedMessage { Codec = Compression.GZip, Key = "key1", Value = "value1", Partition = 1, Offset = 1234 }; actual.ShouldBeEquivalentTo(expected); }
public void WriteToValidSequenceForDefaultConstructor() { byte[] messageBytes = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9 }; Message message = new Message(messageBytes); MemoryStream ms = new MemoryStream(); message.WriteTo(ms); Assert.AreEqual(0, ms.ToArray()[MagicNumberPartOffset]); // default magic number should be 0 byte[] checksumPart = new byte[ChecksumPartLength]; Array.Copy(ms.ToArray(), ChecksumPartOffset, checksumPart, 0, ChecksumPartLength); Assert.AreEqual(Crc32Hasher.Compute(messageBytes), checksumPart); byte[] dataPart = new byte[messageBytes.Length]; Array.Copy(ms.ToArray(), DataPartOffset, dataPart, 0, messageBytes.Length); Assert.AreEqual(messageBytes, dataPart); }
protected bool Equals(Message other) { return this.buffer.Equals(other.Buffer); }
public void SimpleSyncProducerSends2MessagesAndConsumerConnectorGetsThemBack() { // first producing string payload1 = "kafka 1."; byte[] payloadData1 = Encoding.UTF8.GetBytes(payload1); var msg1 = new Message(payloadData1); string payload2 = "kafka 2."; byte[] payloadData2 = Encoding.UTF8.GetBytes(payload2); var msg2 = new Message(payloadData2); var producerConfig = new SyncProducerConfig(clientConfig); var producer = new SyncProducer(producerConfig); var producerRequest = new ProducerRequest(CurrentTestTopic, 0, new List<Message> { msg1, msg2 }); producer.Send(producerRequest); // now consuming var config = new ConsumerConfig(clientConfig) { AutoCommit = false }; var resultMessages = new List<Message>(); using (IConsumerConnector consumerConnector = new ZookeeperConsumerConnector(config, true)) { var topicCount = new Dictionary<string, int> { { CurrentTestTopic, 1 } }; var messages = consumerConnector.CreateMessageStreams(topicCount); var sets = messages[CurrentTestTopic]; try { foreach (var set in sets) { foreach (var message in set) { resultMessages.Add(message); } } } catch (ConsumerTimeoutException) { // do nothing, this is expected } } Assert.AreEqual(2, resultMessages.Count); Assert.AreEqual(msg1.ToString(), resultMessages[0].ToString()); Assert.AreEqual(msg2.ToString(), resultMessages[1].ToString()); }
public void OneMessageIsSentAndReceivedThenExceptionsWhenNoMessageThenAnotherMessageIsSentAndReceived() { // first producing string payload1 = "kafka 1."; byte[] payloadData1 = Encoding.UTF8.GetBytes(payload1); var msg1 = new Message(payloadData1); var producerConfig = new SyncProducerConfig(clientConfig); var producer = new SyncProducer(producerConfig); var producerRequest = new ProducerRequest(CurrentTestTopic, 0, new List<Message> { msg1 }); producer.Send(producerRequest); // now consuming var config = new ConsumerConfig(clientConfig) { AutoCommit = false, Timeout = 5000 }; using (IConsumerConnector consumerConnector = new ZookeeperConsumerConnector(config, true)) { var topicCount = new Dictionary<string, int> { { CurrentTestTopic, 1 } }; var messages = consumerConnector.CreateMessageStreams(topicCount); var sets = messages[CurrentTestTopic]; KafkaMessageStream myStream = sets[0]; var enumerator = myStream.GetEnumerator(); Assert.IsTrue(enumerator.MoveNext()); Assert.AreEqual(msg1.ToString(), enumerator.Current.ToString()); Assert.Throws<ConsumerTimeoutException>(() => enumerator.MoveNext()); Assert.Throws<Exception>(() => enumerator.MoveNext()); // iterator is in failed state enumerator.Reset(); // producing again string payload2 = "kafka 2."; byte[] payloadData2 = Encoding.UTF8.GetBytes(payload2); var msg2 = new Message(payloadData2); var producerRequest2 = new ProducerRequest(CurrentTestTopic, 0, new List<Message> { msg2 }); producer.Send(producerRequest2); Thread.Sleep(3000); Assert.IsTrue(enumerator.MoveNext()); Assert.AreEqual(msg2.ToString(), enumerator.Current.ToString()); } }
public MessageTestVal(byte[] key, byte[] payload, CompressionCodecs codec, Message message) { this.Key = key; this.Payload = payload; this.Codec = codec; this.Message = message; }
public void ConsumerConnectorConsumesTwoDifferentTopics() { string topic1 = CurrentTestTopic + "1"; string topic2 = CurrentTestTopic + "2"; // first producing string payload1 = "kafka 1."; byte[] payloadData1 = Encoding.UTF8.GetBytes(payload1); var msg1 = new Message(payloadData1); string payload2 = "kafka 2."; byte[] payloadData2 = Encoding.UTF8.GetBytes(payload2); var msg2 = new Message(payloadData2); var producerConfig = new SyncProducerConfig(clientConfig); var producer = new SyncProducer(producerConfig); var producerRequest1 = new ProducerRequest(topic1, 0, new List<Message> { msg1 }); producer.Send(producerRequest1); var producerRequest2 = new ProducerRequest(topic2, 0, new List<Message> { msg2 }); producer.Send(producerRequest2); // now consuming var config = new ConsumerConfig(clientConfig) { AutoCommit = false }; var resultMessages1 = new List<Message>(); var resultMessages2 = new List<Message>(); using (IConsumerConnector consumerConnector = new ZookeeperConsumerConnector(config, true)) { var topicCount = new Dictionary<string, int> { { topic1, 1 }, { topic2, 1 } }; var messages = consumerConnector.CreateMessageStreams(topicCount); Assert.IsTrue(messages.ContainsKey(topic1)); Assert.IsTrue(messages.ContainsKey(topic2)); var sets1 = messages[topic1]; try { foreach (var set in sets1) { foreach (var message in set) { resultMessages1.Add(message); } } } catch (ConsumerTimeoutException) { // do nothing, this is expected } var sets2 = messages[topic2]; try { foreach (var set in sets2) { foreach (var message in set) { resultMessages2.Add(message); } } } catch (ConsumerTimeoutException) { // do nothing, this is expected } } Assert.AreEqual(1, resultMessages1.Count); Assert.AreEqual(msg1.ToString(), resultMessages1[0].ToString()); Assert.AreEqual(1, resultMessages2.Count); Assert.AreEqual(msg2.ToString(), resultMessages2[0].ToString()); }
public void ProducerSendsMessage() { string payload1 = "kafka 1."; byte[] payloadData1 = Encoding.UTF8.GetBytes(payload1); Message msg1 = new Message(payloadData1); string payload2 = "kafka 2."; byte[] payloadData2 = Encoding.UTF8.GetBytes(payload2); Message msg2 = new Message(payloadData2); var config = new SyncProducerConfig(clientConfig); var producer = new SyncProducer(config); var producerRequest = new ProducerRequest(CurrentTestTopic, 0, new List<Message>() { msg1, msg2 }); producer.Send(producerRequest); }
public void TestMessageSizeTooLarge() { var props = TestUtils.GetSyncProducerConfig(this.Configs[0].Port); var producer = new SyncProducer(props); AdminUtils.CreateTopic(this.ZkClient, "test", 1, 1, new Dictionary<string, string>()); TestUtils.WaitUntilLeaderIsElectedOrChanged(this.ZkClient, "test", 0, 500); TestUtils.WaitUntilMetadataIsPropagated(Servers, "test", 0, 2000); var message1 = new Message(new byte[Configs[0].MessageMaxBytes + 1]); var messageSet1 = new ByteBufferMessageSet(CompressionCodecs.NoCompressionCodec, new List<Message> { message1 }); var response1 = producer.Send(TestUtils.ProduceRequest("test", 0, messageSet1, acks: 1)); Assert.Equal(1, response1.Status.Count(kvp => kvp.Value.Error != ErrorMapping.NoError)); Assert.Equal(ErrorMapping.MessageSizeTooLargeCode, response1.Status[new TopicAndPartition("test", 0)].Error); Assert.Equal(-1L, response1.Status[new TopicAndPartition("test", 0)].Offset); var safeSize = Configs[0].MessageMaxBytes - Message.MessageOverhead - MessageSet.LogOverhead - 1; var message2 = new Message(new byte[safeSize]); var messageSet2 = new ByteBufferMessageSet( CompressionCodecs.NoCompressionCodec, new List<Message> { message2 }); var response2 = producer.Send(TestUtils.ProduceRequest("test", 0, messageSet2, acks: 1)); Assert.Equal(0, response2.Status.Count(kvp => kvp.Value.Error != ErrorMapping.NoError)); Assert.Equal(ErrorMapping.NoError, response2.Status[new TopicAndPartition("test", 0)].Error); Assert.Equal(0, response2.Status[new TopicAndPartition("test", 0)].Offset); }
public void ProducerSendsMessageWithLongTopic() { var prodConfig = this.SyncProducerConfig1; var msg = new Message(Encoding.UTF8.GetBytes("test message")); string topic = "ThisIsAVeryLongTopicThisIsAVeryLongTopicThisIsAVeryLongTopicThisIsAVeryLongTopicThisIsAVeryLongTopicThisIsAVeryLongTopic"; using (var producer = new SyncProducer(prodConfig)) { var producerRequest = new ProducerRequest(topic, 0, new List<Message> { msg }); producer.Send(producerRequest); } }
public void ProducerSendsAndConsumerReceivesMultiRequest() { string testTopic1 = CurrentTestTopic + "1"; string testTopic2 = CurrentTestTopic + "2"; string testTopic3 = CurrentTestTopic + "3"; Message sourceMessage1 = new Message(Encoding.UTF8.GetBytes("1: TestMessage")); Message sourceMessage2 = new Message(Encoding.UTF8.GetBytes("2: TestMessage")); Message sourceMessage3 = new Message(Encoding.UTF8.GetBytes("3: TestMessage")); Message sourceMessage4 = new Message(Encoding.UTF8.GetBytes("4: TestMessage")); List<ProducerRequest> requests = new List<ProducerRequest> { new ProducerRequest(testTopic1, 0, new List<Message> { sourceMessage1 }), new ProducerRequest(testTopic1, 0, new List<Message> { sourceMessage2 }), new ProducerRequest(testTopic2, 0, new List<Message> { sourceMessage3 }), new ProducerRequest(testTopic3, 0, new List<Message> { sourceMessage4 }) }; var config = new SyncProducerConfig(clientConfig); var producer = new SyncProducer(config); long currentOffset1 = TestHelper.GetCurrentKafkaOffset(testTopic1, clientConfig); long currentOffset2 = TestHelper.GetCurrentKafkaOffset(testTopic2, clientConfig); long currentOffset3 = TestHelper.GetCurrentKafkaOffset(testTopic3, clientConfig); producer.MultiSend(requests); ConsumerConfig consumerConfig = new ConsumerConfig(clientConfig); IConsumer consumer = new Consumers.Consumer(consumerConfig); MultiFetchRequest request = new MultiFetchRequest(new List<FetchRequest> { new FetchRequest(testTopic1, 0, currentOffset1), new FetchRequest(testTopic2, 0, currentOffset2), new FetchRequest(testTopic3, 0, currentOffset3) }); IList<BufferedMessageSet> messageSets; int totalWaitTimeInMiliseconds = 0; int waitSingle = 100; while (true) { Thread.Sleep(waitSingle); messageSets = consumer.MultiFetch(request); if (messageSets.Count > 2 && messageSets[0].Messages.Count() > 0 && messageSets[1].Messages.Count() > 0 && messageSets[2].Messages.Count() > 0) { break; } else { totalWaitTimeInMiliseconds += waitSingle; if (totalWaitTimeInMiliseconds >= MaxTestWaitTimeInMiliseconds) { break; } } } Assert.AreEqual(3, messageSets.Count); Assert.AreEqual(2, messageSets[0].Messages.Count()); Assert.AreEqual(1, messageSets[1].Messages.Count()); Assert.AreEqual(1, messageSets[2].Messages.Count()); Assert.AreEqual(sourceMessage1.ToString(), messageSets[0].Messages.First().ToString()); Assert.AreEqual(sourceMessage2.ToString(), messageSets[0].Messages.Skip(1).First().ToString()); Assert.AreEqual(sourceMessage3.ToString(), messageSets[1].Messages.First().ToString()); Assert.AreEqual(sourceMessage4.ToString(), messageSets[2].Messages.First().ToString()); }
public void ProducerSendsMessage() { var prodConfig = this.SyncProducerConfig1; string payload1 = "kafka 1."; byte[] payloadData1 = Encoding.UTF8.GetBytes(payload1); var msg1 = new Message(payloadData1); string payload2 = "kafka 2."; byte[] payloadData2 = Encoding.UTF8.GetBytes(payload2); var msg2 = new Message(payloadData2); using (var producer = new SyncProducer(prodConfig)) { var producerRequest = new ProducerRequest(CurrentTestTopic, 0, new List<Message> { msg1, msg2 }); producer.Send(producerRequest); } }
public void ZkAwareProducerSends3Messages() { var prodConfig = this.ZooKeeperBasedSyncProdConfig; int totalWaitTimeInMiliseconds = 0; int waitSingle = 100; var originalMessage1 = new Message(Encoding.UTF8.GetBytes("TestData1")); var originalMessage2 = new Message(Encoding.UTF8.GetBytes("TestData2")); var originalMessage3 = new Message(Encoding.UTF8.GetBytes("TestData3")); var originalMessageList = new List<Message> { originalMessage1, originalMessage2, originalMessage3 }; var multipleBrokersHelper = new TestMultipleBrokersHelper(CurrentTestTopic); multipleBrokersHelper.GetCurrentOffsets(new[] { this.SyncProducerConfig1, this.SyncProducerConfig2, this.SyncProducerConfig3 }); var mockPartitioner = new MockAlwaysZeroPartitioner(); using (var producer = new Producer<string, Message>(prodConfig, mockPartitioner, new DefaultEncoder())) { var producerData = new ProducerData<string, Message>(CurrentTestTopic, "somekey", originalMessageList); producer.Send(producerData); while (!multipleBrokersHelper.CheckIfAnyBrokerHasChanged(new[] { this.SyncProducerConfig1, this.SyncProducerConfig2, this.SyncProducerConfig3 })) { totalWaitTimeInMiliseconds += waitSingle; Thread.Sleep(waitSingle); if (totalWaitTimeInMiliseconds > this.maxTestWaitTimeInMiliseconds) { Assert.Fail("None of the brokers changed their offset after sending a message"); } } totalWaitTimeInMiliseconds = 0; var consumerConfig = new ConsumerConfiguration( multipleBrokersHelper.BrokerThatHasChanged.Host, multipleBrokersHelper.BrokerThatHasChanged.Port); IConsumer consumer = new Consumer(consumerConfig); var request = new FetchRequest(CurrentTestTopic, 0, multipleBrokersHelper.OffsetFromBeforeTheChange); BufferedMessageSet response; while (true) { Thread.Sleep(waitSingle); response = consumer.Fetch(request); if (response != null && response.Messages.Count() > 2) { break; } totalWaitTimeInMiliseconds += waitSingle; if (totalWaitTimeInMiliseconds >= this.maxTestWaitTimeInMiliseconds) { break; } } Assert.NotNull(response); Assert.AreEqual(3, response.Messages.Count()); Assert.AreEqual(originalMessage1.ToString(), response.Messages.First().ToString()); Assert.AreEqual(originalMessage2.ToString(), response.Messages.Skip(1).First().ToString()); Assert.AreEqual(originalMessage3.ToString(), response.Messages.Skip(2).First().ToString()); } }
internal static Message ParseFrom(KafkaBinaryReader reader, int size) { Message result; int readed = 0; byte magic = reader.ReadByte(); readed++; byte[] checksum; byte[] payload; if (magic == 1) { byte attributes = reader.ReadByte(); readed++; checksum = reader.ReadBytes(4); readed += 4; payload = reader.ReadBytes(size - (DefaultHeaderSize + 1)); readed += size - (DefaultHeaderSize + 1); result = new Message(payload, checksum, Messages.CompressionCodec.GetCompressionCodec(attributes & CompressionCodeMask)); } else { checksum = reader.ReadBytes(4); readed += 4; payload = reader.ReadBytes(size - DefaultHeaderSize); readed += size - DefaultHeaderSize; result = new Message(payload, checksum); } if (size != readed) { throw new KafkaException(KafkaException.InvalidRetchSizeCode); } return result; }
public void WriteToValidSequenceForCustomConstructor() { byte[] messageBytes = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9 }; byte[] customChecksum = new byte[] { 3, 4, 5, 6 }; Message message = new Message(messageBytes, (byte)33, customChecksum); MemoryStream ms = new MemoryStream(); message.WriteTo(ms); Assert.AreEqual((byte)33, ms.ToArray()[MagicNumberPartOffset]); byte[] checksumPart = new byte[ChecksumPartLength]; Array.Copy(ms.ToArray(), ChecksumPartOffset, checksumPart, 0, ChecksumPartLength); Assert.AreEqual(customChecksum, checksumPart); byte[] dataPart = new byte[messageBytes.Length]; Array.Copy(ms.ToArray(), DataPartOffset, dataPart, 0, messageBytes.Length); Assert.AreEqual(messageBytes, dataPart); }
public void ProducerSendsMessageWithLongTopic() { Message msg = new Message(Encoding.UTF8.GetBytes("test message")); string topic = "ThisIsAVeryLongTopicThisIsAVeryLongTopicThisIsAVeryLongTopicThisIsAVeryLongTopicThisIsAVeryLongTopicThisIsAVeryLongTopic"; var config = new SyncProducerConfig(clientConfig); var producer = new SyncProducer(config); var producerRequest = new ProducerRequest(topic, 0, new List<Message>() { msg }); producer.Send(producerRequest); }
/** * A message. The format of an N byte message is the following: * * 1. 4 byte CRC32 of the message * 2. 1 byte "magic" identifier to allow format changes, value is 2 currently * 3. 1 byte "attributes" identifier to allow annotations on the message independent of the version (e.g. compression enabled, type of codec used) * 4. 4 byte key length, containing length K * 5. K byte key * 6. 4 byte payload length, containing length V * 7. V byte payload * */ internal static Message ParseFrom(KafkaBinaryReader reader, long offset, int size, int partitionID) { Message result; int readed = 0; uint checksum = reader.ReadUInt32(); readed += 4; byte magic = reader.ReadByte(); readed++; byte[] payload; if (magic == 2 || magic == 0) // some producers (CLI) send magic 0 while others have value of 2 { byte attributes = reader.ReadByte(); readed++; var keyLength = reader.ReadInt32(); readed += 4; byte[] key = null; if (keyLength != -1) { key = reader.ReadBytes(keyLength); readed += keyLength; } var payloadSize = reader.ReadInt32(); readed += 4; payload = reader.ReadBytes(payloadSize); readed += payloadSize; result = new Message(payload, key, Messages.CompressionCodec.GetCompressionCodec(attributes & CompressionCodeMask)) { Offset = offset, PartitionId = partitionID }; } else { payload = reader.ReadBytes(size - DefaultHeaderSize); readed += size - DefaultHeaderSize; result = new Message(payload) { Offset = offset, PartitionId = partitionID }; } if (size != readed) { throw new KafkaException(ErrorMapping.InvalidFetchSizeCode); } return result; }
public void SetSizeValid() { byte[] messageBytes = new byte[] { 1, 2, 3, 4, 5 }; Message msg1 = new Message(messageBytes); Message msg2 = new Message(messageBytes); MessageSet messageSet = new BufferedMessageSet(new List<Message>() { msg1, msg2 }); Assert.AreEqual( 2 * (MessageLengthPartLength + MagicNumberPartLength + AttributesPartLength + ChecksumPartLength + messageBytes.Length), messageSet.SetSize); }
/// <summary> /// Gives the size of a size-delimited entry in a message set /// </summary> /// <param name="message"> /// The message. /// </param> /// <returns> /// Size of message /// </returns> public static int GetEntrySize(Message message) { Guard.NotNull(message, "message"); return message.Size + DefaultMessageLengthSize; }
/// <summary> /// Gives the size of a size-delimited entry in a message set /// </summary> /// <param name="message"> /// The message. /// </param> /// <returns> /// Size of message /// </returns> public static int GetEntrySize(Message message) { Guard.Assert<ArgumentNullException>(() => message != null); return message.Size + DefaultMessageLengthSize; }