public void TestConsumerIteratorDeduplicationDeepIterator() { var messageStrings = Enumerable.Range(0, 10).Select(x => x.ToString()).ToList(); var messages = messageStrings.Select(s => new Message(Encoding.UTF8.GetBytes(s))).ToList(); var messageSet = new ByteBufferMessageSet( CompressionCodecs.DefaultCompressionCodec, new AtomicLong(0), messages); this.topicInfos[0].Enqueue(messageSet); Assert.Equal(1, this.queue.Count); this.queue.Add(ZookeeperConsumerConnector.ShutdownCommand); var iter = new ConsumerIterator <string, string>(this.queue, this.consumerConfig.ConsumerTimeoutMs, new StringDecoder(), new StringDecoder(), string.Empty); var receivedMessages = Enumerable.Range(0, 5).Select(_ => iter.Next().Message).ToList(); Assert.False(iter.HasNext()); Assert.Equal(1, this.queue.Count); // This is only shutdown comamnd Assert.Equal(5, receivedMessages.Count); var unconsumed = messageSet.Where(x => x.Offset >= this.consumedOffset).Select(m => Util.ReadString(m.Message.Payload)); Assert.Equal(unconsumed, receivedMessages); }
public void TestOffsetAssigment() { var messages = new ByteBufferMessageSet( CompressionCodecs.NoCompressionCodec, new List <Message> { new Message(Encoding.UTF8.GetBytes("hello")), new Message(Encoding.UTF8.GetBytes("there")), new Message(Encoding.UTF8.GetBytes("beautiful")), }); var compressedMessages = new ByteBufferMessageSet( CompressionCodecs.DefaultCompressionCodec, messages.Select(m => m.Message).ToList()); // check uncompressed offsets this.CheckOffsets(messages, 0); var offset = 1234567; this.CheckOffsets(messages.AssignOffsets(new AtomicLong(offset), CompressionCodecs.NoCompressionCodec), offset); // check compressed offset this.CheckOffsets(compressedMessages, 0); this.CheckOffsets( compressedMessages.AssignOffsets(new AtomicLong(offset), CompressionCodecs.DefaultCompressionCodec), offset); }
public void TestMessageSizeTooLarge() { var props = TestUtils.GetSyncProducerConfig(this.Configs[0].Port); var producer = new SyncProducer(props); AdminUtils.CreateTopic(this.ZkClient, "test", 1, 1, new Dictionary <string, string>()); TestUtils.WaitUntilLeaderIsElectedOrChanged(this.ZkClient, "test", 0, 500); TestUtils.WaitUntilMetadataIsPropagated(Servers, "test", 0, 2000); var message1 = new Message(new byte[Configs[0].MessageMaxBytes + 1]); var messageSet1 = new ByteBufferMessageSet(CompressionCodecs.NoCompressionCodec, new List <Message> { message1 }); var response1 = producer.Send(TestUtils.ProduceRequest("test", 0, messageSet1, acks: 1)); Assert.Equal(1, response1.Status.Count(kvp => kvp.Value.Error != ErrorMapping.NoError)); Assert.Equal(ErrorMapping.MessageSizeTooLargeCode, response1.Status[new TopicAndPartition("test", 0)].Error); Assert.Equal(-1L, response1.Status[new TopicAndPartition("test", 0)].Offset); var safeSize = Configs[0].MessageMaxBytes - Message.MessageOverhead - MessageSet.LogOverhead - 1; var message2 = new Message(new byte[safeSize]); var messageSet2 = new ByteBufferMessageSet( CompressionCodecs.NoCompressionCodec, new List <Message> { message2 }); var response2 = producer.Send(TestUtils.ProduceRequest("test", 0, messageSet2, acks: 1)); Assert.Equal(0, response2.Status.Count(kvp => kvp.Value.Error != ErrorMapping.NoError)); Assert.Equal(ErrorMapping.NoError, response2.Status[new TopicAndPartition("test", 0)].Error); Assert.Equal(0, response2.Status[new TopicAndPartition("test", 0)].Offset); }
internal void CheckOffsets(ByteBufferMessageSet messages, long baseOffset) { var offset = baseOffset; foreach (var entry in messages) { Assert.Equal(offset, entry.Offset); offset++; } }
internal void VerifyShallowIterator(ByteBufferMessageSet messageSet) { // make sure the offsets returned by a shallow iterator is a subset of that of a deep iterator var shallowOffsets = new HashSet <long>( TestUtil.IteratorToArray(messageSet.ShallowIterator()).Select(msgAndOff => msgAndOff.Offset)); var deepOffsets = new HashSet <long>( TestUtil.IteratorToArray(messageSet.Iterator()).Select(msgAndOff => msgAndOff.Offset)); Assert.True(shallowOffsets.IsSubsetOf(deepOffsets)); }
private void TestSimpleCompressDecompressInner(CompressionCodecs compressionCodec) { var messages = new List<Message> { new Message(Encoding.UTF8.GetBytes("hi there")), new Message(Encoding.UTF8.GetBytes("I am fine")), new Message(Encoding.UTF8.GetBytes("I am not so well today")), }; var messageSet = new ByteBufferMessageSet(compressionCodec, messages); Assert.Equal(compressionCodec, messageSet.ShallowIterator().Next().Message.CompressionCodec); var decompressed = messageSet.Iterator().ToEnumerable().Select(x => x.Message).ToList(); Assert.Equal(messages, decompressed); }
public void TestProduceCorrectlyReceivesResponse() { var props = TestUtils.GetSyncProducerConfig(this.Configs[0].Port); var producer = new SyncProducer(props); var messages = new ByteBufferMessageSet( CompressionCodecs.NoCompressionCodec, new List <Message> { new Message(this.messageBytes) }); // #1 - test that we get an error when partition does not belong to broker in response var request = TestUtils.ProduceRequestWithAcks( new List <string> { "topic1", "topic2", "topic3" }, new List <int> { 0 }, messages, 1); var response = producer.Send(request); Assert.NotNull(response); Assert.Equal(request.CorrelationId, response.CorrelationId); Assert.Equal(3, response.Status.Count); foreach (var responseStatus in response.Status.Values) { Assert.Equal(ErrorMapping.UnknownTopicOrPartitionCode, responseStatus.Error); Assert.Equal(-1L, responseStatus.Offset); } // #2 - test that we get correct offsets when partition is owned by broker AdminUtils.CreateTopic(this.ZkClient, "topic1", 1, 1, new Dictionary <string, string>()); AdminUtils.CreateTopic(this.ZkClient, "topic3", 1, 1, new Dictionary <string, string>()); TestUtils.WaitUntilLeaderIsElectedOrChanged(this.ZkClient, "topic3", 0, 5000); TestUtils.WaitUntilMetadataIsPropagated(Servers, "topic3", 0, 2000); var response2 = producer.Send(request); Assert.NotNull(response2); Assert.Equal(request.CorrelationId, response2.CorrelationId); Assert.Equal(3, response2.Status.Count); // the first and last message should have been accepted by broker Assert.Equal(ErrorMapping.NoError, response2.Status[new TopicAndPartition("topic1", 0)].Error); Assert.Equal(ErrorMapping.NoError, response2.Status[new TopicAndPartition("topic3", 0)].Error); Assert.Equal(0, response2.Status[new TopicAndPartition("topic1", 0)].Offset); Assert.Equal(0, response2.Status[new TopicAndPartition("topic3", 0)].Offset); // the middle message should have been rejected because broker doesn't lead partition Assert.Equal(ErrorMapping.UnknownTopicOrPartitionCode, response2.Status[new TopicAndPartition("topic2", 0)].Error); Assert.Equal(-1L, response2.Status[new TopicAndPartition("topic2", 0)].Offset); }
internal void TestWriteToWithMessageSet(MessageSet set) { // do the write twice to ensure the message set is restored to its orginal state for (var i = 0; i < 1; i++) { var stream = ByteBuffer.Allocate(1024); var written = set.WriteTo(stream, 0, 1024); stream.SetLength(written); Assert.Equal(set.SizeInBytes, written); stream.Position = 0; var newSet = new ByteBufferMessageSet(stream); Assert.Equal(TestUtil.IteratorToArray(set.Iterator()), TestUtil.IteratorToArray(newSet.Iterator())); } }
private void TestSimpleCompressDecompressInner(CompressionCodecs compressionCodec) { var messages = new List <Message> { new Message(Encoding.UTF8.GetBytes("hi there")), new Message(Encoding.UTF8.GetBytes("I am fine")), new Message(Encoding.UTF8.GetBytes("I am not so well today")), }; var messageSet = new ByteBufferMessageSet(compressionCodec, messages); Assert.Equal(compressionCodec, messageSet.ShallowIterator().Next().Message.CompressionCodec); var decompressed = messageSet.Iterator().ToEnumerable().Select(x => x.Message).ToList(); Assert.Equal(messages, decompressed); }
public void TestComplexCompressDecompress() { var messages = new List<Message> { new Message(Encoding.UTF8.GetBytes("hi there")), new Message(Encoding.UTF8.GetBytes("I am fine")), new Message(Encoding.UTF8.GetBytes("I am not so well today")), }; var message = new ByteBufferMessageSet(CompressionCodecs.DefaultCompressionCodec, messages.Take(2).ToList()); var complexMessages = new List<Message> { message.ShallowIterator().Next().Message }.Union(messages.Skip(2).Take(1).ToList()) .ToList(); var complexMessage = new ByteBufferMessageSet(CompressionCodecs.DefaultCompressionCodec, complexMessages); var decompressedMessages = complexMessage.Iterator().ToEnumerable().Select(x => x.Message).ToList(); Assert.Equal(messages, decompressedMessages); }
public static ProducerRequest ProduceRequest( string topic, int partition, ByteBufferMessageSet message, int acks = SyncProducerConfig.DefaultRequiredAcks, int timeout = SyncProducerConfig.DefaultAckTimeout, int correlationId = 0, string clientId = SyncProducerConfig.DefaultClientId) { return(ProduceRequestWithAcks( new List <string> { topic }, new List <int> { partition }, message, acks, timeout, correlationId, clientId)); }
public void TestValidBytesWithCompression() { var messages = new ByteBufferMessageSet( CompressionCodecs.DefaultCompressionCodec, new List <Message> { new Message(Encoding.UTF8.GetBytes("hello")), new Message(Encoding.UTF8.GetBytes("there")) }); var buffer = ByteBuffer.Allocate(messages.SizeInBytes + 2); buffer.Put(messages.Buffer); buffer.PutShort(4); var messagesPlus = new ByteBufferMessageSet(buffer); Assert.Equal(messages.ValidBytes, messagesPlus.ValidBytes); }
public static ProducerRequest ProduceRequestWithAcks( List <string> topics, List <int> partitions, ByteBufferMessageSet message, int acks = SyncProducerConfig.DefaultRequiredAcks, int timeout = SyncProducerConfig.DefaultAckTimeout, int correlationId = 0, string clientId = SyncProducerConfig.DefaultClientId) { var data = topics.SelectMany( topic => partitions.Select(partition => Tuple.Create(new TopicAndPartition(topic, partition), message))) .ToList(); return(new ProducerRequest( correlationId, clientId, (short)acks, timeout, data.ToDictionary(x => x.Item1, x => x.Item2))); }
public void TestComplexCompressDecompress() { var messages = new List <Message> { new Message(Encoding.UTF8.GetBytes("hi there")), new Message(Encoding.UTF8.GetBytes("I am fine")), new Message(Encoding.UTF8.GetBytes("I am not so well today")), }; var message = new ByteBufferMessageSet(CompressionCodecs.DefaultCompressionCodec, messages.Take(2).ToList()); var complexMessages = new List <Message> { message.ShallowIterator().Next().Message }.Union(messages.Skip(2).Take(1).ToList()) .ToList(); var complexMessage = new ByteBufferMessageSet(CompressionCodecs.DefaultCompressionCodec, complexMessages); var decompressedMessages = complexMessage.Iterator().ToEnumerable().Select(x => x.Message).ToList(); Assert.Equal(messages, decompressedMessages); }
/// <summary> /// Enqueue a message set for processing. /// </summary> /// <param name="messages"></param> public void Enqueue(ByteBufferMessageSet messages) { var size = messages.ValidBytes; if (size > 0) { var next = messages.ShallowIterator().ToEnumerable().Last().NextOffset; Logger.DebugFormat("Updating fetch offset = {0} to {1}", this.fetchedOffset.Get(), next); this.chunkQueue.Add(new FetchedDataChunk(messages, this, this.fetchedOffset.Get())); this.fetchedOffset.Set(next); Logger.DebugFormat("Updated fetch offset of {0} to {1}", this, next); this.consumerTopicStats.GetConsumerTopicStats(this.Topic).ByteRate.Mark(size); this.consumerTopicStats.GetConsumerAllTopicStats().ByteRate.Mark(size); } else if (messages.SizeInBytes > 0) { this.chunkQueue.Add(new FetchedDataChunk(messages, this, this.fetchedOffset.Get())); } }
public void TestValidBytes() { var messages = new ByteBufferMessageSet( CompressionCodecs.NoCompressionCodec, new List<Message> { new Message(Encoding.UTF8.GetBytes("hello")), new Message(Encoding.UTF8.GetBytes("there")) }); var buffer = ByteBuffer.Allocate(messages.SizeInBytes + 2); buffer.Put(messages.Buffer); buffer.PutShort(4); var messagesPlus = new ByteBufferMessageSet(buffer); Assert.Equal(messages.ValidBytes, messagesPlus.ValidBytes); // test valid bytes on empty ByteBufferMessageSet { Assert.Equal(0, MessageSet.Empty.ValidBytes); } }
public void TestProduceAndFetch() { // send some messages var topic = "test"; var sendMessages = new List <string> { "hello", "there" }; var producerData = sendMessages.Select(m => new KeyedMessage <string, string>(topic, topic, m)).ToArray(); Producer.Send(producerData); TestUtils.WaitUntilMetadataIsPropagated(this.Servers, topic, 0, 1000); ByteBufferMessageSet fetchedMessage = null; while (fetchedMessage == null || fetchedMessage.ValidBytes == 0) { var fetched = Consumer.Fetch(new FetchRequestBuilder().AddFetch(topic, 0, 0, 10000).Build()); fetchedMessage = fetched.MessageSet(topic, 0); } Assert.Equal(sendMessages, fetchedMessage.Select(m => Util.ReadString(m.Message.Payload)).ToList()); // send an invalid offset try { var fetchedWithError = Consumer.Fetch(new FetchRequestBuilder().AddFetch(topic, 0, -1, 10000).Build()); foreach (var pdata in fetchedWithError.Data.Values) { ErrorMapping.MaybeThrowException(pdata.Error); } Assert.True(false, "Expected an OffsetOutOfRangeException exception to be thrown"); } catch (OffsetOutOfRangeException) { } }
public void TestValidBytes() { var messages = new ByteBufferMessageSet( CompressionCodecs.NoCompressionCodec, new List <Message> { new Message(Encoding.UTF8.GetBytes("hello")), new Message(Encoding.UTF8.GetBytes("there")) }); var buffer = ByteBuffer.Allocate(messages.SizeInBytes + 2); buffer.Put(messages.Buffer); buffer.PutShort(4); var messagesPlus = new ByteBufferMessageSet(buffer); Assert.Equal(messages.ValidBytes, messagesPlus.ValidBytes); // test valid bytes on empty ByteBufferMessageSet { Assert.Equal(0, MessageSet.Empty.ValidBytes); } }
public void TestConsumerIteratorDecodingFailure() { var messageStrings = Enumerable.Range(0, 10).Select(x => x.ToString()).ToList(); var messages = messageStrings.Select(s => new Message(Encoding.UTF8.GetBytes(s))).ToList(); var messageSet = new ByteBufferMessageSet( CompressionCodecs.NoCompressionCodec, new AtomicLong(0), messages); this.topicInfos[0].Enqueue(messageSet); Assert.Equal(1, this.queue.Count); var iter = new ConsumerIterator <string, string>( this.queue, ConsumerConfig.DefaultConsumerTimeoutMs, new FailDecoder(), new FailDecoder(), string.Empty); for (var i = 0; i < 5; i++) { Assert.True(iter.HasNext()); var message = iter.Next(); Assert.Equal(message.Offset, i + this.consumedOffset); Assert.Throws <NotSupportedException>(() => message.Message); } }
public void TestEquals() { var messages = new ByteBufferMessageSet( CompressionCodecs.DefaultCompressionCodec, new List <Message> { new Message(Encoding.UTF8.GetBytes("hello")), new Message(Encoding.UTF8.GetBytes("there")) }); var moreMessages = new ByteBufferMessageSet( CompressionCodecs.DefaultCompressionCodec, new List <Message> { new Message(Encoding.UTF8.GetBytes("hello")), new Message(Encoding.UTF8.GetBytes("there")) }); Assert.True(messages.Equals(moreMessages)); messages = new ByteBufferMessageSet( CompressionCodecs.NoCompressionCodec, new List <Message> { new Message(Encoding.UTF8.GetBytes("hello")), new Message(Encoding.UTF8.GetBytes("there")) }); moreMessages = new ByteBufferMessageSet( CompressionCodecs.NoCompressionCodec, new List <Message> { new Message(Encoding.UTF8.GetBytes("hello")), new Message(Encoding.UTF8.GetBytes("there")) }); Assert.True(messages.Equals(moreMessages)); }
public void TestMessageSizeTooLarge() { var props = TestUtils.GetSyncProducerConfig(this.Configs[0].Port); var producer = new SyncProducer(props); AdminUtils.CreateTopic(this.ZkClient, "test", 1, 1, new Dictionary<string, string>()); TestUtils.WaitUntilLeaderIsElectedOrChanged(this.ZkClient, "test", 0, 500); TestUtils.WaitUntilMetadataIsPropagated(Servers, "test", 0, 2000); var message1 = new Message(new byte[Configs[0].MessageMaxBytes + 1]); var messageSet1 = new ByteBufferMessageSet(CompressionCodecs.NoCompressionCodec, new List<Message> { message1 }); var response1 = producer.Send(TestUtils.ProduceRequest("test", 0, messageSet1, acks: 1)); Assert.Equal(1, response1.Status.Count(kvp => kvp.Value.Error != ErrorMapping.NoError)); Assert.Equal(ErrorMapping.MessageSizeTooLargeCode, response1.Status[new TopicAndPartition("test", 0)].Error); Assert.Equal(-1L, response1.Status[new TopicAndPartition("test", 0)].Offset); var safeSize = Configs[0].MessageMaxBytes - Message.MessageOverhead - MessageSet.LogOverhead - 1; var message2 = new Message(new byte[safeSize]); var messageSet2 = new ByteBufferMessageSet( CompressionCodecs.NoCompressionCodec, new List<Message> { message2 }); var response2 = producer.Send(TestUtils.ProduceRequest("test", 0, messageSet2, acks: 1)); Assert.Equal(0, response2.Status.Count(kvp => kvp.Value.Error != ErrorMapping.NoError)); Assert.Equal(ErrorMapping.NoError, response2.Status[new TopicAndPartition("test", 0)].Error); Assert.Equal(0, response2.Status[new TopicAndPartition("test", 0)].Offset); }
public void TestProduceCorrectlyReceivesResponse() { var props = TestUtils.GetSyncProducerConfig(this.Configs[0].Port); var producer = new SyncProducer(props); var messages = new ByteBufferMessageSet( CompressionCodecs.NoCompressionCodec, new List<Message> { new Message(this.messageBytes) }); // #1 - test that we get an error when partition does not belong to broker in response var request = TestUtils.ProduceRequestWithAcks( new List<string> { "topic1", "topic2", "topic3" }, new List<int> { 0 }, messages, 1); var response = producer.Send(request); Assert.NotNull(response); Assert.Equal(request.CorrelationId, response.CorrelationId); Assert.Equal(3, response.Status.Count); foreach (var responseStatus in response.Status.Values) { Assert.Equal(ErrorMapping.UnknownTopicOrPartitionCode, responseStatus.Error); Assert.Equal(-1L, responseStatus.Offset); } // #2 - test that we get correct offsets when partition is owned by broker AdminUtils.CreateTopic(this.ZkClient, "topic1", 1, 1, new Dictionary<string, string>()); AdminUtils.CreateTopic(this.ZkClient, "topic3", 1, 1, new Dictionary<string, string>()); TestUtils.WaitUntilLeaderIsElectedOrChanged(this.ZkClient, "topic3", 0, 5000); TestUtils.WaitUntilMetadataIsPropagated(Servers, "topic3", 0, 2000); var response2 = producer.Send(request); Assert.NotNull(response2); Assert.Equal(request.CorrelationId, response2.CorrelationId); Assert.Equal(3, response2.Status.Count); // the first and last message should have been accepted by broker Assert.Equal(ErrorMapping.NoError, response2.Status[new TopicAndPartition("topic1", 0)].Error); Assert.Equal(ErrorMapping.NoError, response2.Status[new TopicAndPartition("topic3", 0)].Error); Assert.Equal(0, response2.Status[new TopicAndPartition("topic1", 0)].Offset); Assert.Equal(0, response2.Status[new TopicAndPartition("topic3", 0)].Offset); // the middle message should have been rejected because broker doesn't lead partition Assert.Equal(ErrorMapping.UnknownTopicOrPartitionCode, response2.Status[new TopicAndPartition("topic2", 0)].Error); Assert.Equal(-1L, response2.Status[new TopicAndPartition("topic2", 0)].Offset); }
public void TestValidBytesWithCompression() { var messages = new ByteBufferMessageSet( CompressionCodecs.DefaultCompressionCodec, new List<Message> { new Message(Encoding.UTF8.GetBytes("hello")), new Message(Encoding.UTF8.GetBytes("there")) }); var buffer = ByteBuffer.Allocate(messages.SizeInBytes + 2); buffer.Put(messages.Buffer); buffer.PutShort(4); var messagesPlus = new ByteBufferMessageSet(buffer); Assert.Equal(messages.ValidBytes, messagesPlus.ValidBytes); }
public FetchedDataChunk(ByteBufferMessageSet messages, PartitionTopicInfo topicInfo, long fetchOffset) { this.Messages = messages; this.TopicInfo = topicInfo; this.FetchOffset = fetchOffset; }
internal void VerifyShallowIterator(ByteBufferMessageSet messageSet) { // make sure the offsets returned by a shallow iterator is a subset of that of a deep iterator var shallowOffsets = new HashSet<long>( TestUtil.IteratorToArray(messageSet.ShallowIterator()).Select(msgAndOff => msgAndOff.Offset)); var deepOffsets = new HashSet<long>( TestUtil.IteratorToArray(messageSet.Iterator()).Select(msgAndOff => msgAndOff.Offset)); Assert.True(shallowOffsets.IsSubsetOf(deepOffsets)); }
public void TestOffsetAssigment() { var messages = new ByteBufferMessageSet( CompressionCodecs.NoCompressionCodec, new List<Message> { new Message(Encoding.UTF8.GetBytes("hello")), new Message(Encoding.UTF8.GetBytes("there")), new Message(Encoding.UTF8.GetBytes("beautiful")), }); var compressedMessages = new ByteBufferMessageSet( CompressionCodecs.DefaultCompressionCodec, messages.Select(m => m.Message).ToList()); // check uncompressed offsets this.CheckOffsets(messages, 0); var offset = 1234567; this.CheckOffsets(messages.AssignOffsets(new AtomicLong(offset), CompressionCodecs.NoCompressionCodec), offset); // check compressed offset this.CheckOffsets(compressedMessages, 0); this.CheckOffsets( compressedMessages.AssignOffsets(new AtomicLong(offset), CompressionCodecs.DefaultCompressionCodec), offset); }
public void TestEquals() { var messages = new ByteBufferMessageSet( CompressionCodecs.DefaultCompressionCodec, new List<Message> { new Message(Encoding.UTF8.GetBytes("hello")), new Message(Encoding.UTF8.GetBytes("there")) }); var moreMessages = new ByteBufferMessageSet( CompressionCodecs.DefaultCompressionCodec, new List<Message> { new Message(Encoding.UTF8.GetBytes("hello")), new Message(Encoding.UTF8.GetBytes("there")) }); Assert.True(messages.Equals(moreMessages)); messages = new ByteBufferMessageSet( CompressionCodecs.NoCompressionCodec, new List<Message> { new Message(Encoding.UTF8.GetBytes("hello")), new Message(Encoding.UTF8.GetBytes("there")) }); moreMessages = new ByteBufferMessageSet( CompressionCodecs.NoCompressionCodec, new List<Message> { new Message(Encoding.UTF8.GetBytes("hello")), new Message(Encoding.UTF8.GetBytes("there")) }); Assert.True(messages.Equals(moreMessages)); }
public void TestIterator() { var messageList = new List<Message> { new Message(Encoding.UTF8.GetBytes("msg1")), new Message(Encoding.UTF8.GetBytes("msg2")), new Message(Encoding.UTF8.GetBytes("msg3")), }; // test for uncompressed regular Messages { var messageSet = new ByteBufferMessageSet(CompressionCodecs.NoCompressionCodec, messageList); TestUtils.CheckEquals( messageList, TestUtils.GetMessageIterator(messageSet.Iterator()).ToEnumerable()); // make sure ByteBufferMessageSet is re-iterable. TestUtils.CheckEquals( messageList, TestUtils.GetMessageIterator(messageSet.Iterator()).ToEnumerable()); // make sure shallow iterator is the same as deep iterator TestUtils.CheckEquals( TestUtils.GetMessageIterator(messageSet.ShallowIterator()).ToEnumerable(), TestUtils.GetMessageIterator(messageSet.Iterator()).ToEnumerable()); } // test for compressed regular Messages { var messageSet = new ByteBufferMessageSet(CompressionCodecs.DefaultCompressionCodec, messageList); TestUtils.CheckEquals( messageList, TestUtils.GetMessageIterator(messageSet.Iterator()).ToEnumerable()); // make sure ByteBufferMessageSet is re-iterable. TestUtils.CheckEquals( messageList, TestUtils.GetMessageIterator(messageSet.Iterator()).ToEnumerable()); } // test for mixed empty and non-empty messagesets uncompressed { List<Message> emptyMessageList = null; var emptyMessageSet = new ByteBufferMessageSet(CompressionCodecs.NoCompressionCodec, emptyMessageList); var regularMessgeSet = new ByteBufferMessageSet(CompressionCodecs.NoCompressionCodec, messageList); var buffer = ByteBuffer.Allocate(emptyMessageSet.Buffer.Limit() + regularMessgeSet.Buffer.Limit()); buffer.Put(emptyMessageSet.Buffer); buffer.Put(regularMessgeSet.Buffer); buffer.Rewind(); var mixedMessageSet = new ByteBufferMessageSet(buffer); TestUtils.CheckEquals( messageList, TestUtils.GetMessageIterator(mixedMessageSet.Iterator()).ToEnumerable()); // make sure ByteBufferMessageSet is re-iterable. TestUtils.CheckEquals( messageList, TestUtils.GetMessageIterator(mixedMessageSet.Iterator()).ToEnumerable()); // make sure shallow iterator is the same as deep iterator TestUtils.CheckEquals( TestUtils.GetMessageIterator(mixedMessageSet.ShallowIterator()).ToEnumerable(), TestUtils.GetMessageIterator(mixedMessageSet.Iterator()).ToEnumerable()); } // test for mixed empty and non-empty messagesets compressed { List<Message> emptyMessageList = null; var emptyMessageSet = new ByteBufferMessageSet(CompressionCodecs.DefaultCompressionCodec, emptyMessageList); var regularMessgeSet = new ByteBufferMessageSet(CompressionCodecs.DefaultCompressionCodec, messageList); var buffer = ByteBuffer.Allocate(emptyMessageSet.Buffer.Limit() + regularMessgeSet.Buffer.Limit()); buffer.Put(emptyMessageSet.Buffer); buffer.Put(regularMessgeSet.Buffer); buffer.Rewind(); var mixedMessageSet = new ByteBufferMessageSet(buffer); TestUtils.CheckEquals( messageList, TestUtils.GetMessageIterator(mixedMessageSet.Iterator()).ToEnumerable()); // make sure ByteBufferMessageSet is re-iterable. TestUtils.CheckEquals( messageList, TestUtils.GetMessageIterator(mixedMessageSet.Iterator()).ToEnumerable()); this.VerifyShallowIterator(mixedMessageSet); } }
public void TestIterator() { var messageList = new List <Message> { new Message(Encoding.UTF8.GetBytes("msg1")), new Message(Encoding.UTF8.GetBytes("msg2")), new Message(Encoding.UTF8.GetBytes("msg3")), }; // test for uncompressed regular Messages { var messageSet = new ByteBufferMessageSet(CompressionCodecs.NoCompressionCodec, messageList); TestUtils.CheckEquals( messageList, TestUtils.GetMessageIterator(messageSet.Iterator()).ToEnumerable()); // make sure ByteBufferMessageSet is re-iterable. TestUtils.CheckEquals( messageList, TestUtils.GetMessageIterator(messageSet.Iterator()).ToEnumerable()); // make sure shallow iterator is the same as deep iterator TestUtils.CheckEquals( TestUtils.GetMessageIterator(messageSet.ShallowIterator()).ToEnumerable(), TestUtils.GetMessageIterator(messageSet.Iterator()).ToEnumerable()); } // test for compressed regular Messages { var messageSet = new ByteBufferMessageSet(CompressionCodecs.DefaultCompressionCodec, messageList); TestUtils.CheckEquals( messageList, TestUtils.GetMessageIterator(messageSet.Iterator()).ToEnumerable()); // make sure ByteBufferMessageSet is re-iterable. TestUtils.CheckEquals( messageList, TestUtils.GetMessageIterator(messageSet.Iterator()).ToEnumerable()); } // test for mixed empty and non-empty messagesets uncompressed { List <Message> emptyMessageList = null; var emptyMessageSet = new ByteBufferMessageSet(CompressionCodecs.NoCompressionCodec, emptyMessageList); var regularMessgeSet = new ByteBufferMessageSet(CompressionCodecs.NoCompressionCodec, messageList); var buffer = ByteBuffer.Allocate(emptyMessageSet.Buffer.Limit() + regularMessgeSet.Buffer.Limit()); buffer.Put(emptyMessageSet.Buffer); buffer.Put(regularMessgeSet.Buffer); buffer.Rewind(); var mixedMessageSet = new ByteBufferMessageSet(buffer); TestUtils.CheckEquals( messageList, TestUtils.GetMessageIterator(mixedMessageSet.Iterator()).ToEnumerable()); // make sure ByteBufferMessageSet is re-iterable. TestUtils.CheckEquals( messageList, TestUtils.GetMessageIterator(mixedMessageSet.Iterator()).ToEnumerable()); // make sure shallow iterator is the same as deep iterator TestUtils.CheckEquals( TestUtils.GetMessageIterator(mixedMessageSet.ShallowIterator()).ToEnumerable(), TestUtils.GetMessageIterator(mixedMessageSet.Iterator()).ToEnumerable()); } // test for mixed empty and non-empty messagesets compressed { List <Message> emptyMessageList = null; var emptyMessageSet = new ByteBufferMessageSet(CompressionCodecs.DefaultCompressionCodec, emptyMessageList); var regularMessgeSet = new ByteBufferMessageSet(CompressionCodecs.DefaultCompressionCodec, messageList); var buffer = ByteBuffer.Allocate(emptyMessageSet.Buffer.Limit() + regularMessgeSet.Buffer.Limit()); buffer.Put(emptyMessageSet.Buffer); buffer.Put(regularMessgeSet.Buffer); buffer.Rewind(); var mixedMessageSet = new ByteBufferMessageSet(buffer); TestUtils.CheckEquals( messageList, TestUtils.GetMessageIterator(mixedMessageSet.Iterator()).ToEnumerable()); // make sure ByteBufferMessageSet is re-iterable. TestUtils.CheckEquals( messageList, TestUtils.GetMessageIterator(mixedMessageSet.Iterator()).ToEnumerable()); this.VerifyShallowIterator(mixedMessageSet); } }