public void TestConsumerIteratorDeduplicationDeepIterator()
{
var messageStrings = Enumerable.Range(0, 10).Select(x => x.ToString()).ToList();
var messages = messageStrings.Select(s => new Message(Encoding.UTF8.GetBytes(s))).ToList();
var messageSet = new ByteBufferMessageSet(
CompressionCodecs.DefaultCompressionCodec, new AtomicLong(0), messages);
this.topicInfos[0].Enqueue(messageSet);
Assert.Equal(1, this.queue.Count);
this.queue.Add(ZookeeperConsumerConnector.ShutdownCommand);
var iter = new ConsumerIterator <string, string>(this.queue, this.consumerConfig.ConsumerTimeoutMs, new StringDecoder(), new StringDecoder(), string.Empty);
var receivedMessages = Enumerable.Range(0, 5).Select(_ => iter.Next().Message).ToList();
Assert.False(iter.HasNext());
Assert.Equal(1, this.queue.Count); // This is only shutdown comamnd
Assert.Equal(5, receivedMessages.Count);
var unconsumed =
messageSet.Where(x => x.Offset >= this.consumedOffset).Select(m => Util.ReadString(m.Message.Payload));
Assert.Equal(unconsumed, receivedMessages);
}
public void TestOffsetAssigment()
{
var messages = new ByteBufferMessageSet(
CompressionCodecs.NoCompressionCodec,
new List <Message>
{
new Message(Encoding.UTF8.GetBytes("hello")),
new Message(Encoding.UTF8.GetBytes("there")),
new Message(Encoding.UTF8.GetBytes("beautiful")),
});
var compressedMessages = new ByteBufferMessageSet(
CompressionCodecs.DefaultCompressionCodec, messages.Select(m => m.Message).ToList());
// check uncompressed offsets
this.CheckOffsets(messages, 0);
var offset = 1234567;
this.CheckOffsets(messages.AssignOffsets(new AtomicLong(offset), CompressionCodecs.NoCompressionCodec), offset);
// check compressed offset
this.CheckOffsets(compressedMessages, 0);
this.CheckOffsets(
compressedMessages.AssignOffsets(new AtomicLong(offset), CompressionCodecs.DefaultCompressionCodec),
offset);
}
public void TestMessageSizeTooLarge()
{
var props = TestUtils.GetSyncProducerConfig(this.Configs[0].Port);
var producer = new SyncProducer(props);
AdminUtils.CreateTopic(this.ZkClient, "test", 1, 1, new Dictionary <string, string>());
TestUtils.WaitUntilLeaderIsElectedOrChanged(this.ZkClient, "test", 0, 500);
TestUtils.WaitUntilMetadataIsPropagated(Servers, "test", 0, 2000);
var message1 = new Message(new byte[Configs[0].MessageMaxBytes + 1]);
var messageSet1 = new ByteBufferMessageSet(CompressionCodecs.NoCompressionCodec, new List <Message> {
message1
});
var response1 = producer.Send(TestUtils.ProduceRequest("test", 0, messageSet1, acks: 1));
Assert.Equal(1, response1.Status.Count(kvp => kvp.Value.Error != ErrorMapping.NoError));
Assert.Equal(ErrorMapping.MessageSizeTooLargeCode, response1.Status[new TopicAndPartition("test", 0)].Error);
Assert.Equal(-1L, response1.Status[new TopicAndPartition("test", 0)].Offset);
var safeSize = Configs[0].MessageMaxBytes - Message.MessageOverhead - MessageSet.LogOverhead - 1;
var message2 = new Message(new byte[safeSize]);
var messageSet2 = new ByteBufferMessageSet(
CompressionCodecs.NoCompressionCodec, new List <Message> {
message2
});
var response2 = producer.Send(TestUtils.ProduceRequest("test", 0, messageSet2, acks: 1));
Assert.Equal(0, response2.Status.Count(kvp => kvp.Value.Error != ErrorMapping.NoError));
Assert.Equal(ErrorMapping.NoError, response2.Status[new TopicAndPartition("test", 0)].Error);
Assert.Equal(0, response2.Status[new TopicAndPartition("test", 0)].Offset);
}
internal void CheckOffsets(ByteBufferMessageSet messages, long baseOffset)
{
var offset = baseOffset;
foreach (var entry in messages)
{
Assert.Equal(offset, entry.Offset);
offset++;
}
}
internal void VerifyShallowIterator(ByteBufferMessageSet messageSet)
{
// make sure the offsets returned by a shallow iterator is a subset of that of a deep iterator
var shallowOffsets =
new HashSet <long>(
TestUtil.IteratorToArray(messageSet.ShallowIterator()).Select(msgAndOff => msgAndOff.Offset));
var deepOffsets = new HashSet <long>(
TestUtil.IteratorToArray(messageSet.Iterator()).Select(msgAndOff => msgAndOff.Offset));
Assert.True(shallowOffsets.IsSubsetOf(deepOffsets));
}
private void TestSimpleCompressDecompressInner(CompressionCodecs compressionCodec)
{
var messages = new List<Message>
{
new Message(Encoding.UTF8.GetBytes("hi there")),
new Message(Encoding.UTF8.GetBytes("I am fine")),
new Message(Encoding.UTF8.GetBytes("I am not so well today")),
};
var messageSet = new ByteBufferMessageSet(compressionCodec, messages);
Assert.Equal(compressionCodec, messageSet.ShallowIterator().Next().Message.CompressionCodec);
var decompressed = messageSet.Iterator().ToEnumerable().Select(x => x.Message).ToList();
Assert.Equal(messages, decompressed);
}
public void TestProduceCorrectlyReceivesResponse()
{
var props = TestUtils.GetSyncProducerConfig(this.Configs[0].Port);
var producer = new SyncProducer(props);
var messages = new ByteBufferMessageSet(
CompressionCodecs.NoCompressionCodec, new List <Message> {
new Message(this.messageBytes)
});
// #1 - test that we get an error when partition does not belong to broker in response
var request = TestUtils.ProduceRequestWithAcks(
new List <string> {
"topic1", "topic2", "topic3"
}, new List <int> {
0
}, messages, 1);
var response = producer.Send(request);
Assert.NotNull(response);
Assert.Equal(request.CorrelationId, response.CorrelationId);
Assert.Equal(3, response.Status.Count);
foreach (var responseStatus in response.Status.Values)
{
Assert.Equal(ErrorMapping.UnknownTopicOrPartitionCode, responseStatus.Error);
Assert.Equal(-1L, responseStatus.Offset);
}
// #2 - test that we get correct offsets when partition is owned by broker
AdminUtils.CreateTopic(this.ZkClient, "topic1", 1, 1, new Dictionary <string, string>());
AdminUtils.CreateTopic(this.ZkClient, "topic3", 1, 1, new Dictionary <string, string>());
TestUtils.WaitUntilLeaderIsElectedOrChanged(this.ZkClient, "topic3", 0, 5000);
TestUtils.WaitUntilMetadataIsPropagated(Servers, "topic3", 0, 2000);
var response2 = producer.Send(request);
Assert.NotNull(response2);
Assert.Equal(request.CorrelationId, response2.CorrelationId);
Assert.Equal(3, response2.Status.Count);
// the first and last message should have been accepted by broker
Assert.Equal(ErrorMapping.NoError, response2.Status[new TopicAndPartition("topic1", 0)].Error);
Assert.Equal(ErrorMapping.NoError, response2.Status[new TopicAndPartition("topic3", 0)].Error);
Assert.Equal(0, response2.Status[new TopicAndPartition("topic1", 0)].Offset);
Assert.Equal(0, response2.Status[new TopicAndPartition("topic3", 0)].Offset);
// the middle message should have been rejected because broker doesn't lead partition
Assert.Equal(ErrorMapping.UnknownTopicOrPartitionCode, response2.Status[new TopicAndPartition("topic2", 0)].Error);
Assert.Equal(-1L, response2.Status[new TopicAndPartition("topic2", 0)].Offset);
}
internal void TestWriteToWithMessageSet(MessageSet set)
{
// do the write twice to ensure the message set is restored to its orginal state
for (var i = 0; i < 1; i++)
{
var stream = ByteBuffer.Allocate(1024);
var written = set.WriteTo(stream, 0, 1024);
stream.SetLength(written);
Assert.Equal(set.SizeInBytes, written);
stream.Position = 0;
var newSet = new ByteBufferMessageSet(stream);
Assert.Equal(TestUtil.IteratorToArray(set.Iterator()), TestUtil.IteratorToArray(newSet.Iterator()));
}
}
internal void TestWriteToWithMessageSet(MessageSet set)
{
// do the write twice to ensure the message set is restored to its orginal state
for (var i = 0; i < 1; i++)
{
var stream = ByteBuffer.Allocate(1024);
var written = set.WriteTo(stream, 0, 1024);
stream.SetLength(written);
Assert.Equal(set.SizeInBytes, written);
stream.Position = 0;
var newSet = new ByteBufferMessageSet(stream);
Assert.Equal(TestUtil.IteratorToArray(set.Iterator()), TestUtil.IteratorToArray(newSet.Iterator()));
}
}
private void TestSimpleCompressDecompressInner(CompressionCodecs compressionCodec)
{
var messages = new List <Message>
{
new Message(Encoding.UTF8.GetBytes("hi there")),
new Message(Encoding.UTF8.GetBytes("I am fine")),
new Message(Encoding.UTF8.GetBytes("I am not so well today")),
};
var messageSet = new ByteBufferMessageSet(compressionCodec, messages);
Assert.Equal(compressionCodec, messageSet.ShallowIterator().Next().Message.CompressionCodec);
var decompressed = messageSet.Iterator().ToEnumerable().Select(x => x.Message).ToList();
Assert.Equal(messages, decompressed);
}
public void TestComplexCompressDecompress()
{
var messages = new List<Message>
{
new Message(Encoding.UTF8.GetBytes("hi there")),
new Message(Encoding.UTF8.GetBytes("I am fine")),
new Message(Encoding.UTF8.GetBytes("I am not so well today")),
};
var message = new ByteBufferMessageSet(CompressionCodecs.DefaultCompressionCodec, messages.Take(2).ToList());
var complexMessages =
new List<Message> { message.ShallowIterator().Next().Message }.Union(messages.Skip(2).Take(1).ToList())
.ToList();
var complexMessage = new ByteBufferMessageSet(CompressionCodecs.DefaultCompressionCodec, complexMessages);
var decompressedMessages = complexMessage.Iterator().ToEnumerable().Select(x => x.Message).ToList();
Assert.Equal(messages, decompressedMessages);
}
public static ProducerRequest ProduceRequest(
string topic,
int partition,
ByteBufferMessageSet message,
int acks = SyncProducerConfig.DefaultRequiredAcks,
int timeout = SyncProducerConfig.DefaultAckTimeout,
int correlationId = 0,
string clientId = SyncProducerConfig.DefaultClientId)
{
return(ProduceRequestWithAcks(
new List <string> {
topic
}, new List <int> {
partition
}, message, acks, timeout, correlationId, clientId));
}
public void TestValidBytesWithCompression()
{
var messages = new ByteBufferMessageSet(
CompressionCodecs.DefaultCompressionCodec,
new List <Message>
{
new Message(Encoding.UTF8.GetBytes("hello")),
new Message(Encoding.UTF8.GetBytes("there"))
});
var buffer = ByteBuffer.Allocate(messages.SizeInBytes + 2);
buffer.Put(messages.Buffer);
buffer.PutShort(4);
var messagesPlus = new ByteBufferMessageSet(buffer);
Assert.Equal(messages.ValidBytes, messagesPlus.ValidBytes);
}
public static ProducerRequest ProduceRequestWithAcks(
List <string> topics,
List <int> partitions,
ByteBufferMessageSet message,
int acks = SyncProducerConfig.DefaultRequiredAcks,
int timeout = SyncProducerConfig.DefaultAckTimeout,
int correlationId = 0,
string clientId = SyncProducerConfig.DefaultClientId)
{
var data =
topics.SelectMany(
topic =>
partitions.Select(partition => Tuple.Create(new TopicAndPartition(topic, partition), message)))
.ToList();
return(new ProducerRequest(
correlationId, clientId, (short)acks, timeout, data.ToDictionary(x => x.Item1, x => x.Item2)));
}
public void TestComplexCompressDecompress()
{
var messages = new List <Message>
{
new Message(Encoding.UTF8.GetBytes("hi there")),
new Message(Encoding.UTF8.GetBytes("I am fine")),
new Message(Encoding.UTF8.GetBytes("I am not so well today")),
};
var message = new ByteBufferMessageSet(CompressionCodecs.DefaultCompressionCodec, messages.Take(2).ToList());
var complexMessages =
new List <Message> {
message.ShallowIterator().Next().Message
}.Union(messages.Skip(2).Take(1).ToList())
.ToList();
var complexMessage = new ByteBufferMessageSet(CompressionCodecs.DefaultCompressionCodec, complexMessages);
var decompressedMessages = complexMessage.Iterator().ToEnumerable().Select(x => x.Message).ToList();
Assert.Equal(messages, decompressedMessages);
}
/// <summary>
/// Enqueue a message set for processing.
/// </summary>
/// <param name="messages"></param>
public void Enqueue(ByteBufferMessageSet messages)
{
var size = messages.ValidBytes;
if (size > 0)
{
var next = messages.ShallowIterator().ToEnumerable().Last().NextOffset;
Logger.DebugFormat("Updating fetch offset = {0} to {1}", this.fetchedOffset.Get(), next);
this.chunkQueue.Add(new FetchedDataChunk(messages, this, this.fetchedOffset.Get()));
this.fetchedOffset.Set(next);
Logger.DebugFormat("Updated fetch offset of {0} to {1}", this, next);
this.consumerTopicStats.GetConsumerTopicStats(this.Topic).ByteRate.Mark(size);
this.consumerTopicStats.GetConsumerAllTopicStats().ByteRate.Mark(size);
}
else if (messages.SizeInBytes > 0)
{
this.chunkQueue.Add(new FetchedDataChunk(messages, this, this.fetchedOffset.Get()));
}
}
public void TestValidBytes()
{
var messages = new ByteBufferMessageSet(
CompressionCodecs.NoCompressionCodec,
new List<Message>
{
new Message(Encoding.UTF8.GetBytes("hello")),
new Message(Encoding.UTF8.GetBytes("there"))
});
var buffer = ByteBuffer.Allocate(messages.SizeInBytes + 2);
buffer.Put(messages.Buffer);
buffer.PutShort(4);
var messagesPlus = new ByteBufferMessageSet(buffer);
Assert.Equal(messages.ValidBytes, messagesPlus.ValidBytes);
// test valid bytes on empty ByteBufferMessageSet
{
Assert.Equal(0, MessageSet.Empty.ValidBytes);
}
}
public void TestProduceAndFetch()
{
// send some messages
var topic = "test";
var sendMessages = new List <string> {
"hello", "there"
};
var producerData = sendMessages.Select(m => new KeyedMessage <string, string>(topic, topic, m)).ToArray();
Producer.Send(producerData);
TestUtils.WaitUntilMetadataIsPropagated(this.Servers, topic, 0, 1000);
ByteBufferMessageSet fetchedMessage = null;
while (fetchedMessage == null || fetchedMessage.ValidBytes == 0)
{
var fetched = Consumer.Fetch(new FetchRequestBuilder().AddFetch(topic, 0, 0, 10000).Build());
fetchedMessage = fetched.MessageSet(topic, 0);
}
Assert.Equal(sendMessages, fetchedMessage.Select(m => Util.ReadString(m.Message.Payload)).ToList());
// send an invalid offset
try
{
var fetchedWithError = Consumer.Fetch(new FetchRequestBuilder().AddFetch(topic, 0, -1, 10000).Build());
foreach (var pdata in fetchedWithError.Data.Values)
{
ErrorMapping.MaybeThrowException(pdata.Error);
}
Assert.True(false, "Expected an OffsetOutOfRangeException exception to be thrown");
}
catch (OffsetOutOfRangeException)
{
}
}
public void TestValidBytes()
{
var messages = new ByteBufferMessageSet(
CompressionCodecs.NoCompressionCodec,
new List <Message>
{
new Message(Encoding.UTF8.GetBytes("hello")),
new Message(Encoding.UTF8.GetBytes("there"))
});
var buffer = ByteBuffer.Allocate(messages.SizeInBytes + 2);
buffer.Put(messages.Buffer);
buffer.PutShort(4);
var messagesPlus = new ByteBufferMessageSet(buffer);
Assert.Equal(messages.ValidBytes, messagesPlus.ValidBytes);
// test valid bytes on empty ByteBufferMessageSet
{
Assert.Equal(0, MessageSet.Empty.ValidBytes);
}
}
public void TestConsumerIteratorDecodingFailure()
{
var messageStrings = Enumerable.Range(0, 10).Select(x => x.ToString()).ToList();
var messages = messageStrings.Select(s => new Message(Encoding.UTF8.GetBytes(s))).ToList();
var messageSet = new ByteBufferMessageSet(
CompressionCodecs.NoCompressionCodec, new AtomicLong(0), messages);
this.topicInfos[0].Enqueue(messageSet);
Assert.Equal(1, this.queue.Count);
var iter = new ConsumerIterator <string, string>(
this.queue, ConsumerConfig.DefaultConsumerTimeoutMs, new FailDecoder(), new FailDecoder(), string.Empty);
for (var i = 0; i < 5; i++)
{
Assert.True(iter.HasNext());
var message = iter.Next();
Assert.Equal(message.Offset, i + this.consumedOffset);
Assert.Throws <NotSupportedException>(() => message.Message);
}
}
public void TestEquals()
{
var messages = new ByteBufferMessageSet(
CompressionCodecs.DefaultCompressionCodec,
new List <Message>
{
new Message(Encoding.UTF8.GetBytes("hello")),
new Message(Encoding.UTF8.GetBytes("there"))
});
var moreMessages = new ByteBufferMessageSet(
CompressionCodecs.DefaultCompressionCodec,
new List <Message>
{
new Message(Encoding.UTF8.GetBytes("hello")),
new Message(Encoding.UTF8.GetBytes("there"))
});
Assert.True(messages.Equals(moreMessages));
messages = new ByteBufferMessageSet(
CompressionCodecs.NoCompressionCodec,
new List <Message>
{
new Message(Encoding.UTF8.GetBytes("hello")),
new Message(Encoding.UTF8.GetBytes("there"))
});
moreMessages = new ByteBufferMessageSet(
CompressionCodecs.NoCompressionCodec,
new List <Message>
{
new Message(Encoding.UTF8.GetBytes("hello")),
new Message(Encoding.UTF8.GetBytes("there"))
});
Assert.True(messages.Equals(moreMessages));
}
/// <summary>
/// Enqueue a message set for processing.
/// </summary>
/// <param name="messages"></param>
public void Enqueue(ByteBufferMessageSet messages)
{
var size = messages.ValidBytes;
if (size > 0)
{
var next = messages.ShallowIterator().ToEnumerable().Last().NextOffset;
Logger.DebugFormat("Updating fetch offset = {0} to {1}", this.fetchedOffset.Get(), next);
this.chunkQueue.Add(new FetchedDataChunk(messages, this, this.fetchedOffset.Get()));
this.fetchedOffset.Set(next);
Logger.DebugFormat("Updated fetch offset of {0} to {1}", this, next);
this.consumerTopicStats.GetConsumerTopicStats(this.Topic).ByteRate.Mark(size);
this.consumerTopicStats.GetConsumerAllTopicStats().ByteRate.Mark(size);
}
else if (messages.SizeInBytes > 0)
{
this.chunkQueue.Add(new FetchedDataChunk(messages, this, this.fetchedOffset.Get()));
}
}
public void TestMessageSizeTooLarge()
{
var props = TestUtils.GetSyncProducerConfig(this.Configs[0].Port);
var producer = new SyncProducer(props);
AdminUtils.CreateTopic(this.ZkClient, "test", 1, 1, new Dictionary<string, string>());
TestUtils.WaitUntilLeaderIsElectedOrChanged(this.ZkClient, "test", 0, 500);
TestUtils.WaitUntilMetadataIsPropagated(Servers, "test", 0, 2000);
var message1 = new Message(new byte[Configs[0].MessageMaxBytes + 1]);
var messageSet1 = new ByteBufferMessageSet(CompressionCodecs.NoCompressionCodec, new List<Message> { message1 });
var response1 = producer.Send(TestUtils.ProduceRequest("test", 0, messageSet1, acks: 1));
Assert.Equal(1, response1.Status.Count(kvp => kvp.Value.Error != ErrorMapping.NoError));
Assert.Equal(ErrorMapping.MessageSizeTooLargeCode, response1.Status[new TopicAndPartition("test", 0)].Error);
Assert.Equal(-1L, response1.Status[new TopicAndPartition("test", 0)].Offset);
var safeSize = Configs[0].MessageMaxBytes - Message.MessageOverhead - MessageSet.LogOverhead - 1;
var message2 = new Message(new byte[safeSize]);
var messageSet2 = new ByteBufferMessageSet(
CompressionCodecs.NoCompressionCodec, new List<Message> { message2 });
var response2 = producer.Send(TestUtils.ProduceRequest("test", 0, messageSet2, acks: 1));
Assert.Equal(0, response2.Status.Count(kvp => kvp.Value.Error != ErrorMapping.NoError));
Assert.Equal(ErrorMapping.NoError, response2.Status[new TopicAndPartition("test", 0)].Error);
Assert.Equal(0, response2.Status[new TopicAndPartition("test", 0)].Offset);
}
public void TestProduceCorrectlyReceivesResponse()
{
var props = TestUtils.GetSyncProducerConfig(this.Configs[0].Port);
var producer = new SyncProducer(props);
var messages = new ByteBufferMessageSet(
CompressionCodecs.NoCompressionCodec, new List<Message> { new Message(this.messageBytes) });
// #1 - test that we get an error when partition does not belong to broker in response
var request = TestUtils.ProduceRequestWithAcks(
new List<string> { "topic1", "topic2", "topic3" }, new List<int> { 0 }, messages, 1);
var response = producer.Send(request);
Assert.NotNull(response);
Assert.Equal(request.CorrelationId, response.CorrelationId);
Assert.Equal(3, response.Status.Count);
foreach (var responseStatus in response.Status.Values)
{
Assert.Equal(ErrorMapping.UnknownTopicOrPartitionCode, responseStatus.Error);
Assert.Equal(-1L, responseStatus.Offset);
}
// #2 - test that we get correct offsets when partition is owned by broker
AdminUtils.CreateTopic(this.ZkClient, "topic1", 1, 1, new Dictionary<string, string>());
AdminUtils.CreateTopic(this.ZkClient, "topic3", 1, 1, new Dictionary<string, string>());
TestUtils.WaitUntilLeaderIsElectedOrChanged(this.ZkClient, "topic3", 0, 5000);
TestUtils.WaitUntilMetadataIsPropagated(Servers, "topic3", 0, 2000);
var response2 = producer.Send(request);
Assert.NotNull(response2);
Assert.Equal(request.CorrelationId, response2.CorrelationId);
Assert.Equal(3, response2.Status.Count);
// the first and last message should have been accepted by broker
Assert.Equal(ErrorMapping.NoError, response2.Status[new TopicAndPartition("topic1", 0)].Error);
Assert.Equal(ErrorMapping.NoError, response2.Status[new TopicAndPartition("topic3", 0)].Error);
Assert.Equal(0, response2.Status[new TopicAndPartition("topic1", 0)].Offset);
Assert.Equal(0, response2.Status[new TopicAndPartition("topic3", 0)].Offset);
// the middle message should have been rejected because broker doesn't lead partition
Assert.Equal(ErrorMapping.UnknownTopicOrPartitionCode, response2.Status[new TopicAndPartition("topic2", 0)].Error);
Assert.Equal(-1L, response2.Status[new TopicAndPartition("topic2", 0)].Offset);
}
public void TestValidBytesWithCompression()
{
var messages = new ByteBufferMessageSet(
CompressionCodecs.DefaultCompressionCodec,
new List<Message>
{
new Message(Encoding.UTF8.GetBytes("hello")),
new Message(Encoding.UTF8.GetBytes("there"))
});
var buffer = ByteBuffer.Allocate(messages.SizeInBytes + 2);
buffer.Put(messages.Buffer);
buffer.PutShort(4);
var messagesPlus = new ByteBufferMessageSet(buffer);
Assert.Equal(messages.ValidBytes, messagesPlus.ValidBytes);
}
public FetchedDataChunk(ByteBufferMessageSet messages, PartitionTopicInfo topicInfo, long fetchOffset)
{
this.Messages = messages;
this.TopicInfo = topicInfo;
this.FetchOffset = fetchOffset;
}
internal void VerifyShallowIterator(ByteBufferMessageSet messageSet)
{
// make sure the offsets returned by a shallow iterator is a subset of that of a deep iterator
var shallowOffsets =
new HashSet<long>(
TestUtil.IteratorToArray(messageSet.ShallowIterator()).Select(msgAndOff => msgAndOff.Offset));
var deepOffsets = new HashSet<long>(
TestUtil.IteratorToArray(messageSet.Iterator()).Select(msgAndOff => msgAndOff.Offset));
Assert.True(shallowOffsets.IsSubsetOf(deepOffsets));
}
internal void CheckOffsets(ByteBufferMessageSet messages, long baseOffset)
{
var offset = baseOffset;
foreach (var entry in messages)
{
Assert.Equal(offset, entry.Offset);
offset++;
}
}
public void TestOffsetAssigment()
{
var messages = new ByteBufferMessageSet(
CompressionCodecs.NoCompressionCodec,
new List<Message>
{
new Message(Encoding.UTF8.GetBytes("hello")),
new Message(Encoding.UTF8.GetBytes("there")),
new Message(Encoding.UTF8.GetBytes("beautiful")),
});
var compressedMessages = new ByteBufferMessageSet(
CompressionCodecs.DefaultCompressionCodec, messages.Select(m => m.Message).ToList());
// check uncompressed offsets
this.CheckOffsets(messages, 0);
var offset = 1234567;
this.CheckOffsets(messages.AssignOffsets(new AtomicLong(offset), CompressionCodecs.NoCompressionCodec), offset);
// check compressed offset
this.CheckOffsets(compressedMessages, 0);
this.CheckOffsets(
compressedMessages.AssignOffsets(new AtomicLong(offset), CompressionCodecs.DefaultCompressionCodec),
offset);
}
public void TestEquals()
{
var messages = new ByteBufferMessageSet(
CompressionCodecs.DefaultCompressionCodec,
new List<Message>
{
new Message(Encoding.UTF8.GetBytes("hello")),
new Message(Encoding.UTF8.GetBytes("there"))
});
var moreMessages = new ByteBufferMessageSet(
CompressionCodecs.DefaultCompressionCodec,
new List<Message>
{
new Message(Encoding.UTF8.GetBytes("hello")),
new Message(Encoding.UTF8.GetBytes("there"))
});
Assert.True(messages.Equals(moreMessages));
messages = new ByteBufferMessageSet(
CompressionCodecs.NoCompressionCodec,
new List<Message>
{
new Message(Encoding.UTF8.GetBytes("hello")),
new Message(Encoding.UTF8.GetBytes("there"))
});
moreMessages = new ByteBufferMessageSet(
CompressionCodecs.NoCompressionCodec,
new List<Message>
{
new Message(Encoding.UTF8.GetBytes("hello")),
new Message(Encoding.UTF8.GetBytes("there"))
});
Assert.True(messages.Equals(moreMessages));
}
public void TestIterator()
{
var messageList = new List<Message>
{
new Message(Encoding.UTF8.GetBytes("msg1")),
new Message(Encoding.UTF8.GetBytes("msg2")),
new Message(Encoding.UTF8.GetBytes("msg3")),
};
// test for uncompressed regular Messages
{
var messageSet = new ByteBufferMessageSet(CompressionCodecs.NoCompressionCodec, messageList);
TestUtils.CheckEquals(
messageList, TestUtils.GetMessageIterator(messageSet.Iterator()).ToEnumerable());
// make sure ByteBufferMessageSet is re-iterable.
TestUtils.CheckEquals(
messageList, TestUtils.GetMessageIterator(messageSet.Iterator()).ToEnumerable());
// make sure shallow iterator is the same as deep iterator
TestUtils.CheckEquals(
TestUtils.GetMessageIterator(messageSet.ShallowIterator()).ToEnumerable(),
TestUtils.GetMessageIterator(messageSet.Iterator()).ToEnumerable());
}
// test for compressed regular Messages
{
var messageSet = new ByteBufferMessageSet(CompressionCodecs.DefaultCompressionCodec, messageList);
TestUtils.CheckEquals(
messageList, TestUtils.GetMessageIterator(messageSet.Iterator()).ToEnumerable());
// make sure ByteBufferMessageSet is re-iterable.
TestUtils.CheckEquals(
messageList, TestUtils.GetMessageIterator(messageSet.Iterator()).ToEnumerable());
}
// test for mixed empty and non-empty messagesets uncompressed
{
List<Message> emptyMessageList = null;
var emptyMessageSet = new ByteBufferMessageSet(CompressionCodecs.NoCompressionCodec, emptyMessageList);
var regularMessgeSet = new ByteBufferMessageSet(CompressionCodecs.NoCompressionCodec, messageList);
var buffer = ByteBuffer.Allocate(emptyMessageSet.Buffer.Limit() + regularMessgeSet.Buffer.Limit());
buffer.Put(emptyMessageSet.Buffer);
buffer.Put(regularMessgeSet.Buffer);
buffer.Rewind();
var mixedMessageSet = new ByteBufferMessageSet(buffer);
TestUtils.CheckEquals(
messageList, TestUtils.GetMessageIterator(mixedMessageSet.Iterator()).ToEnumerable());
// make sure ByteBufferMessageSet is re-iterable.
TestUtils.CheckEquals(
messageList, TestUtils.GetMessageIterator(mixedMessageSet.Iterator()).ToEnumerable());
// make sure shallow iterator is the same as deep iterator
TestUtils.CheckEquals(
TestUtils.GetMessageIterator(mixedMessageSet.ShallowIterator()).ToEnumerable(),
TestUtils.GetMessageIterator(mixedMessageSet.Iterator()).ToEnumerable());
}
// test for mixed empty and non-empty messagesets compressed
{
List<Message> emptyMessageList = null;
var emptyMessageSet = new ByteBufferMessageSet(CompressionCodecs.DefaultCompressionCodec, emptyMessageList);
var regularMessgeSet = new ByteBufferMessageSet(CompressionCodecs.DefaultCompressionCodec, messageList);
var buffer = ByteBuffer.Allocate(emptyMessageSet.Buffer.Limit() + regularMessgeSet.Buffer.Limit());
buffer.Put(emptyMessageSet.Buffer);
buffer.Put(regularMessgeSet.Buffer);
buffer.Rewind();
var mixedMessageSet = new ByteBufferMessageSet(buffer);
TestUtils.CheckEquals(
messageList, TestUtils.GetMessageIterator(mixedMessageSet.Iterator()).ToEnumerable());
// make sure ByteBufferMessageSet is re-iterable.
TestUtils.CheckEquals(
messageList, TestUtils.GetMessageIterator(mixedMessageSet.Iterator()).ToEnumerable());
this.VerifyShallowIterator(mixedMessageSet);
}
}
public void TestIterator()
{
var messageList = new List <Message>
{
new Message(Encoding.UTF8.GetBytes("msg1")),
new Message(Encoding.UTF8.GetBytes("msg2")),
new Message(Encoding.UTF8.GetBytes("msg3")),
};
// test for uncompressed regular Messages
{
var messageSet = new ByteBufferMessageSet(CompressionCodecs.NoCompressionCodec, messageList);
TestUtils.CheckEquals(
messageList, TestUtils.GetMessageIterator(messageSet.Iterator()).ToEnumerable());
// make sure ByteBufferMessageSet is re-iterable.
TestUtils.CheckEquals(
messageList, TestUtils.GetMessageIterator(messageSet.Iterator()).ToEnumerable());
// make sure shallow iterator is the same as deep iterator
TestUtils.CheckEquals(
TestUtils.GetMessageIterator(messageSet.ShallowIterator()).ToEnumerable(),
TestUtils.GetMessageIterator(messageSet.Iterator()).ToEnumerable());
}
// test for compressed regular Messages
{
var messageSet = new ByteBufferMessageSet(CompressionCodecs.DefaultCompressionCodec, messageList);
TestUtils.CheckEquals(
messageList, TestUtils.GetMessageIterator(messageSet.Iterator()).ToEnumerable());
// make sure ByteBufferMessageSet is re-iterable.
TestUtils.CheckEquals(
messageList, TestUtils.GetMessageIterator(messageSet.Iterator()).ToEnumerable());
}
// test for mixed empty and non-empty messagesets uncompressed
{
List <Message> emptyMessageList = null;
var emptyMessageSet = new ByteBufferMessageSet(CompressionCodecs.NoCompressionCodec, emptyMessageList);
var regularMessgeSet = new ByteBufferMessageSet(CompressionCodecs.NoCompressionCodec, messageList);
var buffer = ByteBuffer.Allocate(emptyMessageSet.Buffer.Limit() + regularMessgeSet.Buffer.Limit());
buffer.Put(emptyMessageSet.Buffer);
buffer.Put(regularMessgeSet.Buffer);
buffer.Rewind();
var mixedMessageSet = new ByteBufferMessageSet(buffer);
TestUtils.CheckEquals(
messageList, TestUtils.GetMessageIterator(mixedMessageSet.Iterator()).ToEnumerable());
// make sure ByteBufferMessageSet is re-iterable.
TestUtils.CheckEquals(
messageList, TestUtils.GetMessageIterator(mixedMessageSet.Iterator()).ToEnumerable());
// make sure shallow iterator is the same as deep iterator
TestUtils.CheckEquals(
TestUtils.GetMessageIterator(mixedMessageSet.ShallowIterator()).ToEnumerable(),
TestUtils.GetMessageIterator(mixedMessageSet.Iterator()).ToEnumerable());
}
// test for mixed empty and non-empty messagesets compressed
{
List <Message> emptyMessageList = null;
var emptyMessageSet = new ByteBufferMessageSet(CompressionCodecs.DefaultCompressionCodec, emptyMessageList);
var regularMessgeSet = new ByteBufferMessageSet(CompressionCodecs.DefaultCompressionCodec, messageList);
var buffer = ByteBuffer.Allocate(emptyMessageSet.Buffer.Limit() + regularMessgeSet.Buffer.Limit());
buffer.Put(emptyMessageSet.Buffer);
buffer.Put(regularMessgeSet.Buffer);
buffer.Rewind();
var mixedMessageSet = new ByteBufferMessageSet(buffer);
TestUtils.CheckEquals(
messageList, TestUtils.GetMessageIterator(mixedMessageSet.Iterator()).ToEnumerable());
// make sure ByteBufferMessageSet is re-iterable.
TestUtils.CheckEquals(
messageList, TestUtils.GetMessageIterator(mixedMessageSet.Iterator()).ToEnumerable());
this.VerifyShallowIterator(mixedMessageSet);
}
}
