internal void VerifyShallowIterator(ByteBufferMessageSet messageSet)
{
// make sure the offsets returned by a shallow iterator is a subset of that of a deep iterator
var shallowOffsets =
new HashSet <long>(
TestUtil.IteratorToArray(messageSet.ShallowIterator()).Select(msgAndOff => msgAndOff.Offset));
var deepOffsets = new HashSet <long>(
TestUtil.IteratorToArray(messageSet.Iterator()).Select(msgAndOff => msgAndOff.Offset));
Assert.True(shallowOffsets.IsSubsetOf(deepOffsets));
}
private void TestSimpleCompressDecompressInner(CompressionCodecs compressionCodec)
{
var messages = new List<Message>
{
new Message(Encoding.UTF8.GetBytes("hi there")),
new Message(Encoding.UTF8.GetBytes("I am fine")),
new Message(Encoding.UTF8.GetBytes("I am not so well today")),
};
var messageSet = new ByteBufferMessageSet(compressionCodec, messages);
Assert.Equal(compressionCodec, messageSet.ShallowIterator().Next().Message.CompressionCodec);
var decompressed = messageSet.Iterator().ToEnumerable().Select(x => x.Message).ToList();
Assert.Equal(messages, decompressed);
}
internal void TestWriteToWithMessageSet(MessageSet set)
{
// do the write twice to ensure the message set is restored to its orginal state
for (var i = 0; i < 1; i++)
{
var stream = ByteBuffer.Allocate(1024);
var written = set.WriteTo(stream, 0, 1024);
stream.SetLength(written);
Assert.Equal(set.SizeInBytes, written);
stream.Position = 0;
var newSet = new ByteBufferMessageSet(stream);
Assert.Equal(TestUtil.IteratorToArray(set.Iterator()), TestUtil.IteratorToArray(newSet.Iterator()));
}
}
private void TestSimpleCompressDecompressInner(CompressionCodecs compressionCodec)
{
var messages = new List <Message>
{
new Message(Encoding.UTF8.GetBytes("hi there")),
new Message(Encoding.UTF8.GetBytes("I am fine")),
new Message(Encoding.UTF8.GetBytes("I am not so well today")),
};
var messageSet = new ByteBufferMessageSet(compressionCodec, messages);
Assert.Equal(compressionCodec, messageSet.ShallowIterator().Next().Message.CompressionCodec);
var decompressed = messageSet.Iterator().ToEnumerable().Select(x => x.Message).ToList();
Assert.Equal(messages, decompressed);
}
public void TestComplexCompressDecompress()
{
var messages = new List<Message>
{
new Message(Encoding.UTF8.GetBytes("hi there")),
new Message(Encoding.UTF8.GetBytes("I am fine")),
new Message(Encoding.UTF8.GetBytes("I am not so well today")),
};
var message = new ByteBufferMessageSet(CompressionCodecs.DefaultCompressionCodec, messages.Take(2).ToList());
var complexMessages =
new List<Message> { message.ShallowIterator().Next().Message }.Union(messages.Skip(2).Take(1).ToList())
.ToList();
var complexMessage = new ByteBufferMessageSet(CompressionCodecs.DefaultCompressionCodec, complexMessages);
var decompressedMessages = complexMessage.Iterator().ToEnumerable().Select(x => x.Message).ToList();
Assert.Equal(messages, decompressedMessages);
}
public void TestComplexCompressDecompress()
{
var messages = new List <Message>
{
new Message(Encoding.UTF8.GetBytes("hi there")),
new Message(Encoding.UTF8.GetBytes("I am fine")),
new Message(Encoding.UTF8.GetBytes("I am not so well today")),
};
var message = new ByteBufferMessageSet(CompressionCodecs.DefaultCompressionCodec, messages.Take(2).ToList());
var complexMessages =
new List <Message> {
message.ShallowIterator().Next().Message
}.Union(messages.Skip(2).Take(1).ToList())
.ToList();
var complexMessage = new ByteBufferMessageSet(CompressionCodecs.DefaultCompressionCodec, complexMessages);
var decompressedMessages = complexMessage.Iterator().ToEnumerable().Select(x => x.Message).ToList();
Assert.Equal(messages, decompressedMessages);
}
internal void VerifyShallowIterator(ByteBufferMessageSet messageSet)
{
// make sure the offsets returned by a shallow iterator is a subset of that of a deep iterator
var shallowOffsets =
new HashSet<long>(
TestUtil.IteratorToArray(messageSet.ShallowIterator()).Select(msgAndOff => msgAndOff.Offset));
var deepOffsets = new HashSet<long>(
TestUtil.IteratorToArray(messageSet.Iterator()).Select(msgAndOff => msgAndOff.Offset));
Assert.True(shallowOffsets.IsSubsetOf(deepOffsets));
}
public void TestIterator()
{
var messageList = new List<Message>
{
new Message(Encoding.UTF8.GetBytes("msg1")),
new Message(Encoding.UTF8.GetBytes("msg2")),
new Message(Encoding.UTF8.GetBytes("msg3")),
};
// test for uncompressed regular Messages
{
var messageSet = new ByteBufferMessageSet(CompressionCodecs.NoCompressionCodec, messageList);
TestUtils.CheckEquals(
messageList, TestUtils.GetMessageIterator(messageSet.Iterator()).ToEnumerable());
// make sure ByteBufferMessageSet is re-iterable.
TestUtils.CheckEquals(
messageList, TestUtils.GetMessageIterator(messageSet.Iterator()).ToEnumerable());
// make sure shallow iterator is the same as deep iterator
TestUtils.CheckEquals(
TestUtils.GetMessageIterator(messageSet.ShallowIterator()).ToEnumerable(),
TestUtils.GetMessageIterator(messageSet.Iterator()).ToEnumerable());
}
// test for compressed regular Messages
{
var messageSet = new ByteBufferMessageSet(CompressionCodecs.DefaultCompressionCodec, messageList);
TestUtils.CheckEquals(
messageList, TestUtils.GetMessageIterator(messageSet.Iterator()).ToEnumerable());
// make sure ByteBufferMessageSet is re-iterable.
TestUtils.CheckEquals(
messageList, TestUtils.GetMessageIterator(messageSet.Iterator()).ToEnumerable());
}
// test for mixed empty and non-empty messagesets uncompressed
{
List<Message> emptyMessageList = null;
var emptyMessageSet = new ByteBufferMessageSet(CompressionCodecs.NoCompressionCodec, emptyMessageList);
var regularMessgeSet = new ByteBufferMessageSet(CompressionCodecs.NoCompressionCodec, messageList);
var buffer = ByteBuffer.Allocate(emptyMessageSet.Buffer.Limit() + regularMessgeSet.Buffer.Limit());
buffer.Put(emptyMessageSet.Buffer);
buffer.Put(regularMessgeSet.Buffer);
buffer.Rewind();
var mixedMessageSet = new ByteBufferMessageSet(buffer);
TestUtils.CheckEquals(
messageList, TestUtils.GetMessageIterator(mixedMessageSet.Iterator()).ToEnumerable());
// make sure ByteBufferMessageSet is re-iterable.
TestUtils.CheckEquals(
messageList, TestUtils.GetMessageIterator(mixedMessageSet.Iterator()).ToEnumerable());
// make sure shallow iterator is the same as deep iterator
TestUtils.CheckEquals(
TestUtils.GetMessageIterator(mixedMessageSet.ShallowIterator()).ToEnumerable(),
TestUtils.GetMessageIterator(mixedMessageSet.Iterator()).ToEnumerable());
}
// test for mixed empty and non-empty messagesets compressed
{
List<Message> emptyMessageList = null;
var emptyMessageSet = new ByteBufferMessageSet(CompressionCodecs.DefaultCompressionCodec, emptyMessageList);
var regularMessgeSet = new ByteBufferMessageSet(CompressionCodecs.DefaultCompressionCodec, messageList);
var buffer = ByteBuffer.Allocate(emptyMessageSet.Buffer.Limit() + regularMessgeSet.Buffer.Limit());
buffer.Put(emptyMessageSet.Buffer);
buffer.Put(regularMessgeSet.Buffer);
buffer.Rewind();
var mixedMessageSet = new ByteBufferMessageSet(buffer);
TestUtils.CheckEquals(
messageList, TestUtils.GetMessageIterator(mixedMessageSet.Iterator()).ToEnumerable());
// make sure ByteBufferMessageSet is re-iterable.
TestUtils.CheckEquals(
messageList, TestUtils.GetMessageIterator(mixedMessageSet.Iterator()).ToEnumerable());
this.VerifyShallowIterator(mixedMessageSet);
}
}
internal void TestWriteToWithMessageSet(MessageSet set)
{
// do the write twice to ensure the message set is restored to its orginal state
for (var i = 0; i < 1; i++)
{
var stream = ByteBuffer.Allocate(1024);
var written = set.WriteTo(stream, 0, 1024);
stream.SetLength(written);
Assert.Equal(set.SizeInBytes, written);
stream.Position = 0;
var newSet = new ByteBufferMessageSet(stream);
Assert.Equal(TestUtil.IteratorToArray(set.Iterator()), TestUtil.IteratorToArray(newSet.Iterator()));
}
}
public void TestIterator()
{
var messageList = new List <Message>
{
new Message(Encoding.UTF8.GetBytes("msg1")),
new Message(Encoding.UTF8.GetBytes("msg2")),
new Message(Encoding.UTF8.GetBytes("msg3")),
};
// test for uncompressed regular Messages
{
var messageSet = new ByteBufferMessageSet(CompressionCodecs.NoCompressionCodec, messageList);
TestUtils.CheckEquals(
messageList, TestUtils.GetMessageIterator(messageSet.Iterator()).ToEnumerable());
// make sure ByteBufferMessageSet is re-iterable.
TestUtils.CheckEquals(
messageList, TestUtils.GetMessageIterator(messageSet.Iterator()).ToEnumerable());
// make sure shallow iterator is the same as deep iterator
TestUtils.CheckEquals(
TestUtils.GetMessageIterator(messageSet.ShallowIterator()).ToEnumerable(),
TestUtils.GetMessageIterator(messageSet.Iterator()).ToEnumerable());
}
// test for compressed regular Messages
{
var messageSet = new ByteBufferMessageSet(CompressionCodecs.DefaultCompressionCodec, messageList);
TestUtils.CheckEquals(
messageList, TestUtils.GetMessageIterator(messageSet.Iterator()).ToEnumerable());
// make sure ByteBufferMessageSet is re-iterable.
TestUtils.CheckEquals(
messageList, TestUtils.GetMessageIterator(messageSet.Iterator()).ToEnumerable());
}
// test for mixed empty and non-empty messagesets uncompressed
{
List <Message> emptyMessageList = null;
var emptyMessageSet = new ByteBufferMessageSet(CompressionCodecs.NoCompressionCodec, emptyMessageList);
var regularMessgeSet = new ByteBufferMessageSet(CompressionCodecs.NoCompressionCodec, messageList);
var buffer = ByteBuffer.Allocate(emptyMessageSet.Buffer.Limit() + regularMessgeSet.Buffer.Limit());
buffer.Put(emptyMessageSet.Buffer);
buffer.Put(regularMessgeSet.Buffer);
buffer.Rewind();
var mixedMessageSet = new ByteBufferMessageSet(buffer);
TestUtils.CheckEquals(
messageList, TestUtils.GetMessageIterator(mixedMessageSet.Iterator()).ToEnumerable());
// make sure ByteBufferMessageSet is re-iterable.
TestUtils.CheckEquals(
messageList, TestUtils.GetMessageIterator(mixedMessageSet.Iterator()).ToEnumerable());
// make sure shallow iterator is the same as deep iterator
TestUtils.CheckEquals(
TestUtils.GetMessageIterator(mixedMessageSet.ShallowIterator()).ToEnumerable(),
TestUtils.GetMessageIterator(mixedMessageSet.Iterator()).ToEnumerable());
}
// test for mixed empty and non-empty messagesets compressed
{
List <Message> emptyMessageList = null;
var emptyMessageSet = new ByteBufferMessageSet(CompressionCodecs.DefaultCompressionCodec, emptyMessageList);
var regularMessgeSet = new ByteBufferMessageSet(CompressionCodecs.DefaultCompressionCodec, messageList);
var buffer = ByteBuffer.Allocate(emptyMessageSet.Buffer.Limit() + regularMessgeSet.Buffer.Limit());
buffer.Put(emptyMessageSet.Buffer);
buffer.Put(regularMessgeSet.Buffer);
buffer.Rewind();
var mixedMessageSet = new ByteBufferMessageSet(buffer);
TestUtils.CheckEquals(
messageList, TestUtils.GetMessageIterator(mixedMessageSet.Iterator()).ToEnumerable());
// make sure ByteBufferMessageSet is re-iterable.
TestUtils.CheckEquals(
messageList, TestUtils.GetMessageIterator(mixedMessageSet.Iterator()).ToEnumerable());
this.VerifyShallowIterator(mixedMessageSet);
}
}