public void GetBytesValidSequence()
{
var payload = Encoding.UTF8.GetBytes("kafka");
Message message = new Message(payload, CompressionCodecs.NoCompressionCodec);
MemoryStream ms = new MemoryStream();
message.WriteTo(ms);
Assert.AreEqual(message.Size, ms.Length);
var crc = Crc32Hasher.ComputeCrcUint32(ms.GetBuffer(), 4, (int)(ms.Length - 4));
// first 4 bytes = the crc
using (var reader = new KafkaBinaryReader(ms))
{
Assert.AreEqual(crc, reader.ReadUInt32());
// magic
Assert.AreEqual(message.Magic, reader.ReadByte());
// attributes
Assert.AreEqual((byte)0, reader.ReadByte());
// key size
Assert.AreEqual(-1, reader.ReadInt32());
// payload size
Assert.AreEqual(payload.Length, reader.ReadInt32());
// remaining bytes = the payload
payload.SequenceEqual(reader.ReadBytes(10)).Should().BeTrue();
}
}
public void BufferedMessageSetWriteToValidSequence()
{
byte[] messageBytes = { 1, 2, 3, 4, 5 };
var msg1 = new Message(messageBytes)
{
Offset = 0
};
var msg2 = new Message(messageBytes);
msg2.Offset = 1;
MessageSet messageSet = new BufferedMessageSet(new List <Message>()
{
msg1, msg2
}, 0);
var ms = new MemoryStream();
messageSet.WriteTo(ms);
var reader = new KafkaBinaryReader(ms);
int baseOffset = 0;
for (int i = 0; i < 2; ++i)
{
reader.ReadInt64().Should().Be(i); // offset
var msgLength = reader.ReadInt32(); // length
msgLength.Should().Be(Message.DefaultHeaderSize + msg1.PayloadSize);
reader.ReadUInt32().Should().Be(Crc32Hasher.ComputeCrcUint32(ms.GetBuffer(), baseOffset + 8 + 4 + 4, msgLength - 4));
reader.ReadByte().Should().Be(0); // magic
reader.ReadByte().Should().Be(msg1.Attributes);
reader.ReadInt32().Should().Be(-1); // key length
reader.ReadInt32().Should().Be(messageBytes.Length); // message length
reader.ReadBytes(messageBytes.Length).SequenceEqual(messageBytes).Should().BeTrue();
baseOffset += 8 + 4 + msgLength;
}
}
public void ParseFromValid()
{
Crc32Hasher crc32 = new Crc32Hasher();
string payload = "kafka";
byte magic = 1;
byte attributes = 0;
byte[] payloadData = Encoding.UTF8.GetBytes(payload);
byte[] payloadSize = BitConverter.GetBytes(payloadData.Length);
byte[] checksum = crc32.ComputeHash(payloadData);
byte[] messageData = new byte[payloadData.Length + 2 + payloadSize.Length + checksum.Length];
Buffer.BlockCopy(payloadSize, 0, messageData, 0, payloadSize.Length);
messageData[4] = magic;
messageData[5] = attributes;
Buffer.BlockCopy(checksum, 0, messageData, payloadSize.Length + 2, checksum.Length);
Buffer.BlockCopy(payloadData, 0, messageData, payloadSize.Length + 2 + checksum.Length, payloadData.Length);
Message message = Message.ParseFrom(messageData);
Assert.IsNotNull(message);
Assert.AreEqual(magic, message.Magic);
Assert.IsTrue(payloadData.SequenceEqual(message.Payload));
Assert.IsTrue(checksum.SequenceEqual(message.Checksum));
}
public void AddString(string key, string value)
{
AddString(new MessageBinString
{
Value = value,
Hash = (int)Crc32Hasher.Crc32Hash(key)
});
}
public void BufferedMessageSetWriteToValidSequence()
{
byte[] messageBytes = new byte[] { 1, 2, 3, 4, 5 };
Message msg1 = new Message(messageBytes);
Message msg2 = new Message(messageBytes);
MessageSet messageSet = new BufferedMessageSet(new List <Message>()
{
msg1, msg2
});
MemoryStream ms = new MemoryStream();
messageSet.WriteTo(ms);
////first message
byte[] messageLength = new byte[MessageLengthPartLength];
Array.Copy(ms.ToArray(), MessageLengthPartOffset, messageLength, 0, MessageLengthPartLength);
if (BitConverter.IsLittleEndian)
{
Array.Reverse(messageLength);
}
Assert.AreEqual(MagicNumberPartLength + AttributesPartLength + ChecksumPartLength + messageBytes.Length, BitConverter.ToInt32(messageLength, 0));
Assert.AreEqual(1, ms.ToArray()[MagicNumberPartOffset]); // default magic number should be 1
byte[] checksumPart = new byte[ChecksumPartLength];
Array.Copy(ms.ToArray(), ChecksumPartOffset, checksumPart, 0, ChecksumPartLength);
Assert.AreEqual(Crc32Hasher.Compute(messageBytes), checksumPart);
byte[] dataPart = new byte[messageBytes.Length];
Array.Copy(ms.ToArray(), DataPartOffset, dataPart, 0, messageBytes.Length);
Assert.AreEqual(messageBytes, dataPart);
////second message
int secondMessageOffset = MessageLengthPartLength + MagicNumberPartLength + AttributesPartLength + ChecksumPartLength +
messageBytes.Length;
messageLength = new byte[MessageLengthPartLength];
Array.Copy(ms.ToArray(), secondMessageOffset + MessageLengthPartOffset, messageLength, 0, MessageLengthPartLength);
if (BitConverter.IsLittleEndian)
{
Array.Reverse(messageLength);
}
Assert.AreEqual(MagicNumberPartLength + AttributesPartLength + ChecksumPartLength + messageBytes.Length, BitConverter.ToInt32(messageLength, 0));
Assert.AreEqual(1, ms.ToArray()[secondMessageOffset + MagicNumberPartOffset]); // default magic number should be 1
checksumPart = new byte[ChecksumPartLength];
Array.Copy(ms.ToArray(), secondMessageOffset + ChecksumPartOffset, checksumPart, 0, ChecksumPartLength);
Assert.AreEqual(Crc32Hasher.Compute(messageBytes), checksumPart);
dataPart = new byte[messageBytes.Length];
Array.Copy(ms.ToArray(), secondMessageOffset + DataPartOffset, dataPart, 0, messageBytes.Length);
Assert.AreEqual(messageBytes, dataPart);
}
private uint GetFilenameHash(string filename)
{
if (filename.StartsWith("0x"))
{
// Filename was made by this class and is simply a representation of the hash itself
return(Convert.ToUInt32(filename.Substring(2), fromBase: 16));
}
return(Crc32Hasher.Crc32Hash(filename));
}
public void CanBuildMessageBin()
{
// Arrange
var messageBin = new MessageBinEntry();
messageBin.AddString("first", "Grookey");
messageBin.AddString("second", "Scorbunny");
messageBin.AddString("third", "Sobble");
// Act
var data = messageBin.ToByteArray();
// Assert
var rebuiltDb = new MessageBinEntry(data);
rebuiltDb.GetStringByHash((int)Crc32Hasher.Crc32Hash("first")).Should().Be("Grookey");
rebuiltDb.GetStringByHash((int)Crc32Hasher.Crc32Hash("second")).Should().Be("Scorbunny");
rebuiltDb.GetStringByHash((int)Crc32Hasher.Crc32Hash("third")).Should().Be("Sobble");
}
public void WriteToValidSequenceForDefaultConstructor()
{
byte[] messageBytes = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9 };
Message message = new Message(messageBytes);
MemoryStream ms = new MemoryStream();
message.WriteTo(ms);
Assert.AreEqual(1, ms.ToArray()[MagicNumberPartOffset]); // default magic number should be 1
byte[] checksumPart = new byte[ChecksumPartLength];
Array.Copy(ms.ToArray(), ChecksumPartOffset, checksumPart, 0, ChecksumPartLength);
Assert.AreEqual(Crc32Hasher.Compute(messageBytes), checksumPart);
message.ToString();
byte[] dataPart = new byte[messageBytes.Length];
Array.Copy(ms.ToArray(), DataPartOffset, dataPart, 0, messageBytes.Length);
Assert.AreEqual(messageBytes, dataPart);
}
private uint ComputeChecksum(byte[] message, int offset, int count)
{
return(Crc32Hasher.ComputeCrcUint32(message, offset, count));
}
/// <summary>
/// Initializes a new instance of the Message class.
/// </summary>
/// <remarks>
/// Initializes the checksum as null. It will be automatically computed.
/// </remarks>
/// <param name="payload">The data for the payload.</param>
/// <param name="magic">The magic identifier.</param>
public Message(byte[] payload, CompressionCodecs compressionCodec)
: this(payload, Crc32Hasher.Compute(payload), compressionCodec)
{
Guard.NotNull(payload, "payload");
}
public void HashesCorrectly(string testString, int expectedHash)
{
var expectedHashUnsigned = BitConverter.ToUInt32(BitConverter.GetBytes(expectedHash), 0);
Crc32Hasher.Crc32Hash(testString).Should().Be(expectedHashUnsigned);
}
