public static byte[] DeserializeBytesWithVarIntSize(ReusableMemoryStream stream)
{
var len = VarIntConverter.ReadInt32(stream);
if (len == -1)
{
return(null);
}
var buffer = new byte[len];
stream.Read(buffer, 0, len);
return(buffer);
}
public void TestSizeOfVarInt()
{
var buffer = new byte[255];
var previousPosition = 0L;
using (var stream = new MemoryStream(buffer))
{
foreach (var value in VarIntNumbers.Keys)
{
VarIntConverter.Write(stream, value);
Assert.AreEqual(stream.Position - previousPosition, VarIntConverter.SizeOfVarInt(value));
previousPosition = stream.Position;
}
}
}
/// <summary>
/// Returns the size of the Serialized object without the attributes.
/// </summary>
/// <returns>Serialized size of the body in bytes</returns>
public long SizeOfBodyInBytes(long offsetDelta, long timestampDelta)
{
var size = 1L; // always 1 byte for attributes
size += VarIntConverter.SizeOfVarInt(offsetDelta);
size += VarIntConverter.SizeOfVarInt(timestampDelta);
if (Key == null)
{
size += Basics.MinusOneVarInt.Length;
}
else
{
var keySize = Basics.SizeOfSerializedObject(Key, KeySerializer);
size += keySize + VarIntConverter.SizeOfVarInt(keySize);
}
if (Value == null)
{
size += Basics.MinusOneVarInt.Length;
}
else
{
var valueSize = Basics.SizeOfSerializedObject(Value, ValueSerializer);
size += valueSize + VarIntConverter.SizeOfVarInt(valueSize);
}
if (Headers == null)
{
size += Basics.ZeroVarInt.Length;
}
else
{
var headersCount = 0;
foreach (KafkaRecordHeader header in Headers)
{
size += (int)SerializedSizeOfHeader(header);
++headersCount;
}
size += VarIntConverter.SizeOfVarInt(headersCount);
}
return(size);
}
public IEnumerable <Record> DeserializeRecordsUncompressed(ReusableMemoryStream input, int numberOfRecords, long endOfBatch,
long firstTimeStamp, Deserializers deserializers)
{
for (var i = 0; i < numberOfRecords; i++)
{
var length = VarIntConverter.ReadInt32(input);
if (input.Length - input.Position < length)
{
throw new ProtocolException(
$"Record said it was of length {length}, but actually only {input.Length - input.Position} bytes remain");
}
var attributes = input.ReadByte(); // ignored for now
var timeStampDelta = VarIntConverter.ReadInt64(input);
var offsetDelta = VarIntConverter.ReadInt32(input);
var keyLength = VarIntConverter.ReadInt32(input);
var key = keyLength == -1 ? null : deserializers.Item1.Deserialize(input, keyLength);
var valueLength = VarIntConverter.ReadInt32(input);
var value = valueLength == -1 ? null : deserializers.Item1.Deserialize(input, valueLength);
var headersCount = VarIntConverter.ReadInt32(input);
var headers = new List <KafkaRecordHeader>(headersCount);
for (var j = 0; j < headersCount; j++)
{
headers.Add(Record.DeserializeHeader(input));
}
yield return(new Record
{
Headers = headers,
Key = key,
Timestamp = firstTimeStamp + timeStampDelta,
Value = value,
Offset = BaseOffset + offsetDelta
});
}
}
public ReusableMemoryStream Serialize(ReusableMemoryStream target, long baseTimestamp, long offsetDelta)
{
long timestampDelta = Timestamp - baseTimestamp;
VarIntConverter.Write(target, SizeOfBodyInBytes(offsetDelta, timestampDelta));
// Record attributes are always null.
target.WriteByte(0x00);
VarIntConverter.Write(target, timestampDelta);
VarIntConverter.Write(target, offsetDelta);
if (SerializedKeyValue == null)
{
Basics.WriteObject(target, Key, KeySerializer);
Basics.WriteObject(target, Value, ValueSerializer);
}
else
{
target.Write(SerializedKeyValue.GetBuffer(), 0, (int)SerializedKeyValue.Length);
}
if (Headers == null)
{
target.Write(Basics.ZeroVarInt, 0, Basics.ZeroVarInt.Length);
}
else
{
VarIntConverter.Write(target, Headers.Count);
foreach (KafkaRecordHeader header in Headers)
{
SerializeHeader(target, header);
}
}
return(target);
}
public static long SerializedSizeOfHeader(KafkaRecordHeader header)
{
header.Validate();
var keySize = Basics.SizeOfSerializedString(header.Key);
return(keySize + header.Value.Length + VarIntConverter.SizeOfVarInt(keySize) + VarIntConverter.SizeOfVarInt(header.Value.Length));
}
public void TestReadInt64Overflows(byte[] value)
{
Assert.Throws <OverflowException>(() => VarIntConverter.ReadAsInt64(new MemoryStream(value)));
}
/// <summary>
/// Write the size of the serialized object as a VarInt then the serialized object on the stream.
///
/// </summary>
/// <param name="stream">target stream</param>
/// <param name="o">object to serialize and write to target</param>
/// <param name="serializer">serializer to use if object is not serializable</param>
public static void WriteObject(ReusableMemoryStream stream, object o, ISerializer serializer)
{
switch (o)
{
case null:
stream.Write(MinusOneVarInt, 0, MinusOneVarInt.Length);
break;
case byte[] asBytes:
SerializeBytesWithVarIntSize(stream, asBytes);
break;
case string asString:
SerializeStringWithVarIntSize(stream, asString);
break;
default:
{
long expectedPosition;
if (o is ISizedMemorySerializable asSizedSerializable)
{
long expectedSize = asSizedSerializable.SerializedSize();
VarIntConverter.Write(stream, expectedSize);
expectedPosition = stream.Position + expectedSize;
asSizedSerializable.Serialize(stream);
}
else if (serializer is ISizableSerializer sizableSerializer)
{
long expectedSize = sizableSerializer.SerializedSize(o);
VarIntConverter.Write(stream, expectedSize);
expectedPosition = stream.Position + expectedSize;
sizableSerializer.Serialize(o, stream);
}
else
{
// If we can not know the size of the serialized object in advance, we need to use an intermediate buffer.
using (var buffer = stream.Pool.Reserve())
{
if (o is IMemorySerializable asSerializable)
{
asSerializable.Serialize(buffer);
}
else
{
serializer.Serialize(o, buffer);
}
WriteObject(stream, buffer, null);
}
return;
}
if (expectedPosition != stream.Position)
{
throw new Exception(
"SerializedSize() returned a different value than the size of the serialized object written");
}
break;
}
}
}
