public void SerializeBody(ReusableMemoryStream stream, object extra)
{
stream.Write(Basics.MinusOne32, 0, 4); // ReplicaId, non clients that are not a broker must use -1
BigEndianConverter.Write(stream, MaxWaitTime);
BigEndianConverter.Write(stream, MinBytes);
Basics.WriteArray(stream, TopicsData, extra);
}
public void Serialize(ReusableMemoryStream stream, object _, Basics.ApiVersion __)
{
BigEndianConverter.Write(stream, (short)ErrorCode);
BigEndianConverter.Write(stream, CoordinatorId);
Basics.SerializeString(stream, CoordinatorHost);
BigEndianConverter.Write(stream, CoordinatorPort);
}
void Flush(byte[] buff, int offset, int count)
{
var compressedSize = SnappyCodec.Compress(buff, offset, count, _compressedBuffer, 0);
BigEndianConverter.Write(_base, compressedSize);
_base.Write(_compressedBuffer, 0, compressedSize);
}
public void Serialize(ReusableMemoryStream stream, object _, Basics.ApiVersion __)
{
BigEndianConverter.Write(stream, Partition);
BigEndianConverter.Write(stream, Offset);
Basics.SerializeString(stream, Metadata);
BigEndianConverter.Write(stream, (short)ErrorCode);
}
public void Serialize(ReusableMemoryStream stream, object _, Basics.ApiVersion __)
{
BigEndianConverter.Write(stream, (short)ErrorCode);
var ma = MemberAssignment;
Basics.WriteWithSize(stream, s => ma.Serialize(s, null, Basics.ApiVersion.Ignored));
}
public void SerializeBody(ReusableMemoryStream stream, object _, Basics.ApiVersion __)
{
Basics.SerializeString(stream, GroupId);
BigEndianConverter.Write(stream, GenerationId);
Basics.SerializeString(stream, MemberId);
Basics.WriteArray(stream, GroupAssignment);
}
static void WriteCompressedMessageSet(Stream stream, MessageData message, CompressionType compresionType)
{
var messageRawSize = 0;
if (message.Key != null)
{
messageRawSize += message.Key.Length;
}
if (message.Value != null)
{
messageRawSize += message.Value.Count;
}
var messageSetSize = _messageSetOverheadSize + messageRawSize;
BigEndianConverter.Write(stream, messageSetSize);
_log.Debug("WriteCompressedMessageSet: size: {0}", messageSetSize);
stream.Write(_zero64, 0, 8); // Producer fake offset
var messageSize = _messageOverheadSize + messageRawSize;
BigEndianConverter.Write(stream, messageSize);
Write(stream, message, compresionType);
}
private static void Write(Stream stream, MessageData message, CompressionType compression = CompressionType.None)
{
var crc = Crc32.Update(_zero32, len: 1);
crc = Crc32.Update(_compressionEnumTable, crc, 1, (int)compression);
if (message.Key == null)
{
crc = Crc32.Update(_minusOne32, crc);
}
else
{
crc = Crc32.Update(message.Key.Length, crc);
crc = Crc32.Update(message.Key, crc);
}
crc = Crc32.Update(message.Value.Count, crc);
crc = Crc32.Update(message.Value, crc);
crc = Crc32.GetHash(crc);
BigEndianConverter.Write(stream, crc);
stream.WriteByte(0); // magic byte
stream.Write(_compressionEnumTable, (int)compression, 1); // attributes
if (message.Key == null)
{
stream.Write(_minusOne32, 0, 4);
}
else
{
BigEndianConverter.Write(stream, message.Key.Length);
stream.Write(message.Key, 0, message.Key.Length);
}
BigEndianConverter.Write(stream, message.Value.Count);
stream.Write(message.Value.Array, 0, message.Value.Count);
}
public void TestBadCompression()
{
using (var serialized = Pool.Reserve())
{
var set = new PartitionData
{
Partition = 42,
CompressionCodec = CompressionCodec.Gzip,
Messages = new[]
{
new Message {
Key = Key, Value = Value
}
}
};
set.Serialize(serialized, SerializationConfig.ByteArraySerializers);
// corrupt compressed data
serialized[37] = 8;
// recompute crc
var crc = (int)Crc32.Compute(serialized, 24, serialized.Length - 24);
serialized.Position = 20;
BigEndianConverter.Write(serialized, crc);
// go
serialized.Position = 4;
Assert.That(() => FetchPartitionResponse.DeserializeMessageSet(serialized, SerializationConfig.ByteArrayDeserializers), Throws.InstanceOf <UncompressException>());
}
}
private static void Write(MemoryStream stream, MessageData message)
{
var crcPos = stream.Position;
stream.Write(_minusOne32, 0, 4); // crc placeholder
var bodyPos = stream.Position;
stream.WriteByte(0); // magic byte
stream.WriteByte(0); // attributes
if (message.Key == null)
{
stream.Write(_minusOne32, 0, 4);
}
else
{
BigEndianConverter.Write(stream, message.Key.Length);
stream.Write(message.Key, 0, message.Key.Length);
}
if (message.Value == null)
{
stream.Write(_minusOne32, 0, 4);
}
else
{
BigEndianConverter.Write(stream, message.Value.Length);
stream.Write(message.Value, 0, message.Value.Length);
}
// update crc
var crc = Crc32.Compute(stream, bodyPos, stream.Position - bodyPos);
Update(stream, crcPos, crc);
}
public void Serialize(ReusableMemoryStream stream, CompressionCodec compressionCodec, Tuple <ISerializer, ISerializer> serializers)
{
var crcPos = stream.Position;
stream.Write(Basics.MinusOne32, 0, 4); // crc placeholder
var bodyPos = stream.Position;
stream.WriteByte(0); // magic byte
stream.WriteByte((byte)compressionCodec); // attributes
if (SerializedKeyValue != null)
{
stream.Write(SerializedKeyValue.GetBuffer(), 0, (int)SerializedKeyValue.Length);
}
else
{
DoSerializeKeyValue(stream, serializers);
}
// update crc
var crc = Crc32.Compute(stream, bodyPos, stream.Position - bodyPos);
var curPos = stream.Position;
stream.Position = crcPos;
BigEndianConverter.Write(stream, (int)crc);
stream.Position = curPos;
}
// Used only in tests
public void Serialize(ReusableMemoryStream stream, object _, Basics.ApiVersion __)
{
BigEndianConverter.Write(stream, (short)ErrorCode);
BigEndianConverter.Write(stream, Id);
BigEndianConverter.Write(stream, Leader);
Basics.WriteArray(stream, Replicas, BigEndianConverter.Write);
Basics.WriteArray(stream, Isr, BigEndianConverter.Write);
}
public void SerializeBody(ReusableMemoryStream stream, object _, Basics.ApiVersion __)
{
Basics.SerializeString(stream, ConsumerGroupId);
BigEndianConverter.Write(stream, ConsumerGroupGenerationId);
Basics.SerializeString(stream, ConsumerId);
BigEndianConverter.Write(stream, RetentionTime);
Basics.WriteArray(stream, TopicsData);
}
private static void WriteRequestHeader(MemoryStream stream, int correlationId, ApiKey requestType)
{
stream.Write(_minusOne32, 0, 4); // reserve space for message size
BigEndianConverter.Write(stream, (short)requestType);
stream.Write(_apiVersion, 0, 2);
BigEndianConverter.Write(stream, correlationId);
stream.Write(_clientId, 0, _clientId.Length);
}
public void Serialize(ReusableMemoryStream stream, object extra, Basics.ApiVersion version)
{
ProducePartitionResponse.Serialize(stream, extra, version);
if (version > Basics.ApiVersion.V0)
{
BigEndianConverter.Write(stream, ThrottleTime);
}
}
private static void WriteArraySize(ReusableMemoryStream stream, long sizePosition, int count)
{
var pos = stream.Position; // update count field
stream.Position = sizePosition;
BigEndianConverter.Write(stream, count);
stream.Position = pos;
}
public static void WriteRequestHeader(ReusableMemoryStream stream, int correlationId, ApiKey requestType, ApiVersion version, byte[] clientId)
{
stream.Write(MinusOne32, 0, 4); // reserve space for message size
BigEndianConverter.Write(stream, (short)requestType);
stream.Write(VersionBytes[(int)version], 0, 2);
BigEndianConverter.Write(stream, correlationId);
BigEndianConverter.Write(stream, (short)clientId.Length);
stream.Write(clientId, 0, clientId.Length);
}
static void WriteHeader(ReusableMemoryStream stream, long initPos)
{
var pos = stream.Position;
var size = pos - initPos;
stream.Position = initPos - 4;
BigEndianConverter.Write(stream, (int)size);
stream.Position = pos;
}
public void Serialize(ReusableMemoryStream stream, object _, Basics.ApiVersion __)
{
BigEndianConverter.Write(stream, (short)ErrorCode);
BigEndianConverter.Write(stream, GenerationId);
Basics.SerializeString(stream, GroupProtocol);
Basics.SerializeString(stream, LeaderId);
Basics.SerializeString(stream, MemberId);
Basics.WriteArray(stream, GroupMembers, (s, m) => m.Serialize(s, null, Basics.ApiVersion.Ignored));
}
public void Serialize(ReusableMemoryStream stream, object _, Basics.ApiVersion version)
{
BigEndianConverter.Write(stream, Partition);
BigEndianConverter.Write(stream, Time);
if (version == Basics.ApiVersion.V0)
{
BigEndianConverter.Write(stream, MaxNumberOfOffsets);
}
}
public void Serialize(ReusableMemoryStream stream, object extra)
{
BigEndianConverter.Write(stream, Partition);
Basics.WriteSizeInBytes(stream, Messages,
new SerializationInfo {
Serializers = extra as Serializers, CompressionCodec = CompressionCodec
},
SerializeMessages);
}
public void SerializeBody(ReusableMemoryStream stream, object extra, Basics.ApiVersion version)
{
if (version >= Basics.ApiVersion.V3)
{
Basics.SerializeString(stream, TransactionalID);
}
BigEndianConverter.Write(stream, RequiredAcks);
BigEndianConverter.Write(stream, Timeout);
Basics.WriteArray(stream, TopicsData, extra, version);
}
public long LogStartOffset; // Required by the protocol, but will always be zero in our case (i.e. we are consumers, not brokers)
#region Serialization
public void Serialize(ReusableMemoryStream stream, object _, Basics.ApiVersion version)
{
BigEndianConverter.Write(stream, Partition);
BigEndianConverter.Write(stream, FetchOffset);
BigEndianConverter.Write(stream, MaxBytes);
if (version >= Basics.ApiVersion.V5)
{
stream.Write(Basics.Zero64, 0, 8); // log_start_offset is 0 for consumer, only used by follower.
}
}
public static byte[] Serialize(ProduceRequest request, int correlationId)
{
var stream = new MemoryStream();
WriteRequestHeader(stream, correlationId, ApiKey.ProduceRequest);
BigEndianConverter.Write(stream, request.RequiredAcks);
BigEndianConverter.Write(stream, request.Timeout);
WriteArray(stream, request.TopicData, t => Write(stream, t));
return(WriteMessageLength(stream));
}
private static byte[] WriteMessageLength(MemoryStream stream)
{
var buff = stream.ToArray();
var len = buff.Length - 4; // -4 because do not count size flag itself
// write message length to the head
// TODO: use seek?
BigEndianConverter.Write(buff, len);
return(buff);
}
public static ReusableMemoryStream WriteMessageLength(ReusableMemoryStream stream)
{
stream.Position = 0;
var len = (int)stream.Length - 4; // -4 because do not count size flag itself
// write message length to the head
// TODO: use seek?
BigEndianConverter.Write(stream, len);
return(stream);
}
// Used only in tests
public void Serialize(ReusableMemoryStream stream, object _, Basics.ApiVersion version)
{
BigEndianConverter.Write(stream, Partition);
BigEndianConverter.Write(stream, (short)ErrorCode);
BigEndianConverter.Write(stream, Offset);
if (version >= Basics.ApiVersion.V2)
{
BigEndianConverter.Write(stream, Timestamp);
}
}
public static void SerializeString(ReusableMemoryStream stream, string s)
{
if (s == null)
{
stream.Write(MinusOne16, 0, 2);
return;
}
BigEndianConverter.Write(stream, (short)s.Length);
_stringSer.Serialize(s, stream);
}
public static void SerializeBytes(ReusableMemoryStream stream, byte[] b)
{
if (b == null)
{
stream.Write(MinusOne32, 0, 4);
return;
}
BigEndianConverter.Write(stream, b.Length);
stream.Write(b, 0, b.Length);
}
private static void Write(Stream stream, IEnumerable <MessageData> messages)
{
foreach (var message in messages)
{
stream.Write(_zero64, 0, 8); // producer does fake offset
var messageSize = _messageOverheadSize + (message.Key?.Length ?? 0) + message.Value.Count;
BigEndianConverter.Write(stream, messageSize);
Write(stream, message);
}
}
