public void Serialize(ReusableMemoryStream stream, CompressionCodec compressionCodec, Tuple <ISerializer, ISerializer> serializers)
{
var crcPos = stream.Position;
stream.Write(Basics.MinusOne32, 0, 4); // crc placeholder
var bodyPos = stream.Position;
stream.WriteByte(0); // magic byte
stream.WriteByte((byte)compressionCodec); // attributes
if (SerializedKeyValue != null)
{
stream.Write(SerializedKeyValue.GetBuffer(), 0, (int)SerializedKeyValue.Length);
}
else
{
DoSerializeKeyValue(stream, serializers);
}
// update crc
var crc = Crc32.Compute(stream, bodyPos, stream.Position - bodyPos);
var curPos = stream.Position;
stream.Position = crcPos;
BigEndianConverter.Write(stream, (int)crc);
stream.Position = curPos;
}
private void HandleInvalidSerializedKeyValue(ReusableMemoryStream stream)
{
stream.Logger?.LogError("Invalid SerializedKeyValue. Length is only " + SerializedKeyValue.Length
+ " bytes. Message cannot be serialized : " + SerializedKeyValue.GetBuffer());
// Simulate an empty key & message to not send a corrupted message
stream.Write(Basics.MinusOne32, 0, 4);
stream.Write(Basics.MinusOne32, 0, 4);
}
public static void WriteRequestHeader(ReusableMemoryStream stream, int correlationId, ApiKey requestType, ApiVersion version, byte[] clientId)
{
stream.Write(MinusOne32, 0, 4); // reserve space for message size
BigEndianConverter.Write(stream, (short)requestType);
stream.Write(VersionBytes[(int)version], 0, 2);
BigEndianConverter.Write(stream, correlationId);
BigEndianConverter.Write(stream, (short)clientId.Length);
stream.Write(clientId, 0, clientId.Length);
}
public static void SerializeBytesWithVarIntSize(ReusableMemoryStream stream, byte[] b)
{
if (b == null)
{
stream.Write(MinusOneVarInt, 0, MinusOneVarInt.Length);
return;
}
VarIntConverter.Write(stream, b.Length);
stream.Write(b, 0, b.Length);
}
public static void SerializeBytes(ReusableMemoryStream stream, byte[] b)
{
if (b == null)
{
stream.Write(MinusOne32, 0, 4);
return;
}
BigEndianConverter.Write(stream, b.Length);
stream.Write(b, 0, b.Length);
}
public ReusableMemoryStream Serialize(ReusableMemoryStream target, long baseTimestamp, long offsetDelta)
{
var timestampDelta = Timestamp - baseTimestamp;
VarIntConverter.Write(target, SizeOfBodyInBytes(offsetDelta, timestampDelta));
// Record attributes are always null.
target.WriteByte(0x00);
VarIntConverter.Write(target, timestampDelta);
VarIntConverter.Write(target, offsetDelta);
Basics.WriteObject(target, Key, KeySerializer);
Basics.WriteObject(target, Value, ValueSerializer);
if (Headers == null)
{
target.Write(Basics.ZeroVarInt, 0, Basics.ZeroVarInt.Length);
}
else
{
VarIntConverter.Write(target, Headers.Count);
foreach (KafkaRecordHeader header in Headers)
{
SerializeHeader(target, header);
}
}
return(target);
}
private static long WriteArrayHeader(ReusableMemoryStream stream)
{
var sizePosition = stream.Position;
stream.Write(MinusOne32, 0, 4); // placeholder for count field
return(sizePosition);
}
public void SerializeBody(ReusableMemoryStream stream, object extra)
{
stream.Write(Basics.MinusOne32, 0, 4); // ReplicaId, non clients that are not a broker must use -1
BigEndianConverter.Write(stream, MaxWaitTime);
BigEndianConverter.Write(stream, MinBytes);
Basics.WriteArray(stream, TopicsData, extra);
}
private static void BlockCompress(ReusableMemoryStream target, byte[] body, int offset, int count)
{
var position = (int)target.Position;
target.SetLength(target.Length + MaxCompressedSize + 4);
target.Position = position + 4;
var size = LZ4Codec.Encode(body, offset, count, target.GetBuffer(), position + 4, MaxCompressedSize);
if (size >= count)
{
// Do not compress block
// => set block header highest bit to 1 to mark no compression
LittleEndianWriteUInt32((uint)(count | 1 << 31), target.GetBuffer(), position);
// Write uncompressed data
target.Write(body, offset, count);
}
else
{
LittleEndianWriteUInt32((uint)size, target.GetBuffer(), position);
// compressed data is already written, just set the position
target.Position += size;
}
target.SetLength(target.Position);
}
private static void _SerializeMessages(ReusableMemoryStream stream, IEnumerable <Message> messages, SerializationInfo info)
{
if (info.CompressionCodec != CompressionCodec.None)
{
stream.Write(Basics.Zero64, 0, 8);
using (var msgsetStream = stream.Pool.Reserve())
{
SerializeMessagesUncompressed(msgsetStream, messages, info.Serializers, info.MessageVersion);
using (var compressed = stream.Pool.Reserve())
{
Basics.CompressStream(msgsetStream, compressed, info.CompressionCodec);
var m = new Message
{
Value = compressed,
TimeStamp = Timestamp.Now
};
Basics.WriteWithSize(stream, m,
new SerializationInfo
{
Serializers = SerializationConfig.ByteArraySerializers,
CompressionCodec = info.CompressionCodec,
MessageVersion = info.MessageVersion
}, SerializeMessageWithCodec);
}
}
}
else
{
SerializeMessagesUncompressed(stream, messages, info.Serializers, info.MessageVersion);
}
}
private static uint UncompressBlock(ReusableMemoryStream target, byte[] body, int dataIndex, bool hasChecksum, int blockSize)
{
var blockHeader = LittleEndianReadUInt32(body, dataIndex);
if (blockHeader == 0) // last frame
{
return(0);
}
var size = blockHeader & 0x7FFFFFFF;
if ((blockHeader & 0x80000000) == 0) // compressed data
{
target.SetLength(target.Length + blockSize);
var dsize = LZ4Codec.Decode(body, dataIndex + 4, (int)size, target.GetBuffer(), (int)target.Position, blockSize);
if (dsize < blockSize)
{
target.SetLength(target.Length - blockSize + dsize);
}
target.Position = target.Length;
}
else // uncompressed data
{
target.Write(body, dataIndex + 4, (int)size);
}
return(size + 4 + (hasChecksum ? 4u : 0));
}
/// <summary>
/// Write the size of the serialized object as a VarInt then the serialized object on the stream.
///
/// </summary>
/// <param name="stream">target stream</param>
/// <param name="o">object to serialize and write to target</param>
/// <param name="serializer">serializer to use if object is not serializable</param>
public static void WriteObject(ReusableMemoryStream stream, object o, ISerializer serializer)
{
if (o is byte[] asBytes)
{
SerializeBytesWithVarIntSize(stream, asBytes);
}
else if (o is string asString)
{
SerializeStringWithVarIntSize(stream, asString);
}
else
{
if (o == null)
{
stream.Write(MinusOneVarInt, 0, MinusOneVarInt.Length);
return;
}
var expectedPosition = -1L;
if (o is ISizedMemorySerializable asSizedSerializable)
{
var expectedSize = asSizedSerializable.SerializedSize();
VarIntConverter.Write(stream, expectedSize);
expectedPosition = stream.Position + expectedSize;
asSizedSerializable.Serialize(stream);
}
else if (serializer is ISizableSerializer sizableSerializer)
{
var expectedSize = sizableSerializer.SerializedSize(o);
VarIntConverter.Write(stream, expectedSize);
expectedPosition = stream.Position + expectedSize;
sizableSerializer.Serialize(o, stream);
}
else
{
// If we can not know the size of the serialized object in advance, we need to use an intermediate buffer.
using (var buffer = stream.Pool.Reserve())
{
if (o is IMemorySerializable asSerializable)
{
asSerializable.Serialize(buffer);
}
else
{
serializer.Serialize(o, buffer);
}
WriteObject(stream, buffer, null);
}
return;
}
if (expectedPosition != stream.Position)
{
throw new Exception(
"SerializedSize() returned a different value than the size of the serialized object written");
}
}
}
public static void Update(ReusableMemoryStream stream, long pos, byte[] buff)
{
var currPos = stream.Position;
stream.Position = pos;
stream.Write(buff, 0, buff.Length);
stream.Position = currPos;
}
public void Serialize(ReusableMemoryStream stream, CompressionCodec compressionCodec,
Tuple <ISerializer, ISerializer> serializers, MessageVersion msgVersion)
{
var crcPos = stream.Position;
stream.Write(Basics.MinusOne32, 0, 4); // crc placeholder
var bodyPos = stream.Position;
// V0 message format
if (msgVersion == MessageVersion.V0)
{
stream.WriteByte(0); // magic byte
stream.WriteByte((byte)compressionCodec); // attributes
}
else // V1 message format
{
stream.WriteByte(1); // magic byte
stream.WriteByte((byte)compressionCodec); // attributes
BigEndianConverter.Write(stream, TimeStamp);
}
if (SerializedKeyValue != null)
{
if (SerializedKeyValue.Length < MinimumValidSizeForSerializedKeyValue)
{
HandleInvalidSerializedKeyValue(stream);
}
else
{
stream.Write(SerializedKeyValue.GetBuffer(), 0, (int)SerializedKeyValue.Length);
}
}
else
{
DoSerializeKeyValue(stream, serializers);
}
// update crc
var crc = Crc32.Compute(stream, bodyPos, stream.Position - bodyPos);
var curPos = stream.Position;
stream.Position = crcPos;
BigEndianConverter.Write(stream, (int)crc);
stream.Position = curPos;
}
public long LogStartOffset; // Required by the protocol, but will always be zero in our case (i.e. we are consumers, not brokers)
#region Serialization
public void Serialize(ReusableMemoryStream stream, object _, Basics.ApiVersion version)
{
BigEndianConverter.Write(stream, Partition);
BigEndianConverter.Write(stream, FetchOffset);
BigEndianConverter.Write(stream, MaxBytes);
if (version >= Basics.ApiVersion.V5)
{
stream.Write(Basics.Zero64, 0, 8); // log_start_offset is 0 for consumer, only used by follower.
}
}
public static void SerializeString(ReusableMemoryStream stream, string s)
{
if (s == null)
{
stream.Write(MinusOne16, 0, 2);
return;
}
BigEndianConverter.Write(stream, (short)s.Length);
_stringSer.Serialize(s, stream);
}
public static void SerializeStringWithVarIntSize(ReusableMemoryStream stream, string s)
{
if (s == null)
{
stream.Write(MinusOneVarInt, 0, MinusOneVarInt.Length);
return;
}
VarIntConverter.Write(stream, _stringSer.SerializedSize(s));
_stringSer.Serialize(s, stream);
}
private void DoSerializeKeyValue(ReusableMemoryStream stream, Tuple <ISerializer, ISerializer> serializers)
{
if (Key == null)
{
stream.Write(Basics.MinusOne32, 0, 4);
}
else
{
SerializeObject(stream, serializers.Item1, Key);
}
if (Value == null)
{
stream.Write(Basics.MinusOne32, 0, 4);
}
else
{
SerializeObject(stream, serializers.Item2, Value);
}
}
private static void SerializeMessagesUncompressed(ReusableMemoryStream stream, IEnumerable <Message> messages, Serializers serializers)
{
foreach (var message in messages)
{
stream.Write(Basics.Zero64, 0, 8); // producer puts a fake offset
Basics.WriteSizeInBytes(stream, message,
new SerializationInfo {
CompressionCodec = CompressionCodec.None, Serializers = serializers
},
SerializeMessageWithCodec);
}
}
private static void _SerializeMessages(ReusableMemoryStream stream, IEnumerable <Message> messages, SerializationInfo info)
{
if (info.CompressionCodec != CompressionCodec.None)
{
stream.Write(Basics.Zero64, 0, 8);
using (var msgsetStream = stream.Pool.Reserve())
{
SerializeMessagesUncompressed(msgsetStream, messages, info.Serializers, info.MessageVersion);
using (var compressed = stream.Pool.Reserve())
{
if (info.CompressionCodec == CompressionCodec.Gzip)
{
using (var gzip = new GZipStream(compressed, CompressionMode.Compress, true))
{
msgsetStream.WriteTo(gzip);
}
}
else // Snappy
{
#if NETSTANDARD1_3
throw new NotImplementedException();
#else
compressed.SetLength(SnappyCodec.GetMaxCompressedLength((int)msgsetStream.Length));
{
int size = SnappyCodec.Compress(msgsetStream.GetBuffer(), 0, (int)msgsetStream.Length,
compressed.GetBuffer(), 0);
compressed.SetLength(size);
}
#endif
}
var m = new Message
{
Value = compressed,
TimeStamp = Timestamp.Now
};
Basics.WriteSizeInBytes(stream, m,
new SerializationInfo
{
Serializers = SerializationConfig.ByteArraySerializers,
CompressionCodec = info.CompressionCodec,
MessageVersion = info.MessageVersion
}, SerializeMessageWithCodec);
}
}
}
else
{
SerializeMessagesUncompressed(stream, messages, info.Serializers, info.MessageVersion);
}
}
private static void SerializeObject(ReusableMemoryStream stream, ISerializer serializer, object theValue)
{
// byte[] are just copied
if (theValue is byte[] bytes)
{
BigEndianConverter.Write(stream, bytes.Length);
stream.Write(bytes, 0, bytes.Length);
}
else
{
Basics.WriteWithSize(stream, theValue, serializer, SerializerWrite);
}
}
public static void Compress(ReusableMemoryStream target, byte[] body, int count)
{
target.Write(FrameDescriptor, 0, FrameDescriptor.Length);
// Blocks
var left = count;
while (left >= BLOCK_SIZE)
{
BlockCompress(target, body, count - left, BLOCK_SIZE);
left -= BLOCK_SIZE;
}
// Last block if any
if (left > 0)
{
BlockCompress(target, body, count - left, left);
}
// EndMark
target.Write(Basics.Zero32, 0, Basics.Zero32.Length);
target.SetLength(target.Position);
}
public void SerializeBody(ReusableMemoryStream stream, object extra, Basics.ApiVersion version)
{
stream.Write(Basics.MinusOne32, 0, 4); // ReplicaId, non clients that are not a broker must use -1
BigEndianConverter.Write(stream, MaxWaitTime);
BigEndianConverter.Write(stream, MinBytes);
if (version >= Basics.ApiVersion.V3)
{
BigEndianConverter.Write(stream, MaxBytes);
}
if (version >= Basics.ApiVersion.V4)
{
stream.WriteByte((byte)IsolationLevel);
}
Basics.WriteArray(stream, TopicsData, extra, version);
}
public void SerializeBody(ReusableMemoryStream stream, object _, Basics.ApiVersion __)
{
if (Topics == null || Topics.Length == 0)
{
stream.Write(Basics.Zero32, 0, 4);
}
else
{
BigEndianConverter.Write(stream, Topics.Length);
foreach (var t in Topics)
{
Basics.SerializeString(stream, t);
}
}
}
private static void SerializeObject(ReusableMemoryStream stream, ISerializer serializer, object theValue)
{
// byte[] are just copied
var bytes = theValue as byte[];
if (bytes != null)
{
byte[] array = bytes;
BigEndianConverter.Write(stream, array.Length);
stream.Write(array, 0, array.Length);
}
else
{
Basics.WriteSizeInBytes(stream, theValue, serializer, SerializerWrite);
}
}
public async Task TestNoAckFlow()
{
const string data = "The cuiqk brwon fox jumps over the zaly god.";
var server = new FakeServer(SimulationMode.Success);
var connection = new Connection(server.EndPoint, Connection.DefaultSocketFactory, BPool, RPool, 16, 16);
const int correlationId = 379821;
const byte ack = 1;
var buffer = new ReusableMemoryStream(null);
BigEndianConverter.Write(buffer, 4 + 4 + 1 + data.Length - 4);
BigEndianConverter.Write(buffer, correlationId);
buffer.WriteByte(ack);
var s = Encoding.UTF8.GetBytes(data);
buffer.Write(s, 0, s.Length);
var bdata = buffer.ToArray();
var p = new TaskCompletionSource <Response>();
connection.Response +=
(con, cor, b) => p.SetResult(new Response {
Connection = con, CorrelationId = cor, Data = b
});
await connection.ConnectAsync();
buffer.Position = 8;
buffer.WriteByte(0);
await connection.SendAsync(correlationId, buffer, false);
buffer = new ReusableMemoryStream(null);
buffer.Write(bdata, 0, bdata.Length);
buffer.Position = 8;
buffer.WriteByte(1);
await connection.SendAsync(correlationId, buffer, true);
var r = await p.Task;
server.Stop();
Assert.AreSame(connection, r.Connection);
Assert.AreEqual(correlationId, r.CorrelationId);
Assert.AreEqual(data, Encoding.UTF8.GetString(r.Data.ToArray()));
}
public static void SerializeString(ReusableMemoryStream stream, string s)
{
if (s == null)
{
stream.Write(MinusOne16, 0, 2);
return;
}
var start = stream.Position;
BigEndianConverter.Write(stream, (short)0);
var length = _stringSer.Serialize(s, stream);
var current = stream.Position;
stream.Position = start;
BigEndianConverter.Write(stream, (short)length);
stream.Position = current;
}
/// <summary>
/// Serializes an <see cref="T:Photon.SocketServer.OperationResponse"/>.
/// The operation response data will be encrypted using the specified <see cref="T:Photon.SocketServer.Security.ICryptoProvider"/>.
/// </summary>
/// <param name="operationResponse"> The response.</param>
/// <param name="cryptoProvider">An <see cref="T:Photon.SocketServer.Security.ICryptoProvider"/> instance used to encrypt the operation response.</param>
/// <returns>The serialized operation response.</returns>
public byte[] SerializeOperationResponseEncrypted(OperationResponse operationResponse, ICryptoProvider cryptoProvider)
{
byte[] buffer;
using (ReusableMemoryStream stream = new ReusableMemoryStream())
{
SerializeOperationResponse(stream, operationResponse);
byte[] data = stream.ToArray();
buffer = cryptoProvider.Encrypt(data);
}
using (ReusableMemoryStream stream2 = new ReusableMemoryStream())
{
stream2.Position = this.headerSize;
stream2.Write(buffer, 0, buffer.Length);
stream2.Position = 0L;
this.headerWriter.WriteHeader(stream2, RtsMessageType.OperationResponse, true);
return(stream2.ToArray());
}
}
public void Test004_DeserializeString()
{
using (var serialized = new ReusableMemoryStream(null))
{
// Non null string
BigEndianConverter.Write(serialized, (short)Value.Length);
serialized.Write(Value, 0, Value.Length);
serialized.Position = 0;
Assert.AreEqual(TheValue, Basics.DeserializeString(serialized));
// Null string
serialized.SetLength(0);
serialized.WriteByte(0xFF);
serialized.WriteByte(0xFF);
serialized.Position = 0;
Assert.IsNull(Basics.DeserializeString(serialized));
}
}
/// <summary>
/// Encrypts an event.
/// </summary>
/// <param name="eventData"> The event data.</param>
/// <param name="cryptoProvider">The crypto provider.</param>
/// <returns>the encrypted event.</returns>
/// <exception cref="T:System.ArgumentNullException">
/// cryptoProvider is null.
/// </exception>
public byte[] SerializeEventDataEncrypted(IEventData eventData, ICryptoProvider cryptoProvider)
{
byte[] buffer;
if (cryptoProvider == null)
{
throw new ArgumentNullException("cryptoProvider");
}
using (ReusableMemoryStream stream = new ReusableMemoryStream())
{
SerializeEventData(stream, eventData);
byte[] data = stream.ToArray();
buffer = cryptoProvider.Encrypt(data, 0, data.Length);
}
using (ReusableMemoryStream stream2 = new ReusableMemoryStream())
{
stream2.WriteByte(0xf3);
stream2.WriteByte(0x84);
stream2.Write(buffer, 0, buffer.Length);
return(stream2.ToArray());
}
}
