public void Serialize(ReusableMemoryStream stream, CompressionCodec compressionCodec, Tuple <ISerializer, ISerializer> serializers) { var crcPos = stream.Position; stream.Write(Basics.MinusOne32, 0, 4); // crc placeholder var bodyPos = stream.Position; stream.WriteByte(0); // magic byte stream.WriteByte((byte)compressionCodec); // attributes if (SerializedKeyValue != null) { stream.Write(SerializedKeyValue.GetBuffer(), 0, (int)SerializedKeyValue.Length); } else { DoSerializeKeyValue(stream, serializers); } // update crc var crc = Crc32.Compute(stream, bodyPos, stream.Position - bodyPos); var curPos = stream.Position; stream.Position = crcPos; BigEndianConverter.Write(stream, (int)crc); stream.Position = curPos; }
private void HandleInvalidSerializedKeyValue(ReusableMemoryStream stream) { stream.Logger?.LogError("Invalid SerializedKeyValue. Length is only " + SerializedKeyValue.Length + " bytes. Message cannot be serialized : " + SerializedKeyValue.GetBuffer()); // Simulate an empty key & message to not send a corrupted message stream.Write(Basics.MinusOne32, 0, 4); stream.Write(Basics.MinusOne32, 0, 4); }
public static void WriteRequestHeader(ReusableMemoryStream stream, int correlationId, ApiKey requestType, ApiVersion version, byte[] clientId) { stream.Write(MinusOne32, 0, 4); // reserve space for message size BigEndianConverter.Write(stream, (short)requestType); stream.Write(VersionBytes[(int)version], 0, 2); BigEndianConverter.Write(stream, correlationId); BigEndianConverter.Write(stream, (short)clientId.Length); stream.Write(clientId, 0, clientId.Length); }
public static void SerializeBytesWithVarIntSize(ReusableMemoryStream stream, byte[] b) { if (b == null) { stream.Write(MinusOneVarInt, 0, MinusOneVarInt.Length); return; } VarIntConverter.Write(stream, b.Length); stream.Write(b, 0, b.Length); }
public static void SerializeBytes(ReusableMemoryStream stream, byte[] b) { if (b == null) { stream.Write(MinusOne32, 0, 4); return; } BigEndianConverter.Write(stream, b.Length); stream.Write(b, 0, b.Length); }
public ReusableMemoryStream Serialize(ReusableMemoryStream target, long baseTimestamp, long offsetDelta) { var timestampDelta = Timestamp - baseTimestamp; VarIntConverter.Write(target, SizeOfBodyInBytes(offsetDelta, timestampDelta)); // Record attributes are always null. target.WriteByte(0x00); VarIntConverter.Write(target, timestampDelta); VarIntConverter.Write(target, offsetDelta); Basics.WriteObject(target, Key, KeySerializer); Basics.WriteObject(target, Value, ValueSerializer); if (Headers == null) { target.Write(Basics.ZeroVarInt, 0, Basics.ZeroVarInt.Length); } else { VarIntConverter.Write(target, Headers.Count); foreach (KafkaRecordHeader header in Headers) { SerializeHeader(target, header); } } return(target); }
private static long WriteArrayHeader(ReusableMemoryStream stream) { var sizePosition = stream.Position; stream.Write(MinusOne32, 0, 4); // placeholder for count field return(sizePosition); }
public void SerializeBody(ReusableMemoryStream stream, object extra) { stream.Write(Basics.MinusOne32, 0, 4); // ReplicaId, non clients that are not a broker must use -1 BigEndianConverter.Write(stream, MaxWaitTime); BigEndianConverter.Write(stream, MinBytes); Basics.WriteArray(stream, TopicsData, extra); }
private static void BlockCompress(ReusableMemoryStream target, byte[] body, int offset, int count) { var position = (int)target.Position; target.SetLength(target.Length + MaxCompressedSize + 4); target.Position = position + 4; var size = LZ4Codec.Encode(body, offset, count, target.GetBuffer(), position + 4, MaxCompressedSize); if (size >= count) { // Do not compress block // => set block header highest bit to 1 to mark no compression LittleEndianWriteUInt32((uint)(count | 1 << 31), target.GetBuffer(), position); // Write uncompressed data target.Write(body, offset, count); } else { LittleEndianWriteUInt32((uint)size, target.GetBuffer(), position); // compressed data is already written, just set the position target.Position += size; } target.SetLength(target.Position); }
private static void _SerializeMessages(ReusableMemoryStream stream, IEnumerable <Message> messages, SerializationInfo info) { if (info.CompressionCodec != CompressionCodec.None) { stream.Write(Basics.Zero64, 0, 8); using (var msgsetStream = stream.Pool.Reserve()) { SerializeMessagesUncompressed(msgsetStream, messages, info.Serializers, info.MessageVersion); using (var compressed = stream.Pool.Reserve()) { Basics.CompressStream(msgsetStream, compressed, info.CompressionCodec); var m = new Message { Value = compressed, TimeStamp = Timestamp.Now }; Basics.WriteWithSize(stream, m, new SerializationInfo { Serializers = SerializationConfig.ByteArraySerializers, CompressionCodec = info.CompressionCodec, MessageVersion = info.MessageVersion }, SerializeMessageWithCodec); } } } else { SerializeMessagesUncompressed(stream, messages, info.Serializers, info.MessageVersion); } }
private static uint UncompressBlock(ReusableMemoryStream target, byte[] body, int dataIndex, bool hasChecksum, int blockSize) { var blockHeader = LittleEndianReadUInt32(body, dataIndex); if (blockHeader == 0) // last frame { return(0); } var size = blockHeader & 0x7FFFFFFF; if ((blockHeader & 0x80000000) == 0) // compressed data { target.SetLength(target.Length + blockSize); var dsize = LZ4Codec.Decode(body, dataIndex + 4, (int)size, target.GetBuffer(), (int)target.Position, blockSize); if (dsize < blockSize) { target.SetLength(target.Length - blockSize + dsize); } target.Position = target.Length; } else // uncompressed data { target.Write(body, dataIndex + 4, (int)size); } return(size + 4 + (hasChecksum ? 4u : 0)); }
/// <summary> /// Write the size of the serialized object as a VarInt then the serialized object on the stream. /// /// </summary> /// <param name="stream">target stream</param> /// <param name="o">object to serialize and write to target</param> /// <param name="serializer">serializer to use if object is not serializable</param> public static void WriteObject(ReusableMemoryStream stream, object o, ISerializer serializer) { if (o is byte[] asBytes) { SerializeBytesWithVarIntSize(stream, asBytes); } else if (o is string asString) { SerializeStringWithVarIntSize(stream, asString); } else { if (o == null) { stream.Write(MinusOneVarInt, 0, MinusOneVarInt.Length); return; } var expectedPosition = -1L; if (o is ISizedMemorySerializable asSizedSerializable) { var expectedSize = asSizedSerializable.SerializedSize(); VarIntConverter.Write(stream, expectedSize); expectedPosition = stream.Position + expectedSize; asSizedSerializable.Serialize(stream); } else if (serializer is ISizableSerializer sizableSerializer) { var expectedSize = sizableSerializer.SerializedSize(o); VarIntConverter.Write(stream, expectedSize); expectedPosition = stream.Position + expectedSize; sizableSerializer.Serialize(o, stream); } else { // If we can not know the size of the serialized object in advance, we need to use an intermediate buffer. using (var buffer = stream.Pool.Reserve()) { if (o is IMemorySerializable asSerializable) { asSerializable.Serialize(buffer); } else { serializer.Serialize(o, buffer); } WriteObject(stream, buffer, null); } return; } if (expectedPosition != stream.Position) { throw new Exception( "SerializedSize() returned a different value than the size of the serialized object written"); } } }
public static void Update(ReusableMemoryStream stream, long pos, byte[] buff) { var currPos = stream.Position; stream.Position = pos; stream.Write(buff, 0, buff.Length); stream.Position = currPos; }
public void Serialize(ReusableMemoryStream stream, CompressionCodec compressionCodec, Tuple <ISerializer, ISerializer> serializers, MessageVersion msgVersion) { var crcPos = stream.Position; stream.Write(Basics.MinusOne32, 0, 4); // crc placeholder var bodyPos = stream.Position; // V0 message format if (msgVersion == MessageVersion.V0) { stream.WriteByte(0); // magic byte stream.WriteByte((byte)compressionCodec); // attributes } else // V1 message format { stream.WriteByte(1); // magic byte stream.WriteByte((byte)compressionCodec); // attributes BigEndianConverter.Write(stream, TimeStamp); } if (SerializedKeyValue != null) { if (SerializedKeyValue.Length < MinimumValidSizeForSerializedKeyValue) { HandleInvalidSerializedKeyValue(stream); } else { stream.Write(SerializedKeyValue.GetBuffer(), 0, (int)SerializedKeyValue.Length); } } else { DoSerializeKeyValue(stream, serializers); } // update crc var crc = Crc32.Compute(stream, bodyPos, stream.Position - bodyPos); var curPos = stream.Position; stream.Position = crcPos; BigEndianConverter.Write(stream, (int)crc); stream.Position = curPos; }
public long LogStartOffset; // Required by the protocol, but will always be zero in our case (i.e. we are consumers, not brokers) #region Serialization public void Serialize(ReusableMemoryStream stream, object _, Basics.ApiVersion version) { BigEndianConverter.Write(stream, Partition); BigEndianConverter.Write(stream, FetchOffset); BigEndianConverter.Write(stream, MaxBytes); if (version >= Basics.ApiVersion.V5) { stream.Write(Basics.Zero64, 0, 8); // log_start_offset is 0 for consumer, only used by follower. } }
public static void SerializeString(ReusableMemoryStream stream, string s) { if (s == null) { stream.Write(MinusOne16, 0, 2); return; } BigEndianConverter.Write(stream, (short)s.Length); _stringSer.Serialize(s, stream); }
public static void SerializeStringWithVarIntSize(ReusableMemoryStream stream, string s) { if (s == null) { stream.Write(MinusOneVarInt, 0, MinusOneVarInt.Length); return; } VarIntConverter.Write(stream, _stringSer.SerializedSize(s)); _stringSer.Serialize(s, stream); }
private void DoSerializeKeyValue(ReusableMemoryStream stream, Tuple <ISerializer, ISerializer> serializers) { if (Key == null) { stream.Write(Basics.MinusOne32, 0, 4); } else { SerializeObject(stream, serializers.Item1, Key); } if (Value == null) { stream.Write(Basics.MinusOne32, 0, 4); } else { SerializeObject(stream, serializers.Item2, Value); } }
private static void SerializeMessagesUncompressed(ReusableMemoryStream stream, IEnumerable <Message> messages, Serializers serializers) { foreach (var message in messages) { stream.Write(Basics.Zero64, 0, 8); // producer puts a fake offset Basics.WriteSizeInBytes(stream, message, new SerializationInfo { CompressionCodec = CompressionCodec.None, Serializers = serializers }, SerializeMessageWithCodec); } }
private static void _SerializeMessages(ReusableMemoryStream stream, IEnumerable <Message> messages, SerializationInfo info) { if (info.CompressionCodec != CompressionCodec.None) { stream.Write(Basics.Zero64, 0, 8); using (var msgsetStream = stream.Pool.Reserve()) { SerializeMessagesUncompressed(msgsetStream, messages, info.Serializers, info.MessageVersion); using (var compressed = stream.Pool.Reserve()) { if (info.CompressionCodec == CompressionCodec.Gzip) { using (var gzip = new GZipStream(compressed, CompressionMode.Compress, true)) { msgsetStream.WriteTo(gzip); } } else // Snappy { #if NETSTANDARD1_3 throw new NotImplementedException(); #else compressed.SetLength(SnappyCodec.GetMaxCompressedLength((int)msgsetStream.Length)); { int size = SnappyCodec.Compress(msgsetStream.GetBuffer(), 0, (int)msgsetStream.Length, compressed.GetBuffer(), 0); compressed.SetLength(size); } #endif } var m = new Message { Value = compressed, TimeStamp = Timestamp.Now }; Basics.WriteSizeInBytes(stream, m, new SerializationInfo { Serializers = SerializationConfig.ByteArraySerializers, CompressionCodec = info.CompressionCodec, MessageVersion = info.MessageVersion }, SerializeMessageWithCodec); } } } else { SerializeMessagesUncompressed(stream, messages, info.Serializers, info.MessageVersion); } }
private static void SerializeObject(ReusableMemoryStream stream, ISerializer serializer, object theValue) { // byte[] are just copied if (theValue is byte[] bytes) { BigEndianConverter.Write(stream, bytes.Length); stream.Write(bytes, 0, bytes.Length); } else { Basics.WriteWithSize(stream, theValue, serializer, SerializerWrite); } }
public static void Compress(ReusableMemoryStream target, byte[] body, int count) { target.Write(FrameDescriptor, 0, FrameDescriptor.Length); // Blocks var left = count; while (left >= BLOCK_SIZE) { BlockCompress(target, body, count - left, BLOCK_SIZE); left -= BLOCK_SIZE; } // Last block if any if (left > 0) { BlockCompress(target, body, count - left, left); } // EndMark target.Write(Basics.Zero32, 0, Basics.Zero32.Length); target.SetLength(target.Position); }
public void SerializeBody(ReusableMemoryStream stream, object extra, Basics.ApiVersion version) { stream.Write(Basics.MinusOne32, 0, 4); // ReplicaId, non clients that are not a broker must use -1 BigEndianConverter.Write(stream, MaxWaitTime); BigEndianConverter.Write(stream, MinBytes); if (version >= Basics.ApiVersion.V3) { BigEndianConverter.Write(stream, MaxBytes); } if (version >= Basics.ApiVersion.V4) { stream.WriteByte((byte)IsolationLevel); } Basics.WriteArray(stream, TopicsData, extra, version); }
public void SerializeBody(ReusableMemoryStream stream, object _, Basics.ApiVersion __) { if (Topics == null || Topics.Length == 0) { stream.Write(Basics.Zero32, 0, 4); } else { BigEndianConverter.Write(stream, Topics.Length); foreach (var t in Topics) { Basics.SerializeString(stream, t); } } }
private static void SerializeObject(ReusableMemoryStream stream, ISerializer serializer, object theValue) { // byte[] are just copied var bytes = theValue as byte[]; if (bytes != null) { byte[] array = bytes; BigEndianConverter.Write(stream, array.Length); stream.Write(array, 0, array.Length); } else { Basics.WriteSizeInBytes(stream, theValue, serializer, SerializerWrite); } }
public async Task TestNoAckFlow() { const string data = "The cuiqk brwon fox jumps over the zaly god."; var server = new FakeServer(SimulationMode.Success); var connection = new Connection(server.EndPoint, Connection.DefaultSocketFactory, BPool, RPool, 16, 16); const int correlationId = 379821; const byte ack = 1; var buffer = new ReusableMemoryStream(null); BigEndianConverter.Write(buffer, 4 + 4 + 1 + data.Length - 4); BigEndianConverter.Write(buffer, correlationId); buffer.WriteByte(ack); var s = Encoding.UTF8.GetBytes(data); buffer.Write(s, 0, s.Length); var bdata = buffer.ToArray(); var p = new TaskCompletionSource <Response>(); connection.Response += (con, cor, b) => p.SetResult(new Response { Connection = con, CorrelationId = cor, Data = b }); await connection.ConnectAsync(); buffer.Position = 8; buffer.WriteByte(0); await connection.SendAsync(correlationId, buffer, false); buffer = new ReusableMemoryStream(null); buffer.Write(bdata, 0, bdata.Length); buffer.Position = 8; buffer.WriteByte(1); await connection.SendAsync(correlationId, buffer, true); var r = await p.Task; server.Stop(); Assert.AreSame(connection, r.Connection); Assert.AreEqual(correlationId, r.CorrelationId); Assert.AreEqual(data, Encoding.UTF8.GetString(r.Data.ToArray())); }
public static void SerializeString(ReusableMemoryStream stream, string s) { if (s == null) { stream.Write(MinusOne16, 0, 2); return; } var start = stream.Position; BigEndianConverter.Write(stream, (short)0); var length = _stringSer.Serialize(s, stream); var current = stream.Position; stream.Position = start; BigEndianConverter.Write(stream, (short)length); stream.Position = current; }
/// <summary> /// Serializes an <see cref="T:Photon.SocketServer.OperationResponse"/>. /// The operation response data will be encrypted using the specified <see cref="T:Photon.SocketServer.Security.ICryptoProvider"/>. /// </summary> /// <param name="operationResponse"> The response.</param> /// <param name="cryptoProvider">An <see cref="T:Photon.SocketServer.Security.ICryptoProvider"/> instance used to encrypt the operation response.</param> /// <returns>The serialized operation response.</returns> public byte[] SerializeOperationResponseEncrypted(OperationResponse operationResponse, ICryptoProvider cryptoProvider) { byte[] buffer; using (ReusableMemoryStream stream = new ReusableMemoryStream()) { SerializeOperationResponse(stream, operationResponse); byte[] data = stream.ToArray(); buffer = cryptoProvider.Encrypt(data); } using (ReusableMemoryStream stream2 = new ReusableMemoryStream()) { stream2.Position = this.headerSize; stream2.Write(buffer, 0, buffer.Length); stream2.Position = 0L; this.headerWriter.WriteHeader(stream2, RtsMessageType.OperationResponse, true); return(stream2.ToArray()); } }
public void Test004_DeserializeString() { using (var serialized = new ReusableMemoryStream(null)) { // Non null string BigEndianConverter.Write(serialized, (short)Value.Length); serialized.Write(Value, 0, Value.Length); serialized.Position = 0; Assert.AreEqual(TheValue, Basics.DeserializeString(serialized)); // Null string serialized.SetLength(0); serialized.WriteByte(0xFF); serialized.WriteByte(0xFF); serialized.Position = 0; Assert.IsNull(Basics.DeserializeString(serialized)); } }
/// <summary> /// Encrypts an event. /// </summary> /// <param name="eventData"> The event data.</param> /// <param name="cryptoProvider">The crypto provider.</param> /// <returns>the encrypted event.</returns> /// <exception cref="T:System.ArgumentNullException"> /// cryptoProvider is null. /// </exception> public byte[] SerializeEventDataEncrypted(IEventData eventData, ICryptoProvider cryptoProvider) { byte[] buffer; if (cryptoProvider == null) { throw new ArgumentNullException("cryptoProvider"); } using (ReusableMemoryStream stream = new ReusableMemoryStream()) { SerializeEventData(stream, eventData); byte[] data = stream.ToArray(); buffer = cryptoProvider.Encrypt(data, 0, data.Length); } using (ReusableMemoryStream stream2 = new ReusableMemoryStream()) { stream2.WriteByte(0xf3); stream2.WriteByte(0x84); stream2.Write(buffer, 0, buffer.Length); return(stream2.ToArray()); } }