/// <summary>
/// Deserialize an object of type <see cref="T"/> from a byte array.
/// </summary>
/// <param name="topic">
/// The topic associated with the data (ignored by this deserializer).
/// </param>
/// <param name="array">
/// A byte array containing the object serialized in the format produced
/// by <see cref="AvroSerializer" />.
/// </param>
/// <returns>
/// The deserialized <see cref="T"/> value.
/// </returns>
public T Deserialize(string topic, byte[] array)
{
// Note: topic is not necessary for deserialization (or knowing if it's a key
// or value) only the schema id is needed.
using (var stream = new MemoryStream(array))
using (var reader = new BinaryReader(stream))
{
var magicByte = reader.ReadByte();
if (magicByte != Constants.MagicByte)
{
// may change in the future.
throw new InvalidDataException($"magic byte should be 0, not {magicByte}");
}
var writerIdBigEndian = reader.ReadInt32();
var writerId = IPAddress.NetworkToHostOrder(writerIdBigEndian);
datumReaderBySchemaId.TryGetValue(writerId, out var datumReader);
if (datumReader == null)
{
// it's a good idea to retain the lock over this blocking call since there
// may be concurrent deserialize calls for the same T, and in that case it's
// best not to hit Schema Registry more than once for the same information.
var writerSchemaJson = SchemaRegistryClient.GetSchemaAsync(writerId).ConfigureAwait(false).GetAwaiter().GetResult();
var writerSchema = Avro.Schema.Parse(writerSchemaJson);
datumReader = new SpecificReader <T>(writerSchema, ReaderSchema);
datumReaderBySchemaId[writerId] = datumReader;
}
return(datumReader.Read(default(T), new BinaryDecoder(stream)));
}
}
public void TestEnumResolution()
{
Schema writerSchema = Schema.Parse("{\"type\":\"record\",\"name\":\"EnumRecord\",\"namespace\":\"Avro.Test\"," +
"\"fields\":[{\"name\":\"enumType\",\"type\": { \"type\": \"enum\", \"name\": \"EnumType\", \"symbols\": [\"FIRST\", \"SECOND\"]} }]}");
Schema readerSchema = Schema.Parse("{\"type\":\"record\",\"name\":\"EnumRecord\",\"namespace\":\"Avro.Test\"," +
"\"fields\":[{\"name\":\"enumType\",\"type\": { \"type\": \"enum\", \"name\": \"EnumType\", \"symbols\": [\"THIRD\", \"FIRST\", \"SECOND\"]} }]}");
EnumRecord testRecord = new EnumRecord();
testRecord.enumType = EnumType.SECOND;
// serialize
var stream = new MemoryStream();
var binEncoder = new BinaryEncoder(stream);
var writer = new SpecificWriter <EnumRecord>(writerSchema);
writer.Write(testRecord, binEncoder);
// deserialize
stream.Position = 0;
var decoder = new BinaryDecoder(stream);
var reader = new SpecificReader <EnumRecord>(writerSchema, readerSchema);
var rec2 = reader.Read(null, decoder);
Assert.AreEqual(EnumType.SECOND, rec2.enumType);
}
public async Task <T> Deserialize(string topic, byte[] array)
{
try
{
// Note: topic is not necessary for deserialization (or knowing if it's a key
// or value) only the schema id is needed.
if (array.Length < 5)
{
throw new InvalidDataException($"Expecting data framing of length 5 bytes or more but total data size is {array.Length} bytes");
}
using (var stream = new MemoryStream(array))
using (var reader = new BinaryReader(stream))
{
var magicByte = reader.ReadByte();
if (magicByte != Constants.MagicByte)
{
throw new InvalidDataException($"Expecting data with Confluent Schema Registry framing. Magic byte was {array[0]}, expecting {Constants.MagicByte}");
}
var writerId = IPAddress.NetworkToHostOrder(reader.ReadInt32());
DatumReader <T> datumReader;
await deserializeMutex.WaitAsync().ConfigureAwait(continueOnCapturedContext: false);
try
{
datumReaderBySchemaId.TryGetValue(writerId, out datumReader);
if (datumReader == null)
{
if (datumReaderBySchemaId.Count > schemaRegistryClient.MaxCachedSchemas)
{
datumReaderBySchemaId.Clear();
}
var writerSchemaJson = await schemaRegistryClient.GetSchemaAsync(writerId).ConfigureAwait(continueOnCapturedContext: false);
var writerSchema = global::Avro.Schema.Parse(writerSchemaJson.SchemaString);
datumReader = new SpecificReader <T>(writerSchema, ReaderSchema);
datumReaderBySchemaId[writerId] = datumReader;
}
}
finally
{
deserializeMutex.Release();
}
return(datumReader.Read(default(T), new BinaryDecoder(stream)));
}
}
catch (AggregateException e)
{
throw e.InnerException;
}
}
private static S deserialize <S>(Stream ms, Schema ws, Schema rs) where S : class, ISpecificRecord
{
long initialPos = ms.Position;
var r = new SpecificReader <S>(ws, rs);
Decoder d = new BinaryDecoder(ms);
S output = r.Read(null, d);
Assert.AreEqual(ms.Length, ms.Position); // Ensure we have read everything.
checkAlternateDeserializers(output, ms, initialPos, ws, rs);
return(output);
}
public async Task <object> DeserializeAsync(ReadOnlyMemory <byte> data, bool isNull, SerializationContext context)
{
try
{
// Note: topic is not necessary for deserialization (or knowing if it's a key
// or value) only the schema id is needed.
using (var stream = new MemoryStream(data.ToArray()))
using (var reader = new BinaryReader(stream))
{
var magicByte = reader.ReadByte();
if (magicByte != ConfluentConstants.MagicByte)
{
// may change in the future.
throw new InvalidDataException($"magic byte should be {ConfluentConstants.MagicByte}, not {magicByte}");
}
var writerId = IPAddress.NetworkToHostOrder(reader.ReadInt32());
DatumReader <object> datumReader;
await _deserializeMutex.WaitAsync().ConfigureAwait(continueOnCapturedContext: false);
try
{
_datumReaderBySchemaId.TryGetValue(writerId, out datumReader);
if (datumReader == null)
{
if (_datumReaderBySchemaId.Count > _schemaRegistryClient.MaxCachedSchemas)
{
_datumReaderBySchemaId.Clear();
}
var writerSchemaJson = await _schemaRegistryClient.GetSchemaAsync(writerId).ConfigureAwait(continueOnCapturedContext: false);
var writerSchema = global::Avro.Schema.Parse(writerSchemaJson);
// Get the ReaderSchema From The Local TopicSubjectSchemaCache
var readerSchema = _cache.GetSchema(writerSchema);
datumReader = new SpecificReader <object>(writerSchema, readerSchema);
_datumReaderBySchemaId[writerId] = datumReader;
}
}
finally
{
_deserializeMutex.Release();
}
return(datumReader.Read(default, new BinaryDecoder(stream)));
public async Task <T> Deserialize(string topic, byte[] array)
{
try
{
// Note: topic is not necessary for deserialization (or knowing if it's a key
// or value) only the schema id is needed.
using (var stream = new MemoryStream(array))
using (var reader = new BinaryReader(stream))
{
var magicByte = reader.ReadByte();
if (magicByte != Constants.MagicByte)
{
// may change in the future.
throw new DeserializationException($"magic byte should be {Constants.MagicByte}, not {magicByte}");
}
var writerId = IPAddress.NetworkToHostOrder(reader.ReadInt32());
DatumReader <T> datumReader;
await deserializeMutex.WaitAsync();
try
{
datumReaderBySchemaId.TryGetValue(writerId, out datumReader);
if (datumReader == null)
{
if (datumReaderBySchemaId.Count > schemaRegistryClient.MaxCachedSchemas)
{
datumReaderBySchemaId.Clear();
}
var writerSchemaJson = await schemaRegistryClient.GetSchemaAsync(writerId).ConfigureAwait(continueOnCapturedContext: false);
var writerSchema = global::Avro.Schema.Parse(writerSchemaJson);
datumReader = new SpecificReader <T>(writerSchema, ReaderSchema);
datumReaderBySchemaId[writerId] = datumReader;
}
}
finally
{
deserializeMutex.Release();
}
return(datumReader.Read(default(T), new BinaryDecoder(stream)));
}
}
catch (AggregateException e)
{
throw e.InnerException;
}
}
public Func <Stream, T> BuildDeserializer(string writerSchemaJson)
{
var writerSchema = Schema.Parse(writerSchemaJson);
var avroReader = new SpecificReader <T>(writerSchema, _schema);
return((Stream stream) =>
{
var result = default(T);
var decoder = new BinaryDecoder(stream);
result = avroReader.Read(result, decoder);
return result;
});
}
public override Exception ReadError(Schema writer, Schema reader, Decoder decoder)
{
var response = new SpecificReader <object>(writer, reader).Read(null, decoder);
var error = response as Exception;
if (error != null)
{
return(error);
}
return(new Exception(response.ToString()));
}
/// <summary>
/// Decodes the message body using the specified decoder.
/// </summary>
/// <typeparam name="T">The type of the message.</typeparam>
/// <param name="decoder">The decoder.</param>
/// <param name="body">The message body.</param>
/// <returns>The decoded message body.</returns>
public static T Decode <T>(this Decoder decoder, string body) where T : ISpecificRecord
{
if (!string.IsNullOrWhiteSpace(body))
{
return(Deserialize <T>(body));
}
var record = Activator.CreateInstance <T>();
var reader = new SpecificReader <T>(new EtpSpecificReader(record.Schema, record.Schema));
reader.Read(record, decoder);
return(record);
}
private static DatumReader <T> CreateDefaultReader(Schema writerSchema, Schema readerSchema)
{
DatumReader <T> reader = null;
Type type = typeof(T);
if (typeof(ISpecificRecord).GetTypeInfo().IsAssignableFrom(type))
{
reader = new SpecificReader <T>(writerSchema, readerSchema);
}
else // generic
{
reader = new GenericReader <T>(writerSchema, readerSchema);
}
return(reader);
}
private DatumReader <T> GetReaderFromSchema()
{
DatumReader <T> reader = null;
Type type = typeof(T);
if (typeof(ISpecificRecord).IsAssignableFrom(type))
{
reader = new SpecificReader <T>(_header.Schema, _readerSchema ?? _header.Schema);
}
else // generic
{
reader = new GenericReader <T>(_header.Schema, _readerSchema ?? _header.Schema);
}
return(reader);
}
public static ISpecificRecord unpackMessage <T>(IBytesMessage message)
{
long len = message.BodyLength;
byte[] data = new byte[(int)len];
int read = message.ReadBytes(data);
MemoryStream ins = new MemoryStream(data);
Avro.IO.Decoder dc = new Avro.IO.BinaryDecoder(ins);
// Decode<T> r = new Decode<T>();
// SpecificReader<T> r = new SpecificReader<T>(c);
SpecificDefaultReader sdr = new SpecificDefaultReader(x, z);
DatumReader <T> reader = new SpecificReader <T>(sdr);
T actual = reader.Read(typeof(T), dc);
//T actual = r(dc);
ins.Close();
return((ISpecificRecord)actual);
}
public T Deserialize(string topic, byte[] array)
{
// Note: topic is not necessary for deserialization (or knowing if it's a key
// or value) only the schema id is needed.
using (var stream = new MemoryStream(array))
using (var reader = new BinaryReader(stream))
{
var magicByte = reader.ReadByte();
if (magicByte != Constants.MagicByte)
{
// may change in the future.
throw new InvalidDataException($"magic byte should be 0, not {magicByte}");
}
var writerId = IPAddress.NetworkToHostOrder(reader.ReadInt32());
DatumReader <T> datumReader;
lock (deserializeLockObj)
{
datumReaderBySchemaId.TryGetValue(writerId, out datumReader);
if (datumReader == null)
{
if (datumReaderBySchemaId.Count > schemaRegistryClient.MaxCachedSchemas)
{
datumReaderBySchemaId.Clear();
}
var writerSchemaJson = schemaRegistryClient.GetSchemaAsync(writerId).ConfigureAwait(false).GetAwaiter().GetResult();
var writerSchema = Avro.Schema.Parse(writerSchemaJson);
datumReader = new SpecificReader <T>(writerSchema, ReaderSchema);
datumReaderBySchemaId[writerId] = datumReader;
}
}
return(datumReader.Read(default(T), new BinaryDecoder(stream)));
}
}
public override Exception ReadError(Schema writer, Schema reader, Decoder decoder)
{
var response = new SpecificReader <object>(writer, reader).Read(null, decoder);
throw (Exception)response;
}
protected void Decode(byte[] data)
{
var receivedTime = DateTime.UtcNow;
using (var inputStream = new MemoryStream(data))
{
// create avro binary decoder to read from memory stream
var decoder = new BinaryDecoder(inputStream);
var record = Activator.CreateInstance <MessageHeader>();
var reader = new SpecificReader <MessageHeader>(new EtpSpecificReader(record.Schema, record.Schema));
MessageHeader header = reader.Read(record, decoder);
// string message = Encoding.UTF8.GetString(inputStream.ToArray());
if (header.Protocol == 0 && header.MessageType == 2)
{
lock (m_ConnectionLock)
{
m_HasConnected = true;
}
var recordSession = Activator.CreateInstance <OpenSession>();
var readerSession = new SpecificReader <OpenSession>(new EtpSpecificReader(recordSession.Schema, recordSession.Schema));
readerSession.Read(recordSession, decoder);
string message = ToString(recordSession);
var timediff = receivedTime - m_Time;
Message?.Invoke(message, timediff.TotalMilliseconds, TraceLevel.Info);
}
else if (header.Protocol == 3 && header.MessageType == 2)
{
var responce = Activator.CreateInstance <GetResourcesResponse>();
var bodyreader = new SpecificReader <GetResourcesResponse>(new EtpSpecificReader(responce.Schema, responce.Schema));
GetResourcesResponse bodyheader = bodyreader.Read(responce, decoder);
RequestInformation parent;
lock (m_RequestInformation)
{
parent = m_RequestInformation[header.CorrelationId];
}
var timediff = receivedTime - parent.RequestTime;
string message = ToString(responce);
Message?.Invoke(message, timediff.TotalMilliseconds, TraceLevel.Info);
if (parent.ChannelItem == null)
{
ChannelItem channelItem = new ChannelItem()
{
Name = responce.Resource.Name,
Uid = responce.Resource.Uuid,
Eml = responce.Resource.Uri,
Level = 0,
ChildrensCount = responce.Resource.HasChildren
};
ChannelItemsReceived?.Invoke(channelItem);
}
else
{
ChannelItem channelItem = new ChannelItem()
{
Name = responce.Resource.Name,
Uid = responce.Resource.Uuid,
Eml = responce.Resource.Uri,
Level = parent.ChannelItem.Level + 1,
ChildrensCount = responce.Resource.HasChildren
};
ChannelChildrensReceived?.Invoke(channelItem, parent.ChannelItem);
}
}
else if (header.Protocol == 1 && header.MessageType == 2)
{
var timediff = receivedTime - m_Time;
string message = "Channels received: [";
var recordMetadata = Activator.CreateInstance <ChannelMetadata>();
var readerMetadata = new SpecificReader <ChannelMetadata>(new EtpSpecificReader(recordMetadata.Schema, recordMetadata.Schema));
readerMetadata.Read(recordMetadata, decoder);
ChannelMetadata metadata = new ChannelMetadata();
metadata.Channels = new List <ChannelMetadataRecord>();
lock (m_ChannelStreamingInfo)
{
foreach (var channel in recordMetadata.Channels)
{
if (m_LogCurveEml.Contains(channel.ChannelUri, StringComparer.InvariantCultureIgnoreCase))
{
metadata.Channels.Add(channel);
ChannelStreamingInfo channelStreamingInfo = new ChannelStreamingInfo()
{
ChannelId = channel.ChannelId,
StartIndex = new StreamingStartIndex()
{
Item = null
},
ReceiveChangeNotification = true
};
m_ChannelStreamingInfo.Add(channelStreamingInfo);
message = message + $"\n{channel.ChannelId} {channel.ChannelName}";
ChannelMetaDataVM channelMetaData_VM = ETPMapper.Instance().Map <ChannelMetaDataVM>(channel);
string json = JsonConvert.SerializeObject(channelMetaData_VM, Formatting.Indented);
Message?.Invoke(json, timediff.TotalMilliseconds, TraceLevel.Info);
}
}
ChannelInfoReceived?.Invoke(metadata);
}
message = message + "\n]";
Message?.Invoke(message, timediff.TotalMilliseconds, TraceLevel.Info);
HasDescribing = false;
}
else if (header.Protocol == 1 && header.MessageType == 3)
{
var recordData = Activator.CreateInstance <ChannelData>();
var readerdata = new SpecificReader <ChannelData>(new EtpSpecificReader(recordData.Schema, recordData.Schema));
readerdata.Read(recordData, decoder);
ChannelDataReceived?.Invoke(recordData.Data);
}
else if (header.MessageType == 1000)
{
var timediff = receivedTime - m_Time;
var bodyrecord = Activator.CreateInstance <ProtocolException>();
var bodyreader = new SpecificReader <ProtocolException>(new EtpSpecificReader(bodyrecord.Schema, bodyrecord.Schema));
ProtocolException bodyheader = bodyreader.Read(bodyrecord, decoder);
string message = $"Error Received ({bodyrecord.ErrorCode}): {bodyrecord.ErrorMessage}";
Message?.Invoke(message, timediff.TotalMilliseconds, TraceLevel.Error);
HasDescribing = false;
}
else
{
HasDescribing = false;
}
}
}
public void TestEnumResolution()
{
Schema writerSchema = Schema.Parse("{\"type\":\"record\",\"name\":\"EnumRecord\",\"namespace\":\"Avro.Test\"," +
"\"fields\":[{\"name\":\"enumType\",\"type\": { \"type\": \"enum\", \"name\": \"EnumType\", \"symbols\": [\"FIRST\", \"SECOND\"]} }]}");
Schema readerSchema = Schema.Parse("{\"type\":\"record\",\"name\":\"EnumRecord\",\"namespace\":\"Avro.Test\"," +
"\"fields\":[{\"name\":\"enumType\",\"type\": { \"type\": \"enum\", \"name\": \"EnumType\", \"symbols\": [\"THIRD\", \"FIRST\", \"SECOND\"]} }]}");
EnumRecord testRecord = new EnumRecord();
testRecord.enumType = EnumType.SECOND;
// serialize
var stream = new MemoryStream();
var binEncoder = new BinaryEncoder(stream);
var writer = new SpecificWriter<EnumRecord>(writerSchema);
writer.Write(testRecord, binEncoder);
// deserialize
stream.Position = 0;
var decoder = new BinaryDecoder(stream);
var reader = new SpecificReader<EnumRecord>(writerSchema, readerSchema);
var rec2 = reader.Read(null, decoder);
Assert.AreEqual( EnumType.SECOND, rec2.enumType );
}
public async Task <T> Deserialize(string topic, byte[] array)
{
try
{
// Note: topic is not necessary for deserialization (or knowing if it's a key
// or value) only the schema id is needed.
if (array.Length < 5)
{
throw new InvalidDataException($"Expecting data framing of length 5 bytes or more but total data size is {array.Length} bytes");
}
using (var stream = new MemoryStream(array))
using (var reader = new BinaryReader(stream))
{
var magicByte = reader.ReadByte();
if (magicByte != Constants.MagicByte)
{
throw new InvalidDataException($"Expecting data with Confluent Schema Registry framing. Magic byte was {array[0]}, expecting {Constants.MagicByte}");
}
var writerId = IPAddress.NetworkToHostOrder(reader.ReadInt32());
DatumReader <T> datumReader;
await deserializeMutex.WaitAsync().ConfigureAwait(continueOnCapturedContext: false);
try
{
datumReaderBySchemaId.TryGetValue(writerId, out datumReader);
if (datumReader == null)
{
if (datumReaderBySchemaId.Count > schemaRegistryClient.MaxCachedSchemas)
{
datumReaderBySchemaId.Clear();
}
var writerSchemaJson = await schemaRegistryClient.GetSchemaAsync(writerId).ConfigureAwait(continueOnCapturedContext: false);
var writerSchema = global::Avro.Schema.Parse(writerSchemaJson.SchemaString);
datumReader = new SpecificReader <T>(writerSchema, ReaderSchema);
datumReaderBySchemaId[writerId] = datumReader;
}
}
finally
{
deserializeMutex.Release();
}
if (typeof(ISpecificRecord).IsAssignableFrom(typeof(T)))
{
// This is a generic deserializer and it knows the type that needs to be serialized into.
// Passing default(T) will result in null value and that will force the datumRead to
// use the schema namespace and name provided in the schema, which may not match (T).
var reuse = Activator.CreateInstance <T>();
return(datumReader.Read(reuse, new BinaryDecoder(stream)));
}
return(datumReader.Read(default(T), new BinaryDecoder(stream)));
}
}
catch (AggregateException e)
{
throw e.InnerException;
}
}
