public void Component_HeadlessList_ResultIsTheSameAsInput()
{
//Arrange
List <int> list = _fixture.Create <List <int> >();
var schema = AvroConvert.GenerateSchema(typeof(List <int>));
//Act
var serialized = AvroConvert.SerializeHeadless(list, schema);
var result = new List <int>();
using (var reader = AvroConvert.OpenDeserializer <int>(new MemoryStream(serialized)))
{
while (reader.HasNext())
{
var item = reader.ReadNext();
result.Add(item);
}
}
//Assert
Assert.NotNull(result);
Assert.NotNull(serialized);
Assert.Equal(list, result);
}
public void Component_AvroAttributeClass_ResultIsEqualToInput()
{
//Arrange
AttributeClass toSerialize = new AttributeClass
{
AndAnotherString = "anotherString",
NullableIntProperty = 1,
NullableIntPropertyWithDefaultValue = null,
NullableStringProperty = "nullableString"
};
string schema = AvroConvert.GenerateSchema(typeof(AttributeClass));
//Act
var result = AvroConvert.SerializeHeadless(toSerialize, schema);
var deserialized = AvroConvert.DeserializeHeadless <AttributeClass>(result, schema);
//Assert
Assert.NotNull(result);
Assert.NotNull(deserialized);
Assert.Equal(toSerialize.NullableIntProperty, deserialized.NullableIntProperty);
Assert.Equal(toSerialize.AndAnotherString, deserialized.AndAnotherString);
Assert.Equal(toSerialize.NullableStringProperty, deserialized.NullableStringProperty);
Assert.Equal(2137, deserialized.NullableIntPropertyWithDefaultValue);
}
public async Task <byte[]> SerializeAsync(T data, SerializationContext context)
{
string subject = $"{context.Topic }-{context.Component}";
int id = 0;
var schema = AvroConvert.GenerateSchema(typeof(T));
if (cache.ContainsKey(subject))
{
id = cache[subject];
}
else
{
var existingSchema = await RegistryClient.GetLatestSchemaAsync(subject);
if (existingSchema.SchemaString == schema)
{
cache.AddOrUpdate(subject, existingSchema.Id, (key, oldValue) => existingSchema.Id);
}
else
{
Confluent.SchemaRegistry.Schema confluentSchema = new Confluent.SchemaRegistry.Schema(
subject,
existingSchema.Version + 1,
0,
schema
);
id = await RegistryClient.RegisterSchemaAsync(subject, confluentSchema.ToString());
cache.AddOrUpdate(subject, id, (key, oldValue) => id);
}
}
using (var stream = new MemoryStream())
{
//Confluent Kafka format:
//https://docs.confluent.io/current/schema-registry/docs/serializer-formatter.html#wire-format
//Magic number
stream.WriteByte(0x00);
//Id
var idAsBytes = BitConverter.GetBytes(id);
stream.Write(idAsBytes, 0, idAsBytes.Length);
//Data
var serializedData = AvroConvert.SerializeHeadless(data, schema);
stream.Write(serializedData, 0, serializedData.Length);
return(stream.ToArray());
}
}
public void Component_SerializeHeadlessBiggerObjectAndReadSmaller_NoError()
{
//Arrange
VeryComplexClass toSerialize = _fixture.Create <VeryComplexClass>();
string schema = AvroConvert.GenerateSchema(typeof(VeryComplexClass));
//Act
var result = AvroConvert.SerializeHeadless(toSerialize, schema);
var deserialized = AvroConvert.DeserializeHeadless <VeryComplexClass>(result, schema);
//Assert
Assert.NotNull(result);
Assert.NotNull(deserialized);
Assert.Equal(toSerialize, deserialized);
}
public void Component_DeserializeWithMissingFields_NoError()
{
//Arrange
BaseTestClass toSerialize = _fixture.Create <BaseTestClass>();
string schema = AvroConvert.GenerateSchema(typeof(BaseTestClass));
//Act
var result = AvroConvert.SerializeHeadless(toSerialize, schema);
var deserialized = AvroConvert.DeserializeHeadless <ReducedBaseTestClass>(result, schema);
//Assert
Assert.NotNull(result);
Assert.NotNull(deserialized);
Assert.Equal(toSerialize.justSomeProperty, deserialized.justSomeProperty);
}
public void Component_SerializeHeadlessBiggerObjectAndReadSmaller_NoError()
{
//Arrange
ExtendedBaseTestClass toSerialize = _fixture.Create <ExtendedBaseTestClass>();
string schema = AvroConvert.GenerateSchema(typeof(BaseTestClass));
//Act
var result = AvroConvert.SerializeHeadless(toSerialize, schema);
var deserialized = AvroConvert.DeserializeHeadless <BaseTestClass>(result, schema);
//Assert
Assert.NotNull(result);
Assert.NotNull(deserialized);
Assert.Equal(toSerialize.andLongProperty, deserialized.andLongProperty);
Assert.Equal(toSerialize.justSomeProperty, deserialized.justSomeProperty);
}
public void Avro2Json_ConvertDouble_ProducedDesiredJson()
{
//Arrange
var doubleTestObject = 21.34;
var expectedJson = JsonConvert.SerializeObject(doubleTestObject);
var schema = AvroConvert.GenerateSchema(doubleTestObject.GetType());
var avroSerialized = AvroConvert.SerializeHeadless(doubleTestObject, schema);
//Act
var resultJson = AvroConvert.Avro2Json(avroSerialized, schema);
//Assert
Assert.Equal(expectedJson, resultJson);
}
public void Avro2Json_ConvertNull_ProducedDesiredJson()
{
//Arrange
string nullTestObject = null;
var expectedJson = JsonConvert.SerializeObject(nullTestObject);
var schema = AvroConvert.GenerateSchema(nullTestObject?.GetType());
var avroSerialized = AvroConvert.SerializeHeadless(nullTestObject, schema);
//Act
var resultJson = AvroConvert.Avro2Json(avroSerialized, schema);
//Assert
Assert.Equal(expectedJson, resultJson);
}
public void Avro2Json_ConvertInt_ProducedDesiredJson()
{
//Arrange
var @int = 2137;
var expectedJson = JsonConvert.SerializeObject(@int);
var schema = AvroConvert.GenerateSchema(@int.GetType());
var avroSerialized = AvroConvert.SerializeHeadless(@int, schema);
//Act
var resultJson = AvroConvert.Avro2Json(avroSerialized, schema);
//Assert
Assert.Equal(expectedJson, resultJson);
}
public void Avro2Json_ConvertString_ProducedDesiredJson()
{
//Arrange
var @string = "I am the serialization string";
var expectedJson = JsonConvert.SerializeObject(@string);
var schema = AvroConvert.GenerateSchema(@string.GetType());
var avroSerialized = AvroConvert.SerializeHeadless(@string, schema);
//Act
var resultJson = AvroConvert.Avro2Json(avroSerialized, schema);
//Assert
Assert.Equal(expectedJson, resultJson);
}
public void Avro2Json_ConvertArray_ProducedDesiredJson()
{
//Arrange
var arrayTestObject = new int[] { 2, 1, 3, 7, 453, 1, 6, };
var expectedJson = JsonConvert.SerializeObject(arrayTestObject);
var schema = AvroConvert.GenerateSchema(arrayTestObject.GetType());
var avroSerialized = AvroConvert.SerializeHeadless(arrayTestObject, schema);
//Act
var resultJson = AvroConvert.Avro2Json(avroSerialized, schema);
//Assert
Assert.Equal(expectedJson, resultJson);
}
public byte[] Serialize(T data, SerializationContext context)
{
using (var stream = new MemoryStream())
{
//Magic number
stream.WriteByte(0x00);
//Id is unknown and not needed - put 0000
stream.WriteByte(0x00);
stream.WriteByte(0x00);
stream.WriteByte(0x00);
stream.WriteByte(0x00);
var serializedData = AvroConvert.SerializeHeadless(data, _schema);
stream.Write(serializedData, 0, serializedData.Length);
return(stream.ToArray());
}
}
public void Avro2Json_ConvertComplexType_ProducedDesiredJson()
{
//Arrange
var user = new User();
user.favorite_color = "blue";
user.favorite_number = 2137;
user.name = "red";
var expectedJson = JsonConvert.SerializeObject(user);
var schema = AvroConvert.GenerateSchema(user.GetType());
var avroSerialized = AvroConvert.SerializeHeadless(user, schema);
//Act
var resultJson = AvroConvert.Avro2Json(avroSerialized, schema);
//Assert
Assert.Equal(expectedJson, resultJson);
}
public static void Invoke()
{
//Arrange
var fixture = new Fixture();
Dataset dataset = fixture.Create <Dataset>();
var schema = AvroConvert.GenerateSchema(typeof(Dataset));
Schema apacheSchema = Schema.Parse(schema);
//AvroConvert to Apache
var avroConvertSerialized = AvroConvert.SerializeHeadless(dataset, schema);
Dataset apacheDeserialized;
using (var ms = new MemoryStream(avroConvertSerialized))
{
var apacheReader = new GenericDatumReader <GenericRecord>(apacheSchema, apacheSchema);
var decoder = new BinaryDecoder(ms);
apacheDeserialized = (ApacheAvroHelpers.Decreate <Dataset>(apacheReader.Read(null, decoder)));
}
Contract.Assert(dataset == apacheDeserialized);
//Apache to AvroConvert
MemoryStream apacheAvroSerializeStream = new MemoryStream();
var encoder = new BinaryEncoder(apacheAvroSerializeStream);
var apacheWriter = new GenericDatumWriter <GenericRecord>(apacheSchema);
apacheWriter.Write(ApacheAvroHelpers.Create(dataset, apacheSchema), encoder);
var apacheSerialized = apacheAvroSerializeStream.ToArray();
var avroConvertDeserialized = AvroConvert.DeserializeHeadless <Dataset>(apacheSerialized);
Contract.Assert(dataset == avroConvertDeserialized);
}
private static BenchmarkResult RunBenchmark(Dataset[] datasets, string schema)
{
var result = new BenchmarkResult();
Stopwatch stopwatch = Stopwatch.StartNew();
//Serialize Apache.Avro
MemoryStream apacheAvroSerializeStream = new MemoryStream();
var encoder = new BinaryEncoder(apacheAvroSerializeStream);
var apacheSchema = Schema.Parse(AvroConvert.GenerateSchema(typeof(Dataset)));
var apacheWriter = new GenericDatumWriter <GenericRecord>(apacheSchema);
foreach (var dataset in datasets)
{
apacheWriter.Write(ApacheAvroHelpers.Create(dataset, apacheSchema), encoder);
}
var apacheAvro = apacheAvroSerializeStream.ToArray();
result.ApacheAvroSerializeTime = stopwatch.ElapsedMilliseconds;
stopwatch.Restart();
//Deserialize Apache.Avro
List <Dataset> apacheResult = new List <Dataset>();
using (var ms = new MemoryStream(apacheAvro))
{
apacheSchema = Schema.Parse(AvroConvert.GenerateSchema(typeof(Dataset)));
var apacheReader = new GenericDatumReader <GenericRecord>(apacheSchema, apacheSchema);
var decoder = new BinaryDecoder(ms);
foreach (var dataset in datasets)
{
apacheResult.Add(ApacheAvroHelpers.Decreate <Dataset>(apacheReader.Read(null, decoder)));
}
}
result.ApacheAvroDeserializeTime = stopwatch.ElapsedMilliseconds;
stopwatch.Restart();
//Serialize AvroConvert Headerless
var avroHeadless = AvroConvert.SerializeHeadless(datasets, schema);
result.AvroConvertHeadlessSerializeTime = stopwatch.ElapsedMilliseconds;
stopwatch.Restart();
//Deserialize AvroConvert Headerless
AvroConvert.DeserializeHeadless <List <Dataset> >(avroHeadless, schema);
result.AvroConvertHeadlessDeserializeTime = stopwatch.ElapsedMilliseconds;
stopwatch.Restart();
//Serialize AvroConvert Gzip
var avroGzip = AvroConvert.Serialize(datasets, CodecType.GZip);
result.AvroConvertGzipSerializeTime = stopwatch.ElapsedMilliseconds;
stopwatch.Restart();
//Deserialize AvroConvert Gzip
AvroConvert.Deserialize <Dataset[]>(avroGzip);
result.AvroConvertGzipDeserializeTime = stopwatch.ElapsedMilliseconds;
stopwatch.Restart();
//Serialize AvroConvert vNext
var newAvro = AvroConvertToUpdate.AvroConvert.SerializeHeadless(datasets, schema);
result.AvroConvertVNextHeadlessSerializeTime = stopwatch.ElapsedMilliseconds;
stopwatch.Restart();
//Deserialize AvroConvert vNext
AvroConvertToUpdate.AvroConvert.DeserializeHeadless <Dataset[]>(newAvro, schema);
result.AvroConvertVNextHeadlessDeserializeTime = stopwatch.ElapsedMilliseconds;
stopwatch.Stop();
//Serialize AvroConvert vNext Gzip
var newAvroGzip = AvroConvertToUpdate.AvroConvert.Serialize(datasets, AvroConvertToUpdate.Codec.CodecType.GZip);
result.AvroConvertVNextGzipSerializeTime = stopwatch.ElapsedMilliseconds;
stopwatch.Restart();
//Deserialize AvroConvert vNext Gzip
AvroConvertToUpdate.AvroConvert.Deserialize <Dataset[]>(newAvroGzip);
result.AvroConvertVNextGzipDeserializeTime = stopwatch.ElapsedMilliseconds;
stopwatch.Stop();
//Size
result.ApacheAvroSize = apacheAvro.Length;
result.AvroConvertHeadlessSize = avroHeadless.Length;
result.AvroConvertGzipSize = avroGzip.Length;
result.AvroConvertVNextSize = newAvro.Length;
result.AvroConvertVNextGzipSize = newAvroGzip.Length;
return(result);
}
static async Task Main(string[] args)
{
double temperature;
var rand = new Random();
using (var security = new SecurityProviderSymmetricKey(registrationId, primaryKey, secondaryKey))
using (var transport = new ProvisioningTransportHandlerMqtt())
{
ProvisioningDeviceClient provClient = ProvisioningDeviceClient.Create(GlobalDeviceEndpoint, idScope, security, transport);
DeviceRegistrationResult result = await provClient.RegisterAsync();
IAuthenticationMethod auth = new DeviceAuthenticationWithRegistrySymmetricKey(result.DeviceId, (security as SecurityProviderSymmetricKey).GetPrimaryKey());
using (DeviceClient iotClient = DeviceClient.Create(result.AssignedHub, auth, TransportType.Mqtt))
{
while (true)
{
if (_temperature.IsAvailable)
{
temperature = Math.Round(_temperature.Temperature.DegreesCelsius, 2);
}
else
{
temperature = rand.Next(15, 35);
}
try
{
msgId++;
Console.WriteLine($"The CPU temperature is {temperature}");
Image img = Image.FromFile(filename); // simulate camera captured image
using (MemoryStream ms = new MemoryStream())
{
img.Save(ms, System.Drawing.Imaging.ImageFormat.Jpeg);
TelemetryAvro telemetryAvro = new TelemetryAvro()
{
Temperature = temperature,
Humidity = 50,
Pressure = 1100,
MsgId = msgId,
Label = "Orange",
Probability = 1.0,
Image = ms.ToArray(),
DeviceId = string.Empty // this is set in the azure function when writing to cosmosdb
};
var serialized = AvroConvert.SerializeHeadless(telemetryAvro, avroTelemetrySchema);
// Deserialise AVRO
// var deserialized = AvroConvert.DeserializeHeadless<TelemetryAvro>(serialized, avroTelemetrySchema);
// using (MemoryStream mStream = new MemoryStream(deserialized.Image))
// {
// Image.FromStream(mStream).Save(appDirectory + "images/orangev2.jpg");
// }
await SendMsgIotHub(iotClient, serialized);
await SendMsgIotHub(iotClient, temperature);
}
}
catch (Exception ex)
{
Console.WriteLine("exception msg: " + ex.Message);
}
Thread.Sleep(4000);
}
}
}
}
