public async Task CanSerializeAndDeserializeWithCompatibleSchema()
{
var client = CreateClient();
var groupName = TestEnvironment.SchemaRegistryGroup;
var employee = new Employee_V2 {
Age = 42, Name = "Caketown", City = "Redmond"
};
var serializer = new SchemaRegistryAvroSerializer(client, groupName, new SchemaRegistryAvroSerializerOptions {
AutoRegisterSchemas = true
});
var content = await serializer.SerializeAsync <MessageContent, Employee_V2>(employee);
// deserialize using the old schema, which is forward compatible with the new schema
// if you swap the old schema and the new schema in your mind, then this can also be thought as a backwards compatible test
var deserializedObject = await serializer.DeserializeAsync <Employee>(content);
var readEmployee = deserializedObject as Employee;
Assert.IsNotNull(readEmployee);
Assert.AreEqual("Caketown", readEmployee.Name);
Assert.AreEqual(42, readEmployee.Age);
// deserialize using the new schema to make sure we are respecting it
var readEmployeeV2 = await serializer.DeserializeAsync <Employee_V2>(content);
Assert.IsNotNull(readEmployee);
Assert.AreEqual("Caketown", readEmployeeV2.Name);
Assert.AreEqual(42, readEmployeeV2.Age);
Assert.AreEqual("Redmond", readEmployeeV2.City);
}
public async Task SerializeWorksWithNoOptions()
{
var mockClient = new Mock <SchemaRegistryClient>();
mockClient
.Setup(
client => client.GetSchemaPropertiesAsync(
"groupName",
Employee._SCHEMA.Fullname,
Employee._SCHEMA.ToString(),
SchemaFormat.Avro,
CancellationToken.None))
.Returns(
Task.FromResult(
Response.FromValue(
SchemaRegistryModelFactory.SchemaProperties(SchemaFormat.Avro, "schemaId"), new MockResponse(200))));
var serializer = new SchemaRegistryAvroSerializer(mockClient.Object, "groupName");
var content = await serializer.SerializeAsync(new Employee { Age = 42, Name = "Caketown" });
Assert.AreEqual("schemaId", content.ContentType.ToString().Split('+')[1]);
// also validate explicitly passing null
serializer = new SchemaRegistryAvroSerializer(mockClient.Object, "groupName", null);
content = await serializer.SerializeAsync(new Employee { Age = 42, Name = "Caketown" });
Assert.AreEqual("schemaId", content.ContentType.ToString().Split('+')[1]);
}
public void SerializingWithoutGroupNameThrows()
{
var client = CreateClient();
var serializer = new SchemaRegistryAvroSerializer(client);
Assert.ThrowsAsync <InvalidOperationException>(
async() => await serializer.SerializeAsync(new Employee {
Age = 42, Name = "Caketown"
}));
}
public void CannotSerializeUnsupportedType()
{
var client = CreateClient();
var groupName = TestEnvironment.SchemaRegistryGroup;
var timeZoneInfo = TimeZoneInfo.Utc;
var serializer = new SchemaRegistryAvroSerializer(client, groupName, new SchemaRegistryAvroSerializerOptions {
AutoRegisterSchemas = true
});
Assert.ThrowsAsync <ArgumentException>(async() => await serializer.SerializeAsync <MessageContent, TimeZoneInfo>(timeZoneInfo));
}
public SchemaRegistryAvroSerializerPerfTestBase(SizeCountOptions options) : base(options)
{
_environment = new SchemaRegistryAvroSerializerTestEnvironment();
_client = new SchemaRegistryClient(_environment.SchemaRegistryEndpoint, _environment.Credential);
Serializer = new SchemaRegistryAvroSerializer(
_client,
_environment.SchemaRegistryGroup,
// the first iteration will register and cache the schema
new SchemaRegistryAvroSerializerOptions {
AutoRegisterSchemas = true
});
}
public void SerializingWithoutAutoRegisterThrowsForNonExistentSchema()
{
var client = CreateClient();
var groupName = TestEnvironment.SchemaRegistryGroup;
var serializer = new SchemaRegistryAvroSerializer(client, groupName);
Assert.ThrowsAsync <RequestFailedException>(
async() => await serializer.SerializeAsync(new Employee_Unregistered {
Age = 42, Name = "Caketown"
}));
}
public async Task CanUseEncoderWithEventDataUsingGenerics()
{
var client = CreateClient();
var groupName = TestEnvironment.SchemaRegistryGroup;
#region Snippet:SchemaRegistryAvroEncodeEventDataGenerics
var serializer = new SchemaRegistryAvroSerializer(client, groupName, new SchemaRegistryAvroSerializerOptions {
AutoRegisterSchemas = true
});
var employee = new Employee {
Age = 42, Name = "Caketown"
};
EventData eventData = await serializer.SerializeAsync <EventData, Employee>(employee);
#if SNIPPET
// the schema Id will be included as a parameter of the content type
Console.WriteLine(eventData.ContentType);
// the serialized Avro data will be stored in the EventBody
Console.WriteLine(eventData.EventBody);
#endif
#endregion
Assert.IsFalse(eventData.IsReadOnly);
string[] contentType = eventData.ContentType.Split('+');
Assert.AreEqual(2, contentType.Length);
Assert.AreEqual("avro/binary", contentType[0]);
Assert.IsNotEmpty(contentType[1]);
#region Snippet:SchemaRegistryAvroDecodeEventDataGenerics
Employee deserialized = await serializer.DeserializeAsync <Employee>(eventData);
#if SNIPPET
Console.WriteLine(deserialized.Age);
Console.WriteLine(deserialized.Name);
#endif
#endregion
// decoding should not alter the message
contentType = eventData.ContentType.Split('+');
Assert.AreEqual(2, contentType.Length);
Assert.AreEqual("avro/binary", contentType[0]);
Assert.IsNotEmpty(contentType[1]);
// verify the payload was decoded correctly
Assert.IsNotNull(deserialized);
Assert.AreEqual("Caketown", deserialized.Name);
Assert.AreEqual(42, deserialized.Age);
}
public void SerializingToMessageTypeWithoutConstructorThrows()
{
var client = CreateClient();
var groupName = TestEnvironment.SchemaRegistryGroup;
var serializer = new SchemaRegistryAvroSerializer(client, groupName, new SchemaRegistryAvroSerializerOptions {
AutoRegisterSchemas = true
});
Assert.ThrowsAsync <InvalidOperationException>(
async() => await serializer.SerializeAsync(new Employee {
Age = 42, Name = "Caketown"
}, messageType: typeof(MessageContentWithNoConstructor)));
}
public void ThrowsAvroSerializationExceptionWhenSerializingWithInvalidAvroSchema()
{
var client = CreateClient();
var groupName = TestEnvironment.SchemaRegistryGroup;
var invalid = new InvalidAvroModel();
var serializer = new SchemaRegistryAvroSerializer(client, groupName, new SchemaRegistryAvroSerializerOptions {
AutoRegisterSchemas = true
});
Assert.That(
async() => await serializer.SerializeAsync <MessageContent, InvalidAvroModel>(invalid),
Throws.InstanceOf <Exception>().And.Property(nameof(Exception.InnerException)).InstanceOf <SchemaParseException>());
}
public void CannotDeserializeWithNullSchemaId()
{
var client = CreateClient();
var groupName = TestEnvironment.SchemaRegistryGroup;
var serializer = new SchemaRegistryAvroSerializer(client, groupName, new SchemaRegistryAvroSerializerOptions {
AutoRegisterSchemas = true
});
var content = new MessageContent
{
Data = new BinaryData(Array.Empty <byte>()),
ContentType = null
};
Assert.ThrowsAsync <ArgumentNullException>(async() => await serializer.DeserializeAsync <TimeZoneInfo>(content));
}
public async Task CannotDeserializeUnsupportedType()
{
var client = CreateClient();
var groupName = TestEnvironment.SchemaRegistryGroup;
var serializer = new SchemaRegistryAvroSerializer(client, groupName, new SchemaRegistryAvroSerializerOptions {
AutoRegisterSchemas = true
});
var content = new BinaryContent
{
Data = new BinaryData(Array.Empty <byte>()),
ContentType = "avro/binary+234234"
};
Assert.ThrowsAsync <ArgumentException>(async() => await serializer.DeserializeAsync <TimeZoneInfo>(content));
await Task.CompletedTask;
}
public async Task ThrowsAvroSerializationExceptionWhenDeserializingWithInvalidAvroSchema()
{
var client = CreateClient();
var groupName = TestEnvironment.SchemaRegistryGroup;
var serializer = new SchemaRegistryAvroSerializer(client, groupName, new SchemaRegistryAvroSerializerOptions {
AutoRegisterSchemas = true
});
var employee = new Employee {
Age = 42, Name = "Caketown"
};
var content = await serializer.SerializeAsync <MessageContent, Employee>(employee);
Assert.That(
async() => await serializer.DeserializeAsync <InvalidAvroModel>(content),
Throws.InstanceOf <Exception>()
.And.Property(nameof(Exception.InnerException)).InstanceOf <SchemaParseException>());
}
public async Task CannotDeserializeIntoNonCompatibleType()
{
var client = CreateClient();
var groupName = TestEnvironment.SchemaRegistryGroup;
var employee = new Employee()
{
Age = 42, Name = "Caketown"
};
var serializer = new SchemaRegistryAvroSerializer(client, groupName, new SchemaRegistryAvroSerializerOptions {
AutoRegisterSchemas = true
});
var content = await serializer.SerializeAsync <MessageContent, Employee>(employee);
var schemaId = content.ContentType.ToString().Split('+')[1];
// deserialize with the new schema, which is NOT backward compatible with the old schema as it adds a new field
Assert.That(
async() => await serializer.DeserializeAsync <Employee_V2>(content),
Throws.InstanceOf <SchemaRegistryAvroException>()
.And.Property(nameof(Exception.InnerException)).InstanceOf <AvroException>()
.And.Property(nameof(SchemaRegistryAvroException.SchemaId)).EqualTo(schemaId));
#region Snippet:SchemaRegistryAvroException
try
{
Employee_V2 employeeV2 = await serializer.DeserializeAsync <Employee_V2>(content);
}
catch (SchemaRegistryAvroException exception)
{
// When this exception occurs when deserializing, the exception message will contain the schema ID that was used to
// serialize the data.
Console.WriteLine(exception);
// We might also want to look up the specific schema from Schema Registry so that we can log the schema definition
if (exception.SchemaId != null)
{
SchemaRegistrySchema schema = await client.GetSchemaAsync(exception.SchemaId);
Console.WriteLine(schema.Definition);
}
}
#endregion
}
public async Task CannotDeserializeIntoNonCompatibleType()
{
var client = CreateClient();
var groupName = TestEnvironment.SchemaRegistryGroup;
var employee = new Employee()
{
Age = 42, Name = "Caketown"
};
var serializer = new SchemaRegistryAvroSerializer(client, groupName, new SchemaRegistryAvroSerializerOptions {
AutoRegisterSchemas = true
});
var content = await serializer.SerializeAsync <BinaryContent, Employee>(employee);
// deserialize with the new schema, which is NOT backward compatible with the old schema as it adds a new field
Assert.That(
async() => await serializer.DeserializeAsync <Employee_V2>(content),
Throws.InstanceOf <AvroException>());
}
public async Task CanSerializeAndDeserializeGenericRecord()
{
var client = CreateClient();
var groupName = TestEnvironment.SchemaRegistryGroup;
var record = new GenericRecord((RecordSchema)Employee._SCHEMA);
record.Add("Name", "Caketown");
record.Add("Age", 42);
var serializer = new SchemaRegistryAvroSerializer(client, groupName, new SchemaRegistryAvroSerializerOptions {
AutoRegisterSchemas = true
});
var content = await serializer.SerializeAsync <MessageContent, GenericRecord>(record);
var deserializedObject = await serializer.DeserializeAsync <GenericRecord>(content);
Assert.IsNotNull(deserializedObject);
Assert.AreEqual("Caketown", deserializedObject.GetValue(0));
Assert.AreEqual(42, deserializedObject.GetValue(1));
}
static void Main(string[] args)
{
Console.WriteLine("-- CONSUMER FOR POSTGRES --");
var consumer = new CustomConsumer <string, MappedCustomerMessage>(new ObjectSerializer <MappedCustomerMessage>(), new ObjectSerializer <string>());
var schemaRegistryClient = new CachedSchemaRegistryClient(_schemaRegistryUri, 200);
var judaSerializer = new SchemaRegistryAvroSerializer(schemaRegistryClient, false);
var valueSerializer = new AvroSerializer <CustomerKafkaMessage>(judaSerializer, _topic, false);
var keySerializer = new AvroSerializer <string>(judaSerializer, _topic, true);
var config = new Dictionary <string, object>
{
["bootstrap.servers"] = _bootstrapServers
};
consumer.Consume("customers", (_, msg) =>
{
using (var producer = new Producer <string, CustomerKafkaMessage>(config, keySerializer, valueSerializer))
{
var kafkaMessage = new CustomerKafkaMessage
{
CustomerId = Int64.Parse(msg.Key),
Value = JsonConvert.SerializeObject(msg.Value)
};
producer.ProduceAsync(_topic, msg.Key, kafkaMessage).ContinueWith(task =>
{
if (task.IsFaulted)
{
Console.WriteLine(task.Exception.Message);
}
else if (task.Result.Error.HasError)
{
Console.WriteLine(task.Result.Error);
}
Console.WriteLine(kafkaMessage.ToString());
});
Console.WriteLine("remaining: " + producer.Flush(10000));
}
});
}
public async Task CanSerializeAndDeserializeWithSeparateInstances()
{
var client = CreateClient();
var groupName = TestEnvironment.SchemaRegistryGroup;
var employee = new Employee {
Age = 42, Name = "Caketown"
};
var serializer = new SchemaRegistryAvroSerializer(client, groupName, new SchemaRegistryAvroSerializerOptions {
AutoRegisterSchemas = true
});
MessageContent content = await serializer.SerializeAsync <MessageContent, Employee>(employee);
// validate that we can use the constructor that only takes the client when deserializing since groupName is not necessary
serializer = new SchemaRegistryAvroSerializer(client);
Employee deserializedEmployee = await serializer.DeserializeAsync <Employee>(content);
Assert.IsNotNull(deserializedEmployee);
Assert.AreEqual("Caketown", deserializedEmployee.Name);
Assert.AreEqual(42, deserializedEmployee.Age);
}
public async Task CanSerializeAndDeserialize()
{
var client = CreateClient();
var groupName = TestEnvironment.SchemaRegistryGroup;
var employee = new Employee {
Age = 42, Name = "Caketown"
};
#region Snippet:SchemaRegistryAvroEncodeDecodeBinaryContent
var serializer = new SchemaRegistryAvroSerializer(client, groupName, new SchemaRegistryAvroSerializerOptions {
AutoRegisterSchemas = true
});
BinaryContent content = await serializer.SerializeAsync <BinaryContent, Employee>(employee);
Employee deserializedEmployee = await serializer.DeserializeAsync <Employee>(content);
#endregion
Assert.IsNotNull(deserializedEmployee);
Assert.AreEqual("Caketown", deserializedEmployee.Name);
Assert.AreEqual(42, deserializedEmployee.Age);
}
public async Task CanDecodePreamble()
{
var client = CreateClient();
var groupName = TestEnvironment.SchemaRegistryGroup;
var serializer = new SchemaRegistryAvroSerializer(client, groupName, new SchemaRegistryAvroSerializerOptions {
AutoRegisterSchemas = true
});
var employee = new Employee {
Age = 42, Name = "Caketown"
};
EventData eventData = await serializer.SerializeAsync <EventData, Employee>(employee);
string schemaId = eventData.ContentType.Split('+')[1];
eventData.ContentType = "avro/binary";
using var stream = new MemoryStream();
stream.Write(new byte[] { 0, 0, 0, 0 }, 0, 4);
var encoding = new UTF8Encoding(false);
stream.Write(encoding.GetBytes(schemaId), 0, 32);
stream.Write(eventData.Body.ToArray(), 0, eventData.Body.Length);
stream.Position = 0;
eventData.EventBody = BinaryData.FromStream(stream);
Employee deserialized = await serializer.DeserializeAsync <Employee>(eventData);
// decoding should not alter the message
Assert.AreEqual("avro/binary", eventData.ContentType);
// verify the payload was decoded correctly
Assert.IsNotNull(deserialized);
Assert.AreEqual("Caketown", deserialized.Name);
Assert.AreEqual(42, deserialized.Age);
}
[LiveOnly] // due to Event Hubs integration
public async Task CanUseEncoderWithEventData()
{
var client = CreateClient();
var groupName = TestEnvironment.SchemaRegistryGroup;
#region Snippet:SchemaRegistryAvroEncodeEventData
var serializer = new SchemaRegistryAvroSerializer(client, groupName, new SchemaRegistryAvroSerializerOptions {
AutoRegisterSchemas = true
});
var employee = new Employee {
Age = 42, Name = "Caketown"
};
EventData eventData = (EventData)await serializer.SerializeAsync(employee, messageType : typeof(EventData));
#if SNIPPET
// the schema Id will be included as a parameter of the content type
Console.WriteLine(eventData.ContentType);
// the serialized Avro data will be stored in the EventBody
Console.WriteLine(eventData.EventBody);
#endif
// construct a publisher and publish the events to our event hub
#if SNIPPET
var fullyQualifiedNamespace = "<< FULLY-QUALIFIED EVENT HUBS NAMESPACE (like something.servicebus.windows.net) >>";
var eventHubName = "<< NAME OF THE EVENT HUB >>";
var credential = new DefaultAzureCredential();
#else
var fullyQualifiedNamespace = TestEnvironment.SchemaRegistryEndpoint;
var eventHubName = TestEnvironment.SchemaRegistryEventHubName;
var credential = TestEnvironment.Credential;
#endif
await using var producer = new EventHubProducerClient(fullyQualifiedNamespace, eventHubName, credential);
await producer.SendAsync(new EventData[] { eventData });
#endregion
Assert.IsFalse(eventData.IsReadOnly);
string[] contentType = eventData.ContentType.Split('+');
Assert.AreEqual(2, contentType.Length);
Assert.AreEqual("avro/binary", contentType[0]);
Assert.IsNotEmpty(contentType[1]);
#region Snippet:SchemaRegistryAvroDecodeEventData
// construct a consumer and consume the event from our event hub
await using var consumer = new EventHubConsumerClient(EventHubConsumerClient.DefaultConsumerGroupName, fullyQualifiedNamespace, eventHubName, credential);
await foreach (PartitionEvent receivedEvent in consumer.ReadEventsAsync())
{
Employee deserialized = (Employee)await serializer.DeserializeAsync(eventData, typeof(Employee));
#if SNIPPET
Console.WriteLine(deserialized.Age);
Console.WriteLine(deserialized.Name);
#else
// decoding should not alter the message
contentType = eventData.ContentType.Split('+');
Assert.AreEqual(2, contentType.Length);
Assert.AreEqual("avro/binary", contentType[0]);
Assert.IsNotEmpty(contentType[1]);
// verify the payload was decoded correctly
Assert.IsNotNull(deserialized);
Assert.AreEqual("Caketown", deserialized.Name);
Assert.AreEqual(42, deserialized.Age);
#endif
break;
}
#endregion
}
public async Task UpdatingCacheLogsEvents()
{
using var logger = AzureEventSourceListener.CreateConsoleLogger(EventLevel.Verbose);
var client = CreateClient();
var groupName = TestEnvironment.SchemaRegistryGroup;
var serializer = new SchemaRegistryAvroSerializer(client, groupName, new SchemaRegistryAvroSerializerOptions {
AutoRegisterSchemas = true
});
var employee = new Employee {
Age = 42, Name = "Caketown"
};
EventData eventData = await serializer.SerializeAsync <EventData, Employee>(employee);
Assert.IsFalse(eventData.IsReadOnly);
string[] contentType = eventData.ContentType.Split('+');
Assert.AreEqual(2, contentType.Length);
Assert.AreEqual("avro/binary", contentType[0]);
Assert.IsNotEmpty(contentType[1]);
Employee deserialized = await serializer.DeserializeAsync <Employee>(eventData);
// decoding should not alter the message
contentType = eventData.ContentType.Split('+');
Assert.AreEqual(2, contentType.Length);
Assert.AreEqual("avro/binary", contentType[0]);
Assert.IsNotEmpty(contentType[1]);
// verify the payload was decoded correctly
Assert.IsNotNull(deserialized);
Assert.AreEqual("Caketown", deserialized.Name);
Assert.AreEqual(42, deserialized.Age);
// Use different schema so we can see the cache updated
eventData = await serializer.SerializeAsync <EventData, Employee_V2>(new Employee_V2 { Age = 42, Name = "Caketown", City = "Redmond" });
Assert.IsFalse(eventData.IsReadOnly);
eventData.ContentType.Split('+');
Assert.AreEqual(2, contentType.Length);
Assert.AreEqual("avro/binary", contentType[0]);
Assert.IsNotEmpty(contentType[1]);
await serializer.DeserializeAsync <Employee>(eventData);
// decoding should not alter the message
contentType = eventData.ContentType.Split('+');
Assert.AreEqual(2, contentType.Length);
Assert.AreEqual("avro/binary", contentType[0]);
Assert.IsNotEmpty(contentType[1]);
// verify the payload was decoded correctly
Assert.IsNotNull(deserialized);
Assert.AreEqual("Caketown", deserialized.Name);
Assert.AreEqual(42, deserialized.Age);
var events = _listener.EventsById(SchemaRegistryAvroEventSource.CacheUpdatedEvent).ToArray();
Assert.AreEqual(2, events.Length);
// first log entry should have 2 as the total number of entries as we maintain two caches for each schema
Assert.AreEqual(2, events[0].Payload[0]);
// the second payload element is the total schema length
Assert.AreEqual(334, events[0].Payload[1]);
// second entry will include both V1 and V2 schemas - so 4 total entries
Assert.AreEqual(4, events[1].Payload[0]);
Assert.AreEqual(732, events[1].Payload[1]);
}