public KafkaAvroSerializer(string schemaRegistryUrl, TokenCredential credential, string schemaGroup, Boolean autoRegisterSchemas = false)
{
this.serializer = new SchemaRegistryAvroObjectSerializer(
new SchemaRegistryClient(
schemaRegistryUrl,
credential),
schemaGroup,
new SchemaRegistryAvroObjectSerializerOptions()
{
AutoRegisterSchemas = autoRegisterSchemas
});
}
public async Task CannotDeserializeUnsupportedType()
{
var client = CreateClient();
var groupName = TestEnvironment.SchemaRegistryGroup;
using var memoryStream = new MemoryStream();
var serializer = new SchemaRegistryAvroObjectSerializer(client, groupName, new SchemaRegistryAvroObjectSerializerOptions {
AutoRegisterSchemas = true
});
Assert.ThrowsAsync <ArgumentException>(async() => await serializer.DeserializeAsync(memoryStream, typeof(TimeZoneInfo), CancellationToken.None));
await Task.CompletedTask;
}
public void Deserialize()
{
var client = new SchemaRegistryClient(TestEnvironment.SchemaRegistryUri, TestEnvironment.Credential);
using var memoryStream = new MemoryStream();
#region Snippet:Deserialize
string groupName = "<schema_group_name>";
var serializer = new SchemaRegistryAvroObjectSerializer(client, groupName, new SchemaRegistryAvroObjectSerializerOptions {
AutoRegisterSchemas = true
});
memoryStream.Position = 0;
Employee employee = (Employee)serializer.Deserialize(memoryStream, typeof(Employee), CancellationToken.None);
#endregion
}
public void Deserialize()
{
using var memoryStream = new MemoryStream(_memoryStreamBytes);
string groupName = TestEnvironment.SchemaRegistryGroup;
#region Snippet:SchemaRegistryAvroDeserialize
var serializer = new SchemaRegistryAvroObjectSerializer(schemaRegistryClient, groupName, new SchemaRegistryAvroObjectSerializerOptions {
AutoRegisterSchemas = true
});
memoryStream.Position = 0;
Employee employee = (Employee)serializer.Deserialize(memoryStream, typeof(Employee), CancellationToken.None);
#endregion
Assert.AreEqual(42, employee.Age);
Assert.AreEqual("John Doe", employee.Name);
}
public void Serialize()
{
var client = new SchemaRegistryClient(TestEnvironment.SchemaRegistryUri, TestEnvironment.Credential);
#region Snippet:Serialize
var employee = new Employee {
Age = 42, Name = "John Doe"
};
string groupName = "<schema_group_name>";
using var memoryStream = new MemoryStream();
var serializer = new SchemaRegistryAvroObjectSerializer(client, groupName, new SchemaRegistryAvroObjectSerializerOptions {
AutoRegisterSchemas = true
});
serializer.Serialize(memoryStream, employee, typeof(Employee), CancellationToken.None);
#endregion
}
public void Serialize()
{
string groupName = TestEnvironment.SchemaRegistryGroup;
#region Snippet:SchemaRegistryAvroSerialize
var employee = new Employee {
Age = 42, Name = "John Doe"
};
using var memoryStream = new MemoryStream();
var serializer = new SchemaRegistryAvroObjectSerializer(schemaRegistryClient, groupName, new SchemaRegistryAvroObjectSerializerOptions {
AutoRegisterSchemas = true
});
serializer.Serialize(memoryStream, employee, typeof(Employee), CancellationToken.None);
#endregion
Assert.IsTrue(memoryStream.Length > 0);
_memoryStreamBytes = memoryStream.ToArray();
}
public async Task CanSerializeAndDeserialize()
{
var client = CreateClient();
var groupName = TestEnvironment.SchemaRegistryGroup;
var employee = new Employee {
Age = 42, Name = "Caketown"
};
using var memoryStream = new MemoryStream();
var serializer = new SchemaRegistryAvroObjectSerializer(client, groupName, new SchemaRegistryAvroObjectSerializerOptions {
AutoRegisterSchemas = true
});
await serializer.SerializeAsync(memoryStream, employee, typeof(Employee), CancellationToken.None);
memoryStream.Position = 0;
var deserializedObject = await serializer.DeserializeAsync(memoryStream, typeof(Employee), CancellationToken.None);
var readEmployee = deserializedObject as Employee;
Assert.IsNotNull(readEmployee);
Assert.AreEqual("Caketown", readEmployee.Name);
Assert.AreEqual(42, readEmployee.Age);
}
public async Task CanSerializeAndDeserializeGenericRecord()
{
var client = CreateClient();
var groupName = TestEnvironment.SchemaRegistryGroup;
var record = new GenericRecord((RecordSchema)Employee._SCHEMA);
record.Add("Name", "Caketown");
record.Add("Age", 42);
using var memoryStream = new MemoryStream();
var serializer = new SchemaRegistryAvroObjectSerializer(client, groupName, new SchemaRegistryAvroObjectSerializerOptions {
AutoRegisterSchemas = true
});
await serializer.SerializeAsync(memoryStream, record, typeof(GenericRecord), CancellationToken.None);
memoryStream.Position = 0;
var deserializedObject = await serializer.DeserializeAsync(memoryStream, typeof(GenericRecord), CancellationToken.None);
var readRecord = deserializedObject as GenericRecord;
Assert.IsNotNull(readRecord);
Assert.AreEqual("Caketown", readRecord.GetValue(0));
Assert.AreEqual(42, readRecord.GetValue(1));
}
/// <summary>
/// Runs the sample using the specified Event Hubs connection information.
/// </summary>
///
/// <param name="fullyQualifiedNamespace">The fully qualified Event Hubs namespace. This is likely to be similar to <c>{yournamespace}.servicebus.windows.net</c>.</param>
/// <param name="eventHubName">The name of the Event Hub, sometimes known as its path, that the sample should run against.</param>
/// <param name="schemaGroupName">The name of the schema group in the Schema Registry.</param>
/// <param name="tenantId">The Azure Active Directory tenant that holds the service principal.</param>
/// <param name="clientId">The Azure Active Directory client identifier of the service principal.</param>
/// <param name="secret">The Azure Active Directory secret of the service principal.</param>
///
public async Task RunAsync(string fullyQualifiedNamespace,
string eventHubName,
string schemaGroupName,
string tenantId,
string clientId,
string secret)
{
// Service principal authentication is a means for applications to authenticate against Azure Active
// Directory and consume Azure services. This is advantageous compared to using a connection string for
// authorization, as it offers a far more robust mechanism for transparently updating credentials in place,
// without an application being explicitly aware or involved.
//
// For this example, we'll take advantage of a service principal to publish and receive events. To do so, we'll make
// use of the ClientSecretCredential from the Azure.Identity library to enable the Event Hubs clients to perform authorization
// using a service principal.
ClientSecretCredential credential = new ClientSecretCredential(tenantId, clientId, secret);
// This client is used to connect to Azure Schema Registry. This will allow us to serialize Avro messages without encoding
// the schema into the message. Instead, the schema ID from the Azure Schema Registry will be used.
SchemaRegistryClient schemaRegistryClient = new SchemaRegistryClient(fullyQualifiedNamespace, credential);
// Creating the Avro serializer requires the SchemaRegistryClient. Setting AutoRegisterSchemas to true allows any schema
// provided to Azure Schema Registry to be added to the registry automatically.
SchemaRegistryAvroObjectSerializer avroSerializer = new SchemaRegistryAvroObjectSerializer(schemaRegistryClient, schemaGroupName,
new SchemaRegistryAvroObjectSerializerOptions {
AutoRegisterSchemas = true
});
// This defines the Message schema a single time to be used for all of our generic records.
RecordSchema exampleSchema = (RecordSchema)Schema.Parse(
"{\"type\":\"record\",\"name\":\"Message\",\"namespace\":\"ExampleSchema\",\"fields\":[{\"name\":\"Message\",\"type\":\"string\"}]}");
// To start, we'll publish a small number of events using a producer client. To ensure that our client is appropriately closed, we'll
// take advantage of the asynchronous dispose when we are done or when an exception is encountered.
await using (var producerClient = new EventHubProducerClient(fullyQualifiedNamespace, eventHubName, credential))
{
using EventDataBatch eventBatch = await producerClient.CreateBatchAsync();
GenericRecord firstEventRecord = new GenericRecord(exampleSchema);
firstEventRecord.Add("Message", "Hello, Event Hubs!");
EventData firstEvent = new EventData(avroSerializer.SerializeToBinaryData(firstEventRecord, typeof(GenericRecord)));
eventBatch.TryAdd(firstEvent);
GenericRecord secondEventRecord = new GenericRecord(exampleSchema);
secondEventRecord.Add("Message", "The middle event is this one");
EventData secondEvent = new EventData(avroSerializer.SerializeToBinaryData(secondEventRecord, typeof(GenericRecord)));
eventBatch.TryAdd(secondEvent);
GenericRecord thirdEventRecord = new GenericRecord(exampleSchema);
thirdEventRecord.Add("Message", "Goodbye, Event Hubs!");
EventData thirdEvent = new EventData(avroSerializer.SerializeToBinaryData(thirdEventRecord, typeof(GenericRecord)));
eventBatch.TryAdd(thirdEvent);
await producerClient.SendAsync(eventBatch);
Console.WriteLine("The event batch has been published.");
}
// Now that the events have been published, we'll read back all events from the Event Hub using a consumer client.
// It's important to note that because events are not removed from the partition when consuming, that if you're using
// an existing Event Hub for the sample, you will see events that were published prior to running this sample as well
// as those from the batch that we just sent.
//
// An Event Hub consumer is associated with a specific Event Hub and consumer group. The consumer group is
// a label that identifies one or more consumers as a set. Often, consumer groups are named after the responsibility
// of the consumer in an application, such as "Telemetry" or "OrderProcessing". When an Event Hub is created, a default
// consumer group is created with it, called "$Default."
//
// Each consumer has a unique view of the events in a partition that it reads from, meaning that events are available to all
// consumers and are not removed from the partition when a consumer reads them. This allows for one or more consumers to read and
// process events from the partition at different speeds and beginning with different events without interfering with
// one another.
//
// When events are published, they will continue to exist in the partition and be available for consuming until they
// reach an age where they are older than the retention period.
// (see: https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-faq#what-is-the-maximum-retention-period-for-events)
//
// In this example, we will create our consumer client using the default consumer group that is created with an Event Hub.
// Our consumer will begin watching the partition at the very end, reading only new events that we will publish for it.
await using (var consumerClient = new EventHubConsumerClient(EventHubConsumerClient.DefaultConsumerGroupName, fullyQualifiedNamespace, eventHubName, credential))
{
// To ensure that we do not wait for an indeterminate length of time, we'll stop reading after we receive three events. For a
// fresh Event Hub, those will be the three that we had published. We'll also ask for cancellation after 30 seconds, just to be
// safe.
using CancellationTokenSource cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(TimeSpan.FromSeconds(90));
int eventsRead = 0;
int maximumEvents = 3;
await foreach (PartitionEvent partitionEvent in consumerClient.ReadEventsAsync(cancellationSource.Token))
{
BinaryData binaryData = partitionEvent.Data.EventBody;
GenericRecord eventRecord = binaryData.ToObject <GenericRecord>(avroSerializer);
string messageText = (string)eventRecord["Message"];
Console.WriteLine($"Event Read: { messageText }");
eventsRead++;
if (eventsRead >= maximumEvents)
{
break;
}
}
}
// At this point, our clients have both passed their "using" scopes and have safely been disposed of. We
// have no further obligations.
Console.WriteLine();
}
/// <summary>
/// Constructor for KafkaAvroDeserializer.
/// </summary>
/// <param name="schemaRegistryUrl"></param> URL endpoint for Azure Schema Registry instance
/// <param name="credential"></param> TokenCredential implementation for OAuth2 authentication
public KafkaAvroDeserializer(string schemaRegistryUrl, TokenCredential credential)
{
this.serializer = new SchemaRegistryAvroObjectSerializer(new SchemaRegistryClient(schemaRegistryUrl, credential), "$default");
}
