protected override async Task <int> OnExecute(CommandLineApplication app)
{
//Topic = "cc_payments";
//Key = false;
if (SchemaId.HasValue)
{
var schema = await SchemaRegistryClient.GetSchemaAsync(SchemaId.Value);
Output($"SCHM: {schema}");
return(1);
}
else
{
if (string.IsNullOrEmpty(Topic))
{
return(-1);
}
var schemaName = Key ? SchemaRegistryClient.ConstructKeySubjectName(Topic) : SchemaRegistryClient.ConstructValueSubjectName(Topic);
var schema = await SchemaRegistryClient.GetLatestSchemaAsync(schemaName);
var header = $"Subject: {schema.Subject} | ID: {schema.Id} | Version: {schema.Version}";
Output($"INFO: {header}");
Output($"SCHM: {schema.SchemaString}");
return(1);
}
}
/// <summary>
/// Releases any unmanaged resources owned by the serializer.
/// </summary>
public void Dispose()
{
if (disposeClientOnDispose)
{
SchemaRegistryClient.Dispose();
}
}
/// <summary>
/// Serialize an instance of type <see cref="T"/> to a byte array in avro format. The serialized
/// data is preceeded by a "magic byte" (1 byte) and the id of the schema as registered
/// in Confluent's Schema Registry (4 bytes, network byte order). This call may block or throw
/// on first use for a particular topic during schema registration.
/// </summary>
/// <param name="topic">
/// The topic associated wih the data.
/// </param>
/// <param name="data">
/// The object to serialize.
/// </param>
/// <returns>
/// <paramref name="data" /> serialized as a byte array.
/// </returns>
public byte[] Serialize(string topic, T data)
{
if (!topicsRegistered.Contains(topic))
{
string subject = IsKey
? SchemaRegistryClient.ConstructKeySubjectName(topic)
: SchemaRegistryClient.ConstructValueSubjectName(topic);
// first usage: register/get schema to check compatibility
if (AutoRegisterSchema)
{
SchemaId = SchemaRegistryClient.RegisterSchemaAsync(subject, writerSchemaString).Result;
}
else
{
SchemaId = SchemaRegistryClient.GetSchemaIdAsync(subject, writerSchemaString).Result;
}
topicsRegistered.Add(topic);
}
using (var stream = new MemoryStream(InitialBufferSize))
using (var writer = new BinaryWriter(stream))
{
stream.WriteByte(Constants.MagicByte);
writer.Write(IPAddress.HostToNetworkOrder(SchemaId.Value));
avroWriter.Write(data, new BinaryEncoder(stream));
// TODO: maybe change the ISerializer interface so that this copy isn't necessary.
return(stream.ToArray());
}
}
public static void AddKafkaProducer(this IServiceCollection services, JsonSerializerOptions jsonSerializerOptions = null)
{
if (jsonSerializerOptions == null)
{
jsonSerializerOptions = GetDefaultJsonSerializerOptions();
}
AddLogging(services);
services.AddSingleton <KafkaProducerConnection>();
services.AddSingleton <ISchemaRegistryClient>(serviceProvider =>
{
var httpClientFactory = serviceProvider.GetService <IHttpClientFactory>();
var httpClient = httpClientFactory.CreateClient();
var schemaRegistryConfiguration = serviceProvider.GetService <KafkaSchemaRegistryConfiguration>();
var httpSchemaClient = new SchemaRegistryClient(httpClient, schemaRegistryConfiguration);
return(schemaRegistryConfiguration.CacheTimeout.TotalSeconds > 0
? (ISchemaRegistryClient) new CachedSchemaRegistryClient(httpSchemaClient, schemaRegistryConfiguration)
: httpSchemaClient);
});
services.AddScoped <IKafkaSerializerFactory>(serviceProvider =>
{
var schemaRegistryClient = serviceProvider.GetService <ISchemaRegistryClient>();
return(new KafkaSerializerFactory(schemaRegistryClient, jsonSerializerOptions));
});
services.AddScoped(typeof(IKafkaProducer <,>), typeof(KafkaProducer <,>));
}
public KafkaProducer(string bootstrapServers,
RecordConfig recordConfig,
string topic,
//int partition,
//int offset,
Action <string> errorHandler)
{
if (errorHandler == null)
{
throw new Exception("Empty handler");
}
//1 var schemaRegistry = new CachedSchemaRegistryClient(new SchemaRegistryConfig
//{
// SchemaRegistryUrl = schemaRegistryUrl,
// SchemaRegistryRequestTimeoutMs = 5000,
//});
//var schemaRegistry = new SchemaRegistryClient(new Schema(recordConfig.Subject, recordConfig.Version, recordConfig.Id, recordConfig.SchemaString)); //1
var schemaRegistry = new SchemaRegistryClient(new Schema(recordConfig.Subject, recordConfig.Version, recordConfig.Id, recordConfig.SchemaString)); //1
_avroSerializer = new AvroSerializer <GenericRecord>(schemaRegistry);
_producer =
new ProducerBuilder <string, byte[]>(new ProducerConfig {
BootstrapServers = bootstrapServers
})
.SetKeySerializer(Serializers.Utf8)
.SetValueSerializer(Serializers.ByteArray /*new AvroSerializer<T>(schemaRegistry)*/)
.Build();
_topic = topic;
}
public void RetrieveSchema()
{
var client = new SchemaRegistryClient(TestEnvironment.SchemaRegistryUri, TestEnvironment.Credential);
#region Snippet:SchemaRegistryRetrieveSchema
string schemaId = "<schema_id>";
Response <SchemaProperties> schemaProperties = client.GetSchema(schemaId);
string schemaContent = schemaProperties.Value.Content;
#endregion
}
public SchemaRegistryAvroSerializerPerfTestBase(SizeCountOptions options) : base(options)
{
_environment = new SchemaRegistryAvroSerializerTestEnvironment();
_client = new SchemaRegistryClient(_environment.SchemaRegistryEndpoint, _environment.Credential);
Serializer = new SchemaRegistryAvroSerializer(
_client,
_environment.SchemaRegistryGroup,
// the first iteration will register and cache the schema
new SchemaRegistryAvroSerializerOptions {
AutoRegisterSchemas = true
});
}
public void CreateSchemaRegistryClient()
{
#region Snippet:CreateSchemaRegistryClient
string endpoint = "<event_hubs_namespace_hostname>";
var credentials = new ClientSecretCredential(
"<tenant_id>",
"<client_id>",
"<client_secret>"
);
var client = new SchemaRegistryClient(endpoint, credentials);
#endregion
}
public void CreateSchemaRegistryClient()
{
string endpoint = TestEnvironment.SchemaRegistryEndpoint;
#region Snippet:SchemaRegistryCreateSchemaRegistryClient
// Create a new SchemaRegistry client using the default credential from Azure.Identity using environment variables previously set,
// including AZURE_CLIENT_ID, AZURE_CLIENT_SECRET, and AZURE_TENANT_ID.
// For more information on Azure.Identity usage, see: https://github.com/Azure/azure-sdk-for-net/blob/main/sdk/identity/Azure.Identity/README.md
var client = new SchemaRegistryClient(endpoint: endpoint, credential: new DefaultAzureCredential());
#endregion
this.client = client;
}
public void Deserialize()
{
var client = new SchemaRegistryClient(TestEnvironment.SchemaRegistryUri, TestEnvironment.Credential);
using var memoryStream = new MemoryStream();
#region Snippet:Deserialize
string groupName = "<schema_group_name>";
var serializer = new SchemaRegistryAvroObjectSerializer(client, groupName, new SchemaRegistryAvroObjectSerializerOptions {
AutoRegisterSchemas = true
});
memoryStream.Position = 0;
Employee employee = (Employee)serializer.Deserialize(memoryStream, typeof(Employee), CancellationToken.None);
#endregion
}
public void Serialize()
{
var client = new SchemaRegistryClient(TestEnvironment.SchemaRegistryUri, TestEnvironment.Credential);
#region Snippet:Serialize
var employee = new Employee {
Age = 42, Name = "John Doe"
};
string groupName = "<schema_group_name>";
using var memoryStream = new MemoryStream();
var serializer = new SchemaRegistryAvroObjectSerializer(client, groupName, new SchemaRegistryAvroObjectSerializerOptions {
AutoRegisterSchemas = true
});
serializer.Serialize(memoryStream, employee, typeof(Employee), CancellationToken.None);
#endregion
}
public KafkaConsumer(string bootstrapServers,
RecordConfig recordConfig,
string topic,
string groupId,
int partition,
int offset,
Action <string, dynamic, DateTime> consumeResultHandler,
Action <string> errorHandler)
{
if (consumeResultHandler == null || errorHandler == null)
{
throw new Exception("Empty handler");
}
_consumeResultHandler = consumeResultHandler;
_cts = new CancellationTokenSource();
//1 var schemaRegistry = new CachedSchemaRegistryClient(new SchemaRegistryConfig { SchemaRegistryUrl = schemaRegistryUrl });
//var schemaRegistry = new SchemaRegistryClient(new Schema(recordConfig.Subject, recordConfig.Version, recordConfig.Id, recordConfig.SchemaString)); //1
var schemaRegistry = new SchemaRegistryClient(new Schema(recordConfig.Subject, recordConfig.Version, recordConfig.Id, recordConfig.SchemaString)); //1
_avroDeserializer = new AvroDeserializer <GenericRecord>(schemaRegistry);
_consumer = new ConsumerBuilder <string, byte[]>(
new ConsumerConfig {
BootstrapServers = bootstrapServers, GroupId = groupId, AutoOffsetReset = AutoOffsetReset.Earliest
})
.SetKeyDeserializer(Deserializers.Utf8)
.SetValueDeserializer(Deserializers.ByteArray /*new AvroDeserializer<T>(schemaRegistry).AsSyncOverAsync()*/)
.SetErrorHandler((_, e) => errorHandler(e.Reason))
.Build();
_consumer.Assign(new List <TopicPartitionOffset> {
new TopicPartitionOffset(topic, partition, offset)
});
_topic = topic;
}
public void RegisterSchema()
{
var client = new SchemaRegistryClient(TestEnvironment.SchemaRegistryUri, TestEnvironment.Credential);
#region Snippet:SchemaRegistryRegisterSchema
string schemaName = "<schema_name>";
string groupName = "<schema_group_name>";
SerializationType schemaType = SerializationType.Avro;
// Example schema's content
string schemaContent = @"
{
""type"" : ""record"",
""namespace"" : ""TestSchema"",
""name"" : ""Employee"",
""fields"" : [
{ ""name"" : ""Name"" , ""type"" : ""string"" },
{ ""name"" : ""Age"", ""type"" : ""int"" }
]
}";
Response <SchemaProperties> schemaProperties = client.RegisterSchema(groupName, schemaName, schemaType, schemaContent);
#endregion
}
/// <summary>
/// Initializes a new instance of the <see cref="SchemaRegistryAvroSerializer"/>. This constructor can be used to create an instance
/// that will work for both serialization and deserialization.
/// </summary>
/// <param name="client">The <see cref="SchemaRegistryClient"/> instance to use for looking up schemas.</param>
/// <param name="groupName">The Schema Registry group name that contains the schemas that will be used to serialize.</param>
/// <param name="options">The set of options to customize the <see cref="SchemaRegistryAvroSerializer"/>.</param>
/// <exception cref="ArgumentNullException"></exception>
public SchemaRegistryAvroSerializer(SchemaRegistryClient client, string groupName, SchemaRegistryAvroSerializerOptions options)
{
_client = client ?? throw new ArgumentNullException(nameof(client));
_groupName = groupName;
_options = options?.Clone() ?? new SchemaRegistryAvroSerializerOptions();
}
/// <summary>
/// Runs the sample using the specified Event Hubs connection information.
/// </summary>
///
/// <param name="fullyQualifiedNamespace">The fully qualified Event Hubs namespace. This is likely to be similar to <c>{yournamespace}.servicebus.windows.net</c>.</param>
/// <param name="eventHubName">The name of the Event Hub, sometimes known as its path, that the sample should run against.</param>
/// <param name="schemaGroupName">The name of the schema group in the Schema Registry.</param>
/// <param name="tenantId">The Azure Active Directory tenant that holds the service principal.</param>
/// <param name="clientId">The Azure Active Directory client identifier of the service principal.</param>
/// <param name="secret">The Azure Active Directory secret of the service principal.</param>
///
public async Task RunAsync(string fullyQualifiedNamespace,
string eventHubName,
string schemaGroupName,
string tenantId,
string clientId,
string secret)
{
// Service principal authentication is a means for applications to authenticate against Azure Active
// Directory and consume Azure services. This is advantageous compared to using a connection string for
// authorization, as it offers a far more robust mechanism for transparently updating credentials in place,
// without an application being explicitly aware or involved.
//
// For this example, we'll take advantage of a service principal to publish and receive events. To do so, we'll make
// use of the ClientSecretCredential from the Azure.Identity library to enable the Event Hubs clients to perform authorization
// using a service principal.
ClientSecretCredential credential = new ClientSecretCredential(tenantId, clientId, secret);
// This client is used to connect to Azure Schema Registry. This will allow us to serialize Avro messages without encoding
// the schema into the message. Instead, the schema ID from the Azure Schema Registry will be used.
SchemaRegistryClient schemaRegistryClient = new SchemaRegistryClient(fullyQualifiedNamespace, credential);
// Creating the Avro serializer requires the SchemaRegistryClient. Setting AutoRegisterSchemas to true allows any schema
// provided to Azure Schema Registry to be added to the registry automatically.
SchemaRegistryAvroObjectSerializer avroSerializer = new SchemaRegistryAvroObjectSerializer(schemaRegistryClient, schemaGroupName,
new SchemaRegistryAvroObjectSerializerOptions {
AutoRegisterSchemas = true
});
// This defines the Message schema a single time to be used for all of our generic records.
RecordSchema exampleSchema = (RecordSchema)Schema.Parse(
"{\"type\":\"record\",\"name\":\"Message\",\"namespace\":\"ExampleSchema\",\"fields\":[{\"name\":\"Message\",\"type\":\"string\"}]}");
// To start, we'll publish a small number of events using a producer client. To ensure that our client is appropriately closed, we'll
// take advantage of the asynchronous dispose when we are done or when an exception is encountered.
await using (var producerClient = new EventHubProducerClient(fullyQualifiedNamespace, eventHubName, credential))
{
using EventDataBatch eventBatch = await producerClient.CreateBatchAsync();
GenericRecord firstEventRecord = new GenericRecord(exampleSchema);
firstEventRecord.Add("Message", "Hello, Event Hubs!");
EventData firstEvent = new EventData(avroSerializer.SerializeToBinaryData(firstEventRecord, typeof(GenericRecord)));
eventBatch.TryAdd(firstEvent);
GenericRecord secondEventRecord = new GenericRecord(exampleSchema);
secondEventRecord.Add("Message", "The middle event is this one");
EventData secondEvent = new EventData(avroSerializer.SerializeToBinaryData(secondEventRecord, typeof(GenericRecord)));
eventBatch.TryAdd(secondEvent);
GenericRecord thirdEventRecord = new GenericRecord(exampleSchema);
thirdEventRecord.Add("Message", "Goodbye, Event Hubs!");
EventData thirdEvent = new EventData(avroSerializer.SerializeToBinaryData(thirdEventRecord, typeof(GenericRecord)));
eventBatch.TryAdd(thirdEvent);
await producerClient.SendAsync(eventBatch);
Console.WriteLine("The event batch has been published.");
}
// Now that the events have been published, we'll read back all events from the Event Hub using a consumer client.
// It's important to note that because events are not removed from the partition when consuming, that if you're using
// an existing Event Hub for the sample, you will see events that were published prior to running this sample as well
// as those from the batch that we just sent.
//
// An Event Hub consumer is associated with a specific Event Hub and consumer group. The consumer group is
// a label that identifies one or more consumers as a set. Often, consumer groups are named after the responsibility
// of the consumer in an application, such as "Telemetry" or "OrderProcessing". When an Event Hub is created, a default
// consumer group is created with it, called "$Default."
//
// Each consumer has a unique view of the events in a partition that it reads from, meaning that events are available to all
// consumers and are not removed from the partition when a consumer reads them. This allows for one or more consumers to read and
// process events from the partition at different speeds and beginning with different events without interfering with
// one another.
//
// When events are published, they will continue to exist in the partition and be available for consuming until they
// reach an age where they are older than the retention period.
// (see: https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-faq#what-is-the-maximum-retention-period-for-events)
//
// In this example, we will create our consumer client using the default consumer group that is created with an Event Hub.
// Our consumer will begin watching the partition at the very end, reading only new events that we will publish for it.
await using (var consumerClient = new EventHubConsumerClient(EventHubConsumerClient.DefaultConsumerGroupName, fullyQualifiedNamespace, eventHubName, credential))
{
// To ensure that we do not wait for an indeterminate length of time, we'll stop reading after we receive three events. For a
// fresh Event Hub, those will be the three that we had published. We'll also ask for cancellation after 30 seconds, just to be
// safe.
using CancellationTokenSource cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(TimeSpan.FromSeconds(90));
int eventsRead = 0;
int maximumEvents = 3;
await foreach (PartitionEvent partitionEvent in consumerClient.ReadEventsAsync(cancellationSource.Token))
{
BinaryData binaryData = partitionEvent.Data.EventBody;
GenericRecord eventRecord = binaryData.ToObject <GenericRecord>(avroSerializer);
string messageText = (string)eventRecord["Message"];
Console.WriteLine($"Event Read: { messageText }");
eventsRead++;
if (eventsRead >= maximumEvents)
{
break;
}
}
}
// At this point, our clients have both passed their "using" scopes and have safely been disposed of. We
// have no further obligations.
Console.WriteLine();
}
/// <summary>
/// Initializes a new instance of the <see cref="SchemaRegistryAvroSerializer"/>. This constructor can only be used to create an
/// instance which will be used for deserialization. In order to serialize (or both serialize and deserialize) you will need to use
/// one of the constructors that have a <code>groupName</code> parameter.
/// </summary>
/// <param name="client">The <see cref="SchemaRegistryClient"/> instance to use for looking up schemas.</param>
/// <exception cref="ArgumentNullException"></exception>
public SchemaRegistryAvroSerializer(SchemaRegistryClient client)
: this(client, null, new SchemaRegistryAvroSerializerOptions())
{
}
/// <summary>
/// Initializes a new instance of the <see cref="SchemaRegistryAvroSerializer"/>. This constructor can be used to create an instance
/// that will work for both serialization and deserialization.
/// </summary>
/// <param name="client">The <see cref="SchemaRegistryClient"/> instance to use for looking up schemas.</param>
/// <param name="groupName">The Schema Registry group name that contains the schemas that will be used to serialize.</param>
/// <exception cref="ArgumentNullException"></exception>
public SchemaRegistryAvroSerializer(SchemaRegistryClient client, string groupName)
: this(client, groupName, new SchemaRegistryAvroSerializerOptions())
{
}
/// <summary>
/// Initializes new instance of <see cref="SchemaRegistryAvroSerializer"/>.
/// </summary>
public SchemaRegistryAvroSerializer(SchemaRegistryClient client, string groupName, SchemaRegistryAvroSerializerOptions options = null)
{
_client = client ?? throw new ArgumentNullException(nameof(client));
_groupName = groupName ?? throw new ArgumentNullException(nameof(groupName));
_options = options;
}
