protected override async Task <int> OnExecute(CommandLineApplication app) { //Topic = "cc_payments"; //Key = false; if (SchemaId.HasValue) { var schema = await SchemaRegistryClient.GetSchemaAsync(SchemaId.Value); Output($"SCHM: {schema}"); return(1); } else { if (string.IsNullOrEmpty(Topic)) { return(-1); } var schemaName = Key ? SchemaRegistryClient.ConstructKeySubjectName(Topic) : SchemaRegistryClient.ConstructValueSubjectName(Topic); var schema = await SchemaRegistryClient.GetLatestSchemaAsync(schemaName); var header = $"Subject: {schema.Subject} | ID: {schema.Id} | Version: {schema.Version}"; Output($"INFO: {header}"); Output($"SCHM: {schema.SchemaString}"); return(1); } }
/// <summary> /// Releases any unmanaged resources owned by the serializer. /// </summary> public void Dispose() { if (disposeClientOnDispose) { SchemaRegistryClient.Dispose(); } }
/// <summary> /// Serialize an instance of type <see cref="T"/> to a byte array in avro format. The serialized /// data is preceeded by a "magic byte" (1 byte) and the id of the schema as registered /// in Confluent's Schema Registry (4 bytes, network byte order). This call may block or throw /// on first use for a particular topic during schema registration. /// </summary> /// <param name="topic"> /// The topic associated wih the data. /// </param> /// <param name="data"> /// The object to serialize. /// </param> /// <returns> /// <paramref name="data" /> serialized as a byte array. /// </returns> public byte[] Serialize(string topic, T data) { if (!topicsRegistered.Contains(topic)) { string subject = IsKey ? SchemaRegistryClient.ConstructKeySubjectName(topic) : SchemaRegistryClient.ConstructValueSubjectName(topic); // first usage: register/get schema to check compatibility if (AutoRegisterSchema) { SchemaId = SchemaRegistryClient.RegisterSchemaAsync(subject, writerSchemaString).Result; } else { SchemaId = SchemaRegistryClient.GetSchemaIdAsync(subject, writerSchemaString).Result; } topicsRegistered.Add(topic); } using (var stream = new MemoryStream(InitialBufferSize)) using (var writer = new BinaryWriter(stream)) { stream.WriteByte(Constants.MagicByte); writer.Write(IPAddress.HostToNetworkOrder(SchemaId.Value)); avroWriter.Write(data, new BinaryEncoder(stream)); // TODO: maybe change the ISerializer interface so that this copy isn't necessary. return(stream.ToArray()); } }
public static void AddKafkaProducer(this IServiceCollection services, JsonSerializerOptions jsonSerializerOptions = null) { if (jsonSerializerOptions == null) { jsonSerializerOptions = GetDefaultJsonSerializerOptions(); } AddLogging(services); services.AddSingleton <KafkaProducerConnection>(); services.AddSingleton <ISchemaRegistryClient>(serviceProvider => { var httpClientFactory = serviceProvider.GetService <IHttpClientFactory>(); var httpClient = httpClientFactory.CreateClient(); var schemaRegistryConfiguration = serviceProvider.GetService <KafkaSchemaRegistryConfiguration>(); var httpSchemaClient = new SchemaRegistryClient(httpClient, schemaRegistryConfiguration); return(schemaRegistryConfiguration.CacheTimeout.TotalSeconds > 0 ? (ISchemaRegistryClient) new CachedSchemaRegistryClient(httpSchemaClient, schemaRegistryConfiguration) : httpSchemaClient); }); services.AddScoped <IKafkaSerializerFactory>(serviceProvider => { var schemaRegistryClient = serviceProvider.GetService <ISchemaRegistryClient>(); return(new KafkaSerializerFactory(schemaRegistryClient, jsonSerializerOptions)); }); services.AddScoped(typeof(IKafkaProducer <,>), typeof(KafkaProducer <,>)); }
public KafkaProducer(string bootstrapServers, RecordConfig recordConfig, string topic, //int partition, //int offset, Action <string> errorHandler) { if (errorHandler == null) { throw new Exception("Empty handler"); } //1 var schemaRegistry = new CachedSchemaRegistryClient(new SchemaRegistryConfig //{ // SchemaRegistryUrl = schemaRegistryUrl, // SchemaRegistryRequestTimeoutMs = 5000, //}); //var schemaRegistry = new SchemaRegistryClient(new Schema(recordConfig.Subject, recordConfig.Version, recordConfig.Id, recordConfig.SchemaString)); //1 var schemaRegistry = new SchemaRegistryClient(new Schema(recordConfig.Subject, recordConfig.Version, recordConfig.Id, recordConfig.SchemaString)); //1 _avroSerializer = new AvroSerializer <GenericRecord>(schemaRegistry); _producer = new ProducerBuilder <string, byte[]>(new ProducerConfig { BootstrapServers = bootstrapServers }) .SetKeySerializer(Serializers.Utf8) .SetValueSerializer(Serializers.ByteArray /*new AvroSerializer<T>(schemaRegistry)*/) .Build(); _topic = topic; }
public void RetrieveSchema() { var client = new SchemaRegistryClient(TestEnvironment.SchemaRegistryUri, TestEnvironment.Credential); #region Snippet:SchemaRegistryRetrieveSchema string schemaId = "<schema_id>"; Response <SchemaProperties> schemaProperties = client.GetSchema(schemaId); string schemaContent = schemaProperties.Value.Content; #endregion }
public SchemaRegistryAvroSerializerPerfTestBase(SizeCountOptions options) : base(options) { _environment = new SchemaRegistryAvroSerializerTestEnvironment(); _client = new SchemaRegistryClient(_environment.SchemaRegistryEndpoint, _environment.Credential); Serializer = new SchemaRegistryAvroSerializer( _client, _environment.SchemaRegistryGroup, // the first iteration will register and cache the schema new SchemaRegistryAvroSerializerOptions { AutoRegisterSchemas = true }); }
public void CreateSchemaRegistryClient() { #region Snippet:CreateSchemaRegistryClient string endpoint = "<event_hubs_namespace_hostname>"; var credentials = new ClientSecretCredential( "<tenant_id>", "<client_id>", "<client_secret>" ); var client = new SchemaRegistryClient(endpoint, credentials); #endregion }
public void CreateSchemaRegistryClient() { string endpoint = TestEnvironment.SchemaRegistryEndpoint; #region Snippet:SchemaRegistryCreateSchemaRegistryClient // Create a new SchemaRegistry client using the default credential from Azure.Identity using environment variables previously set, // including AZURE_CLIENT_ID, AZURE_CLIENT_SECRET, and AZURE_TENANT_ID. // For more information on Azure.Identity usage, see: https://github.com/Azure/azure-sdk-for-net/blob/main/sdk/identity/Azure.Identity/README.md var client = new SchemaRegistryClient(endpoint: endpoint, credential: new DefaultAzureCredential()); #endregion this.client = client; }
public void Deserialize() { var client = new SchemaRegistryClient(TestEnvironment.SchemaRegistryUri, TestEnvironment.Credential); using var memoryStream = new MemoryStream(); #region Snippet:Deserialize string groupName = "<schema_group_name>"; var serializer = new SchemaRegistryAvroObjectSerializer(client, groupName, new SchemaRegistryAvroObjectSerializerOptions { AutoRegisterSchemas = true }); memoryStream.Position = 0; Employee employee = (Employee)serializer.Deserialize(memoryStream, typeof(Employee), CancellationToken.None); #endregion }
public void Serialize() { var client = new SchemaRegistryClient(TestEnvironment.SchemaRegistryUri, TestEnvironment.Credential); #region Snippet:Serialize var employee = new Employee { Age = 42, Name = "John Doe" }; string groupName = "<schema_group_name>"; using var memoryStream = new MemoryStream(); var serializer = new SchemaRegistryAvroObjectSerializer(client, groupName, new SchemaRegistryAvroObjectSerializerOptions { AutoRegisterSchemas = true }); serializer.Serialize(memoryStream, employee, typeof(Employee), CancellationToken.None); #endregion }
public KafkaConsumer(string bootstrapServers, RecordConfig recordConfig, string topic, string groupId, int partition, int offset, Action <string, dynamic, DateTime> consumeResultHandler, Action <string> errorHandler) { if (consumeResultHandler == null || errorHandler == null) { throw new Exception("Empty handler"); } _consumeResultHandler = consumeResultHandler; _cts = new CancellationTokenSource(); //1 var schemaRegistry = new CachedSchemaRegistryClient(new SchemaRegistryConfig { SchemaRegistryUrl = schemaRegistryUrl }); //var schemaRegistry = new SchemaRegistryClient(new Schema(recordConfig.Subject, recordConfig.Version, recordConfig.Id, recordConfig.SchemaString)); //1 var schemaRegistry = new SchemaRegistryClient(new Schema(recordConfig.Subject, recordConfig.Version, recordConfig.Id, recordConfig.SchemaString)); //1 _avroDeserializer = new AvroDeserializer <GenericRecord>(schemaRegistry); _consumer = new ConsumerBuilder <string, byte[]>( new ConsumerConfig { BootstrapServers = bootstrapServers, GroupId = groupId, AutoOffsetReset = AutoOffsetReset.Earliest }) .SetKeyDeserializer(Deserializers.Utf8) .SetValueDeserializer(Deserializers.ByteArray /*new AvroDeserializer<T>(schemaRegistry).AsSyncOverAsync()*/) .SetErrorHandler((_, e) => errorHandler(e.Reason)) .Build(); _consumer.Assign(new List <TopicPartitionOffset> { new TopicPartitionOffset(topic, partition, offset) }); _topic = topic; }
public void RegisterSchema() { var client = new SchemaRegistryClient(TestEnvironment.SchemaRegistryUri, TestEnvironment.Credential); #region Snippet:SchemaRegistryRegisterSchema string schemaName = "<schema_name>"; string groupName = "<schema_group_name>"; SerializationType schemaType = SerializationType.Avro; // Example schema's content string schemaContent = @" { ""type"" : ""record"", ""namespace"" : ""TestSchema"", ""name"" : ""Employee"", ""fields"" : [ { ""name"" : ""Name"" , ""type"" : ""string"" }, { ""name"" : ""Age"", ""type"" : ""int"" } ] }"; Response <SchemaProperties> schemaProperties = client.RegisterSchema(groupName, schemaName, schemaType, schemaContent); #endregion }
/// <summary> /// Initializes a new instance of the <see cref="SchemaRegistryAvroSerializer"/>. This constructor can be used to create an instance /// that will work for both serialization and deserialization. /// </summary> /// <param name="client">The <see cref="SchemaRegistryClient"/> instance to use for looking up schemas.</param> /// <param name="groupName">The Schema Registry group name that contains the schemas that will be used to serialize.</param> /// <param name="options">The set of options to customize the <see cref="SchemaRegistryAvroSerializer"/>.</param> /// <exception cref="ArgumentNullException"></exception> public SchemaRegistryAvroSerializer(SchemaRegistryClient client, string groupName, SchemaRegistryAvroSerializerOptions options) { _client = client ?? throw new ArgumentNullException(nameof(client)); _groupName = groupName; _options = options?.Clone() ?? new SchemaRegistryAvroSerializerOptions(); }
/// <summary> /// Runs the sample using the specified Event Hubs connection information. /// </summary> /// /// <param name="fullyQualifiedNamespace">The fully qualified Event Hubs namespace. This is likely to be similar to <c>{yournamespace}.servicebus.windows.net</c>.</param> /// <param name="eventHubName">The name of the Event Hub, sometimes known as its path, that the sample should run against.</param> /// <param name="schemaGroupName">The name of the schema group in the Schema Registry.</param> /// <param name="tenantId">The Azure Active Directory tenant that holds the service principal.</param> /// <param name="clientId">The Azure Active Directory client identifier of the service principal.</param> /// <param name="secret">The Azure Active Directory secret of the service principal.</param> /// public async Task RunAsync(string fullyQualifiedNamespace, string eventHubName, string schemaGroupName, string tenantId, string clientId, string secret) { // Service principal authentication is a means for applications to authenticate against Azure Active // Directory and consume Azure services. This is advantageous compared to using a connection string for // authorization, as it offers a far more robust mechanism for transparently updating credentials in place, // without an application being explicitly aware or involved. // // For this example, we'll take advantage of a service principal to publish and receive events. To do so, we'll make // use of the ClientSecretCredential from the Azure.Identity library to enable the Event Hubs clients to perform authorization // using a service principal. ClientSecretCredential credential = new ClientSecretCredential(tenantId, clientId, secret); // This client is used to connect to Azure Schema Registry. This will allow us to serialize Avro messages without encoding // the schema into the message. Instead, the schema ID from the Azure Schema Registry will be used. SchemaRegistryClient schemaRegistryClient = new SchemaRegistryClient(fullyQualifiedNamespace, credential); // Creating the Avro serializer requires the SchemaRegistryClient. Setting AutoRegisterSchemas to true allows any schema // provided to Azure Schema Registry to be added to the registry automatically. SchemaRegistryAvroObjectSerializer avroSerializer = new SchemaRegistryAvroObjectSerializer(schemaRegistryClient, schemaGroupName, new SchemaRegistryAvroObjectSerializerOptions { AutoRegisterSchemas = true }); // This defines the Message schema a single time to be used for all of our generic records. RecordSchema exampleSchema = (RecordSchema)Schema.Parse( "{\"type\":\"record\",\"name\":\"Message\",\"namespace\":\"ExampleSchema\",\"fields\":[{\"name\":\"Message\",\"type\":\"string\"}]}"); // To start, we'll publish a small number of events using a producer client. To ensure that our client is appropriately closed, we'll // take advantage of the asynchronous dispose when we are done or when an exception is encountered. await using (var producerClient = new EventHubProducerClient(fullyQualifiedNamespace, eventHubName, credential)) { using EventDataBatch eventBatch = await producerClient.CreateBatchAsync(); GenericRecord firstEventRecord = new GenericRecord(exampleSchema); firstEventRecord.Add("Message", "Hello, Event Hubs!"); EventData firstEvent = new EventData(avroSerializer.SerializeToBinaryData(firstEventRecord, typeof(GenericRecord))); eventBatch.TryAdd(firstEvent); GenericRecord secondEventRecord = new GenericRecord(exampleSchema); secondEventRecord.Add("Message", "The middle event is this one"); EventData secondEvent = new EventData(avroSerializer.SerializeToBinaryData(secondEventRecord, typeof(GenericRecord))); eventBatch.TryAdd(secondEvent); GenericRecord thirdEventRecord = new GenericRecord(exampleSchema); thirdEventRecord.Add("Message", "Goodbye, Event Hubs!"); EventData thirdEvent = new EventData(avroSerializer.SerializeToBinaryData(thirdEventRecord, typeof(GenericRecord))); eventBatch.TryAdd(thirdEvent); await producerClient.SendAsync(eventBatch); Console.WriteLine("The event batch has been published."); } // Now that the events have been published, we'll read back all events from the Event Hub using a consumer client. // It's important to note that because events are not removed from the partition when consuming, that if you're using // an existing Event Hub for the sample, you will see events that were published prior to running this sample as well // as those from the batch that we just sent. // // An Event Hub consumer is associated with a specific Event Hub and consumer group. The consumer group is // a label that identifies one or more consumers as a set. Often, consumer groups are named after the responsibility // of the consumer in an application, such as "Telemetry" or "OrderProcessing". When an Event Hub is created, a default // consumer group is created with it, called "$Default." // // Each consumer has a unique view of the events in a partition that it reads from, meaning that events are available to all // consumers and are not removed from the partition when a consumer reads them. This allows for one or more consumers to read and // process events from the partition at different speeds and beginning with different events without interfering with // one another. // // When events are published, they will continue to exist in the partition and be available for consuming until they // reach an age where they are older than the retention period. // (see: https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-faq#what-is-the-maximum-retention-period-for-events) // // In this example, we will create our consumer client using the default consumer group that is created with an Event Hub. // Our consumer will begin watching the partition at the very end, reading only new events that we will publish for it. await using (var consumerClient = new EventHubConsumerClient(EventHubConsumerClient.DefaultConsumerGroupName, fullyQualifiedNamespace, eventHubName, credential)) { // To ensure that we do not wait for an indeterminate length of time, we'll stop reading after we receive three events. For a // fresh Event Hub, those will be the three that we had published. We'll also ask for cancellation after 30 seconds, just to be // safe. using CancellationTokenSource cancellationSource = new CancellationTokenSource(); cancellationSource.CancelAfter(TimeSpan.FromSeconds(90)); int eventsRead = 0; int maximumEvents = 3; await foreach (PartitionEvent partitionEvent in consumerClient.ReadEventsAsync(cancellationSource.Token)) { BinaryData binaryData = partitionEvent.Data.EventBody; GenericRecord eventRecord = binaryData.ToObject <GenericRecord>(avroSerializer); string messageText = (string)eventRecord["Message"]; Console.WriteLine($"Event Read: { messageText }"); eventsRead++; if (eventsRead >= maximumEvents) { break; } } } // At this point, our clients have both passed their "using" scopes and have safely been disposed of. We // have no further obligations. Console.WriteLine(); }
/// <summary> /// Initializes a new instance of the <see cref="SchemaRegistryAvroSerializer"/>. This constructor can only be used to create an /// instance which will be used for deserialization. In order to serialize (or both serialize and deserialize) you will need to use /// one of the constructors that have a <code>groupName</code> parameter. /// </summary> /// <param name="client">The <see cref="SchemaRegistryClient"/> instance to use for looking up schemas.</param> /// <exception cref="ArgumentNullException"></exception> public SchemaRegistryAvroSerializer(SchemaRegistryClient client) : this(client, null, new SchemaRegistryAvroSerializerOptions()) { }
/// <summary> /// Initializes a new instance of the <see cref="SchemaRegistryAvroSerializer"/>. This constructor can be used to create an instance /// that will work for both serialization and deserialization. /// </summary> /// <param name="client">The <see cref="SchemaRegistryClient"/> instance to use for looking up schemas.</param> /// <param name="groupName">The Schema Registry group name that contains the schemas that will be used to serialize.</param> /// <exception cref="ArgumentNullException"></exception> public SchemaRegistryAvroSerializer(SchemaRegistryClient client, string groupName) : this(client, groupName, new SchemaRegistryAvroSerializerOptions()) { }
/// <summary> /// Initializes new instance of <see cref="SchemaRegistryAvroSerializer"/>. /// </summary> public SchemaRegistryAvroSerializer(SchemaRegistryClient client, string groupName, SchemaRegistryAvroSerializerOptions options = null) { _client = client ?? throw new ArgumentNullException(nameof(client)); _groupName = groupName ?? throw new ArgumentNullException(nameof(groupName)); _options = options; }