public async Task It_should_deserialize_the_envelope_correctly() { var schemaRegistry = new CachedSchemaRegistryClient ( new SchemaRegistryConfig { SchemaRegistryUrl = _kafkaConfig.SchemaRegistryUrl } ); //var type = AvroMultipleDeserializer.Get(schema.Name); var subjects = await schemaRegistry.GetAllSubjectsAsync(); subjects.Count.ShouldBePositive(); foreach (var subject in subjects) { var schemas = await schemaRegistry.GetLatestSchemaAsync(subject); var schemaSubject = await schemaRegistry.GetSchemaAsync(1); var schema = global::Avro.Schema.Parse(schemaSubject); } }
public static void JsonWithReferences(Config config) { var srInitial = new CachedSchemaRegistryClient(new SchemaRegistryConfig { Url = config.Server }); var sr = new CachedSchemaRegistryClient(new SchemaRegistryConfig { Url = config.Server }); var topicName = Guid.NewGuid().ToString(); var subjectInitial = SubjectNameStrategy.Topic.ConstructValueSubjectName(topicName, null); var subject1 = SubjectNameStrategy.Topic.ConstructValueSubjectName(topicName + "1", null); var subject2 = SubjectNameStrategy.Topic.ConstructValueSubjectName(topicName + "2", null); // Test there are no errors (exceptions) registering a schema that references another. var id1 = srInitial.RegisterSchemaAsync(subject1, new Schema(S1, SchemaType.Json)).Result; var s1 = srInitial.GetLatestSchemaAsync(subject1).Result; var refs = new List <SchemaReference> { new SchemaReference("https://example.com/geographical-location.schema.json", subject1, s1.Version) }; var id2 = srInitial.RegisterSchemaAsync(subject2, new Schema(S2, refs, SchemaType.Json)).Result; // In fact, it seems references are not checked server side. var latestSchema = sr.GetLatestSchemaAsync(subject2).Result; }
/// <summary> /// Creates a serializer. /// </summary> /// <param name="registryConfiguration"> /// Schema Registry configuration. Using the <see cref="SchemaRegistryConfig" /> class is /// highly recommended. /// </param> /// <param name="registerAutomatically"> /// When to automatically register schemas that match the type being serialized. /// </param> /// <param name="schemaBuilder"> /// The schema builder to use to create a schema for a C# type when registering automatically. /// If none is provided, the default schema builder will be used. /// </param> /// <param name="schemaReader"> /// The JSON schema reader to use to convert schemas received from the registry into abstract /// representations. If none is provided, the default schema reader will be used. /// </param> /// <param name="schemaWriter"> /// The JSON schema writer to use to convert abstract schema representations when registering /// automatically. If none is provided, the default schema writer will be used. /// </param> /// <param name="serializerBuilder"> /// The deserializer builder to use to build serialization functions for C# types. If none /// is provided, the default serializer builder will be used. /// </param> /// <param name="subjectNameBuilder"> /// A function that determines the subject name given the topic name and a component type /// (key or value). If none is provided, the default "{topic name}-{component}" naming /// convention will be used. /// </param> /// <param name="tombstoneBehavior"> /// The behavior of the serializer on tombstone records. /// </param> /// <exception cref="ArgumentNullException"> /// Thrown when the registry configuration is null. /// </exception> public AsyncSchemaRegistrySerializer( IEnumerable <KeyValuePair <string, string> > registryConfiguration, AutomaticRegistrationBehavior registerAutomatically = AutomaticRegistrationBehavior.Never, Abstract.ISchemaBuilder schemaBuilder = null, IJsonSchemaReader schemaReader = null, IJsonSchemaWriter schemaWriter = null, IBinarySerializerBuilder serializerBuilder = null, Func <SerializationContext, string> subjectNameBuilder = null, TombstoneBehavior tombstoneBehavior = TombstoneBehavior.None ) { if (registryConfiguration == null) { throw new ArgumentNullException(nameof(registryConfiguration)); } if (tombstoneBehavior != TombstoneBehavior.None && default(T) != null) { throw new UnsupportedTypeException(typeof(T), $"{typeof(T)} cannot represent tombstone values."); } RegisterAutomatically = registerAutomatically; SchemaBuilder = schemaBuilder ?? new Abstract.SchemaBuilder(); SchemaReader = schemaReader ?? new JsonSchemaReader(); SchemaWriter = schemaWriter ?? new JsonSchemaWriter(); SerializerBuilder = serializerBuilder ?? new BinarySerializerBuilder(); SubjectNameBuilder = subjectNameBuilder ?? (c => $"{c.Topic}-{(c.Component == MessageComponentType.Key ? "key" : "value")}"); TombstoneBehavior = tombstoneBehavior; _cache = new Dictionary <string, Task <Func <T, byte[]> > >(); _register = async(subject, json) => { using (var registry = new CachedSchemaRegistryClient(registryConfiguration)) { return(await registry.RegisterSchemaAsync(subject, new Schema(json, SchemaType.Avro)).ConfigureAwait(false)); } }; _resolve = async subject => { using (var registry = new CachedSchemaRegistryClient(registryConfiguration)) { var schema = await registry.GetLatestSchemaAsync(subject).ConfigureAwait(false); if (schema.SchemaType != SchemaType.Avro) { throw new UnsupportedSchemaException(null, $"The latest schema with subject {subject} is not an Avro schema."); } return(schema); } }; }
public static void SslAuth(Config config) { var testSchema1 = "{\"type\":\"record\",\"name\":\"User\",\"namespace\":\"Confluent.Kafka.Examples.AvroSpecific" + "\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"favorite_number\",\"type\":[\"i" + "nt\",\"null\"]},{\"name\":\"favorite_color\",\"type\":[\"string\",\"null\"]}]}"; // 1. valid configuration cases // 1.1. using SSL valid certificate. var conf = new SchemaRegistryConfig { Url = config.ServerWithSsl, SslKeystoreLocation = config.KeystoreLocation, SslKeystorePassword = config.KeystorePassword, SslCaLocation = config.CaLocation, EnableSslCertificateVerification = bool.Parse(config.EnableSslCertificateVerification), }; // some sanity checking of strongly typed config property name mappings. Assert.Equal(config.ServerWithSsl, conf.Get("schema.registry.url")); using (var sr = new CachedSchemaRegistryClient(conf)) { var topicName = Guid.NewGuid().ToString(); var subject = SubjectNameStrategy.Topic.ToDelegate()(new SerializationContext(MessageComponentType.Value, topicName), null); var id = sr.RegisterSchemaAsync(subject, testSchema1).Result; var schema = sr.GetLatestSchemaAsync(subject).Result; Assert.Equal(schema.Id, id); } // try to connect with invalid SSL config. shouldn't work. Assert.Throws <HttpRequestException>(() => { var sr = new CachedSchemaRegistryClient(new SchemaRegistryConfig { Url = config.ServerWithSsl }); var topicName = Guid.NewGuid().ToString(); var subject = SubjectNameStrategy.Topic.ConstructValueSubjectName(topicName, null); try { var id = sr.RegisterSchemaAsync(subject, testSchema1).Result; } catch (Exception e) { throw e.InnerException; } }); }
private static void GetSchemaBySubjectAndVersion(Dictionary <string, object> config) { var topicName = Guid.NewGuid().ToString(); var testSchema1 = "{\"type\":\"record\",\"name\":\"User\",\"namespace\":\"Confluent.Kafka.Examples.AvroSpecific" + "\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"favorite_number\",\"type\":[\"i" + "nt\",\"null\"]},{\"name\":\"favorite_color\",\"type\":[\"string\",\"null\"]}]}"; var sr = new CachedSchemaRegistryClient(config); var subject = sr.ConstructValueSubjectName(topicName, "Confluent.Kafka.Examples.AvroSpecific.User"); var id = sr.RegisterSchemaAsync(subject, testSchema1).Result; var schema = sr.GetLatestSchemaAsync(subject).Result; var schemaString = sr.GetSchemaAsync(subject, schema.Version).Result; Assert.Equal(schemaString, testSchema1); }
public async Task AvroExampleAsync(string ksqlDbUrl) { var schemaRegistryConfig = new SchemaRegistryConfig { Url = "http://localhost:8081" }; using var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig); var schema = IoTSensor._SCHEMA.ToString(); var subject = "IoTSensors-value"; var registrationResult = await schemaRegistry.RegisterSchemaAsync(subject, schema); //http://localhost:8081/subjects/IoTSensors-value/versions/latest/schema var latestSchema = await schemaRegistry.GetLatestSchemaAsync(subject); var httpClientFactory = new HttpClientFactory(new Uri(ksqlDbUrl)); var restApiClient = new KSqlDbRestApiProvider(httpClientFactory); EntityCreationMetadata metadata = new() { EntityName = "avroSensors", KafkaTopic = "IoTSensors", ValueFormat = SerializationFormats.Avro, Partitions = 1, Replicas = 1 }; var httpResponseMessage = await restApiClient.CreateStreamAsync <IoTSensor>(metadata, ifNotExists : false) .ConfigureAwait(false); var httpResponse = await httpResponseMessage.Content.ReadAsStringAsync(); Console.WriteLine(httpResponse); // Stream Name | Kafka Topic | Key Format | Value Format | Windowed // ------------------------------------------------------------------------------------------------------ // AVROSENSORS | IoTSensors | KAFKA | AVRO | false } }
/// <summary> /// Creates a serializer. /// </summary> /// <param name="registryConfiguration"> /// Schema Registry configuration. Using the <see cref="SchemaRegistryConfig" /> class is /// highly recommended. /// </param> /// <param name="registerAutomatically"> /// Whether to automatically register schemas that match the type being serialized. /// </param> /// <param name="schemaBuilder"> /// The schema builder to use to create a schema for a C# type when registering automatically. /// If none is provided, the default schema builder will be used. /// </param> /// <param name="schemaReader"> /// The JSON schema reader to use to convert schemas received from the registry into abstract /// representations. If none is provided, the default schema reader will be used. /// </param> /// <param name="schemaWriter"> /// The JSON schema writer to use to convert abstract schema representations when registering /// automatically. If none is provided, the default schema writer will be used. /// </param> /// <param name="serializerBuilder"> /// The deserializer builder to use to build serialization functions for C# types. If none /// is provided, the default serializer builder will be used. /// </param> /// <param name="subjectNameBuilder"> /// A function that determines the subject name given the topic name and a component type /// (key or value). If none is provided, the default "{topic name}-{component}" naming /// convention will be used. /// </param> /// <exception cref="ArgumentNullException"> /// Thrown when the registry configuration is null. /// </exception> public AsyncSchemaRegistrySerializer( IEnumerable <KeyValuePair <string, string> > registryConfiguration, bool registerAutomatically = false, Abstract.ISchemaBuilder?schemaBuilder = null, IJsonSchemaReader?schemaReader = null, IJsonSchemaWriter?schemaWriter = null, IBinarySerializerBuilder?serializerBuilder = null, Func <SerializationContext, string>?subjectNameBuilder = null ) { if (registryConfiguration == null) { throw new ArgumentNullException(nameof(registryConfiguration)); } RegisterAutomatically = registerAutomatically; SchemaBuilder = schemaBuilder ?? new Abstract.SchemaBuilder(); SchemaReader = schemaReader ?? new JsonSchemaReader(); SchemaWriter = schemaWriter ?? new JsonSchemaWriter(); SerializerBuilder = serializerBuilder ?? new BinarySerializerBuilder(); SubjectNameBuilder = subjectNameBuilder ?? (c => $"{c.Topic}-{(c.Component == MessageComponentType.Key ? "key" : "value")}"); _cache = new ConcurrentDictionary <string, Task <Func <T, byte[]> > >(); _register = async(subject, json) => { using (var registry = new CachedSchemaRegistryClient(registryConfiguration)) { return(await registry.RegisterSchemaAsync(subject, json).ConfigureAwait(false)); } }; _resolve = async subject => { using (var registry = new CachedSchemaRegistryClient(registryConfiguration)) { return(await registry.GetLatestSchemaAsync(subject).ConfigureAwait(false)); } }; }
/// <summary> /// Creates a serializer. /// </summary> /// <param name="registryConfiguration"> /// Schema Registry configuration. Using the <see cref="SchemaRegistryConfig" /> class is /// highly recommended. /// </param> /// <param name="registerAutomatically"> /// Whether to automatically register schemas that match the type being serialized. /// </param> /// <param name="schemaBuilder"> /// A schema builder (used to build a schema for a C# type when registering automatically). /// If none is provided, the default schema builder will be used. /// </param> /// <param name="schemaReader"> /// A JSON schema reader (used to convert schemas received from the registry into abstract /// representations). If none is provided, the default schema reader will be used. /// </param> /// <param name="schemaWriter"> /// A JSON schema writer (used to convert abstract schema representations when registering /// automatically). If none is provided, the default schema writer will be used. /// </param> /// <param name="serializerBuilder"> /// A deserializer builder (used to build serialization functions for C# types). If none is /// provided, the default serializer builder will be used. /// </param> /// <param name="subjectNameBuilder"> /// A function that determines the subject name given the topic name and a component type /// (key or value). If none is provided, the default "{topic name}-{component}" naming /// convention will be used. /// </param> /// <exception cref="ArgumentNullException"> /// Thrown when the registry configuration is null. /// </exception> public AsyncSchemaRegistrySerializer( IEnumerable <KeyValuePair <string, string> > registryConfiguration, bool registerAutomatically = false, Abstract.ISchemaBuilder schemaBuilder = null, IJsonSchemaReader schemaReader = null, IJsonSchemaWriter schemaWriter = null, IBinarySerializerBuilder serializerBuilder = null, Func <SerializationContext, string> subjectNameBuilder = null ) : this( registerAutomatically, schemaBuilder, schemaReader, schemaWriter, serializerBuilder, subjectNameBuilder ) { if (registryConfiguration == null) { throw new ArgumentNullException(nameof(registryConfiguration)); } _register = async(subject, json) => { using (var registry = new CachedSchemaRegistryClient(registryConfiguration)) { return(await registry.RegisterSchemaAsync(subject, json).ConfigureAwait(false)); } }; _resolve = async subject => { using (var registry = new CachedSchemaRegistryClient(registryConfiguration)) { return(await registry.GetLatestSchemaAsync(subject).ConfigureAwait(false)); } }; }
public static void GetSchemaBySubjectAndVersion(Config config) { var topicName = Guid.NewGuid().ToString(); var testSchema1 = "{\"type\":\"record\",\"name\":\"User\",\"namespace\":\"Confluent.Kafka.Examples.AvroSpecific" + "\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"favorite_number\",\"type\":[\"i" + "nt\",\"null\"]},{\"name\":\"favorite_color\",\"type\":[\"string\",\"null\"]}]}"; var sr = new CachedSchemaRegistryClient(new SchemaRegistryConfig { Url = config.Server }); var subject = SubjectNameStrategy.Topic.ConstructValueSubjectName(topicName, null); var id = sr.RegisterSchemaAsync(subject, testSchema1).Result; var latestSchema = sr.GetLatestSchemaAsync(subject).Result; var schema = sr.GetRegisteredSchemaAsync(subject, latestSchema.Version).Result; Assert.Equal(schema.SchemaString, testSchema1); Assert.Equal(schema.SchemaType, SchemaType.Avro); Assert.Empty(schema.References); }
public static void GetLatestSchema(Config config) { var topicName = Guid.NewGuid().ToString(); var testSchema1 = "{\"type\":\"record\",\"name\":\"User\",\"namespace\":\"Confluent.Kafka.Examples.AvroSpecific" + "\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"favorite_number\",\"type\":[\"i" + "nt\",\"null\"]},{\"name\":\"favorite_color\",\"type\":[\"string\",\"null\"]}]}"; var sr = new CachedSchemaRegistryClient(new SchemaRegistryConfig { SchemaRegistryUrl = config.Server }); var subject = sr.ConstructValueSubjectName(topicName); var id = sr.RegisterSchemaAsync(subject, testSchema1).Result; var schema = sr.GetLatestSchemaAsync(subject).Result; Assert.Equal(schema.Id, id); Assert.Equal(schema.Subject, subject); Assert.Equal(schema.Version, 1); Assert.Equal(schema.SchemaString, testSchema1); }
static async Task Main(string[] args) { if (args.Length != 3) { Console.WriteLine("Usage: .. bootstrapServers schemaRegistryUrl topicName"); return; } string bootstrapServers = args[0]; string schemaRegistryUrl = args[1]; string topicName = args[2]; var producerConfig = new ProducerConfig { BootstrapServers = bootstrapServers }; var schemaRegistryConfig = new SchemaRegistryConfig { // Note: you can specify more than one schema registry url using the // schema.registry.url property for redundancy (comma separated list). // The property name is not plural to follow the convention set by // the Java implementation. Url = schemaRegistryUrl }; var consumerConfig = new ConsumerConfig { BootstrapServers = bootstrapServers, GroupId = "json-example-consumer-group" }; // Note: Specifying json serializer configuration is optional. var jsonSerializerConfig = new JsonSerializerConfig { BufferBytes = 100 }; CancellationTokenSource cts = new CancellationTokenSource(); var consumeTask = Task.Run(() => { using (var consumer = new ConsumerBuilder <string, User>(consumerConfig) .SetKeyDeserializer(Deserializers.Utf8) .SetValueDeserializer(new JsonDeserializer <User>().AsSyncOverAsync()) .SetErrorHandler((_, e) => Console.WriteLine($"Error: {e.Reason}")) .Build()) { consumer.Subscribe(topicName); try { while (true) { try { var cr = consumer.Consume(cts.Token); var user = cr.Message.Value; Console.WriteLine($"user name: {user.Name}, favorite number: {user.FavoriteNumber}, favorite color: {user.FavoriteColor}"); } catch (ConsumeException e) { Console.WriteLine($"Consume error: {e.Error.Reason}"); } } } catch (OperationCanceledException) { consumer.Close(); } } }); using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig)) using (var producer = new ProducerBuilder <string, User>(producerConfig) .SetValueSerializer(new JsonSerializer <User>(schemaRegistry, jsonSerializerConfig)) .Build()) { Console.WriteLine($"{producer.Name} producing on {topicName}. Enter first names, q to exit."); long i = 1; string text; while ((text = Console.ReadLine()) != "q") { User user = new User { Name = text, FavoriteColor = "blue", FavoriteNumber = i++ }; await producer .ProduceAsync(topicName, new Message <string, User> { Value = user }) .ContinueWith(task => task.IsFaulted ? $"error producing message: {task.Exception.Message}" : $"produced to: {task.Result.TopicPartitionOffset}"); } } cts.Cancel(); using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig)) { // Note: a subject name strategy was not configured, so the default "Topic" was used. var schema = await schemaRegistry.GetLatestSchemaAsync(SubjectNameStrategy.Topic.ConstructValueSubjectName(topicName)); Console.WriteLine("\nThe JSON schema corresponding to the written data:"); Console.WriteLine(schema.SchemaString); } }
public static void RegularAndAvro(string bootstrapServers, string schemaRegistryServers) { using (var topic1 = new TemporaryTopic(bootstrapServers, 1)) using (var topic2 = new TemporaryTopic(bootstrapServers, 1)) { var producerConfig = new ProducerConfig { BootstrapServers = bootstrapServers }; var consumerConfig = new ConsumerConfig { BootstrapServers = bootstrapServers, GroupId = Guid.NewGuid().ToString(), SessionTimeoutMs = 6000, AutoOffsetReset = AutoOffsetReset.Earliest }; var schemaRegistryConfig = new SchemaRegistryConfig { SchemaRegistryUrl = schemaRegistryServers }; using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig)) using (var producer = new ProducerBuilder <string, string>(producerConfig) .SetKeySerializer(Serializers.Utf8) .SetValueSerializer(new AvroSerializer <string>(schemaRegistry)) .Build()) { // implicit check that this does not fail. producer.ProduceAsync(topic1.Name, new Message <string, string> { Key = "hello", Value = "world" }).Wait(); // check that the value type was registered with SR, and the key was not. Assert.Throws <SchemaRegistryException>(() => { try { schemaRegistry.GetLatestSchemaAsync(schemaRegistry.ConstructKeySubjectName(topic1.Name)).Wait(); } catch (AggregateException e) { throw e.InnerException; } }); var s2 = schemaRegistry.GetLatestSchemaAsync(schemaRegistry.ConstructValueSubjectName(topic1.Name)).Result; } using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig)) using (var producer = new ProducerBuilder <string, string>(producerConfig) .SetKeySerializer(new AvroSerializer <string>(schemaRegistry)) .SetValueSerializer(Serializers.Utf8) .Build()) { // implicit check that this does not fail. producer.ProduceAsync(topic2.Name, new Message <string, string> { Key = "hello", Value = "world" }).Wait(); // check that the key type was registered with SR, and the value was not. Assert.Throws <SchemaRegistryException>(() => { try { schemaRegistry.GetLatestSchemaAsync(schemaRegistry.ConstructValueSubjectName(topic2.Name)).Wait(); } catch (AggregateException e) { throw e.InnerException; } }); var s2 = schemaRegistry.GetLatestSchemaAsync(schemaRegistry.ConstructKeySubjectName(topic2.Name)).Result; } // check the above can be consumed (using regular / Avro serializers as appropriate) using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig)) { using (var consumer = new ConsumerBuilder <string, string>(consumerConfig) .SetKeyDeserializer(Deserializers.Utf8) .SetValueDeserializer(new AvroDeserializer <string>(schemaRegistry)) .Build()) { consumer.Assign(new TopicPartitionOffset(topic1.Name, 0, 0)); var cr = consumer.Consume(); Assert.Equal("hello", cr.Key); Assert.Equal("world", cr.Value); } using (var consumer = new ConsumerBuilder <string, string>(consumerConfig) .SetKeyDeserializer(new AvroDeserializer <string>(schemaRegistry)) .SetValueDeserializer(Deserializers.Utf8).Build()) { consumer.Assign(new TopicPartitionOffset(topic2.Name, 0, 0)); var cr = consumer.Consume(); Assert.Equal("hello", cr.Key); Assert.Equal("world", cr.Value); } using (var consumer = new ConsumerBuilder <string, string>(consumerConfig) .SetKeyDeserializer(Deserializers.Utf8) .SetValueDeserializer(new AvroDeserializer <string>(schemaRegistry)) .Build()) { consumer.Assign(new TopicPartitionOffset(topic2.Name, 0, 0)); Assert.ThrowsAny <ConsumeException>(() => { try { consumer.Consume(); } catch (AggregateException e) { throw e.InnerException; } }); } using (var consumer = new ConsumerBuilder <string, string>(consumerConfig) .SetKeyDeserializer(new AvroDeserializer <string>(schemaRegistry)) .SetValueDeserializer(Deserializers.Utf8) .Build()) { consumer.Assign(new TopicPartitionOffset(topic1.Name, 0, 0)); Assert.ThrowsAny <ConsumeException>(() => { try { consumer.Consume(); } catch (AggregateException e) { throw e.InnerException; } }); } } } }
public static void BasicAuth(Config config) { var testSchema1 = "{\"type\":\"record\",\"name\":\"User\",\"namespace\":\"Confluent.Kafka.Examples.AvroSpecific" + "\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"favorite_number\",\"type\":[\"i" + "nt\",\"null\"]},{\"name\":\"favorite_color\",\"type\":[\"string\",\"null\"]}]}"; // 1. valid configuration cases // 1.1. credentials specified as USER_INFO. var conf = new Dictionary <string, string> { { "schema.registry.url", config.ServerWithAuth }, { "schema.registry.basic.auth.credentials.source", "USER_INFO" }, { "schema.registry.basic.auth.user.info", $"{config.Username}:{config.Password}" } }; using (var sr = new CachedSchemaRegistryClient(conf)) { var topicName = Guid.NewGuid().ToString(); var subject = sr.ConstructValueSubjectName(topicName); var id = sr.RegisterSchemaAsync(subject, testSchema1).Result; var schema = sr.GetLatestSchemaAsync(subject).Result; Assert.Equal(schema.Id, id); } // 1.2. credentials specified as USER_INFO implicitly (and using strongly typed SchemaRegistryConfig) var conf2 = new SchemaRegistryConfig { SchemaRegistryUrl = config.ServerWithAuth, SchemaRegistryBasicAuthUserInfo = $"{config.Username}:{config.Password}" }; using (var sr = new CachedSchemaRegistryClient(conf2)) { var topicName = Guid.NewGuid().ToString(); var subject = sr.ConstructValueSubjectName(topicName); var id = sr.RegisterSchemaAsync(subject, testSchema1).Result; var schema = sr.GetLatestSchemaAsync(subject).Result; Assert.Equal(schema.Id, id); } // 1.3. credentials specified as SASL_INHERIT. using (var sr = new CachedSchemaRegistryClient( new Dictionary <string, string> { { "schema.registry.url", config.ServerWithAuth }, { "schema.registry.basic.auth.credentials.source", "SASL_INHERIT" }, { "sasl.username", config.Username }, { "sasl.password", config.Password } })) { var topicName = Guid.NewGuid().ToString(); var subject = sr.ConstructValueSubjectName(topicName); var id = sr.RegisterSchemaAsync(subject, testSchema1).Result; var schema = sr.GetLatestSchemaAsync(subject).Result; Assert.Equal(schema.Id, id); } // 1.4. credentials specified as SASL_INHERIT via strongly typed config. var conf3 = new SchemaRegistryConfig { SchemaRegistryUrl = config.ServerWithAuth }; conf3.Set(SchemaRegistryConfig.PropertyNames.SchemaRegistryBasicAuthCredentialsSource, "SASL_INHERIT"); conf3.Set("sasl.username", config.Username); conf3.Set("sasl.password", config.Password); using (var sr = new CachedSchemaRegistryClient(conf3)) { var topicName = Guid.NewGuid().ToString(); var subject = sr.ConstructValueSubjectName(topicName); var id = sr.RegisterSchemaAsync(subject, testSchema1).Result; var schema = sr.GetLatestSchemaAsync(subject).Result; Assert.Equal(schema.Id, id); } // 2. invalid configuration cases Assert.Throws <ArgumentException>(() => { var sr = new CachedSchemaRegistryClient(new Dictionary <string, string> { { "schema.registry.url", config.ServerWithAuth }, { "schema.registry.basic.auth.credentials.source", "SASL_INHERIT" }, { "schema.registry.basic.auth.user.info", $"{config.Username:config.Password}" } }); }); Assert.Throws <ArgumentException>(() => { var sr = new CachedSchemaRegistryClient(new Dictionary <string, string> { { "schema.registry.url", config.ServerWithAuth }, { "schema.registry.basic.auth.credentials.source", "USER_INFO" }, { "sasl.username", config.Username }, { "sasl.password", config.Password } }); }); // connect to authenticating without credentials. shouldn't work. Assert.Throws <HttpRequestException>(() => { var sr = new CachedSchemaRegistryClient(new SchemaRegistryConfig { SchemaRegistryUrl = config.ServerWithAuth }); var topicName = Guid.NewGuid().ToString(); var subject = sr.ConstructValueSubjectName(topicName); try { var id = sr.RegisterSchemaAsync(subject, testSchema1).Result; } catch (Exception e) { throw e.InnerException; } }); }
private static void BMSpecificProduceConsume(string bootstrapServers, string schemaRegistryServers) { var producerConfig = new ProducerConfig { BootstrapServers = bootstrapServers }; var schemaRegistryConfig = new SchemaRegistryConfig { Url = schemaRegistryServers }; var adminClientConfig = new AdminClientConfig { BootstrapServers = bootstrapServers }; var topic = Guid.NewGuid().ToString(); Console.Error.WriteLine($"topic: {topic}"); using (var adminClient = new AdminClientBuilder(adminClientConfig).Build()) { adminClient.CreateTopicsAsync( new List <TopicSpecification> { new TopicSpecification { Name = topic, NumPartitions = 1, ReplicationFactor = 1 } }).Wait(); } var srClient = new CachedSchemaRegistryClient(schemaRegistryConfig); Schema schema1 = new Schema(EventA._SCHEMA.ToString(), SchemaType.Avro); Schema schema2 = new Schema(EventB._SCHEMA.ToString(), SchemaType.Avro); var id1 = srClient.RegisterSchemaAsync("events-a", schema1).Result; var id2 = srClient.RegisterSchemaAsync("events-b", schema2).Result; var avroUnion = @"[""Confluent.Kafka.Examples.AvroSpecific.EventA"",""Confluent.Kafka.Examples.AvroSpecific.EventB""]"; Schema unionSchema = new Schema(avroUnion, SchemaType.Avro); SchemaReference reference = new SchemaReference( "Confluent.Kafka.Examples.AvroSpecific.EventA", "events-a", srClient.GetLatestSchemaAsync("events-a").Result.Version); unionSchema.References.Add(reference); reference = new SchemaReference( "Confluent.Kafka.Examples.AvroSpecific.EventB", "events-b", srClient.GetLatestSchemaAsync("events-b").Result.Version); unionSchema.References.Add(reference); var id3 = srClient.RegisterSchemaAsync($"{topic}-value", unionSchema).Result; AvroSerializerConfig avroSerializerConfig = new AvroSerializerConfig { AutoRegisterSchemas = false, UseLatestSchemaVersion = true }; using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig)) using (var producer = new ProducerBuilder <string, ISpecificRecord>(producerConfig) .SetKeySerializer(new AvroSerializer <string>(schemaRegistry)) .SetValueSerializer(new BmSpecificSerializerImpl <ISpecificRecord>( schemaRegistry, false, 1024, SubjectNameStrategy.Topic.ToDelegate(), true)) .Build()) { for (int i = 0; i < 3; ++i) { var eventA = new EventA { A = "I'm event A", EventId = Guid.NewGuid().ToString(), EventType = "EventType-A", OccuredOn = DateTime.UtcNow.Ticks, }; producer.ProduceAsync(topic, new Message <string, ISpecificRecord> { Key = "DomainEvent", Value = eventA }).Wait(); } for (int i = 0; i < 3; ++i) { var eventB = new EventB { B = 123456987, EventId = Guid.NewGuid().ToString(), EventType = "EventType-B", OccuredOn = DateTime.UtcNow.Ticks, }; producer.ProduceAsync(topic, new Message <string, ISpecificRecord> { Key = "DomainEvent", Value = eventB }).Wait(); } Assert.Equal(0, producer.Flush(TimeSpan.FromSeconds(10))); } var consumerConfig = new ConsumerConfig { BootstrapServers = bootstrapServers, GroupId = Guid.NewGuid().ToString(), SessionTimeoutMs = 6000, AutoOffsetReset = AutoOffsetReset.Earliest, EnablePartitionEof = true }; using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig)) using (var consumer = new ConsumerBuilder <string, GenericRecord>(consumerConfig) .SetKeyDeserializer(new AvroDeserializer <string>(schemaRegistry).AsSyncOverAsync()) .SetValueDeserializer(new BmGenericDeserializerImpl(schemaRegistry).AsSyncOverAsync()) .SetErrorHandler((_, e) => Assert.True(false, e.Reason)) .Build()) { consumer.Subscribe(topic); int i = 0; while (true) { var record = consumer.Consume(TimeSpan.FromMilliseconds(100)); if (record == null) { continue; } if (record.IsPartitionEOF) { break; } Console.WriteLine(record.Message.Value["EventType"]); i += 1; } Assert.Equal(6, i); consumer.Close(); } }
public static void ProduceConsumeMixedJson(string bootstrapServers, string schemaRegistryServers) { var producerConfig = new ProducerConfig { BootstrapServers = bootstrapServers }; var schemaRegistryConfig = new SchemaRegistryConfig { Url = schemaRegistryServers }; using (var topic = new TemporaryTopic(bootstrapServers, 1)) using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig)) using (var producer = new ProducerBuilder <string, PersonPoco>(producerConfig) .SetValueSerializer(new JsonSerializer <PersonPoco>(schemaRegistry)) .Build()) { // test that a System.ComponentModel.DataAnnotations constraint is effective. { var p = new PersonPoco { FirstName = "Test", LastName = "User", NumberWithRange = 7 // range should be between 2 and 5. }; Assert.Throws <ProduceException <string, PersonPoco> >(() => { try { producer.ProduceAsync(topic.Name, new Message <string, PersonPoco> { Key = "test1", Value = p }).Wait(); } catch (AggregateException ax) { // what would be thrown if the call was awaited. throw ax.InnerException; } }); } // test that a Newtonsoft.Json constraint is effective. { var p = new PersonPoco { FirstName = "Test", // Omit LastName NumberWithRange = 3 }; Assert.Throws <AggregateException>(() => { producer.ProduceAsync(topic.Name, new Message <string, PersonPoco> { Key = "test1", Value = p }).Wait(); }); } // test all fields valid. { var p = new PersonPoco { FirstName = "A", MiddleName = "Test", LastName = "User", Gender = GenderEnum.Male, NumberWithRange = 3, Birthday = new DateTime(2010, 11, 1), Company = new CompanyPoco { Name = "Confluent" }, Cars = new List <CarPoco> { new CarPoco { Name = "Volvo", Manufacturer = new CompanyPoco { Name = "Confluent" } } } }; producer.ProduceAsync(topic.Name, new Message <string, PersonPoco> { Key = "test1", Value = p }).Wait(); var schema = schemaRegistry.GetLatestSchemaAsync(SubjectNameStrategy.Topic.ConstructValueSubjectName(topic.Name, null)).Result.SchemaString; var consumerConfig = new ConsumerConfig { BootstrapServers = bootstrapServers, GroupId = Guid.NewGuid().ToString(), AutoOffsetReset = AutoOffsetReset.Earliest }; using (var consumer = new ConsumerBuilder <string, PersonPoco>(consumerConfig) .SetValueDeserializer(new JsonDeserializer <PersonPoco>().AsSyncOverAsync()) .Build()) { consumer.Subscribe(topic.Name); var cr = consumer.Consume(); Assert.Equal(p.FirstName, cr.Message.Value.FirstName); Assert.Equal(p.MiddleName, cr.Message.Value.MiddleName); Assert.Equal(p.LastName, cr.Message.Value.LastName); Assert.Equal(p.Gender, cr.Message.Value.Gender); Assert.Equal(p.NumberWithRange, cr.Message.Value.NumberWithRange); Assert.Equal(p.Birthday, cr.Message.Value.Birthday); Assert.Equal(p.Company.Name, cr.Message.Value.Company.Name); Assert.Single(cr.Message.Value.Cars); Assert.Equal(cr.Message.Value.Cars[0].Manufacturer.Name, p.Cars[0].Manufacturer.Name); Assert.Equal(cr.Message.Value.Cars[0].Name, p.Cars[0].Name); } } } }
public static void BasicAuth(Config config) { var testSchema1 = "{\"type\":\"record\",\"name\":\"User\",\"namespace\":\"Confluent.Kafka.Examples.AvroSpecific" + "\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"favorite_number\",\"type\":[\"i" + "nt\",\"null\"]},{\"name\":\"favorite_color\",\"type\":[\"string\",\"null\"]}]}"; // 1. valid configuration cases // 1.1. credentials specified as USER_INFO. var conf = new SchemaRegistryConfig { Url = config.ServerWithAuth, BasicAuthCredentialsSource = AuthCredentialsSource.UserInfo, BasicAuthUserInfo = $"{config.Username}:{config.Password}" }; // some sanity checking of strongly typed config property name mappings. Assert.Equal(config.ServerWithAuth, conf.Get("schema.registry.url")); Assert.Equal("USER_INFO", conf.Get("schema.registry.basic.auth.credentials.source")); Assert.Equal($"{config.Username}:{config.Password}", conf.Get("schema.registry.basic.auth.user.info")); using (var sr = new CachedSchemaRegistryClient(conf)) { var topicName = Guid.NewGuid().ToString(); var subject = SubjectNameStrategy.Topic.ToDelegate()(new SerializationContext(MessageComponentType.Value, topicName), null); var id = sr.RegisterSchemaAsync(subject, testSchema1).Result; var schema = sr.GetLatestSchemaAsync(subject).Result; Assert.Equal(schema.Id, id); } // 1.2. credentials specified as USER_INFO implicitly (and using strongly typed SchemaRegistryConfig) var conf2 = new SchemaRegistryConfig { Url = config.ServerWithAuth, BasicAuthUserInfo = $"{config.Username}:{config.Password}" }; using (var sr = new CachedSchemaRegistryClient(conf2)) { var topicName = Guid.NewGuid().ToString(); var subject = SubjectNameStrategy.Topic.ConstructValueSubjectName(topicName, null); var id = sr.RegisterSchemaAsync(subject, testSchema1).Result; var schema = sr.GetLatestSchemaAsync(subject).Result; Assert.Equal(schema.Id, id); } // 1.3. credentials specified as SASL_INHERIT. using (var sr = new CachedSchemaRegistryClient( new Dictionary <string, string> { { "schema.registry.url", config.ServerWithAuth }, { "schema.registry.basic.auth.credentials.source", "SASL_INHERIT" }, { "sasl.username", config.Username }, { "sasl.password", config.Password } })) { var topicName = Guid.NewGuid().ToString(); var subject = SubjectNameStrategy.Topic.ConstructValueSubjectName(topicName, null); var id = sr.RegisterSchemaAsync(subject, testSchema1).Result; var schema = sr.GetLatestSchemaAsync(subject).Result; Assert.Equal(schema.Id, id); } // 1.4. credentials specified as SASL_INHERIT via strongly typed config. var conf3 = new SchemaRegistryConfig { Url = config.ServerWithAuth }; conf3.BasicAuthCredentialsSource = AuthCredentialsSource.SaslInherit; conf3.Set("sasl.username", config.Username); conf3.Set("sasl.password", config.Password); using (var sr = new CachedSchemaRegistryClient(conf3)) { var topicName = Guid.NewGuid().ToString(); var subject = SubjectNameStrategy.Topic.ConstructValueSubjectName(topicName, null); var id = sr.RegisterSchemaAsync(subject, testSchema1).Result; var schema = sr.GetLatestSchemaAsync(subject).Result; Assert.Equal(schema.Id, id); } // 2. invalid configuration cases Assert.Throws <ArgumentException>(() => { var sr = new CachedSchemaRegistryClient(new Dictionary <string, string> { { "schema.registry.url", config.ServerWithAuth }, { "schema.registry.basic.auth.credentials.source", "SASL_INHERIT" }, { "schema.registry.basic.auth.user.info", $"{config.Username:config.Password}" } }); }); Assert.Throws <ArgumentException>(() => { var sr = new CachedSchemaRegistryClient(new Dictionary <string, string> { { "schema.registry.url", config.ServerWithAuth }, { "schema.registry.basic.auth.credentials.source", "UBUTE_SOURCE" } }); }); Assert.Throws <ArgumentException>(() => { var sr = new CachedSchemaRegistryClient(new Dictionary <string, string> { { "schema.registry.url", config.ServerWithAuth }, { "schema.registry.basic.auth.credentials.source", "NONE" }, { "schema.registry.basic.auth.user.info", $"{config.Username:config.Password}" } }); }); // connect to authenticating without credentials. shouldn't work. // SR <= 5.3.4 returns Unauthorized with empty Content (HttpRequestException) // 5.3.4 < SR <= 5.3.8 returns Unauthorized with message but without error_code (SchemaRegistryException) // SR >= 5.40 returns Unauthorized with message and error_code (SchemaRegistryException) var schemaRegistryException = Assert.Throws <SchemaRegistryException>(() => { var sr = new CachedSchemaRegistryClient(new SchemaRegistryConfig { Url = config.ServerWithAuth }); var topicName = Guid.NewGuid().ToString(); var subject = SubjectNameStrategy.Topic.ConstructValueSubjectName(topicName, null); try { var id = sr.RegisterSchemaAsync(subject, testSchema1).Result; } catch (Exception e) { throw e.InnerException; } }); Assert.Equal(401, schemaRegistryException.ErrorCode); Assert.Equal("Unauthorized; error code: 401", schemaRegistryException.Message); }
public static void Protobuf(Config config) { var srInitial = new CachedSchemaRegistryClient(new SchemaRegistryConfig { Url = config.Server }); var sr = new CachedSchemaRegistryClient(new SchemaRegistryConfig { Url = config.Server }); var testSchemaBase64 = Confluent.Kafka.Examples.Protobuf.User.Descriptor.File.SerializedData.ToBase64(); var topicName = Guid.NewGuid().ToString(); var subjectInitial = SubjectNameStrategy.Topic.ConstructValueSubjectName(topicName, null); var subject = SubjectNameStrategy.Topic.ConstructValueSubjectName(topicName + "2", null); // check that registering a base64 protobuf schema works. var id1 = srInitial.RegisterSchemaAsync(subjectInitial, new Schema(testSchemaBase64, SchemaType.Protobuf)).Result; var schema1 = sr.GetSchemaAsync(id1, "serialized").Result; // use a different sr instance to ensure a cached value is not read. Assert.Equal(SchemaType.Protobuf, schema1.SchemaType); Assert.NotNull(schema1.SchemaString); // SR slightly munges the schema as a result of moving to a text representation and back so can't check for equality. // check that the id of the schema just registered can be retrieved. var id = sr.GetSchemaIdAsync(subjectInitial, new Schema(schema1.SchemaString, SchemaType.Protobuf)).Result; Assert.Equal(id1, id); // re-register the munged schema (to a different subject) and check that it is not re-munged. var id2 = sr.RegisterSchemaAsync(subject, schema1).Result; var schema2 = sr.GetSchemaAsync(id2, "serialized").Result; Assert.Equal(schema1.SchemaString, schema2.SchemaString); Assert.Equal(schema1.SchemaType, schema2.SchemaType); // This sequence of operations is designed to test caching behavior (and two alternat schema getting methods). var schemaAsText = sr.GetSchemaAsync(id2).Result; var schemaAsSerialized = sr.GetSchemaAsync(id2, "serialized").Result; var schemaAsText2 = sr.GetSchemaAsync(id2).Result; var schemaAsSerialized2 = sr.GetSchemaAsync(id2, "serialized").Result; var latestSchema = sr.GetLatestSchemaAsync(subject).Result; // should come back as text. var schemaAsSerialized3 = sr.GetSchemaAsync(id2, "serialized").Result; var latestSchema2 = sr.GetRegisteredSchemaAsync(subject, latestSchema.Version).Result; // should come back as text. Assert.Equal(schema1.SchemaString, schemaAsSerialized.SchemaString); Assert.Equal(SchemaType.Protobuf, schemaAsSerialized.SchemaType); Assert.Empty(schemaAsSerialized.References); Assert.Equal(schema1.SchemaString, schemaAsSerialized2.SchemaString); Assert.Equal(schema1.SchemaString, schemaAsSerialized3.SchemaString); Assert.NotEqual(schema1.SchemaString, schemaAsText.SchemaString); Assert.Equal(schemaAsText.SchemaString, schemaAsText2.SchemaString); Assert.Equal(schemaAsText.SchemaString, latestSchema.SchemaString); Assert.Equal(schemaAsText.SchemaString, latestSchema2.SchemaString); Assert.Equal(SchemaType.Protobuf, schemaAsText.SchemaType); Assert.Empty(schemaAsText2.References); // compatibility var compat = sr.IsCompatibleAsync(subject, schema2).Result; Assert.True(compat); var avroSchema = "{\"type\":\"record\",\"name\":\"User\",\"namespace\":\"Confluent.Kafka.Examples.AvroSpecific" + "\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"favorite_number\",\"type\":[\"i" + "nt\",\"null\"]},{\"name\":\"favorite_color\",\"type\":[\"string\",\"null\"]}]}"; var compat2 = sr.IsCompatibleAsync(subject, avroSchema).Result; Assert.False(compat2); var compat3 = sr.IsCompatibleAsync(subject, new Schema(avroSchema, SchemaType.Avro)).Result; Assert.False(compat3); // invalid type Assert.ThrowsAny <Exception>(() => { sr.RegisterSchemaAsync(SubjectNameStrategy.Topic.ConstructKeySubjectName(topicName + "3", null), new Schema(avroSchema, SchemaType.Protobuf)).Wait(); }); }