public static void FillTheCache(Config config) { const int capacity = 16; const string testSchema = "{\"type\":\"record\",\"name\":\"User\",\"namespace\":\"Confluent.Kafka.Examples.AvroSpecific" + "\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"favorite_number\",\"type\":[\"i" + "nt\",\"null\"]},{\"name\":\"favorite_color\",\"type\":[\"string\",\"null\"]}]}"; var srConfig = new SchemaRegistryConfig { Url = config.Server, RequestTimeoutMs = 3000, MaxCachedSchemas = capacity }; var sr = new CachedSchemaRegistryClient(srConfig); var registerCount = capacity + 10; var subjects = new List <string>(); var ids = new List <int>(); // test is that this does not throw. Also, inspect in debugger. for (int i = 0; i < registerCount; ++i) { var topicName = Guid.NewGuid().ToString(); var subject = sr.ConstructValueSubjectName(topicName); subjects.Add(subject); var id = sr.RegisterSchemaAsync(subject, testSchema).Result; ids.Add(id); } }
public static void FillTheCache(string server) { const int capacity = 16; const string testSchema = "{\"type\":\"record\",\"name\":\"User\",\"namespace\":\"Confluent.Kafka.Examples.AvroSpecific" + "\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"favorite_number\",\"type\":[\"i" + "nt\",\"null\"]},{\"name\":\"favorite_color\",\"type\":[\"string\",\"null\"]}]}"; var config = new Dictionary <string, object> { { "schema.registry.url", server }, { "schema.registry.connection.timeout.ms", 3000 }, { "schema.registry.max.cached.schemas", capacity } }; var sr = new CachedSchemaRegistryClient(config); var registerCount = capacity + 10; var subjects = new List <string>(); var ids = new List <int>(); // test is that this does not throw. Also, inspect in debugger. for (int i = 0; i < registerCount; ++i) { var topicName = Guid.NewGuid().ToString(); var subject = sr.ConstructValueSubjectName(topicName); subjects.Add(subject); var id = sr.RegisterSchemaAsync(subject, testSchema).Result; ids.Add(id); } }
public static void GetSubjectVersions(Config config) { var topicName = Guid.NewGuid().ToString(); var testSchema1 = "{\"type\":\"record\",\"name\":\"User\",\"namespace\":\"Confluent.Kafka.Examples.AvroSpecific" + "\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"favorite_number\",\"type\":[\"i" + "nt\",\"null\"]},{\"name\":\"favorite_color\",\"type\":[\"string\",\"null\"]}]}"; var testSchema2 = // compatible with testSchema1 "{\"type\":\"record\",\"name\":\"User\",\"namespace\":\"Confluent.Kafka.Examples.AvroSpecific" + "\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"favorite_number\",\"type\":[\"i" + "nt\",\"null\"]},{\"name\":\"favorite_color\",\"type\":[\"string\",\"null\"]}," + "{\"name\":\"favorite_shape\",\"type\":[\"string\",\"null\"], \"default\": \"square\"}]}"; var sr = new CachedSchemaRegistryClient(new SchemaRegistryConfig { Url = config.Server }); var subject = sr.ConstructValueSubjectName(topicName); var id1 = sr.RegisterSchemaAsync(subject, testSchema1).Result; var id2 = sr.RegisterSchemaAsync(subject, testSchema2).Result; var versions = sr.GetSubjectVersionsAsync(subject).Result; Assert.Equal(versions.Count, 2); }
public void ConstructValueSubjectName_Topic1() { var config = new SchemaRegistryConfig { Url = "irrelevanthost:8081" }; var src = new CachedSchemaRegistryClient(config); Assert.Equal("mytopic-value", src.ConstructValueSubjectName("mytopic", "myschemaname")); }
public void ConstructValueSubjectName() { var config = new Dictionary <string, object> { { "schema.registry.url", "irrelevanthost:8081" } }; CachedSchemaRegistryClient src = new CachedSchemaRegistryClient(config); Assert.Equal("mytopic-value", src.ConstructValueSubjectName("mytopic")); }
public void ConstructValueSubjectName_Record() { var config = new SchemaRegistryConfig { Url = "irrelevanthost:8081", ValueSubjectNameStrategy = SubjectNameStrategy.Record }; var src = new CachedSchemaRegistryClient(config); Assert.Equal("myschemaname", src.ConstructValueSubjectName("mytopic", "myschemaname")); }
private static void GetSchemaBySubjectAndVersion(Dictionary <string, object> config) { var topicName = Guid.NewGuid().ToString(); var testSchema1 = "{\"type\":\"record\",\"name\":\"User\",\"namespace\":\"Confluent.Kafka.Examples.AvroSpecific" + "\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"favorite_number\",\"type\":[\"i" + "nt\",\"null\"]},{\"name\":\"favorite_color\",\"type\":[\"string\",\"null\"]}]}"; var sr = new CachedSchemaRegistryClient(config); var subject = sr.ConstructValueSubjectName(topicName, "Confluent.Kafka.Examples.AvroSpecific.User"); var id = sr.RegisterSchemaAsync(subject, testSchema1).Result; var schema = sr.GetLatestSchemaAsync(subject).Result; var schemaString = sr.GetSchemaAsync(subject, schema.Version).Result; Assert.Equal(schemaString, testSchema1); }
public static void GetSchemaById(string server) { var topicName = Guid.NewGuid().ToString(); var testSchema1 = "{\"type\":\"record\",\"name\":\"User\",\"namespace\":\"Confluent.Kafka.Examples.AvroSpecific" + "\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"favorite_number\",\"type\":[\"i" + "nt\",\"null\"]},{\"name\":\"favorite_color\",\"type\":[\"string\",\"null\"]}]}"; var sr = new CachedSchemaRegistryClient(new Dictionary <string, object> { { "schema.registry.url", server } }); var subject = sr.ConstructValueSubjectName(topicName); var id = sr.RegisterSchemaAsync(subject, testSchema1).Result; var schema = sr.GetSchemaAsync(id).Result; Assert.Equal(schema, testSchema1); }
public void ConstructValueSubjectName() { var config = new Dictionary <string, object> { { "schema.registry.url", "irrelevanthost:8081" } }; CachedSchemaRegistryClient src = new CachedSchemaRegistryClient(config); Assert.Equal("mytopic-value", src.ConstructValueSubjectName("mytopic", "myschemaname")); var configTopicName = new Dictionary <string, object> { { "schema.registry.url", "irrelevanthost:8081" }, { "schema.registry.subject.name.strategy", "topic_name_strategy" } }; CachedSchemaRegistryClient srcTopicName = new CachedSchemaRegistryClient(configTopicName); Assert.Equal("mytopic-value", srcTopicName.ConstructValueSubjectName("mytopic", "myschemaname")); var configRecordName = new Dictionary <string, object> { { "schema.registry.url", "irrelevanthost:8081" }, { "schema.registry.subject.name.strategy", "record_name_strategy" } }; CachedSchemaRegistryClient srcRecordName = new CachedSchemaRegistryClient(configRecordName); Assert.Equal("myschemaname", srcRecordName.ConstructValueSubjectName("mytopic", "myschemaname")); var configTopicRecordName = new Dictionary <string, object> { { "schema.registry.url", "irrelevanthost:8081" }, { "schema.registry.subject.name.strategy", "topic_record_name_strategy" } }; CachedSchemaRegistryClient srcTopicRecordName = new CachedSchemaRegistryClient(configTopicRecordName); Assert.Equal("mytopic-myschemaname", srcTopicRecordName.ConstructValueSubjectName("mytopic", "myschemaname")); }
public static void GetLatestSchema(Config config) { var topicName = Guid.NewGuid().ToString(); var testSchema1 = "{\"type\":\"record\",\"name\":\"User\",\"namespace\":\"Confluent.Kafka.Examples.AvroSpecific" + "\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"favorite_number\",\"type\":[\"i" + "nt\",\"null\"]},{\"name\":\"favorite_color\",\"type\":[\"string\",\"null\"]}]}"; var sr = new CachedSchemaRegistryClient(new SchemaRegistryConfig { SchemaRegistryUrl = config.Server }); var subject = sr.ConstructValueSubjectName(topicName); var id = sr.RegisterSchemaAsync(subject, testSchema1).Result; var schema = sr.GetLatestSchemaAsync(subject).Result; Assert.Equal(schema.Id, id); Assert.Equal(schema.Subject, subject); Assert.Equal(schema.Version, 1); Assert.Equal(schema.SchemaString, testSchema1); }
public static void RegularAndAvro(string bootstrapServers, string schemaRegistryServers) { using (var topic1 = new TemporaryTopic(bootstrapServers, 1)) using (var topic2 = new TemporaryTopic(bootstrapServers, 1)) { var producerConfig = new ProducerConfig { BootstrapServers = bootstrapServers }; var consumerConfig = new ConsumerConfig { BootstrapServers = bootstrapServers, GroupId = Guid.NewGuid().ToString(), SessionTimeoutMs = 6000, AutoOffsetReset = AutoOffsetReset.Earliest }; var schemaRegistryConfig = new SchemaRegistryConfig { SchemaRegistryUrl = schemaRegistryServers }; using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig)) using (var producer = new ProducerBuilder <string, string>(producerConfig) .SetKeySerializer(Serializers.Utf8) .SetValueSerializer(new AvroSerializer <string>(schemaRegistry)) .Build()) { // implicit check that this does not fail. producer.ProduceAsync(topic1.Name, new Message <string, string> { Key = "hello", Value = "world" }).Wait(); // check that the value type was registered with SR, and the key was not. Assert.Throws <SchemaRegistryException>(() => { try { schemaRegistry.GetLatestSchemaAsync(schemaRegistry.ConstructKeySubjectName(topic1.Name)).Wait(); } catch (AggregateException e) { throw e.InnerException; } }); var s2 = schemaRegistry.GetLatestSchemaAsync(schemaRegistry.ConstructValueSubjectName(topic1.Name)).Result; } using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig)) using (var producer = new ProducerBuilder <string, string>(producerConfig) .SetKeySerializer(new AvroSerializer <string>(schemaRegistry)) .SetValueSerializer(Serializers.Utf8) .Build()) { // implicit check that this does not fail. producer.ProduceAsync(topic2.Name, new Message <string, string> { Key = "hello", Value = "world" }).Wait(); // check that the key type was registered with SR, and the value was not. Assert.Throws <SchemaRegistryException>(() => { try { schemaRegistry.GetLatestSchemaAsync(schemaRegistry.ConstructValueSubjectName(topic2.Name)).Wait(); } catch (AggregateException e) { throw e.InnerException; } }); var s2 = schemaRegistry.GetLatestSchemaAsync(schemaRegistry.ConstructKeySubjectName(topic2.Name)).Result; } // check the above can be consumed (using regular / Avro serializers as appropriate) using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig)) { using (var consumer = new ConsumerBuilder <string, string>(consumerConfig) .SetKeyDeserializer(Deserializers.Utf8) .SetValueDeserializer(new AvroDeserializer <string>(schemaRegistry)) .Build()) { consumer.Assign(new TopicPartitionOffset(topic1.Name, 0, 0)); var cr = consumer.Consume(); Assert.Equal("hello", cr.Key); Assert.Equal("world", cr.Value); } using (var consumer = new ConsumerBuilder <string, string>(consumerConfig) .SetKeyDeserializer(new AvroDeserializer <string>(schemaRegistry)) .SetValueDeserializer(Deserializers.Utf8).Build()) { consumer.Assign(new TopicPartitionOffset(topic2.Name, 0, 0)); var cr = consumer.Consume(); Assert.Equal("hello", cr.Key); Assert.Equal("world", cr.Value); } using (var consumer = new ConsumerBuilder <string, string>(consumerConfig) .SetKeyDeserializer(Deserializers.Utf8) .SetValueDeserializer(new AvroDeserializer <string>(schemaRegistry)) .Build()) { consumer.Assign(new TopicPartitionOffset(topic2.Name, 0, 0)); Assert.ThrowsAny <ConsumeException>(() => { try { consumer.Consume(); } catch (AggregateException e) { throw e.InnerException; } }); } using (var consumer = new ConsumerBuilder <string, string>(consumerConfig) .SetKeyDeserializer(new AvroDeserializer <string>(schemaRegistry)) .SetValueDeserializer(Deserializers.Utf8) .Build()) { consumer.Assign(new TopicPartitionOffset(topic1.Name, 0, 0)); Assert.ThrowsAny <ConsumeException>(() => { try { consumer.Consume(); } catch (AggregateException e) { throw e.InnerException; } }); } } } }
public static void BasicAuth(Config config) { var testSchema1 = "{\"type\":\"record\",\"name\":\"User\",\"namespace\":\"Confluent.Kafka.Examples.AvroSpecific" + "\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"favorite_number\",\"type\":[\"i" + "nt\",\"null\"]},{\"name\":\"favorite_color\",\"type\":[\"string\",\"null\"]}]}"; // 1. valid configuration cases // 1.1. credentials specified as USER_INFO. var conf = new Dictionary <string, string> { { "schema.registry.url", config.ServerWithAuth }, { "schema.registry.basic.auth.credentials.source", "USER_INFO" }, { "schema.registry.basic.auth.user.info", $"{config.Username}:{config.Password}" } }; using (var sr = new CachedSchemaRegistryClient(conf)) { var topicName = Guid.NewGuid().ToString(); var subject = sr.ConstructValueSubjectName(topicName); var id = sr.RegisterSchemaAsync(subject, testSchema1).Result; var schema = sr.GetLatestSchemaAsync(subject).Result; Assert.Equal(schema.Id, id); } // 1.2. credentials specified as USER_INFO implicitly (and using strongly typed SchemaRegistryConfig) var conf2 = new SchemaRegistryConfig { SchemaRegistryUrl = config.ServerWithAuth, SchemaRegistryBasicAuthUserInfo = $"{config.Username}:{config.Password}" }; using (var sr = new CachedSchemaRegistryClient(conf2)) { var topicName = Guid.NewGuid().ToString(); var subject = sr.ConstructValueSubjectName(topicName); var id = sr.RegisterSchemaAsync(subject, testSchema1).Result; var schema = sr.GetLatestSchemaAsync(subject).Result; Assert.Equal(schema.Id, id); } // 1.3. credentials specified as SASL_INHERIT. using (var sr = new CachedSchemaRegistryClient( new Dictionary <string, string> { { "schema.registry.url", config.ServerWithAuth }, { "schema.registry.basic.auth.credentials.source", "SASL_INHERIT" }, { "sasl.username", config.Username }, { "sasl.password", config.Password } })) { var topicName = Guid.NewGuid().ToString(); var subject = sr.ConstructValueSubjectName(topicName); var id = sr.RegisterSchemaAsync(subject, testSchema1).Result; var schema = sr.GetLatestSchemaAsync(subject).Result; Assert.Equal(schema.Id, id); } // 1.4. credentials specified as SASL_INHERIT via strongly typed config. var conf3 = new SchemaRegistryConfig { SchemaRegistryUrl = config.ServerWithAuth }; conf3.Set(SchemaRegistryConfig.PropertyNames.SchemaRegistryBasicAuthCredentialsSource, "SASL_INHERIT"); conf3.Set("sasl.username", config.Username); conf3.Set("sasl.password", config.Password); using (var sr = new CachedSchemaRegistryClient(conf3)) { var topicName = Guid.NewGuid().ToString(); var subject = sr.ConstructValueSubjectName(topicName); var id = sr.RegisterSchemaAsync(subject, testSchema1).Result; var schema = sr.GetLatestSchemaAsync(subject).Result; Assert.Equal(schema.Id, id); } // 2. invalid configuration cases Assert.Throws <ArgumentException>(() => { var sr = new CachedSchemaRegistryClient(new Dictionary <string, string> { { "schema.registry.url", config.ServerWithAuth }, { "schema.registry.basic.auth.credentials.source", "SASL_INHERIT" }, { "schema.registry.basic.auth.user.info", $"{config.Username:config.Password}" } }); }); Assert.Throws <ArgumentException>(() => { var sr = new CachedSchemaRegistryClient(new Dictionary <string, string> { { "schema.registry.url", config.ServerWithAuth }, { "schema.registry.basic.auth.credentials.source", "USER_INFO" }, { "sasl.username", config.Username }, { "sasl.password", config.Password } }); }); // connect to authenticating without credentials. shouldn't work. Assert.Throws <HttpRequestException>(() => { var sr = new CachedSchemaRegistryClient(new SchemaRegistryConfig { SchemaRegistryUrl = config.ServerWithAuth }); var topicName = Guid.NewGuid().ToString(); var subject = sr.ConstructValueSubjectName(topicName); try { var id = sr.RegisterSchemaAsync(subject, testSchema1).Result; } catch (Exception e) { throw e.InnerException; } }); }