public static void GetAllSubjects(Config config)
{
var topicName = Guid.NewGuid().ToString();
var testSchema1 =
"{\"type\":\"record\",\"name\":\"User\",\"namespace\":\"Confluent.Kafka.Examples.AvroSpecific" +
"\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"favorite_number\",\"type\":[\"i" +
"nt\",\"null\"]},{\"name\":\"favorite_color\",\"type\":[\"string\",\"null\"]}]}";
var sr = new CachedSchemaRegistryClient(new SchemaRegistryConfig {
Url = config.Server
});
var subjectsBefore = sr.GetAllSubjectsAsync().Result;
var subject = SubjectNameStrategy.Topic.ConstructKeySubjectName(topicName, null);
var id = sr.RegisterSchemaAsync(subject, testSchema1).Result;
var subjectsAfter = sr.GetAllSubjectsAsync().Result;
Assert.Equal(1, subjectsAfter.Count - subjectsBefore.Count);
sr.RegisterSchemaAsync(subject, testSchema1).Wait();
var subjectsAfter2 = sr.GetAllSubjectsAsync().Result;
Assert.Equal(subjectsAfter.Count, subjectsAfter2.Count);
Assert.True(subjectsAfter2.Contains(subject));
}
public static void RegisterIncompatibleSchema(string server)
{
var topicName = Guid.NewGuid().ToString();
var testSchema1 =
"{\"type\":\"record\",\"name\":\"User\",\"namespace\":\"Confluent.Kafka.Examples.AvroSpecific" +
"\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"favorite_number\",\"type\":[\"i" +
"nt\",\"null\"]},{\"name\":\"favorite_color\",\"type\":[\"string\",\"null\"]}]}";
var sr = new CachedSchemaRegistryClient(new Dictionary <string, object> {
{ "schema.registry.url", server }
});
var subject = sr.ConstructKeySubjectName(topicName, "Confluent.Kafka.Examples.AvroSpecific.User");
var id = sr.RegisterSchemaAsync(subject, testSchema1).Result;
var testSchema2 = // incompatible with testSchema1
"{\"type\":\"record\",\"name\":\"User\",\"namespace\":\"Confluent.Kafka.Examples.AvroSpecific" +
"\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"favorite_number\",\"type\":[\"i" +
"nt\",\"null\"]},{\"name\":\"favorite_shape\",\"type\":[\"string\",\"null\"]}]}";
Assert.False(sr.IsCompatibleAsync(subject, testSchema2).Result);
Assert.Throws <AggregateException>(() => sr.RegisterSchemaAsync(subject, testSchema2).Result);
Assert.True(sr.GetAllSubjectsAsync().Result.Contains(subject));
}
public async Task It_should_deserialize_the_envelope_correctly()
{
var schemaRegistry = new CachedSchemaRegistryClient
(
new SchemaRegistryConfig {
SchemaRegistryUrl = _kafkaConfig.SchemaRegistryUrl
}
);
//var type = AvroMultipleDeserializer.Get(schema.Name);
var subjects = await schemaRegistry.GetAllSubjectsAsync();
subjects.Count.ShouldBePositive();
foreach (var subject in subjects)
{
var schemas = await schemaRegistry.GetLatestSchemaAsync(subject);
var schemaSubject = await schemaRegistry.GetSchemaAsync(1);
var schema = global::Avro.Schema.Parse(schemaSubject);
}
}
public async void testSchemaRegistry()
{
try{
var lst = await schemaRegistry.GetAllSubjectsAsync();
}
catch (Exception e) {
log.Fatal("Problem in initializing Confluent Schema Registry: " + e.Message);
cancel.Cancel();
}
}
Task CheckBrokerReady()
{
return(Retry.Interval(10, 5000).Retry(async() =>
{
using var client = new CachedSchemaRegistryClient(new Dictionary <string, string>
{
{ "schema.registry.url", "localhost:8081" },
});
await client.GetAllSubjectsAsync();
var clientConfig = new ClientConfig {
BootstrapServers = "localhost:9092"
};
using var adminClient = new AdminClientBuilder(clientConfig).Build();
adminClient.GetMetadata(TimeSpan.FromSeconds(60));
}));
}
public static void RegisterSameSchemaTwice(string server)
{
var topicName = Guid.NewGuid().ToString();
var testSchema1 =
"{\"type\":\"record\",\"name\":\"User\",\"namespace\":\"Confluent.Kafka.Examples.AvroSpecific" +
"\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"favorite_number\",\"type\":[\"i" +
"nt\",\"null\"]},{\"name\":\"favorite_color\",\"type\":[\"string\",\"null\"]}]}";
var sr = new CachedSchemaRegistryClient(new Dictionary <string, object> {
{ "schema.registry.url", server }
});
var subject = sr.ConstructKeySubjectName(topicName);
Assert.Equal(topicName + "-key", subject);
var id1 = sr.RegisterSchemaAsync(subject, testSchema1).Result;
var id2 = sr.RegisterSchemaAsync(subject, testSchema1).Result;
Assert.Equal(id1, id2);
Assert.True(sr.GetAllSubjectsAsync().Result.Contains(subject));
}
public static void SubjectNameStrategiesProtobuf(string bootstrapServers, string schemaRegistryServers)
{
var producerConfig = new ProducerConfig {
BootstrapServers = bootstrapServers
};
var schemaRegistryConfig = new SchemaRegistryConfig {
Url = schemaRegistryServers
};
using (var topic = new TemporaryTopic(bootstrapServers, 1))
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
{
using (var producer =
new ProducerBuilder <string, UInt32Value>(producerConfig)
.SetValueSerializer(new ProtobufSerializer <UInt32Value>(schemaRegistry, new ProtobufSerializerConfig {
SubjectNameStrategy = SubjectNameStrategy.TopicRecord
}))
.Build())
{
var u = new UInt32Value();
u.Value = 42;
producer.ProduceAsync(topic.Name, new Message <string, UInt32Value> {
Key = "test1", Value = u
}).Wait();
var subjects = schemaRegistry.GetAllSubjectsAsync().Result;
Assert.Contains(topic.Name + "-UInt32Value", subjects);
Assert.DoesNotContain(topic.Name + "-value", subjects);
// May contain the record name subject from a previous test.
}
using (var producer =
new ProducerBuilder <string, UInt32Value>(producerConfig)
.SetValueSerializer(new ProtobufSerializer <UInt32Value>(schemaRegistry, new ProtobufSerializerConfig {
SubjectNameStrategy = SubjectNameStrategy.Record
}))
.Build())
{
var u = new UInt32Value();
u.Value = 42;
producer.ProduceAsync(topic.Name, new Message <string, UInt32Value> {
Key = "test1", Value = u
}).Wait();
var subjects = schemaRegistry.GetAllSubjectsAsync().Result;
// Note: If this value is in SR by any means (even if not via this test),
// it implies what is being tested here is functional.
Assert.Contains("UInt32Value", subjects);
Assert.DoesNotContain(topic.Name + "-value", subjects);
}
using (var producer =
new ProducerBuilder <string, UInt32Value>(producerConfig)
.SetValueSerializer(new ProtobufSerializer <UInt32Value>(schemaRegistry))
.Build())
{
var u = new UInt32Value();
u.Value = 42;
producer.ProduceAsync(topic.Name, new Message <string, UInt32Value> {
Key = "test1", Value = u
}).Wait();
var subjects = schemaRegistry.GetAllSubjectsAsync().Result;
Assert.Contains(topic.Name + "-value", subjects);
}
}
}
private static void ProduceConsume(string bootstrapServers, string schemaRegistryServers, SubjectNameStrategy nameStrategy)
{
var producerConfig = new ProducerConfig
{
BootstrapServers = bootstrapServers
};
var consumerConfig = new ConsumerConfig
{
BootstrapServers = bootstrapServers,
GroupId = Guid.NewGuid().ToString(),
SessionTimeoutMs = 6000,
AutoOffsetReset = AutoOffsetReset.Earliest,
EnablePartitionEof = true
};
var schemaRegistryConfig = new SchemaRegistryConfig
{
Url = schemaRegistryServers,
// Test ValueSubjectNameStrategy here,
// and KeySubjectNameStrategy in ProduceConsumeGeneric.
ValueSubjectNameStrategy = nameStrategy
};
var adminClientConfig = new AdminClientConfig
{
BootstrapServers = bootstrapServers
};
string topic = Guid.NewGuid().ToString();
using (var adminClient = new AdminClientBuilder(adminClientConfig).Build())
{
adminClient.CreateTopicsAsync(
new List <TopicSpecification> {
new TopicSpecification {
Name = topic, NumPartitions = 1, ReplicationFactor = 1
}
}).Wait();
}
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
using (var producer =
new ProducerBuilder <string, ProduceConsumeUser>(producerConfig)
.SetKeySerializer(new AvroSerializer <string>(schemaRegistry))
.SetValueSerializer(new AvroSerializer <ProduceConsumeUser>(schemaRegistry))
.Build())
{
for (int i = 0; i < 100; ++i)
{
var user = new ProduceConsumeUser
{
name = i.ToString(),
favorite_number = i,
favorite_color = "blue"
};
producer
.ProduceAsync(topic, new Message <string, ProduceConsumeUser> {
Key = user.name, Value = user
})
.Wait();
}
Assert.Equal(0, producer.Flush(TimeSpan.FromSeconds(10)));
}
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
using (var consumer =
new ConsumerBuilder <string, ProduceConsumeUser>(consumerConfig)
.SetKeyDeserializer(new AvroDeserializer <string>(schemaRegistry).AsSyncOverAsync())
.SetValueDeserializer(new AvroDeserializer <ProduceConsumeUser>(schemaRegistry).AsSyncOverAsync())
.SetErrorHandler((_, e) => Assert.True(false, e.Reason))
.Build())
{
consumer.Subscribe(topic);
int i = 0;
while (true)
{
var record = consumer.Consume(TimeSpan.FromMilliseconds(100));
if (record == null)
{
continue;
}
if (record.IsPartitionEOF)
{
break;
}
Assert.Equal(i.ToString(), record.Message.Key);
Assert.Equal(i.ToString(), record.Message.Value.name);
Assert.Equal(i, record.Message.Value.favorite_number);
Assert.Equal("blue", record.Message.Value.favorite_color);
i += 1;
}
Assert.Equal(100, i);
consumer.Close();
}
// Check that what's in schema registry is what's expected.
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
{
var subjects = schemaRegistry.GetAllSubjectsAsync().Result;
if (nameStrategy == SubjectNameStrategy.TopicRecord)
{
Assert.Equal(2, (int)subjects.Where(s => s.Contains(topic)).Count());
Assert.Single(subjects.Where(s => s == $"{topic}-key"));
Assert.Single(subjects.Where(s => s == $"{topic}-{((Avro.RecordSchema)ProduceConsumeUser._SCHEMA).Fullname}"));
}
if (nameStrategy == SubjectNameStrategy.Topic)
{
Assert.Equal(2, (int)subjects.Where(s => s.Contains(topic)).Count());
Assert.Single(subjects.Where(s => s == $"{topic}-key"));
Assert.Single(subjects.Where(s => s == $"{topic}-value"));
}
if (nameStrategy == SubjectNameStrategy.Record)
{
Assert.Single(subjects.Where(s => s.Contains(topic))); // the string key.
Assert.Single(subjects.Where(s => s == $"{topic}-key"));
Assert.Single(subjects.Where(s => s == $"{((Avro.RecordSchema)ProduceConsumeUser._SCHEMA).Fullname}"));
}
}
}
private static void ProduceConsumeGeneric(string bootstrapServers, string schemaRegistryServers, SubjectNameStrategy nameStrategy)
{
var rs = (RecordSchema)RecordSchema.Parse(
@"{
""namespace"": ""Confluent.Kafka.Examples.AvroSpecific"",
""type"": ""record"",
""name"": ""ProduceConsumeUser2"",
""fields"": [
{""name"": ""name"", ""type"": ""string""},
{""name"": ""favorite_number"", ""type"": [""int"", ""null""]},
{""name"": ""favorite_color"", ""type"": [""string"", ""null""]}
]
}"
);
var config = new ProducerConfig {
BootstrapServers = bootstrapServers
};
var schemaRegistryConfig = new SchemaRegistryConfig
{
Url = schemaRegistryServers,
};
var avroSerializerConfig = new AvroSerializerConfig
{
SubjectNameStrategy = nameStrategy
};
var topic = Guid.NewGuid().ToString();
DeliveryResult <GenericRecord, Null> dr;
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
using (var p =
new ProducerBuilder <GenericRecord, Null>(config)
// Test KeySubjectNameStrategy here,
// and ValueSubjectNameStrategy in ProduceConsume.
.SetKeySerializer(new AvroSerializer <GenericRecord>(schemaRegistry, avroSerializerConfig))
.SetValueSerializer(Serializers.Null)
.Build())
{
var record = new GenericRecord(rs);
record.Add("name", "my name 2");
record.Add("favorite_number", 44);
record.Add("favorite_color", null);
dr = p.ProduceAsync(topic, new Message <GenericRecord, Null> {
Key = record
}).Result;
}
// produce a specific record (to later consume back as a generic record).
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
using (var p =
new ProducerBuilder <ProduceConsumeUser2, Null>(config)
.SetKeySerializer(new AvroSerializer <ProduceConsumeUser2>(schemaRegistry, avroSerializerConfig))
.SetValueSerializer(Serializers.Null)
.Build())
{
var user = new ProduceConsumeUser2
{
name = "my name 3",
favorite_number = 47,
favorite_color = "orange"
};
p.ProduceAsync(topic, new Message <ProduceConsumeUser2, Null> {
Key = user
}).Wait();
}
Assert.Null(dr.Message.Value);
Assert.NotNull(dr.Message.Key);
dr.Message.Key.TryGetValue("name", out object name);
dr.Message.Key.TryGetValue("favorite_number", out object number);
dr.Message.Key.TryGetValue("favorite_color", out object color);
Assert.IsType <string>(name);
Assert.IsType <int>(number);
Assert.Equal("my name 2", name);
Assert.Equal(44, number);
Assert.Null(color);
var cconfig = new ConsumerConfig {
GroupId = Guid.NewGuid().ToString(), BootstrapServers = bootstrapServers
};
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
using (var consumer =
new ConsumerBuilder <GenericRecord, Null>(cconfig)
.SetKeyDeserializer(new AvroDeserializer <GenericRecord>(schemaRegistry).AsSyncOverAsync())
.SetValueDeserializer(Deserializers.Null)
.Build())
{
// consume generic record produced as a generic record.
consumer.Assign(new List <TopicPartitionOffset> {
new TopicPartitionOffset(topic, 0, dr.Offset)
});
var record = consumer.Consume(new CancellationTokenSource(TimeSpan.FromSeconds(10)).Token);
record.Message.Key.TryGetValue("name", out object msgName);
record.Message.Key.TryGetValue("favorite_number", out object msgNumber);
record.Message.Key.TryGetValue("favorite_color", out object msgColor);
Assert.IsType <string>(msgName);
Assert.IsType <int>(msgNumber);
Assert.Equal("my name 2", msgName);
Assert.Equal(44, msgNumber);
Assert.Null(msgColor);
// consume generic record produced as a specific record.
record = consumer.Consume(new CancellationTokenSource(TimeSpan.FromSeconds(10)).Token);
record.Message.Key.TryGetValue("name", out msgName);
record.Message.Key.TryGetValue("favorite_number", out msgNumber);
record.Message.Key.TryGetValue("favorite_color", out msgColor);
Assert.IsType <string>(msgName);
Assert.IsType <int>(msgNumber);
Assert.IsType <string>(msgColor);
Assert.Equal("my name 3", msgName);
Assert.Equal(47, msgNumber);
Assert.Equal("orange", msgColor);
}
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
using (var consumer =
new ConsumerBuilder <ProduceConsumeUser2, Null>(cconfig)
.SetKeyDeserializer(new AvroDeserializer <ProduceConsumeUser2>(schemaRegistry).AsSyncOverAsync())
.SetValueDeserializer(Deserializers.Null)
.Build())
{
consumer.Assign(new List <TopicPartitionOffset> {
new TopicPartitionOffset(topic, 0, dr.Offset)
});
var record = consumer.Consume(new CancellationTokenSource(TimeSpan.FromSeconds(10)).Token);
Assert.Equal("my name 2", record.Message.Key.name);
Assert.Equal(44, record.Message.Key.favorite_number);
Assert.Null(record.Message.Key.favorite_color);
}
// Check that what's in schema registry is what's expected.
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
{
var subjects = schemaRegistry.GetAllSubjectsAsync().Result;
if (nameStrategy == SubjectNameStrategy.TopicRecord)
{
Assert.Single(subjects.Where(s => s.Contains(topic)));
Assert.Single(subjects.Where(s => s == $"{topic}-{((Avro.RecordSchema)ProduceConsumeUser2._SCHEMA).Fullname}"));
}
if (nameStrategy == SubjectNameStrategy.Topic)
{
Assert.Single(subjects.Where(s => s.Contains(topic)));
Assert.Single(subjects.Where(s => s == $"{topic}-key"));
}
if (nameStrategy == SubjectNameStrategy.Record)
{
Assert.Single(subjects.Where(s => s.Contains(topic))); // the string key.
Assert.Single(subjects.Where(s => s == $"{((Avro.RecordSchema)ProduceConsumeUser2._SCHEMA).Fullname}"));
}
}
}
public static void ProduceConsumeExternalRefProtobuf(string bootstrapServers, string schemaRegistryServers)
{
var producerConfig = new ProducerConfig {
BootstrapServers = bootstrapServers
};
var schemaRegistryConfig = new SchemaRegistryConfig {
Url = schemaRegistryServers
};
using (var topic = new TemporaryTopic(bootstrapServers, 1))
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
using (var producer =
new ProducerBuilder <string, WithExternalReference>(producerConfig)
.SetValueSerializer(new ProtobufSerializer <WithExternalReference>(schemaRegistry))
.Build())
{
var u = new WithExternalReference();
u.Value1 = new RefByAnother {
Value = 111
};
u.Value2 = 123;
producer.ProduceAsync(topic.Name, new Message <string, WithExternalReference> {
Key = "test1", Value = u
}).Wait();
var consumerConfig = new ConsumerConfig
{
BootstrapServers = bootstrapServers,
GroupId = Guid.NewGuid().ToString(),
AutoOffsetReset = AutoOffsetReset.Earliest
};
// Test the protobuf deserializer can read this message
using (var consumer =
new ConsumerBuilder <string, WithExternalReference>(consumerConfig)
.SetValueDeserializer(new ProtobufDeserializer <WithExternalReference>().AsSyncOverAsync())
.Build())
{
consumer.Subscribe(topic.Name);
var cr = consumer.Consume();
Assert.Equal(u.Value2, cr.Message.Value.Value2);
Assert.Equal(u.Value1.Value, cr.Message.Value.Value1.Value);
}
// Check the pre-data bytes are as expected.
using (var consumer = new ConsumerBuilder <string, byte[]>(consumerConfig).Build())
{
consumer.Subscribe(topic.Name);
var cr = consumer.Consume();
// magic byte + schema id + expected array index length + at least one data byte.
Assert.True(cr.Message.Value.Length >= 1 + 4 + 1 + 1);
// magic byte
Assert.Equal(0, cr.Message.Value[0]);
// array index (special value as an optimization)
Assert.Equal(0, cr.Message.Value[5]);
}
// Check the referenced schema is in schema registry.
var subjects = schemaRegistry.GetAllSubjectsAsync().Result;
Assert.Contains("RefByAnother.proto", subjects);
}
}
public static void ProduceConsumeGoogleRefProtobuf(string bootstrapServers, string schemaRegistryServers)
{
var producerConfig = new ProducerConfig {
BootstrapServers = bootstrapServers
};
var schemaRegistryConfig = new SchemaRegistryConfig {
Url = schemaRegistryServers
};
using (var topic = new TemporaryTopic(bootstrapServers, 1))
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
using (var producer =
new ProducerBuilder <string, WithGoogleRefs.TheRecord>(producerConfig)
.SetValueSerializer(new ProtobufSerializer <WithGoogleRefs.TheRecord>(schemaRegistry))
.Build())
{
var u = new WithGoogleRefs.TheRecord();
u.ListType = 41;
u.ReceivedTime = new Google.Protobuf.WellKnownTypes.Timestamp();
u.ReceivedTime.Seconds = 1591364591;
producer.ProduceAsync(topic.Name, new Message <string, WithGoogleRefs.TheRecord> {
Key = "test1", Value = u
}).Wait();
var consumerConfig = new ConsumerConfig
{
BootstrapServers = bootstrapServers,
GroupId = Guid.NewGuid().ToString(),
AutoOffsetReset = AutoOffsetReset.Earliest
};
// Test the protobuf deserializer can read this message
using (var consumer =
new ConsumerBuilder <string, WithGoogleRefs.TheRecord>(consumerConfig)
.SetValueDeserializer(new ProtobufDeserializer <WithGoogleRefs.TheRecord>().AsSyncOverAsync())
.Build())
{
consumer.Subscribe(topic.Name);
var cr = consumer.Consume();
Assert.Equal(u.ListType.Value, cr.Message.Value.ListType.Value);
Assert.Equal(u.ReceivedTime.Seconds, cr.Message.Value.ReceivedTime.Seconds);
}
// Check the pre-data bytes are as expected.
using (var consumer = new ConsumerBuilder <string, byte[]>(consumerConfig).Build())
{
consumer.Subscribe(topic.Name);
var cr = consumer.Consume();
// magic byte + schema id + expected array index length + at least one data byte.
Assert.True(cr.Message.Value.Length >= 1 + 4 + 1 + 1);
// magic byte
Assert.Equal(0, cr.Message.Value[0]);
// array index (special value as an optimization)
Assert.Equal(0, cr.Message.Value[5]);
}
// Check the referenced schemas are in schema registry.
var subjects = schemaRegistry.GetAllSubjectsAsync().Result;
Assert.Contains("google/protobuf/timestamp.proto", subjects);
Assert.Contains("google/protobuf/wrappers.proto", subjects);
}
}
