public async Task It_should_deserialize_the_envelope_correctly()
{
var schemaRegistry = new CachedSchemaRegistryClient
(
new SchemaRegistryConfig {
SchemaRegistryUrl = _kafkaConfig.SchemaRegistryUrl
}
);
//var type = AvroMultipleDeserializer.Get(schema.Name);
var subjects = await schemaRegistry.GetAllSubjectsAsync();
subjects.Count.ShouldBePositive();
foreach (var subject in subjects)
{
var schemas = await schemaRegistry.GetLatestSchemaAsync(subject);
var schemaSubject = await schemaRegistry.GetSchemaAsync(1);
var schema = global::Avro.Schema.Parse(schemaSubject);
}
}
public static void JsonWithReferences(Config config)
{
var srInitial = new CachedSchemaRegistryClient(new SchemaRegistryConfig {
Url = config.Server
});
var sr = new CachedSchemaRegistryClient(new SchemaRegistryConfig {
Url = config.Server
});
var topicName = Guid.NewGuid().ToString();
var subjectInitial = SubjectNameStrategy.Topic.ConstructValueSubjectName(topicName, null);
var subject1 = SubjectNameStrategy.Topic.ConstructValueSubjectName(topicName + "1", null);
var subject2 = SubjectNameStrategy.Topic.ConstructValueSubjectName(topicName + "2", null);
// Test there are no errors (exceptions) registering a schema that references another.
var id1 = srInitial.RegisterSchemaAsync(subject1, new Schema(S1, SchemaType.Json)).Result;
var s1 = srInitial.GetLatestSchemaAsync(subject1).Result;
var refs = new List <SchemaReference> {
new SchemaReference("https://example.com/geographical-location.schema.json", subject1, s1.Version)
};
var id2 = srInitial.RegisterSchemaAsync(subject2, new Schema(S2, refs, SchemaType.Json)).Result;
// In fact, it seems references are not checked server side.
var latestSchema = sr.GetLatestSchemaAsync(subject2).Result;
}
/// <summary>
/// Creates a serializer.
/// </summary>
/// <param name="registryConfiguration">
/// Schema Registry configuration. Using the <see cref="SchemaRegistryConfig" /> class is
/// highly recommended.
/// </param>
/// <param name="registerAutomatically">
/// When to automatically register schemas that match the type being serialized.
/// </param>
/// <param name="schemaBuilder">
/// The schema builder to use to create a schema for a C# type when registering automatically.
/// If none is provided, the default schema builder will be used.
/// </param>
/// <param name="schemaReader">
/// The JSON schema reader to use to convert schemas received from the registry into abstract
/// representations. If none is provided, the default schema reader will be used.
/// </param>
/// <param name="schemaWriter">
/// The JSON schema writer to use to convert abstract schema representations when registering
/// automatically. If none is provided, the default schema writer will be used.
/// </param>
/// <param name="serializerBuilder">
/// The deserializer builder to use to build serialization functions for C# types. If none
/// is provided, the default serializer builder will be used.
/// </param>
/// <param name="subjectNameBuilder">
/// A function that determines the subject name given the topic name and a component type
/// (key or value). If none is provided, the default "{topic name}-{component}" naming
/// convention will be used.
/// </param>
/// <param name="tombstoneBehavior">
/// The behavior of the serializer on tombstone records.
/// </param>
/// <exception cref="ArgumentNullException">
/// Thrown when the registry configuration is null.
/// </exception>
public AsyncSchemaRegistrySerializer(
IEnumerable <KeyValuePair <string, string> > registryConfiguration,
AutomaticRegistrationBehavior registerAutomatically = AutomaticRegistrationBehavior.Never,
Abstract.ISchemaBuilder schemaBuilder = null,
IJsonSchemaReader schemaReader = null,
IJsonSchemaWriter schemaWriter = null,
IBinarySerializerBuilder serializerBuilder = null,
Func <SerializationContext, string> subjectNameBuilder = null,
TombstoneBehavior tombstoneBehavior = TombstoneBehavior.None
)
{
if (registryConfiguration == null)
{
throw new ArgumentNullException(nameof(registryConfiguration));
}
if (tombstoneBehavior != TombstoneBehavior.None && default(T) != null)
{
throw new UnsupportedTypeException(typeof(T), $"{typeof(T)} cannot represent tombstone values.");
}
RegisterAutomatically = registerAutomatically;
SchemaBuilder = schemaBuilder ?? new Abstract.SchemaBuilder();
SchemaReader = schemaReader ?? new JsonSchemaReader();
SchemaWriter = schemaWriter ?? new JsonSchemaWriter();
SerializerBuilder = serializerBuilder ?? new BinarySerializerBuilder();
SubjectNameBuilder = subjectNameBuilder ??
(c => $"{c.Topic}-{(c.Component == MessageComponentType.Key ? "key" : "value")}");
TombstoneBehavior = tombstoneBehavior;
_cache = new Dictionary <string, Task <Func <T, byte[]> > >();
_register = async(subject, json) =>
{
using (var registry = new CachedSchemaRegistryClient(registryConfiguration))
{
return(await registry.RegisterSchemaAsync(subject, new Schema(json, SchemaType.Avro)).ConfigureAwait(false));
}
};
_resolve = async subject =>
{
using (var registry = new CachedSchemaRegistryClient(registryConfiguration))
{
var schema = await registry.GetLatestSchemaAsync(subject).ConfigureAwait(false);
if (schema.SchemaType != SchemaType.Avro)
{
throw new UnsupportedSchemaException(null, $"The latest schema with subject {subject} is not an Avro schema.");
}
return(schema);
}
};
}
public static void SslAuth(Config config)
{
var testSchema1 =
"{\"type\":\"record\",\"name\":\"User\",\"namespace\":\"Confluent.Kafka.Examples.AvroSpecific" +
"\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"favorite_number\",\"type\":[\"i" +
"nt\",\"null\"]},{\"name\":\"favorite_color\",\"type\":[\"string\",\"null\"]}]}";
// 1. valid configuration cases
// 1.1. using SSL valid certificate.
var conf = new SchemaRegistryConfig
{
Url = config.ServerWithSsl,
SslKeystoreLocation = config.KeystoreLocation,
SslKeystorePassword = config.KeystorePassword,
SslCaLocation = config.CaLocation,
EnableSslCertificateVerification = bool.Parse(config.EnableSslCertificateVerification),
};
// some sanity checking of strongly typed config property name mappings.
Assert.Equal(config.ServerWithSsl, conf.Get("schema.registry.url"));
using (var sr = new CachedSchemaRegistryClient(conf))
{
var topicName = Guid.NewGuid().ToString();
var subject = SubjectNameStrategy.Topic.ToDelegate()(new SerializationContext(MessageComponentType.Value, topicName), null);
var id = sr.RegisterSchemaAsync(subject, testSchema1).Result;
var schema = sr.GetLatestSchemaAsync(subject).Result;
Assert.Equal(schema.Id, id);
}
// try to connect with invalid SSL config. shouldn't work.
Assert.Throws <HttpRequestException>(() =>
{
var sr = new CachedSchemaRegistryClient(new SchemaRegistryConfig {
Url = config.ServerWithSsl
});
var topicName = Guid.NewGuid().ToString();
var subject = SubjectNameStrategy.Topic.ConstructValueSubjectName(topicName, null);
try
{
var id = sr.RegisterSchemaAsync(subject, testSchema1).Result;
}
catch (Exception e)
{
throw e.InnerException;
}
});
}
private static void GetSchemaBySubjectAndVersion(Dictionary <string, object> config)
{
var topicName = Guid.NewGuid().ToString();
var testSchema1 =
"{\"type\":\"record\",\"name\":\"User\",\"namespace\":\"Confluent.Kafka.Examples.AvroSpecific" +
"\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"favorite_number\",\"type\":[\"i" +
"nt\",\"null\"]},{\"name\":\"favorite_color\",\"type\":[\"string\",\"null\"]}]}";
var sr = new CachedSchemaRegistryClient(config);
var subject = sr.ConstructValueSubjectName(topicName, "Confluent.Kafka.Examples.AvroSpecific.User");
var id = sr.RegisterSchemaAsync(subject, testSchema1).Result;
var schema = sr.GetLatestSchemaAsync(subject).Result;
var schemaString = sr.GetSchemaAsync(subject, schema.Version).Result;
Assert.Equal(schemaString, testSchema1);
}
public async Task AvroExampleAsync(string ksqlDbUrl)
{
var schemaRegistryConfig = new SchemaRegistryConfig
{
Url = "http://localhost:8081"
};
using var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig);
var schema = IoTSensor._SCHEMA.ToString();
var subject = "IoTSensors-value";
var registrationResult = await schemaRegistry.RegisterSchemaAsync(subject, schema);
//http://localhost:8081/subjects/IoTSensors-value/versions/latest/schema
var latestSchema = await schemaRegistry.GetLatestSchemaAsync(subject);
var httpClientFactory = new HttpClientFactory(new Uri(ksqlDbUrl));
var restApiClient = new KSqlDbRestApiProvider(httpClientFactory);
EntityCreationMetadata metadata = new()
{
EntityName = "avroSensors",
KafkaTopic = "IoTSensors",
ValueFormat = SerializationFormats.Avro,
Partitions = 1,
Replicas = 1
};
var httpResponseMessage = await restApiClient.CreateStreamAsync <IoTSensor>(metadata, ifNotExists : false)
.ConfigureAwait(false);
var httpResponse = await httpResponseMessage.Content.ReadAsStringAsync();
Console.WriteLine(httpResponse);
// Stream Name | Kafka Topic | Key Format | Value Format | Windowed
// ------------------------------------------------------------------------------------------------------
// AVROSENSORS | IoTSensors | KAFKA | AVRO | false
}
}
/// <summary>
/// Creates a serializer.
/// </summary>
/// <param name="registryConfiguration">
/// Schema Registry configuration. Using the <see cref="SchemaRegistryConfig" /> class is
/// highly recommended.
/// </param>
/// <param name="registerAutomatically">
/// Whether to automatically register schemas that match the type being serialized.
/// </param>
/// <param name="schemaBuilder">
/// The schema builder to use to create a schema for a C# type when registering automatically.
/// If none is provided, the default schema builder will be used.
/// </param>
/// <param name="schemaReader">
/// The JSON schema reader to use to convert schemas received from the registry into abstract
/// representations. If none is provided, the default schema reader will be used.
/// </param>
/// <param name="schemaWriter">
/// The JSON schema writer to use to convert abstract schema representations when registering
/// automatically. If none is provided, the default schema writer will be used.
/// </param>
/// <param name="serializerBuilder">
/// The deserializer builder to use to build serialization functions for C# types. If none
/// is provided, the default serializer builder will be used.
/// </param>
/// <param name="subjectNameBuilder">
/// A function that determines the subject name given the topic name and a component type
/// (key or value). If none is provided, the default "{topic name}-{component}" naming
/// convention will be used.
/// </param>
/// <exception cref="ArgumentNullException">
/// Thrown when the registry configuration is null.
/// </exception>
public AsyncSchemaRegistrySerializer(
IEnumerable <KeyValuePair <string, string> > registryConfiguration,
bool registerAutomatically = false,
Abstract.ISchemaBuilder?schemaBuilder = null,
IJsonSchemaReader?schemaReader = null,
IJsonSchemaWriter?schemaWriter = null,
IBinarySerializerBuilder?serializerBuilder = null,
Func <SerializationContext, string>?subjectNameBuilder = null
)
{
if (registryConfiguration == null)
{
throw new ArgumentNullException(nameof(registryConfiguration));
}
RegisterAutomatically = registerAutomatically;
SchemaBuilder = schemaBuilder ?? new Abstract.SchemaBuilder();
SchemaReader = schemaReader ?? new JsonSchemaReader();
SchemaWriter = schemaWriter ?? new JsonSchemaWriter();
SerializerBuilder = serializerBuilder ?? new BinarySerializerBuilder();
SubjectNameBuilder = subjectNameBuilder ??
(c => $"{c.Topic}-{(c.Component == MessageComponentType.Key ? "key" : "value")}");
_cache = new ConcurrentDictionary <string, Task <Func <T, byte[]> > >();
_register = async(subject, json) =>
{
using (var registry = new CachedSchemaRegistryClient(registryConfiguration))
{
return(await registry.RegisterSchemaAsync(subject, json).ConfigureAwait(false));
}
};
_resolve = async subject =>
{
using (var registry = new CachedSchemaRegistryClient(registryConfiguration))
{
return(await registry.GetLatestSchemaAsync(subject).ConfigureAwait(false));
}
};
}
/// <summary>
/// Creates a serializer.
/// </summary>
/// <param name="registryConfiguration">
/// Schema Registry configuration. Using the <see cref="SchemaRegistryConfig" /> class is
/// highly recommended.
/// </param>
/// <param name="registerAutomatically">
/// Whether to automatically register schemas that match the type being serialized.
/// </param>
/// <param name="schemaBuilder">
/// A schema builder (used to build a schema for a C# type when registering automatically).
/// If none is provided, the default schema builder will be used.
/// </param>
/// <param name="schemaReader">
/// A JSON schema reader (used to convert schemas received from the registry into abstract
/// representations). If none is provided, the default schema reader will be used.
/// </param>
/// <param name="schemaWriter">
/// A JSON schema writer (used to convert abstract schema representations when registering
/// automatically). If none is provided, the default schema writer will be used.
/// </param>
/// <param name="serializerBuilder">
/// A deserializer builder (used to build serialization functions for C# types). If none is
/// provided, the default serializer builder will be used.
/// </param>
/// <param name="subjectNameBuilder">
/// A function that determines the subject name given the topic name and a component type
/// (key or value). If none is provided, the default "{topic name}-{component}" naming
/// convention will be used.
/// </param>
/// <exception cref="ArgumentNullException">
/// Thrown when the registry configuration is null.
/// </exception>
public AsyncSchemaRegistrySerializer(
IEnumerable <KeyValuePair <string, string> > registryConfiguration,
bool registerAutomatically = false,
Abstract.ISchemaBuilder schemaBuilder = null,
IJsonSchemaReader schemaReader = null,
IJsonSchemaWriter schemaWriter = null,
IBinarySerializerBuilder serializerBuilder = null,
Func <SerializationContext, string> subjectNameBuilder = null
) : this(
registerAutomatically,
schemaBuilder,
schemaReader,
schemaWriter,
serializerBuilder,
subjectNameBuilder
)
{
if (registryConfiguration == null)
{
throw new ArgumentNullException(nameof(registryConfiguration));
}
_register = async(subject, json) =>
{
using (var registry = new CachedSchemaRegistryClient(registryConfiguration))
{
return(await registry.RegisterSchemaAsync(subject, json).ConfigureAwait(false));
}
};
_resolve = async subject =>
{
using (var registry = new CachedSchemaRegistryClient(registryConfiguration))
{
return(await registry.GetLatestSchemaAsync(subject).ConfigureAwait(false));
}
};
}
public static void GetSchemaBySubjectAndVersion(Config config)
{
var topicName = Guid.NewGuid().ToString();
var testSchema1 =
"{\"type\":\"record\",\"name\":\"User\",\"namespace\":\"Confluent.Kafka.Examples.AvroSpecific" +
"\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"favorite_number\",\"type\":[\"i" +
"nt\",\"null\"]},{\"name\":\"favorite_color\",\"type\":[\"string\",\"null\"]}]}";
var sr = new CachedSchemaRegistryClient(new SchemaRegistryConfig {
Url = config.Server
});
var subject = SubjectNameStrategy.Topic.ConstructValueSubjectName(topicName, null);
var id = sr.RegisterSchemaAsync(subject, testSchema1).Result;
var latestSchema = sr.GetLatestSchemaAsync(subject).Result;
var schema = sr.GetRegisteredSchemaAsync(subject, latestSchema.Version).Result;
Assert.Equal(schema.SchemaString, testSchema1);
Assert.Equal(schema.SchemaType, SchemaType.Avro);
Assert.Empty(schema.References);
}
public static void GetLatestSchema(Config config)
{
var topicName = Guid.NewGuid().ToString();
var testSchema1 =
"{\"type\":\"record\",\"name\":\"User\",\"namespace\":\"Confluent.Kafka.Examples.AvroSpecific" +
"\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"favorite_number\",\"type\":[\"i" +
"nt\",\"null\"]},{\"name\":\"favorite_color\",\"type\":[\"string\",\"null\"]}]}";
var sr = new CachedSchemaRegistryClient(new SchemaRegistryConfig {
SchemaRegistryUrl = config.Server
});
var subject = sr.ConstructValueSubjectName(topicName);
var id = sr.RegisterSchemaAsync(subject, testSchema1).Result;
var schema = sr.GetLatestSchemaAsync(subject).Result;
Assert.Equal(schema.Id, id);
Assert.Equal(schema.Subject, subject);
Assert.Equal(schema.Version, 1);
Assert.Equal(schema.SchemaString, testSchema1);
}
static async Task Main(string[] args)
{
if (args.Length != 3)
{
Console.WriteLine("Usage: .. bootstrapServers schemaRegistryUrl topicName");
return;
}
string bootstrapServers = args[0];
string schemaRegistryUrl = args[1];
string topicName = args[2];
var producerConfig = new ProducerConfig
{
BootstrapServers = bootstrapServers
};
var schemaRegistryConfig = new SchemaRegistryConfig
{
// Note: you can specify more than one schema registry url using the
// schema.registry.url property for redundancy (comma separated list).
// The property name is not plural to follow the convention set by
// the Java implementation.
Url = schemaRegistryUrl
};
var consumerConfig = new ConsumerConfig
{
BootstrapServers = bootstrapServers,
GroupId = "json-example-consumer-group"
};
// Note: Specifying json serializer configuration is optional.
var jsonSerializerConfig = new JsonSerializerConfig
{
BufferBytes = 100
};
CancellationTokenSource cts = new CancellationTokenSource();
var consumeTask = Task.Run(() =>
{
using (var consumer =
new ConsumerBuilder <string, User>(consumerConfig)
.SetKeyDeserializer(Deserializers.Utf8)
.SetValueDeserializer(new JsonDeserializer <User>().AsSyncOverAsync())
.SetErrorHandler((_, e) => Console.WriteLine($"Error: {e.Reason}"))
.Build())
{
consumer.Subscribe(topicName);
try
{
while (true)
{
try
{
var cr = consumer.Consume(cts.Token);
var user = cr.Message.Value;
Console.WriteLine($"user name: {user.Name}, favorite number: {user.FavoriteNumber}, favorite color: {user.FavoriteColor}");
}
catch (ConsumeException e)
{
Console.WriteLine($"Consume error: {e.Error.Reason}");
}
}
}
catch (OperationCanceledException)
{
consumer.Close();
}
}
});
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
using (var producer =
new ProducerBuilder <string, User>(producerConfig)
.SetValueSerializer(new JsonSerializer <User>(schemaRegistry, jsonSerializerConfig))
.Build())
{
Console.WriteLine($"{producer.Name} producing on {topicName}. Enter first names, q to exit.");
long i = 1;
string text;
while ((text = Console.ReadLine()) != "q")
{
User user = new User {
Name = text, FavoriteColor = "blue", FavoriteNumber = i++
};
await producer
.ProduceAsync(topicName, new Message <string, User> {
Value = user
})
.ContinueWith(task => task.IsFaulted
? $"error producing message: {task.Exception.Message}"
: $"produced to: {task.Result.TopicPartitionOffset}");
}
}
cts.Cancel();
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
{
// Note: a subject name strategy was not configured, so the default "Topic" was used.
var schema = await schemaRegistry.GetLatestSchemaAsync(SubjectNameStrategy.Topic.ConstructValueSubjectName(topicName));
Console.WriteLine("\nThe JSON schema corresponding to the written data:");
Console.WriteLine(schema.SchemaString);
}
}
public static void RegularAndAvro(string bootstrapServers, string schemaRegistryServers)
{
using (var topic1 = new TemporaryTopic(bootstrapServers, 1))
using (var topic2 = new TemporaryTopic(bootstrapServers, 1))
{
var producerConfig = new ProducerConfig
{
BootstrapServers = bootstrapServers
};
var consumerConfig = new ConsumerConfig
{
BootstrapServers = bootstrapServers,
GroupId = Guid.NewGuid().ToString(),
SessionTimeoutMs = 6000,
AutoOffsetReset = AutoOffsetReset.Earliest
};
var schemaRegistryConfig = new SchemaRegistryConfig
{
SchemaRegistryUrl = schemaRegistryServers
};
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
using (var producer =
new ProducerBuilder <string, string>(producerConfig)
.SetKeySerializer(Serializers.Utf8)
.SetValueSerializer(new AvroSerializer <string>(schemaRegistry))
.Build())
{
// implicit check that this does not fail.
producer.ProduceAsync(topic1.Name, new Message <string, string> {
Key = "hello", Value = "world"
}).Wait();
// check that the value type was registered with SR, and the key was not.
Assert.Throws <SchemaRegistryException>(() =>
{
try
{
schemaRegistry.GetLatestSchemaAsync(schemaRegistry.ConstructKeySubjectName(topic1.Name)).Wait();
}
catch (AggregateException e)
{
throw e.InnerException;
}
});
var s2 = schemaRegistry.GetLatestSchemaAsync(schemaRegistry.ConstructValueSubjectName(topic1.Name)).Result;
}
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
using (var producer =
new ProducerBuilder <string, string>(producerConfig)
.SetKeySerializer(new AvroSerializer <string>(schemaRegistry))
.SetValueSerializer(Serializers.Utf8)
.Build())
{
// implicit check that this does not fail.
producer.ProduceAsync(topic2.Name, new Message <string, string> {
Key = "hello", Value = "world"
}).Wait();
// check that the key type was registered with SR, and the value was not.
Assert.Throws <SchemaRegistryException>(() =>
{
try
{
schemaRegistry.GetLatestSchemaAsync(schemaRegistry.ConstructValueSubjectName(topic2.Name)).Wait();
}
catch (AggregateException e)
{
throw e.InnerException;
}
});
var s2 = schemaRegistry.GetLatestSchemaAsync(schemaRegistry.ConstructKeySubjectName(topic2.Name)).Result;
}
// check the above can be consumed (using regular / Avro serializers as appropriate)
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
{
using (var consumer =
new ConsumerBuilder <string, string>(consumerConfig)
.SetKeyDeserializer(Deserializers.Utf8)
.SetValueDeserializer(new AvroDeserializer <string>(schemaRegistry))
.Build())
{
consumer.Assign(new TopicPartitionOffset(topic1.Name, 0, 0));
var cr = consumer.Consume();
Assert.Equal("hello", cr.Key);
Assert.Equal("world", cr.Value);
}
using (var consumer =
new ConsumerBuilder <string, string>(consumerConfig)
.SetKeyDeserializer(new AvroDeserializer <string>(schemaRegistry))
.SetValueDeserializer(Deserializers.Utf8).Build())
{
consumer.Assign(new TopicPartitionOffset(topic2.Name, 0, 0));
var cr = consumer.Consume();
Assert.Equal("hello", cr.Key);
Assert.Equal("world", cr.Value);
}
using (var consumer =
new ConsumerBuilder <string, string>(consumerConfig)
.SetKeyDeserializer(Deserializers.Utf8)
.SetValueDeserializer(new AvroDeserializer <string>(schemaRegistry))
.Build())
{
consumer.Assign(new TopicPartitionOffset(topic2.Name, 0, 0));
Assert.ThrowsAny <ConsumeException>(() =>
{
try
{
consumer.Consume();
}
catch (AggregateException e)
{
throw e.InnerException;
}
});
}
using (var consumer =
new ConsumerBuilder <string, string>(consumerConfig)
.SetKeyDeserializer(new AvroDeserializer <string>(schemaRegistry))
.SetValueDeserializer(Deserializers.Utf8)
.Build())
{
consumer.Assign(new TopicPartitionOffset(topic1.Name, 0, 0));
Assert.ThrowsAny <ConsumeException>(() =>
{
try
{
consumer.Consume();
}
catch (AggregateException e)
{
throw e.InnerException;
}
});
}
}
}
}
public static void BasicAuth(Config config)
{
var testSchema1 =
"{\"type\":\"record\",\"name\":\"User\",\"namespace\":\"Confluent.Kafka.Examples.AvroSpecific" +
"\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"favorite_number\",\"type\":[\"i" +
"nt\",\"null\"]},{\"name\":\"favorite_color\",\"type\":[\"string\",\"null\"]}]}";
// 1. valid configuration cases
// 1.1. credentials specified as USER_INFO.
var conf = new Dictionary <string, string>
{
{ "schema.registry.url", config.ServerWithAuth },
{ "schema.registry.basic.auth.credentials.source", "USER_INFO" },
{ "schema.registry.basic.auth.user.info", $"{config.Username}:{config.Password}" }
};
using (var sr = new CachedSchemaRegistryClient(conf))
{
var topicName = Guid.NewGuid().ToString();
var subject = sr.ConstructValueSubjectName(topicName);
var id = sr.RegisterSchemaAsync(subject, testSchema1).Result;
var schema = sr.GetLatestSchemaAsync(subject).Result;
Assert.Equal(schema.Id, id);
}
// 1.2. credentials specified as USER_INFO implicitly (and using strongly typed SchemaRegistryConfig)
var conf2 = new SchemaRegistryConfig
{
SchemaRegistryUrl = config.ServerWithAuth,
SchemaRegistryBasicAuthUserInfo = $"{config.Username}:{config.Password}"
};
using (var sr = new CachedSchemaRegistryClient(conf2))
{
var topicName = Guid.NewGuid().ToString();
var subject = sr.ConstructValueSubjectName(topicName);
var id = sr.RegisterSchemaAsync(subject, testSchema1).Result;
var schema = sr.GetLatestSchemaAsync(subject).Result;
Assert.Equal(schema.Id, id);
}
// 1.3. credentials specified as SASL_INHERIT.
using (var sr = new CachedSchemaRegistryClient(
new Dictionary <string, string>
{
{ "schema.registry.url", config.ServerWithAuth },
{ "schema.registry.basic.auth.credentials.source", "SASL_INHERIT" },
{ "sasl.username", config.Username },
{ "sasl.password", config.Password }
}))
{
var topicName = Guid.NewGuid().ToString();
var subject = sr.ConstructValueSubjectName(topicName);
var id = sr.RegisterSchemaAsync(subject, testSchema1).Result;
var schema = sr.GetLatestSchemaAsync(subject).Result;
Assert.Equal(schema.Id, id);
}
// 1.4. credentials specified as SASL_INHERIT via strongly typed config.
var conf3 = new SchemaRegistryConfig {
SchemaRegistryUrl = config.ServerWithAuth
};
conf3.Set(SchemaRegistryConfig.PropertyNames.SchemaRegistryBasicAuthCredentialsSource, "SASL_INHERIT");
conf3.Set("sasl.username", config.Username);
conf3.Set("sasl.password", config.Password);
using (var sr = new CachedSchemaRegistryClient(conf3))
{
var topicName = Guid.NewGuid().ToString();
var subject = sr.ConstructValueSubjectName(topicName);
var id = sr.RegisterSchemaAsync(subject, testSchema1).Result;
var schema = sr.GetLatestSchemaAsync(subject).Result;
Assert.Equal(schema.Id, id);
}
// 2. invalid configuration cases
Assert.Throws <ArgumentException>(() =>
{
var sr = new CachedSchemaRegistryClient(new Dictionary <string, string>
{
{ "schema.registry.url", config.ServerWithAuth },
{ "schema.registry.basic.auth.credentials.source", "SASL_INHERIT" },
{ "schema.registry.basic.auth.user.info", $"{config.Username:config.Password}" }
});
});
Assert.Throws <ArgumentException>(() =>
{
var sr = new CachedSchemaRegistryClient(new Dictionary <string, string>
{
{ "schema.registry.url", config.ServerWithAuth },
{ "schema.registry.basic.auth.credentials.source", "USER_INFO" },
{ "sasl.username", config.Username },
{ "sasl.password", config.Password }
});
});
// connect to authenticating without credentials. shouldn't work.
Assert.Throws <HttpRequestException>(() =>
{
var sr = new CachedSchemaRegistryClient(new SchemaRegistryConfig {
SchemaRegistryUrl = config.ServerWithAuth
});
var topicName = Guid.NewGuid().ToString();
var subject = sr.ConstructValueSubjectName(topicName);
try
{
var id = sr.RegisterSchemaAsync(subject, testSchema1).Result;
}
catch (Exception e)
{
throw e.InnerException;
}
});
}
private static void BMSpecificProduceConsume(string bootstrapServers, string schemaRegistryServers)
{
var producerConfig = new ProducerConfig {
BootstrapServers = bootstrapServers
};
var schemaRegistryConfig = new SchemaRegistryConfig
{
Url = schemaRegistryServers
};
var adminClientConfig = new AdminClientConfig
{
BootstrapServers = bootstrapServers
};
var topic = Guid.NewGuid().ToString();
Console.Error.WriteLine($"topic: {topic}");
using (var adminClient = new AdminClientBuilder(adminClientConfig).Build())
{
adminClient.CreateTopicsAsync(
new List <TopicSpecification> {
new TopicSpecification {
Name = topic, NumPartitions = 1, ReplicationFactor = 1
}
}).Wait();
}
var srClient = new CachedSchemaRegistryClient(schemaRegistryConfig);
Schema schema1 = new Schema(EventA._SCHEMA.ToString(), SchemaType.Avro);
Schema schema2 = new Schema(EventB._SCHEMA.ToString(), SchemaType.Avro);
var id1 = srClient.RegisterSchemaAsync("events-a", schema1).Result;
var id2 = srClient.RegisterSchemaAsync("events-b", schema2).Result;
var avroUnion = @"[""Confluent.Kafka.Examples.AvroSpecific.EventA"",""Confluent.Kafka.Examples.AvroSpecific.EventB""]";
Schema unionSchema = new Schema(avroUnion, SchemaType.Avro);
SchemaReference reference = new SchemaReference(
"Confluent.Kafka.Examples.AvroSpecific.EventA",
"events-a",
srClient.GetLatestSchemaAsync("events-a").Result.Version);
unionSchema.References.Add(reference);
reference = new SchemaReference(
"Confluent.Kafka.Examples.AvroSpecific.EventB",
"events-b",
srClient.GetLatestSchemaAsync("events-b").Result.Version);
unionSchema.References.Add(reference);
var id3 = srClient.RegisterSchemaAsync($"{topic}-value", unionSchema).Result;
AvroSerializerConfig avroSerializerConfig = new AvroSerializerConfig {
AutoRegisterSchemas = false, UseLatestSchemaVersion = true
};
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
using (var producer =
new ProducerBuilder <string, ISpecificRecord>(producerConfig)
.SetKeySerializer(new AvroSerializer <string>(schemaRegistry))
.SetValueSerializer(new BmSpecificSerializerImpl <ISpecificRecord>(
schemaRegistry,
false,
1024,
SubjectNameStrategy.Topic.ToDelegate(),
true))
.Build())
{
for (int i = 0; i < 3; ++i)
{
var eventA = new EventA
{
A = "I'm event A",
EventId = Guid.NewGuid().ToString(),
EventType = "EventType-A",
OccuredOn = DateTime.UtcNow.Ticks,
};
producer.ProduceAsync(topic, new Message <string, ISpecificRecord> {
Key = "DomainEvent", Value = eventA
}).Wait();
}
for (int i = 0; i < 3; ++i)
{
var eventB = new EventB
{
B = 123456987,
EventId = Guid.NewGuid().ToString(),
EventType = "EventType-B",
OccuredOn = DateTime.UtcNow.Ticks,
};
producer.ProduceAsync(topic, new Message <string, ISpecificRecord> {
Key = "DomainEvent", Value = eventB
}).Wait();
}
Assert.Equal(0, producer.Flush(TimeSpan.FromSeconds(10)));
}
var consumerConfig = new ConsumerConfig
{
BootstrapServers = bootstrapServers,
GroupId = Guid.NewGuid().ToString(),
SessionTimeoutMs = 6000,
AutoOffsetReset = AutoOffsetReset.Earliest,
EnablePartitionEof = true
};
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
using (var consumer =
new ConsumerBuilder <string, GenericRecord>(consumerConfig)
.SetKeyDeserializer(new AvroDeserializer <string>(schemaRegistry).AsSyncOverAsync())
.SetValueDeserializer(new BmGenericDeserializerImpl(schemaRegistry).AsSyncOverAsync())
.SetErrorHandler((_, e) => Assert.True(false, e.Reason))
.Build())
{
consumer.Subscribe(topic);
int i = 0;
while (true)
{
var record = consumer.Consume(TimeSpan.FromMilliseconds(100));
if (record == null)
{
continue;
}
if (record.IsPartitionEOF)
{
break;
}
Console.WriteLine(record.Message.Value["EventType"]);
i += 1;
}
Assert.Equal(6, i);
consumer.Close();
}
}
public static void ProduceConsumeMixedJson(string bootstrapServers, string schemaRegistryServers)
{
var producerConfig = new ProducerConfig {
BootstrapServers = bootstrapServers
};
var schemaRegistryConfig = new SchemaRegistryConfig {
Url = schemaRegistryServers
};
using (var topic = new TemporaryTopic(bootstrapServers, 1))
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
using (var producer =
new ProducerBuilder <string, PersonPoco>(producerConfig)
.SetValueSerializer(new JsonSerializer <PersonPoco>(schemaRegistry))
.Build())
{
// test that a System.ComponentModel.DataAnnotations constraint is effective.
{
var p = new PersonPoco
{
FirstName = "Test",
LastName = "User",
NumberWithRange = 7 // range should be between 2 and 5.
};
Assert.Throws <ProduceException <string, PersonPoco> >(() => {
try
{
producer.ProduceAsync(topic.Name, new Message <string, PersonPoco> {
Key = "test1", Value = p
}).Wait();
}
catch (AggregateException ax)
{
// what would be thrown if the call was awaited.
throw ax.InnerException;
}
});
}
// test that a Newtonsoft.Json constraint is effective.
{
var p = new PersonPoco
{
FirstName = "Test",
// Omit LastName
NumberWithRange = 3
};
Assert.Throws <AggregateException>(() => {
producer.ProduceAsync(topic.Name, new Message <string, PersonPoco> {
Key = "test1", Value = p
}).Wait();
});
}
// test all fields valid.
{
var p = new PersonPoco
{
FirstName = "A",
MiddleName = "Test",
LastName = "User",
Gender = GenderEnum.Male,
NumberWithRange = 3,
Birthday = new DateTime(2010, 11, 1),
Company = new CompanyPoco
{
Name = "Confluent"
},
Cars = new List <CarPoco>
{
new CarPoco
{
Name = "Volvo",
Manufacturer = new CompanyPoco
{
Name = "Confluent"
}
}
}
};
producer.ProduceAsync(topic.Name, new Message <string, PersonPoco> {
Key = "test1", Value = p
}).Wait();
var schema = schemaRegistry.GetLatestSchemaAsync(SubjectNameStrategy.Topic.ConstructValueSubjectName(topic.Name, null)).Result.SchemaString;
var consumerConfig = new ConsumerConfig
{
BootstrapServers = bootstrapServers,
GroupId = Guid.NewGuid().ToString(),
AutoOffsetReset = AutoOffsetReset.Earliest
};
using (var consumer =
new ConsumerBuilder <string, PersonPoco>(consumerConfig)
.SetValueDeserializer(new JsonDeserializer <PersonPoco>().AsSyncOverAsync())
.Build())
{
consumer.Subscribe(topic.Name);
var cr = consumer.Consume();
Assert.Equal(p.FirstName, cr.Message.Value.FirstName);
Assert.Equal(p.MiddleName, cr.Message.Value.MiddleName);
Assert.Equal(p.LastName, cr.Message.Value.LastName);
Assert.Equal(p.Gender, cr.Message.Value.Gender);
Assert.Equal(p.NumberWithRange, cr.Message.Value.NumberWithRange);
Assert.Equal(p.Birthday, cr.Message.Value.Birthday);
Assert.Equal(p.Company.Name, cr.Message.Value.Company.Name);
Assert.Single(cr.Message.Value.Cars);
Assert.Equal(cr.Message.Value.Cars[0].Manufacturer.Name, p.Cars[0].Manufacturer.Name);
Assert.Equal(cr.Message.Value.Cars[0].Name, p.Cars[0].Name);
}
}
}
}
public static void BasicAuth(Config config)
{
var testSchema1 =
"{\"type\":\"record\",\"name\":\"User\",\"namespace\":\"Confluent.Kafka.Examples.AvroSpecific" +
"\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"favorite_number\",\"type\":[\"i" +
"nt\",\"null\"]},{\"name\":\"favorite_color\",\"type\":[\"string\",\"null\"]}]}";
// 1. valid configuration cases
// 1.1. credentials specified as USER_INFO.
var conf = new SchemaRegistryConfig
{
Url = config.ServerWithAuth,
BasicAuthCredentialsSource = AuthCredentialsSource.UserInfo,
BasicAuthUserInfo = $"{config.Username}:{config.Password}"
};
// some sanity checking of strongly typed config property name mappings.
Assert.Equal(config.ServerWithAuth, conf.Get("schema.registry.url"));
Assert.Equal("USER_INFO", conf.Get("schema.registry.basic.auth.credentials.source"));
Assert.Equal($"{config.Username}:{config.Password}", conf.Get("schema.registry.basic.auth.user.info"));
using (var sr = new CachedSchemaRegistryClient(conf))
{
var topicName = Guid.NewGuid().ToString();
var subject = SubjectNameStrategy.Topic.ToDelegate()(new SerializationContext(MessageComponentType.Value, topicName), null);
var id = sr.RegisterSchemaAsync(subject, testSchema1).Result;
var schema = sr.GetLatestSchemaAsync(subject).Result;
Assert.Equal(schema.Id, id);
}
// 1.2. credentials specified as USER_INFO implicitly (and using strongly typed SchemaRegistryConfig)
var conf2 = new SchemaRegistryConfig
{
Url = config.ServerWithAuth,
BasicAuthUserInfo = $"{config.Username}:{config.Password}"
};
using (var sr = new CachedSchemaRegistryClient(conf2))
{
var topicName = Guid.NewGuid().ToString();
var subject = SubjectNameStrategy.Topic.ConstructValueSubjectName(topicName, null);
var id = sr.RegisterSchemaAsync(subject, testSchema1).Result;
var schema = sr.GetLatestSchemaAsync(subject).Result;
Assert.Equal(schema.Id, id);
}
// 1.3. credentials specified as SASL_INHERIT.
using (var sr = new CachedSchemaRegistryClient(
new Dictionary <string, string>
{
{ "schema.registry.url", config.ServerWithAuth },
{ "schema.registry.basic.auth.credentials.source", "SASL_INHERIT" },
{ "sasl.username", config.Username },
{ "sasl.password", config.Password }
}))
{
var topicName = Guid.NewGuid().ToString();
var subject = SubjectNameStrategy.Topic.ConstructValueSubjectName(topicName, null);
var id = sr.RegisterSchemaAsync(subject, testSchema1).Result;
var schema = sr.GetLatestSchemaAsync(subject).Result;
Assert.Equal(schema.Id, id);
}
// 1.4. credentials specified as SASL_INHERIT via strongly typed config.
var conf3 = new SchemaRegistryConfig {
Url = config.ServerWithAuth
};
conf3.BasicAuthCredentialsSource = AuthCredentialsSource.SaslInherit;
conf3.Set("sasl.username", config.Username);
conf3.Set("sasl.password", config.Password);
using (var sr = new CachedSchemaRegistryClient(conf3))
{
var topicName = Guid.NewGuid().ToString();
var subject = SubjectNameStrategy.Topic.ConstructValueSubjectName(topicName, null);
var id = sr.RegisterSchemaAsync(subject, testSchema1).Result;
var schema = sr.GetLatestSchemaAsync(subject).Result;
Assert.Equal(schema.Id, id);
}
// 2. invalid configuration cases
Assert.Throws <ArgumentException>(() =>
{
var sr = new CachedSchemaRegistryClient(new Dictionary <string, string>
{
{ "schema.registry.url", config.ServerWithAuth },
{ "schema.registry.basic.auth.credentials.source", "SASL_INHERIT" },
{ "schema.registry.basic.auth.user.info", $"{config.Username:config.Password}" }
});
});
Assert.Throws <ArgumentException>(() =>
{
var sr = new CachedSchemaRegistryClient(new Dictionary <string, string>
{
{ "schema.registry.url", config.ServerWithAuth },
{ "schema.registry.basic.auth.credentials.source", "UBUTE_SOURCE" }
});
});
Assert.Throws <ArgumentException>(() =>
{
var sr = new CachedSchemaRegistryClient(new Dictionary <string, string>
{
{ "schema.registry.url", config.ServerWithAuth },
{ "schema.registry.basic.auth.credentials.source", "NONE" },
{ "schema.registry.basic.auth.user.info", $"{config.Username:config.Password}" }
});
});
// connect to authenticating without credentials. shouldn't work.
// SR <= 5.3.4 returns Unauthorized with empty Content (HttpRequestException)
// 5.3.4 < SR <= 5.3.8 returns Unauthorized with message but without error_code (SchemaRegistryException)
// SR >= 5.40 returns Unauthorized with message and error_code (SchemaRegistryException)
var schemaRegistryException = Assert.Throws <SchemaRegistryException>(() =>
{
var sr = new CachedSchemaRegistryClient(new SchemaRegistryConfig {
Url = config.ServerWithAuth
});
var topicName = Guid.NewGuid().ToString();
var subject = SubjectNameStrategy.Topic.ConstructValueSubjectName(topicName, null);
try
{
var id = sr.RegisterSchemaAsync(subject, testSchema1).Result;
}
catch (Exception e)
{
throw e.InnerException;
}
});
Assert.Equal(401, schemaRegistryException.ErrorCode);
Assert.Equal("Unauthorized; error code: 401", schemaRegistryException.Message);
}
public static void Protobuf(Config config)
{
var srInitial = new CachedSchemaRegistryClient(new SchemaRegistryConfig {
Url = config.Server
});
var sr = new CachedSchemaRegistryClient(new SchemaRegistryConfig {
Url = config.Server
});
var testSchemaBase64 = Confluent.Kafka.Examples.Protobuf.User.Descriptor.File.SerializedData.ToBase64();
var topicName = Guid.NewGuid().ToString();
var subjectInitial = SubjectNameStrategy.Topic.ConstructValueSubjectName(topicName, null);
var subject = SubjectNameStrategy.Topic.ConstructValueSubjectName(topicName + "2", null);
// check that registering a base64 protobuf schema works.
var id1 = srInitial.RegisterSchemaAsync(subjectInitial, new Schema(testSchemaBase64, SchemaType.Protobuf)).Result;
var schema1 = sr.GetSchemaAsync(id1, "serialized").Result; // use a different sr instance to ensure a cached value is not read.
Assert.Equal(SchemaType.Protobuf, schema1.SchemaType);
Assert.NotNull(schema1.SchemaString); // SR slightly munges the schema as a result of moving to a text representation and back so can't check for equality.
// check that the id of the schema just registered can be retrieved.
var id = sr.GetSchemaIdAsync(subjectInitial, new Schema(schema1.SchemaString, SchemaType.Protobuf)).Result;
Assert.Equal(id1, id);
// re-register the munged schema (to a different subject) and check that it is not re-munged.
var id2 = sr.RegisterSchemaAsync(subject, schema1).Result;
var schema2 = sr.GetSchemaAsync(id2, "serialized").Result;
Assert.Equal(schema1.SchemaString, schema2.SchemaString);
Assert.Equal(schema1.SchemaType, schema2.SchemaType);
// This sequence of operations is designed to test caching behavior (and two alternat schema getting methods).
var schemaAsText = sr.GetSchemaAsync(id2).Result;
var schemaAsSerialized = sr.GetSchemaAsync(id2, "serialized").Result;
var schemaAsText2 = sr.GetSchemaAsync(id2).Result;
var schemaAsSerialized2 = sr.GetSchemaAsync(id2, "serialized").Result;
var latestSchema = sr.GetLatestSchemaAsync(subject).Result; // should come back as text.
var schemaAsSerialized3 = sr.GetSchemaAsync(id2, "serialized").Result;
var latestSchema2 = sr.GetRegisteredSchemaAsync(subject, latestSchema.Version).Result; // should come back as text.
Assert.Equal(schema1.SchemaString, schemaAsSerialized.SchemaString);
Assert.Equal(SchemaType.Protobuf, schemaAsSerialized.SchemaType);
Assert.Empty(schemaAsSerialized.References);
Assert.Equal(schema1.SchemaString, schemaAsSerialized2.SchemaString);
Assert.Equal(schema1.SchemaString, schemaAsSerialized3.SchemaString);
Assert.NotEqual(schema1.SchemaString, schemaAsText.SchemaString);
Assert.Equal(schemaAsText.SchemaString, schemaAsText2.SchemaString);
Assert.Equal(schemaAsText.SchemaString, latestSchema.SchemaString);
Assert.Equal(schemaAsText.SchemaString, latestSchema2.SchemaString);
Assert.Equal(SchemaType.Protobuf, schemaAsText.SchemaType);
Assert.Empty(schemaAsText2.References);
// compatibility
var compat = sr.IsCompatibleAsync(subject, schema2).Result;
Assert.True(compat);
var avroSchema =
"{\"type\":\"record\",\"name\":\"User\",\"namespace\":\"Confluent.Kafka.Examples.AvroSpecific" +
"\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"favorite_number\",\"type\":[\"i" +
"nt\",\"null\"]},{\"name\":\"favorite_color\",\"type\":[\"string\",\"null\"]}]}";
var compat2 = sr.IsCompatibleAsync(subject, avroSchema).Result;
Assert.False(compat2);
var compat3 = sr.IsCompatibleAsync(subject, new Schema(avroSchema, SchemaType.Avro)).Result;
Assert.False(compat3);
// invalid type
Assert.ThrowsAny <Exception>(() => {
sr.RegisterSchemaAsync(SubjectNameStrategy.Topic.ConstructKeySubjectName(topicName + "3", null), new Schema(avroSchema, SchemaType.Protobuf)).Wait();
});
}