/// <summary>
/// Initialize a new instance of the JsonSerializer class.
/// </summary>
public JsonSerializer(ISchemaRegistryClient schemaRegistryClient, JsonSerializerConfig config = null)
{
this.schemaRegistryClient = schemaRegistryClient;
this.schema = NJsonSchema.JsonSchema.FromType <T>();
this.schemaFullname = schema.Title;
this.schemaText = schema.ToJson();
if (config == null)
{
return;
}
var nonJsonConfig = config.Where(item => !item.Key.StartsWith("json."));
if (nonJsonConfig.Count() > 0)
{
throw new ArgumentException($"JsonSerializer: unknown configuration parameter {nonJsonConfig.First().Key}");
}
if (config.BufferBytes != null)
{
this.initialBufferSize = config.BufferBytes.Value;
}
if (config.AutoRegisterSchemas != null)
{
this.autoRegisterSchema = config.AutoRegisterSchemas.Value;
}
if (config.SubjectNameStrategy != null)
{
this.subjectNameStrategy = config.SubjectNameStrategy.Value.ToDelegate();
}
}
/// <include file='../Confluent.Kafka/include_docs.xml' path='API/Member[@name="IDeserializer_Configure"]/*' />
public IEnumerable <KeyValuePair <string, object> > Configure(IEnumerable <KeyValuePair <string, object> > config, bool isKey)
{
var keyOrValue = isKey ? "Key" : "Value";
var srConfig = config.Where(item => item.Key.StartsWith("schema.registry."));
var avroConfig = config.Where(item => item.Key.StartsWith("avro."));
if (avroConfig.Count() != 0)
{
throw new ArgumentException($"{keyOrValue} AvroDeserializer: unexpected configuration parameter {avroConfig.First().Key}");
}
if (srConfig.Count() != 0)
{
if (schemaRegistryClient != null)
{
throw new ArgumentException($"{keyOrValue} AvroDeserializer schema registry client was configured via both the constructor and configuration parameters.");
}
schemaRegistryClient = new CachedSchemaRegistryClient(config);
}
if (schemaRegistryClient == null)
{
throw new ArgumentException($"{keyOrValue} AvroDeserializer schema registry client was not supplied or configured.");
}
return(config.Where(item => !item.Key.StartsWith("schema.registry.") && !item.Key.StartsWith("avro.")));
}
private void SetupProducer()
{
if (producer != null)
{
return;
}
lock (myLock)
{
Dispose();
}
var config = new ProducerConfig {
BootstrapServers = kafkaOptions.BootstrapServerUri
};
var schemaRegistryConfig = new SchemaRegistryConfig {
Url = kafkaOptions.SchemaRegistryUri
};
var jsonSerializerConfig = new JsonSerializerConfig();
schemaRegistryClient = new CachedSchemaRegistryClient(schemaRegistryConfig);
producer = new ProducerBuilder <Null, T>(config)
.SetValueSerializer(new JsonSerializer <T>(schemaRegistryClient, jsonSerializerConfig))
.Build();
}
/// <summary>
/// Set the message key deserializer.
/// </summary>
/// <param name="consumerBuilder">
/// The <see cref="ConsumerBuilder{TKey, TValue}" /> instance to be configured.
/// </param>
/// <param name="registryClient">
/// The client to use for Schema Registry operations. The client should only be disposed
/// after the consumer; the deserializer will use it to request schemas as messages are
/// being consumed.
/// </param>
public static ConsumerBuilder <TKey, TValue> SetAvroKeyDeserializer <TKey, TValue>(
this ConsumerBuilder <TKey, TValue> consumerBuilder,
ISchemaRegistryClient registryClient
) => consumerBuilder.SetKeyDeserializer(
new AsyncSchemaRegistryDeserializer <TKey>(
registryClient
).AsSyncOverAsync());
/// <summary>
/// Creates a serializer.
/// </summary>
/// <param name="registryClient">
/// The client to use for Schema Registry operations. (The client will not be disposed.)
/// </param>
/// <param name="registerAutomatically">
/// Whether to automatically register schemas that match the type being serialized.
/// </param>
/// <param name="schemaBuilder">
/// The schema builder to use to create a schema for a C# type when registering automatically.
/// If none is provided, the default schema builder will be used.
/// </param>
/// <param name="schemaReader">
/// The JSON schema reader to use to convert schemas received from the registry into abstract
/// representations. If none is provided, the default schema reader will be used.
/// </param>
/// <param name="schemaWriter">
/// The JSON schema writer to use to convert abstract schema representations when registering
/// automatically. If none is provided, the default schema writer will be used.
/// </param>
/// <param name="serializerBuilder">
/// The deserializer builder to use to build serialization functions for C# types. If none
/// is provided, the default serializer builder will be used.
/// </param>
/// <param name="subjectNameBuilder">
/// A function that determines the subject name given the topic name and a component type
/// (key or value). If none is provided, the default "{topic name}-{component}" naming
/// convention will be used.
/// </param>
/// <exception cref="ArgumentNullException">
/// Thrown when the registry client is null.
/// </exception>
public AsyncSchemaRegistrySerializer(
ISchemaRegistryClient registryClient,
bool registerAutomatically = false,
Abstract.ISchemaBuilder?schemaBuilder = null,
IJsonSchemaReader?schemaReader = null,
IJsonSchemaWriter?schemaWriter = null,
IBinarySerializerBuilder?serializerBuilder = null,
Func <SerializationContext, string>?subjectNameBuilder = null
)
{
if (registryClient == null)
{
throw new ArgumentNullException(nameof(registryClient));
}
RegisterAutomatically = registerAutomatically;
SchemaBuilder = schemaBuilder ?? new Abstract.SchemaBuilder();
SchemaReader = schemaReader ?? new JsonSchemaReader();
SchemaWriter = schemaWriter ?? new JsonSchemaWriter();
SerializerBuilder = serializerBuilder ?? new BinarySerializerBuilder();
SubjectNameBuilder = subjectNameBuilder ??
(c => $"{c.Topic}-{(c.Component == MessageComponentType.Key ? "key" : "value")}");
_cache = new ConcurrentDictionary <string, Task <Func <T, byte[]> > >();
_register = (subject, json) => registryClient.RegisterSchemaAsync(subject, json);
_resolve = subject => registryClient.GetLatestSchemaAsync(subject);
}
private static async Task <IProducer <OrderKey, OrderEventRecord> > CreateProducer(
ISchemaRegistryClient registryClient,
AutomaticRegistrationBehavior automaticRegistrationBehavior)
{
var schemaBuilder = new SchemaBuilder(
SchemaBuilder.CreateDefaultCaseBuilders()
.Prepend(builder => new OrderEventUnionSchemaBuilderCase(builder)));
using var serializerBuilder = new SchemaRegistrySerializerBuilder(
registryClient,
schemaBuilder,
serializerBuilder: new BinarySerializerBuilder(
BinarySerializerBuilder.CreateDefaultCaseBuilders()
.Prepend(builder => new OrderEventUnionSerializerBuilderCase(builder))));
var producerBuilder = new ProducerBuilder <OrderKey, OrderEventRecord>(
new ProducerConfig
{
BootstrapServers = BootstrapServers,
});
await producerBuilder.SetAvroKeySerializer(
registryClient,
SubjectNameStrategy.Topic.ConstructKeySubjectName(Topic),
automaticRegistrationBehavior);
await producerBuilder.SetAvroValueSerializer(
serializerBuilder,
SubjectNameStrategy.Topic.ConstructKeySubjectName(Topic),
automaticRegistrationBehavior);
return(producerBuilder.Build());
}
public KafkaProducer(IOptions <KafkaProducerOptions> options, ISemanticLog log, IJsonSerializer jsonSerializer)
{
this.jsonSerializer = jsonSerializer;
textProducer = new ProducerBuilder <string, string>(options.Value)
.SetErrorHandler((p, error) =>
{
LogError(log, error);
})
.SetLogHandler((p, message) =>
{
LogMessage(log, message);
})
.SetKeySerializer(Serializers.Utf8)
.SetValueSerializer(Serializers.Utf8)
.Build();
if (options.Value.IsSchemaRegistryConfigured())
{
schemaRegistry = new CachedSchemaRegistryClient(options.Value.SchemaRegistry);
avroProducer = new ProducerBuilder <string, GenericRecord>(options.Value)
.SetErrorHandler((p, error) =>
{
LogError(log, error);
})
.SetLogHandler((p, message) =>
{
LogMessage(log, message);
})
.SetKeySerializer(Serializers.Utf8)
.SetValueSerializer(new AvroSerializer <GenericRecord>(schemaRegistry, options.Value.AvroSerializer))
.Build();
}
}
/// <summary>
/// Constructor.
/// </summary>
/// <param name="settings">the settings dictionary, which follows
/// the norm defined by Kafka settings.</param>
/// <exception cref="ArgumentNullException">if any of the method parameters is null.</exception>
/// <exception cref="ArgumentException">if any of the mandatory Kafka parameters is not set.</exception>
public GojulMQKafkaMessageConsumer(Dictionary <string, string> settings)
{
Condition.Requires(settings, "settings").IsNotNull();
Condition.Requires((string)settings[BootstrapServers], BootstrapServers)
.IsNotNull()
.IsNotEmpty();
Condition.Requires((string)settings[GroupId], GroupId)
.IsNotNull()
.IsNotEmpty();
Condition.Requires((string)settings[SchemaRegistryUrl], SchemaRegistryUrl)
.IsNotNull()
.IsNotEmpty();
_disposed = false;
_schemaRegistry = new CachedSchemaRegistryClient(settings);
_consumer = new ConsumerBuilder <string, T>(KafkaSettingsList.SanitizeConfiguration(settings))
.SetKeyDeserializer(Deserializers.Utf8)
.SetValueDeserializer(new AvroDeserializer <T>(_schemaRegistry).AsSyncOverAsync())
.SetErrorHandler((_, error) =>
{
_log.Error(string.Format("Error while processing message %s - Skipping this message !", error));
})
.Build();
}
/// <summary>
/// Initializes a new instance of the <see cref="AsyncSchemaRegistrySerializer{T}" />
/// class with a Schema Registry client.
/// </summary>
/// <param name="registryClient">
/// A Schema Registry client to use for Registry operations. (The client will not be
/// disposed.)
/// </param>
/// <param name="registerAutomatically">
/// Whether the serializer should automatically register schemas that match the type being
/// serialized.
/// </param>
/// <param name="schemaBuilder">
/// A schema builder instance that should be used to create a schema for <typeparamref name="T" />
/// when registering automatically. If none is provided, the default <see cref="SchemaBuilder" />
/// will be used.
/// </param>
/// <param name="schemaReader">
/// A schema reader instance that should be used to convert schemas received from the
/// Registry into abstract representations. If none is provided, the default
/// <see cref="JsonSchemaReader" /> will be used.
/// </param>
/// <param name="schemaWriter">
/// A schema writer instance that should be used to convert abstract schema representations
/// when registering automatically. If none is provided, the default <see cref="JsonSchemaWriter" />
/// will be used.
/// </param>
/// <param name="serializerBuilder">
/// A serializer builder instance that should be used to generate serialization functions
/// for .NET <see cref="Type" />s. If none is provided, the default <see cref="BinarySerializerBuilder" />
/// will be used.
/// </param>
/// <param name="subjectNameBuilder">
/// A function that determines a subject name given a topic name and a component type (key
/// or value). If none is provided, the default <c>{topic name}-{component}</c> naming
/// convention will be used.
/// </param>
/// <param name="tombstoneBehavior">
/// How the serializer should handle tombstone records.
/// </param>
/// <exception cref="ArgumentNullException">
/// Thrown when <paramref name="registryClient" /> is <c>null</c>.
/// </exception>
/// <exception cref="UnsupportedTypeException">
/// Thrown when <paramref name="tombstoneBehavior" /> is incompatible with
/// <typeparamref name="T" />.
/// </exception>
public AsyncSchemaRegistrySerializer(
ISchemaRegistryClient registryClient,
AutomaticRegistrationBehavior registerAutomatically = AutomaticRegistrationBehavior.Never,
Abstract.ISchemaBuilder schemaBuilder = null,
IJsonSchemaReader schemaReader = null,
IJsonSchemaWriter schemaWriter = null,
IBinarySerializerBuilder serializerBuilder = null,
Func <SerializationContext, string> subjectNameBuilder = null,
TombstoneBehavior tombstoneBehavior = TombstoneBehavior.None)
{
if (tombstoneBehavior != TombstoneBehavior.None && default(T) != null)
{
throw new UnsupportedTypeException(typeof(T), $"{typeof(T)} cannot represent tombstone values.");
}
RegisterAutomatically = registerAutomatically;
RegistryClient = registryClient ?? throw new ArgumentNullException(nameof(registryClient));
SchemaBuilder = schemaBuilder ?? new Abstract.SchemaBuilder();
SchemaReader = schemaReader ?? new JsonSchemaReader();
SchemaWriter = schemaWriter ?? new JsonSchemaWriter();
SerializerBuilder = serializerBuilder ?? new BinarySerializerBuilder();
SubjectNameBuilder = subjectNameBuilder ??
(c => $"{c.Topic}-{(c.Component == MessageComponentType.Key ? "key" : "value")}");
TombstoneBehavior = tombstoneBehavior;
cache = new Dictionary <string, Task <Func <T, byte[]> > >();
disposeRegistryClient = false;
}
/// <summary>
/// Creates a deserializer.
/// </summary>
/// <param name="registryClient">
/// The client to use for Schema Registry operations. (The client will not be disposed.)
/// </param>
/// <param name="deserializerBuilder">
/// The deserializer builder used to generate deserialization functions for C# types. If
/// none is provided, the default deserializer builder will be used.
/// </param>
/// <param name="schemaReader">
/// The JSON schema reader used to convert schemas received from the registry into abstract
/// representations. If none is provided, the default schema reader will be used.
/// </param>
/// <param name="tombstoneBehavior">
/// The behavior of the deserializer on tombstone records.
/// </param>
/// <exception cref="ArgumentNullException">
/// Thrown when the registry client is null.
/// </exception>
public AsyncSchemaRegistryDeserializer(
ISchemaRegistryClient registryClient,
IBinaryDeserializerBuilder deserializerBuilder = null,
IJsonSchemaReader schemaReader = null,
TombstoneBehavior tombstoneBehavior = TombstoneBehavior.None
)
{
if (registryClient == null)
{
throw new ArgumentNullException(nameof(registryClient));
}
if (tombstoneBehavior != TombstoneBehavior.None && default(T) != null)
{
throw new UnsupportedTypeException(typeof(T), $"{typeof(T)} cannot represent tombstone values.");
}
DeserializerBuilder = deserializerBuilder ?? new BinaryDeserializerBuilder();
RegistryClient = registryClient;
SchemaReader = schemaReader ?? new JsonSchemaReader();
TombstoneBehavior = tombstoneBehavior;
_cache = new Dictionary <int, Task <Func <Stream, T> > >();
_disposeRegistryClient = false;
}
/// <summary>
/// Creates a serializer.
/// </summary>
/// <param name="registryClient">
/// A client to use for Schema Registry operations. (The client will not be disposed.)
/// </param>
/// <param name="registerAutomatically">
/// Whether to automatically register schemas that match the type being serialized.
/// </param>
/// <param name="schemaBuilder">
/// A schema builder (used to build a schema for a C# type when registering automatically).
/// If none is provided, the default schema builder will be used.
/// </param>
/// <param name="schemaReader">
/// A JSON schema reader (used to convert schemas received from the registry into abstract
/// representations). If none is provided, the default schema reader will be used.
/// </param>
/// <param name="schemaWriter">
/// A JSON schema writer (used to convert abstract schema representations when registering
/// automatically). If none is provided, the default schema writer will be used.
/// </param>
/// <param name="serializerBuilder">
/// A deserializer builder (used to build serialization functions for C# types). If none is
/// provided, the default serializer builder will be used.
/// </param>
/// <param name="subjectNameBuilder">
/// A function that determines the subject name given the topic name and a component type
/// (key or value). If none is provided, the default "{topic name}-{component}" naming
/// convention will be used.
/// </param>
/// <exception cref="ArgumentNullException">
/// Thrown when the registry client is null.
/// </exception>
public AsyncSchemaRegistrySerializer(
ISchemaRegistryClient registryClient,
bool registerAutomatically = false,
Abstract.ISchemaBuilder schemaBuilder = null,
IJsonSchemaReader schemaReader = null,
IJsonSchemaWriter schemaWriter = null,
IBinarySerializerBuilder serializerBuilder = null,
Func <SerializationContext, string> subjectNameBuilder = null
) : this(
registerAutomatically,
schemaBuilder,
schemaReader,
schemaWriter,
serializerBuilder,
subjectNameBuilder
)
{
if (registryClient == null)
{
throw new ArgumentNullException(nameof(registryClient));
}
_register = (subject, json) => registryClient.RegisterSchemaAsync(subject, json);
_resolve = subject => registryClient.GetLatestSchemaAsync(subject);
}
/// <summary>
/// Initialize a new instance of the ProtobufSerializer class.
/// </summary>
public ProtobufSerializer(ISchemaRegistryClient schemaRegistryClient, ProtobufSerializerConfig config = null)
{
this.schemaRegistryClient = schemaRegistryClient;
if (config == null)
{
this.referenceSubjectNameStrategy = ReferenceSubjectNameStrategy.ReferenceName.ToDelegate();
return;
}
var nonProtobufConfig = config.Where(item => !item.Key.StartsWith("protobuf."));
if (nonProtobufConfig.Count() > 0)
{
throw new ArgumentException($"ProtobufSerializer: unknown configuration parameter {nonProtobufConfig.First().Key}");
}
if (config.BufferBytes != null)
{
this.initialBufferSize = config.BufferBytes.Value;
}
if (config.AutoRegisterSchemas != null)
{
this.autoRegisterSchema = config.AutoRegisterSchemas.Value;
}
if (config.SubjectNameStrategy != null)
{
this.subjectNameStrategy = config.SubjectNameStrategy.Value.ToDelegate();
}
this.referenceSubjectNameStrategy = config.ReferenceSubjectNameStrategy == null
? ReferenceSubjectNameStrategy.ReferenceName.ToDelegate()
: config.ReferenceSubjectNameStrategy.Value.ToDelegate();
}
public AvroExternalStreamDeserializer(ISchemaRegistryClient schemaRegistry)
{
_registry = schemaRegistry;
_deserializers = new ConcurrentDictionary <Type, AvroDeserializerWrapper>();
_getOrCreateMethodInfo = typeof(AvroExternalStreamDeserializer)
.GetMethod(nameof(GetOrCreate), BindingFlags.NonPublic | BindingFlags.Instance);
}
public AvroSubjectNameStrategySerializer
(
ISchemaRegistryClient schemaRegistryClient
)
{
_schemaRegistryClient = schemaRegistryClient;
}
public KafkaProducer(ILogger logger, ISchemaRegistryClient schemaRegistryClient, ProducerConfig config)
{
_logger = logger;
Producer = new ProducerBuilder <TKey, TValue>(config)
.SetKeySerializer(new AvroSerializer <TKey>(schemaRegistryClient))
.SetValueSerializer(new AvroSerializer <TValue>(schemaRegistryClient))
.Build();
}
internal ProducerContext(ProducerConfig config, ISchemaRegistryClient registry)
{
_registry = registry;
Producer = new ProducerBuilder <string, T>(config)
.SetValueSerializer(new ProtobufSerializer <T>(_registry))
.Build();
}
internal ConsumerContext(ConsumerConfig config, ISchemaRegistryClient registry)
{
_registry = registry;
Consumer = new ConsumerBuilder <string, T>(config)
.SetValueDeserializer(new ProtobufDeserializer <T>().AsSyncOverAsync())
.Build();
}
public CachedSchemaRegistryClient(ISchemaRegistryClient innerClient, KafkaSchemaRegistryConfiguration configuration)
{
_innerClient = innerClient;
_configuration = configuration;
_schemaVersionListCache = new ConcurrentDictionary <string, CacheItem <int[]> >();
_schemaDetailsCache = new ConcurrentDictionary <string, CacheItem <SchemaDetails> >();
}
public AvroSubjectNameStrategyDeserializer
(
ISchemaRegistryClient schemaRegistryClient,
SubjectNameSchemaCache cache
)
{
_schemaRegistryClient = schemaRegistryClient;
_cache = cache;
}
/// <summary>
/// Initializes a new instance of the <see cref="ConfluentProtobufSerializer"/> class.
/// </summary>
/// <param name="resolver">An instance of <see cref="IDependencyResolver"/></param>
/// <param name="serializerConfig">An instance of <see cref="ProtobufSerializerConfig"/></param>
public ConfluentProtobufSerializer(IDependencyResolver resolver, ProtobufSerializerConfig serializerConfig)
{
this.schemaRegistryClient =
resolver.Resolve <ISchemaRegistryClient>() ??
throw new InvalidOperationException(
$"No schema registry configuration was found. Set it using {nameof(ClusterConfigurationBuilderExtensions.WithSchemaRegistry)} on cluster configuration");
this.serializerConfig = serializerConfig;
}
/// <summary>
/// Set the message value deserializer.
/// </summary>
/// <param name="consumerBuilder">
/// The <see cref="ConsumerBuilder{TKey, TValue}" /> instance to be configured.
/// </param>
/// <param name="registryClient">
/// The client to use for Schema Registry operations. The client should only be disposed
/// after the consumer; the deserializer will use it to request schemas as messages are
/// being consumed.
/// </param>
/// <param name="tombstoneBehavior">
/// The behavior of the deserializer on tombstone records.
/// </param>
public static ConsumerBuilder <TKey, TValue> SetAvroValueDeserializer <TKey, TValue>(
this ConsumerBuilder <TKey, TValue> consumerBuilder,
ISchemaRegistryClient registryClient,
TombstoneBehavior tombstoneBehavior = TombstoneBehavior.None
) => consumerBuilder.SetValueDeserializer(
new AsyncSchemaRegistryDeserializer <TValue>(
registryClient,
tombstoneBehavior: tombstoneBehavior
).AsSyncOverAsync());
public GenericSerializerImpl(
ISchemaRegistryClient schemaRegistryClient,
bool autoRegisterSchema,
int initialBufferSize)
{
this.schemaRegistryClient = schemaRegistryClient;
this.autoRegisterSchema = autoRegisterSchema;
this.initialBufferSize = initialBufferSize;
}
/// <summary>
/// Set the message key serializer.
/// </summary>
/// <param name="producerBuilder">
/// The <see cref="ProducerBuilder{TKey, TValue}" /> instance to be configured.
/// </param>
/// <param name="registryClient">
/// A client to use for Schema Registry operations. The client should only be disposed
/// after the producer; the serializer will use it to request schemas as messages are being
/// produced.
/// </param>
/// <param name="registerAutomatically">
/// Whether to automatically register schemas that match the type being serialized.
/// </param>
/// <param name="subjectNameBuilder">
/// A function that determines the subject name given the topic name and a component type
/// (key or value). If none is provided, the default "{topic name}-{component}" naming
/// convention will be used.
/// </param>
public static ProducerBuilder <TKey, TValue> SetAvroKeySerializer <TKey, TValue>(
this ProducerBuilder <TKey, TValue> producerBuilder,
ISchemaRegistryClient registryClient,
bool registerAutomatically = false,
Func <SerializationContext, string> subjectNameBuilder = null
) => producerBuilder.SetKeySerializer(new AsyncSchemaRegistrySerializer <TKey>(
registryClient,
registerAutomatically: registerAutomatically,
subjectNameBuilder: subjectNameBuilder
));
/// <summary>
/// Initialize a new DynamicSpecificRecordDeserializer instance.
/// </summary>
/// <param name="schemaRegistryClient">
/// An implementation of ISchemaRegistryClient used for
/// communication with Confluent Schema Registry.
/// </param>
/// <param name="config">
/// Deserializer configuration properties (refer to
/// <see cref="AvroDeserializerConfig" />).
/// </param>
public DynamicSpecificRecordDeserializer(ISchemaRegistryClient schemaRegistryClient,
IReadOnlyCollection <KeyValuePair <string, string> > config = null)
{
// There is some config validation that happens so make sure we can construct one
// ReSharper disable once ObjectCreationAsStatement
new AvroDeserializer <ISpecificRecord>(schemaRegistryClient, config);
_schemaRegistryClient = schemaRegistryClient;
_config = config;
}
/// <summary>
/// Initialize a new instance of the AvroSerializer class.
/// When passed as a parameter to the Confluent.Kafka.Producer constructor,
/// the following configuration properties will be extracted from the producer's
/// configuration property collection:
///
/// avro.serializer.buffer.bytes (default: 128) - Initial size (in bytes) of the buffer
/// used for message serialization. Use a value high enough to avoid resizing
/// the buffer, but small enough to avoid excessive memory use. Inspect the size of
/// the byte array returned by the Serialize method to estimate an appropriate value.
/// Note: each call to serialize creates a new buffer.
///
/// avro.serializer.auto.register.schemas (default: true) - true if the serializer should
/// attempt to auto-register unrecognized schemas with Confluent Schema Registry,
/// false if not.
/// </summary>
/// <param name="schemaRegistryClient">
/// An implementation of ISchemaRegistryClient used for
/// communication with Confluent Schema Registry.
/// </param>
/// <param name="config">
/// Serializer configuration properties (refer to
/// <see cref="AvroSerializerConfig" />)
/// </param>
public AvroSerializer(ISchemaRegistryClient schemaRegistryClient, AvroSerializerConfig config = null)
{
this.schemaRegistryClient = schemaRegistryClient;
if (config == null)
{
return;
}
var nonAvroConfig = config.Where(item => !item.Key.StartsWith("avro."));
if (nonAvroConfig.Count() > 0)
{
throw new ArgumentException($"AvroSerializer: unknown configuration parameter {nonAvroConfig.First().Key}");
}
var avroConfig = config.Where(item => item.Key.StartsWith("avro."));
foreach (var property in avroConfig)
{
if (property.Key != AvroSerializerConfig.PropertyNames.AutoRegisterSchemas &&
property.Key != AvroSerializerConfig.PropertyNames.UseLatestVersion &&
property.Key != AvroSerializerConfig.PropertyNames.BufferBytes &&
property.Key != AvroSerializerConfig.PropertyNames.SubjectNameStrategy)
{
throw new ArgumentException($"AvroSerializer: unknown configuration property {property.Key}");
}
}
if (config.BufferBytes != null)
{
this.initialBufferSize = config.BufferBytes.Value;
}
if (config.AutoRegisterSchemas != null)
{
this.autoRegisterSchema = config.AutoRegisterSchemas.Value;
}
if (config.NormalizeSchemas != null)
{
this.normalizeSchemas = config.NormalizeSchemas.Value;
}
if (config.UseLatestVersion != null)
{
this.useLatestVersion = config.UseLatestVersion.Value;
}
if (config.SubjectNameStrategy != null)
{
this.subjectNameStrategy = config.SubjectNameStrategy.Value.ToDelegate();
}
if (this.useLatestVersion && this.autoRegisterSchema)
{
throw new ArgumentException($"AvroSerializer: cannot enable both use.latest.version and auto.register.schemas");
}
}
public GenericSerializerImpl(
ISchemaRegistryClient schemaRegistryClient,
bool autoRegisterSchema,
int initialBufferSize,
SubjectNameStrategyDelegate subjectNameStrategy)
{
this.schemaRegistryClient = schemaRegistryClient;
this.autoRegisterSchema = autoRegisterSchema;
this.initialBufferSize = initialBufferSize;
this.subjectNameStrategy = subjectNameStrategy;
}
public SubjectNameSerializer
(
ISchemaRegistryClient schemaRegistryClient,
bool autoRegisterSchema,
int initialBufferSize
)
{
_schemaRegistryClient = schemaRegistryClient;
_autoRegisterSchema = autoRegisterSchema;
_initialBufferSize = initialBufferSize;
}
/// <summary>
/// Sets an Avro deserializer for keys.
/// </summary>
/// <typeparam name="TKey">
/// The type of key to be deserialized.
/// </typeparam>
/// <typeparam name="TValue">
/// The type of value to be deserialized.
/// </typeparam>
/// <param name="consumerBuilder">
/// A <see cref="ConsumerBuilder{TKey, TValue}" /> instance to be configured.
/// </param>
/// <param name="registryClient">
/// A Schema Registry client to use to resolve the schema. (The client will not be
/// disposed.)
/// </param>
/// <param name="id">
/// The ID of the schema that should be used to deserialize keys.
/// </param>
/// <returns>
/// <paramref name="consumerBuilder" /> with an Avro deserializer configured for
/// <typeparamref name="TKey" />.
/// </returns>
public static async Task <ConsumerBuilder <TKey, TValue> > SetAvroKeyDeserializer <TKey, TValue>(
this ConsumerBuilder <TKey, TValue> consumerBuilder,
ISchemaRegistryClient registryClient,
int id)
{
using var deserializerBuilder = new SchemaRegistryDeserializerBuilder(registryClient);
return(await consumerBuilder
.SetAvroKeyDeserializer(deserializerBuilder, id)
.ConfigureAwait(false));
}
/// <summary>
/// Set the message key serializer.
/// </summary>
/// <param name="producerBuilder">
/// The <see cref="ProducerBuilder{TKey, TValue}" /> instance to be configured.
/// </param>
/// <param name="registryClient">
/// A client to use to resolve the schema. (The client will not be disposed.)
/// </param>
/// <param name="id">
/// The ID of the schema that should be used to serialize keys.
/// </param>
public static async Task <ProducerBuilder <TKey, TValue> > SetAvroKeySerializer <TKey, TValue>(
this ProducerBuilder <TKey, TValue> producerBuilder,
ISchemaRegistryClient registryClient,
int id
)
{
using (var serializerBuilder = new SchemaRegistrySerializerBuilder(registryClient))
{
return(await producerBuilder.SetAvroKeySerializer(serializerBuilder, id));
}
}
/// <summary>
/// Set the message value deserializer.
/// </summary>
/// <param name="consumerBuilder">
/// The <see cref="ConsumerBuilder{TKey, TValue}" /> instance to be configured.
/// </param>
/// <param name="registryClient">
/// The client to use to resolve the schema. (The client will not be disposed.)
/// </param>
/// <param name="subject">
/// The subject of the schema that should be used to deserialize values. The latest version
/// of the subject will be resolved.
/// </param>
public static async Task <ConsumerBuilder <TKey, TValue> > SetAvroValueDeserializer <TKey, TValue>(
this ConsumerBuilder <TKey, TValue> consumerBuilder,
ISchemaRegistryClient registryClient,
string subject
)
{
using (var deserializerBuilder = new SchemaRegistryDeserializerBuilder(registryClient))
{
return(await consumerBuilder.SetAvroValueDeserializer(deserializerBuilder, subject));
}
}