/// <summary>
/// Initialize a new instance of the ProtobufSerializer class.
/// </summary>
public ProtobufSerializer(ISchemaRegistryClient schemaRegistryClient, ProtobufSerializerConfig config = null)
{
this.schemaRegistryClient = schemaRegistryClient;
if (config == null)
{
this.referenceSubjectNameStrategy = ReferenceSubjectNameStrategy.ReferenceName.ToDelegate();
return;
}
var nonProtobufConfig = config.Where(item => !item.Key.StartsWith("protobuf."));
if (nonProtobufConfig.Count() > 0)
{
throw new ArgumentException($"ProtobufSerializer: unknown configuration parameter {nonProtobufConfig.First().Key}");
}
if (config.BufferBytes != null)
{
this.initialBufferSize = config.BufferBytes.Value;
}
if (config.AutoRegisterSchemas != null)
{
this.autoRegisterSchema = config.AutoRegisterSchemas.Value;
}
if (config.SubjectNameStrategy != null)
{
this.subjectNameStrategy = config.SubjectNameStrategy.Value.ToDelegate();
}
this.referenceSubjectNameStrategy = config.ReferenceSubjectNameStrategy == null
? ReferenceSubjectNameStrategy.ReferenceName.ToDelegate()
: config.ReferenceSubjectNameStrategy.Value.ToDelegate();
}
/// <summary>
/// Initialize a new instance of the JsonSerializer class.
/// </summary>
public JsonSerializer(ISchemaRegistryClient schemaRegistryClient, JsonSerializerConfig config = null)
{
this.schemaRegistryClient = schemaRegistryClient;
this.schema = NJsonSchema.JsonSchema.FromType <T>();
this.schemaFullname = schema.Title;
this.schemaText = schema.ToJson();
if (config == null)
{
return;
}
var nonJsonConfig = config.Where(item => !item.Key.StartsWith("json."));
if (nonJsonConfig.Count() > 0)
{
throw new ArgumentException($"JsonSerializer: unknown configuration parameter {nonJsonConfig.First().Key}");
}
if (config.BufferBytes != null)
{
this.initialBufferSize = config.BufferBytes.Value;
}
if (config.AutoRegisterSchemas != null)
{
this.autoRegisterSchema = config.AutoRegisterSchemas.Value;
}
if (config.SubjectNameStrategy != null)
{
this.subjectNameStrategy = config.SubjectNameStrategy.Value.ToDelegate();
}
}
/// <summary>
/// Initialize a new instance of the AvroSerializer class.
/// When passed as a parameter to the Confluent.Kafka.Producer constructor,
/// the following configuration properties will be extracted from the producer's
/// configuration property collection:
///
/// avro.serializer.buffer.bytes (default: 128) - Initial size (in bytes) of the buffer
/// used for message serialization. Use a value high enough to avoid resizing
/// the buffer, but small enough to avoid excessive memory use. Inspect the size of
/// the byte array returned by the Serialize method to estimate an appropriate value.
/// Note: each call to serialize creates a new buffer.
///
/// avro.serializer.auto.register.schemas (default: true) - true if the serializer should
/// attempt to auto-register unrecognized schemas with Confluent Schema Registry,
/// false if not.
/// </summary>
/// <param name="schemaRegistryClient">
/// An implementation of ISchemaRegistryClient used for
/// communication with Confluent Schema Registry.
/// </param>
/// <param name="config">
/// Serializer configuration properties (refer to
/// <see cref="AvroSerializerConfig" />)
/// </param>
public AvroSerializer(ISchemaRegistryClient schemaRegistryClient, AvroSerializerConfig config = null)
{
this.schemaRegistryClient = schemaRegistryClient;
if (config == null)
{
return;
}
var nonAvroConfig = config.Where(item => !item.Key.StartsWith("avro."));
if (nonAvroConfig.Count() > 0)
{
throw new ArgumentException($"AvroSerializer: unknown configuration parameter {nonAvroConfig.First().Key}");
}
var avroConfig = config.Where(item => item.Key.StartsWith("avro."));
foreach (var property in avroConfig)
{
if (property.Key != AvroSerializerConfig.PropertyNames.AutoRegisterSchemas &&
property.Key != AvroSerializerConfig.PropertyNames.UseLatestVersion &&
property.Key != AvroSerializerConfig.PropertyNames.BufferBytes &&
property.Key != AvroSerializerConfig.PropertyNames.SubjectNameStrategy)
{
throw new ArgumentException($"AvroSerializer: unknown configuration property {property.Key}");
}
}
if (config.BufferBytes != null)
{
this.initialBufferSize = config.BufferBytes.Value;
}
if (config.AutoRegisterSchemas != null)
{
this.autoRegisterSchema = config.AutoRegisterSchemas.Value;
}
if (config.NormalizeSchemas != null)
{
this.normalizeSchemas = config.NormalizeSchemas.Value;
}
if (config.UseLatestVersion != null)
{
this.useLatestVersion = config.UseLatestVersion.Value;
}
if (config.SubjectNameStrategy != null)
{
this.subjectNameStrategy = config.SubjectNameStrategy.Value.ToDelegate();
}
if (this.useLatestVersion && this.autoRegisterSchema)
{
throw new ArgumentException($"AvroSerializer: cannot enable both use.latest.version and auto.register.schemas");
}
}
public GenericSerializerImpl(
ISchemaRegistryClient schemaRegistryClient,
bool autoRegisterSchema,
int initialBufferSize,
SubjectNameStrategyDelegate subjectNameStrategy)
{
this.schemaRegistryClient = schemaRegistryClient;
this.autoRegisterSchema = autoRegisterSchema;
this.initialBufferSize = initialBufferSize;
this.subjectNameStrategy = subjectNameStrategy;
}
/// <summary>
/// Initialize a new instance of the ProtobufSerializer class.
/// </summary>
public ProtobufSerializer(ISchemaRegistryClient schemaRegistryClient, ProtobufSerializerConfig config = null)
{
this.schemaRegistryClient = schemaRegistryClient;
if (config == null)
{
this.referenceSubjectNameStrategy = ReferenceSubjectNameStrategy.ReferenceName.ToDelegate();
return;
}
var nonProtobufConfig = config.Where(item => !item.Key.StartsWith("protobuf."));
if (nonProtobufConfig.Count() > 0)
{
throw new ArgumentException($"ProtobufSerializer: unknown configuration parameter {nonProtobufConfig.First().Key}");
}
if (config.BufferBytes != null)
{
this.initialBufferSize = config.BufferBytes.Value;
}
if (config.AutoRegisterSchemas != null)
{
this.autoRegisterSchema = config.AutoRegisterSchemas.Value;
}
if (config.NormalizeSchemas != null)
{
this.normalizeSchemas = config.NormalizeSchemas.Value;
}
if (config.UseLatestVersion != null)
{
this.useLatestVersion = config.UseLatestVersion.Value;
}
if (config.SkipKnownTypes != null)
{
this.skipKnownTypes = config.SkipKnownTypes.Value;
}
if (config.UseDeprecatedFormat != null)
{
this.useDeprecatedFormat = config.UseDeprecatedFormat.Value;
}
if (config.SubjectNameStrategy != null)
{
this.subjectNameStrategy = config.SubjectNameStrategy.Value.ToDelegate();
}
this.referenceSubjectNameStrategy = config.ReferenceSubjectNameStrategy == null
? ReferenceSubjectNameStrategy.ReferenceName.ToDelegate()
: config.ReferenceSubjectNameStrategy.Value.ToDelegate();
if (this.useLatestVersion && this.autoRegisterSchema)
{
throw new ArgumentException($"ProtobufSerializer: cannot enable both use.latest.version and auto.register.schemas");
}
}
/// <summary>
/// Initialize a new instance of the JsonSerializer class.
/// </summary>
/// <param name="schemaRegistryClient">
/// Confluent Schema Registry client instance.
/// </param>
/// <param name="config">
/// Serializer configuration.
/// </param>
/// <param name="jsonSchemaGeneratorSettings">
/// JSON schema generator settings.
/// </param>
public JsonSerializer(ISchemaRegistryClient schemaRegistryClient, JsonSerializerConfig config = null, JsonSchemaGeneratorSettings jsonSchemaGeneratorSettings = null)
{
this.schemaRegistryClient = schemaRegistryClient;
this.jsonSchemaGeneratorSettings = jsonSchemaGeneratorSettings;
this.schema = this.jsonSchemaGeneratorSettings == null
? NJsonSchema.JsonSchema.FromType <T>()
: NJsonSchema.JsonSchema.FromType <T>(this.jsonSchemaGeneratorSettings);
this.schemaFullname = schema.Title;
this.schemaText = schema.ToJson();
if (config == null)
{
return;
}
var nonJsonConfig = config.Where(item => !item.Key.StartsWith("json."));
if (nonJsonConfig.Count() > 0)
{
throw new ArgumentException($"JsonSerializer: unknown configuration parameter {nonJsonConfig.First().Key}");
}
if (config.BufferBytes != null)
{
this.initialBufferSize = config.BufferBytes.Value;
}
if (config.AutoRegisterSchemas != null)
{
this.autoRegisterSchema = config.AutoRegisterSchemas.Value;
}
if (config.NormalizeSchemas != null)
{
this.normalizeSchemas = config.NormalizeSchemas.Value;
}
if (config.UseLatestVersion != null)
{
this.useLatestVersion = config.UseLatestVersion.Value;
}
if (config.SubjectNameStrategy != null)
{
this.subjectNameStrategy = config.SubjectNameStrategy.Value.ToDelegate();
}
if (this.useLatestVersion && this.autoRegisterSchema)
{
throw new ArgumentException($"JsonSerializer: cannot enable both use.latest.version and auto.register.schemas");
}
}
public GenericSerializerImpl(
ISchemaRegistryClient schemaRegistryClient,
bool autoRegisterSchema,
bool normalizeSchemas,
bool useLatestVersion,
int initialBufferSize,
SubjectNameStrategyDelegate subjectNameStrategy)
{
this.schemaRegistryClient = schemaRegistryClient;
this.autoRegisterSchema = autoRegisterSchema;
this.normalizeSchemas = normalizeSchemas;
this.useLatestVersion = useLatestVersion;
this.initialBufferSize = initialBufferSize;
this.subjectNameStrategy = subjectNameStrategy;
}
/// <summary>
/// Initialize a new instance of the SchemaRegistryClient class.
/// </summary>
/// <param name="config">
/// Configuration properties.
/// </param>
public CachedSchemaRegistryClient(IEnumerable <KeyValuePair <string, string> > config)
{
if (config == null)
{
throw new ArgumentNullException("config properties must be specified.");
}
keySubjectNameStrategy = GetKeySubjectNameStrategy(config);
valueSubjectNameStrategy = GetValueSubjectNameStrategy(config);
var schemaRegistryUrisMaybe = config.FirstOrDefault(prop => prop.Key.ToLower() == SchemaRegistryConfig.PropertyNames.SchemaRegistryUrl);
if (schemaRegistryUrisMaybe.Value == null)
{
throw new ArgumentException($"{SchemaRegistryConfig.PropertyNames.SchemaRegistryUrl} configuration property must be specified.");
}
var schemaRegistryUris = (string)schemaRegistryUrisMaybe.Value;
var timeoutMsMaybe = config.FirstOrDefault(prop => prop.Key.ToLower() == SchemaRegistryConfig.PropertyNames.SchemaRegistryRequestTimeoutMs);
int timeoutMs;
try { timeoutMs = timeoutMsMaybe.Value == null ? DefaultTimeout : Convert.ToInt32(timeoutMsMaybe.Value); }
catch (FormatException) { throw new ArgumentException($"Configured value for {SchemaRegistryConfig.PropertyNames.SchemaRegistryRequestTimeoutMs} must be an integer."); }
var identityMapCapacityMaybe = config.FirstOrDefault(prop => prop.Key.ToLower() == SchemaRegistryConfig.PropertyNames.SchemaRegistryMaxCachedSchemas);
try { this.identityMapCapacity = identityMapCapacityMaybe.Value == null ? DefaultMaxCachedSchemas : Convert.ToInt32(identityMapCapacityMaybe.Value); }
catch (FormatException) { throw new ArgumentException($"Configured value for {SchemaRegistryConfig.PropertyNames.SchemaRegistryMaxCachedSchemas} must be an integer."); }
var basicAuthSource = config.FirstOrDefault(prop => prop.Key.ToLower() == SchemaRegistryConfig.PropertyNames.SchemaRegistryBasicAuthCredentialsSource).Value ?? "";
var basicAuthInfo = config.FirstOrDefault(prop => prop.Key.ToLower() == SchemaRegistryConfig.PropertyNames.SchemaRegistryBasicAuthUserInfo).Value ?? "";
string username = null;
string password = null;
if (basicAuthSource == "USER_INFO" || basicAuthSource == "")
{
if (basicAuthInfo != "")
{
var userPass = (basicAuthInfo).Split(':');
if (userPass.Length != 2)
{
throw new ArgumentException($"Configuration property {SchemaRegistryConfig.PropertyNames.SchemaRegistryBasicAuthUserInfo} must be of the form 'username:password'.");
}
username = userPass[0];
password = userPass[1];
}
}
else if (basicAuthSource == "SASL_INHERIT")
{
if (basicAuthInfo != "")
{
throw new ArgumentException($"{SchemaRegistryConfig.PropertyNames.SchemaRegistryBasicAuthCredentialsSource} set to 'SASL_INHERIT', but {SchemaRegistryConfig.PropertyNames.SchemaRegistryBasicAuthUserInfo} as also specified.");
}
var saslUsername = config.FirstOrDefault(prop => prop.Key == "sasl.username");
var saslPassword = config.FirstOrDefault(prop => prop.Key == "sasl.password");
if (saslUsername.Value == null)
{
throw new ArgumentException($"{SchemaRegistryConfig.PropertyNames.SchemaRegistryBasicAuthCredentialsSource} set to 'SASL_INHERIT', but 'sasl.username' property not specified.");
}
if (saslPassword.Value == null)
{
throw new ArgumentException($"{SchemaRegistryConfig.PropertyNames.SchemaRegistryBasicAuthCredentialsSource} set to 'SASL_INHERIT', but 'sasl.password' property not specified.");
}
username = saslUsername.Value;
password = saslPassword.Value;
}
else
{
throw new ArgumentException($"Invalid value '{basicAuthSource}' specified for property '{SchemaRegistryConfig.PropertyNames.SchemaRegistryBasicAuthCredentialsSource}'");
}
foreach (var property in config)
{
if (!property.Key.StartsWith("schema.registry."))
{
continue;
}
if (property.Key != SchemaRegistryConfig.PropertyNames.SchemaRegistryUrl &&
property.Key != SchemaRegistryConfig.PropertyNames.SchemaRegistryRequestTimeoutMs &&
property.Key != SchemaRegistryConfig.PropertyNames.SchemaRegistryMaxCachedSchemas &&
property.Key != SchemaRegistryConfig.PropertyNames.SchemaRegistryBasicAuthCredentialsSource &&
property.Key != SchemaRegistryConfig.PropertyNames.SchemaRegistryBasicAuthUserInfo &&
property.Key != SchemaRegistryConfig.PropertyNames.SchemaRegistryKeySubjectNameStrategy &&
property.Key != SchemaRegistryConfig.PropertyNames.SchemaRegistryValueSubjectNameStrategy &&
property.Key != SchemaRegistryConfig.PropertyNames.SslCaLocation &&
property.Key != SchemaRegistryConfig.PropertyNames.SslKeystoreLocation &&
property.Key != SchemaRegistryConfig.PropertyNames.SslKeystorePassword &&
property.Key != SchemaRegistryConfig.PropertyNames.EnableSslCertificateVerification)
{
throw new ArgumentException($"Unknown configuration parameter {property.Key}");
}
}
var sslVerificationMaybe = config.FirstOrDefault(prop => prop.Key.ToLower() == SchemaRegistryConfig.PropertyNames.EnableSslCertificateVerification);
bool sslVerify;
try { sslVerify = sslVerificationMaybe.Value == null ? DefaultEnableSslCertificateVerification : bool.Parse(sslVerificationMaybe.Value); }
catch (FormatException) { throw new ArgumentException($"Configured value for {SchemaRegistryConfig.PropertyNames.EnableSslCertificateVerification} must be a bool."); }
this.restService = new RestService(schemaRegistryUris, timeoutMs, username, password, SetSslConfig(config), sslVerify);
}
public SpecificSerializerImpl(
ISchemaRegistryClient schemaRegistryClient,
bool autoRegisterSchema,
int initialBufferSize,
SubjectNameStrategyDelegate subjectNameStrategy)
{
this.schemaRegistryClient = schemaRegistryClient;
this.autoRegisterSchema = autoRegisterSchema;
this.initialBufferSize = initialBufferSize;
this.subjectNameStrategy = subjectNameStrategy;
Type writerType = typeof(T);
if (typeof(ISpecificRecord).IsAssignableFrom(writerType))
{
writerSchema = (global::Avro.Schema) typeof(T).GetField("_SCHEMA", BindingFlags.Public | BindingFlags.Static).GetValue(null);
}
else if (writerType.Equals(typeof(int)))
{
writerSchema = global::Avro.Schema.Parse("int");
}
else if (writerType.Equals(typeof(bool)))
{
writerSchema = global::Avro.Schema.Parse("boolean");
}
else if (writerType.Equals(typeof(double)))
{
writerSchema = global::Avro.Schema.Parse("double");
}
else if (writerType.Equals(typeof(string)))
{
// Note: It would arguably be better to make this a union with null, to
// exactly match the .NET string type, however we don't for consistency
// with the Java Avro serializer.
writerSchema = global::Avro.Schema.Parse("string");
}
else if (writerType.Equals(typeof(float)))
{
writerSchema = global::Avro.Schema.Parse("float");
}
else if (writerType.Equals(typeof(long)))
{
writerSchema = global::Avro.Schema.Parse("long");
}
else if (writerType.Equals(typeof(byte[])))
{
// Note: It would arguably be better to make this a union with null, to
// exactly match the .NET byte[] type, however we don't for consistency
// with the Java Avro serializer.
writerSchema = global::Avro.Schema.Parse("bytes");
}
else if (writerType.Equals(typeof(Null)))
{
writerSchema = global::Avro.Schema.Parse("null");
}
else
{
throw new InvalidOperationException(
$"AvroSerializer only accepts type parameters of int, bool, double, string, float, " +
"long, byte[], instances of ISpecificRecord and subclasses of SpecificFixed."
);
}
avroWriter = new SpecificWriter <T>(writerSchema);
writerSchemaString = writerSchema.ToString();
}
