/// <summary> /// Initialize a new instance of the ProtobufSerializer class. /// </summary> public ProtobufSerializer(ISchemaRegistryClient schemaRegistryClient, ProtobufSerializerConfig config = null) { this.schemaRegistryClient = schemaRegistryClient; if (config == null) { this.referenceSubjectNameStrategy = ReferenceSubjectNameStrategy.ReferenceName.ToDelegate(); return; } var nonProtobufConfig = config.Where(item => !item.Key.StartsWith("protobuf.")); if (nonProtobufConfig.Count() > 0) { throw new ArgumentException($"ProtobufSerializer: unknown configuration parameter {nonProtobufConfig.First().Key}"); } if (config.BufferBytes != null) { this.initialBufferSize = config.BufferBytes.Value; } if (config.AutoRegisterSchemas != null) { this.autoRegisterSchema = config.AutoRegisterSchemas.Value; } if (config.SubjectNameStrategy != null) { this.subjectNameStrategy = config.SubjectNameStrategy.Value.ToDelegate(); } this.referenceSubjectNameStrategy = config.ReferenceSubjectNameStrategy == null ? ReferenceSubjectNameStrategy.ReferenceName.ToDelegate() : config.ReferenceSubjectNameStrategy.Value.ToDelegate(); }
/// <summary> /// Initialize a new instance of the JsonSerializer class. /// </summary> public JsonSerializer(ISchemaRegistryClient schemaRegistryClient, JsonSerializerConfig config = null) { this.schemaRegistryClient = schemaRegistryClient; this.schema = NJsonSchema.JsonSchema.FromType <T>(); this.schemaFullname = schema.Title; this.schemaText = schema.ToJson(); if (config == null) { return; } var nonJsonConfig = config.Where(item => !item.Key.StartsWith("json.")); if (nonJsonConfig.Count() > 0) { throw new ArgumentException($"JsonSerializer: unknown configuration parameter {nonJsonConfig.First().Key}"); } if (config.BufferBytes != null) { this.initialBufferSize = config.BufferBytes.Value; } if (config.AutoRegisterSchemas != null) { this.autoRegisterSchema = config.AutoRegisterSchemas.Value; } if (config.SubjectNameStrategy != null) { this.subjectNameStrategy = config.SubjectNameStrategy.Value.ToDelegate(); } }
/// <summary> /// Initialize a new instance of the AvroSerializer class. /// When passed as a parameter to the Confluent.Kafka.Producer constructor, /// the following configuration properties will be extracted from the producer's /// configuration property collection: /// /// avro.serializer.buffer.bytes (default: 128) - Initial size (in bytes) of the buffer /// used for message serialization. Use a value high enough to avoid resizing /// the buffer, but small enough to avoid excessive memory use. Inspect the size of /// the byte array returned by the Serialize method to estimate an appropriate value. /// Note: each call to serialize creates a new buffer. /// /// avro.serializer.auto.register.schemas (default: true) - true if the serializer should /// attempt to auto-register unrecognized schemas with Confluent Schema Registry, /// false if not. /// </summary> /// <param name="schemaRegistryClient"> /// An implementation of ISchemaRegistryClient used for /// communication with Confluent Schema Registry. /// </param> /// <param name="config"> /// Serializer configuration properties (refer to /// <see cref="AvroSerializerConfig" />) /// </param> public AvroSerializer(ISchemaRegistryClient schemaRegistryClient, AvroSerializerConfig config = null) { this.schemaRegistryClient = schemaRegistryClient; if (config == null) { return; } var nonAvroConfig = config.Where(item => !item.Key.StartsWith("avro.")); if (nonAvroConfig.Count() > 0) { throw new ArgumentException($"AvroSerializer: unknown configuration parameter {nonAvroConfig.First().Key}"); } var avroConfig = config.Where(item => item.Key.StartsWith("avro.")); foreach (var property in avroConfig) { if (property.Key != AvroSerializerConfig.PropertyNames.AutoRegisterSchemas && property.Key != AvroSerializerConfig.PropertyNames.UseLatestVersion && property.Key != AvroSerializerConfig.PropertyNames.BufferBytes && property.Key != AvroSerializerConfig.PropertyNames.SubjectNameStrategy) { throw new ArgumentException($"AvroSerializer: unknown configuration property {property.Key}"); } } if (config.BufferBytes != null) { this.initialBufferSize = config.BufferBytes.Value; } if (config.AutoRegisterSchemas != null) { this.autoRegisterSchema = config.AutoRegisterSchemas.Value; } if (config.NormalizeSchemas != null) { this.normalizeSchemas = config.NormalizeSchemas.Value; } if (config.UseLatestVersion != null) { this.useLatestVersion = config.UseLatestVersion.Value; } if (config.SubjectNameStrategy != null) { this.subjectNameStrategy = config.SubjectNameStrategy.Value.ToDelegate(); } if (this.useLatestVersion && this.autoRegisterSchema) { throw new ArgumentException($"AvroSerializer: cannot enable both use.latest.version and auto.register.schemas"); } }
public GenericSerializerImpl( ISchemaRegistryClient schemaRegistryClient, bool autoRegisterSchema, int initialBufferSize, SubjectNameStrategyDelegate subjectNameStrategy) { this.schemaRegistryClient = schemaRegistryClient; this.autoRegisterSchema = autoRegisterSchema; this.initialBufferSize = initialBufferSize; this.subjectNameStrategy = subjectNameStrategy; }
/// <summary> /// Initialize a new instance of the ProtobufSerializer class. /// </summary> public ProtobufSerializer(ISchemaRegistryClient schemaRegistryClient, ProtobufSerializerConfig config = null) { this.schemaRegistryClient = schemaRegistryClient; if (config == null) { this.referenceSubjectNameStrategy = ReferenceSubjectNameStrategy.ReferenceName.ToDelegate(); return; } var nonProtobufConfig = config.Where(item => !item.Key.StartsWith("protobuf.")); if (nonProtobufConfig.Count() > 0) { throw new ArgumentException($"ProtobufSerializer: unknown configuration parameter {nonProtobufConfig.First().Key}"); } if (config.BufferBytes != null) { this.initialBufferSize = config.BufferBytes.Value; } if (config.AutoRegisterSchemas != null) { this.autoRegisterSchema = config.AutoRegisterSchemas.Value; } if (config.NormalizeSchemas != null) { this.normalizeSchemas = config.NormalizeSchemas.Value; } if (config.UseLatestVersion != null) { this.useLatestVersion = config.UseLatestVersion.Value; } if (config.SkipKnownTypes != null) { this.skipKnownTypes = config.SkipKnownTypes.Value; } if (config.UseDeprecatedFormat != null) { this.useDeprecatedFormat = config.UseDeprecatedFormat.Value; } if (config.SubjectNameStrategy != null) { this.subjectNameStrategy = config.SubjectNameStrategy.Value.ToDelegate(); } this.referenceSubjectNameStrategy = config.ReferenceSubjectNameStrategy == null ? ReferenceSubjectNameStrategy.ReferenceName.ToDelegate() : config.ReferenceSubjectNameStrategy.Value.ToDelegate(); if (this.useLatestVersion && this.autoRegisterSchema) { throw new ArgumentException($"ProtobufSerializer: cannot enable both use.latest.version and auto.register.schemas"); } }
/// <summary> /// Initialize a new instance of the JsonSerializer class. /// </summary> /// <param name="schemaRegistryClient"> /// Confluent Schema Registry client instance. /// </param> /// <param name="config"> /// Serializer configuration. /// </param> /// <param name="jsonSchemaGeneratorSettings"> /// JSON schema generator settings. /// </param> public JsonSerializer(ISchemaRegistryClient schemaRegistryClient, JsonSerializerConfig config = null, JsonSchemaGeneratorSettings jsonSchemaGeneratorSettings = null) { this.schemaRegistryClient = schemaRegistryClient; this.jsonSchemaGeneratorSettings = jsonSchemaGeneratorSettings; this.schema = this.jsonSchemaGeneratorSettings == null ? NJsonSchema.JsonSchema.FromType <T>() : NJsonSchema.JsonSchema.FromType <T>(this.jsonSchemaGeneratorSettings); this.schemaFullname = schema.Title; this.schemaText = schema.ToJson(); if (config == null) { return; } var nonJsonConfig = config.Where(item => !item.Key.StartsWith("json.")); if (nonJsonConfig.Count() > 0) { throw new ArgumentException($"JsonSerializer: unknown configuration parameter {nonJsonConfig.First().Key}"); } if (config.BufferBytes != null) { this.initialBufferSize = config.BufferBytes.Value; } if (config.AutoRegisterSchemas != null) { this.autoRegisterSchema = config.AutoRegisterSchemas.Value; } if (config.NormalizeSchemas != null) { this.normalizeSchemas = config.NormalizeSchemas.Value; } if (config.UseLatestVersion != null) { this.useLatestVersion = config.UseLatestVersion.Value; } if (config.SubjectNameStrategy != null) { this.subjectNameStrategy = config.SubjectNameStrategy.Value.ToDelegate(); } if (this.useLatestVersion && this.autoRegisterSchema) { throw new ArgumentException($"JsonSerializer: cannot enable both use.latest.version and auto.register.schemas"); } }
public GenericSerializerImpl( ISchemaRegistryClient schemaRegistryClient, bool autoRegisterSchema, bool normalizeSchemas, bool useLatestVersion, int initialBufferSize, SubjectNameStrategyDelegate subjectNameStrategy) { this.schemaRegistryClient = schemaRegistryClient; this.autoRegisterSchema = autoRegisterSchema; this.normalizeSchemas = normalizeSchemas; this.useLatestVersion = useLatestVersion; this.initialBufferSize = initialBufferSize; this.subjectNameStrategy = subjectNameStrategy; }
/// <summary> /// Initialize a new instance of the SchemaRegistryClient class. /// </summary> /// <param name="config"> /// Configuration properties. /// </param> public CachedSchemaRegistryClient(IEnumerable <KeyValuePair <string, string> > config) { if (config == null) { throw new ArgumentNullException("config properties must be specified."); } keySubjectNameStrategy = GetKeySubjectNameStrategy(config); valueSubjectNameStrategy = GetValueSubjectNameStrategy(config); var schemaRegistryUrisMaybe = config.FirstOrDefault(prop => prop.Key.ToLower() == SchemaRegistryConfig.PropertyNames.SchemaRegistryUrl); if (schemaRegistryUrisMaybe.Value == null) { throw new ArgumentException($"{SchemaRegistryConfig.PropertyNames.SchemaRegistryUrl} configuration property must be specified."); } var schemaRegistryUris = (string)schemaRegistryUrisMaybe.Value; var timeoutMsMaybe = config.FirstOrDefault(prop => prop.Key.ToLower() == SchemaRegistryConfig.PropertyNames.SchemaRegistryRequestTimeoutMs); int timeoutMs; try { timeoutMs = timeoutMsMaybe.Value == null ? DefaultTimeout : Convert.ToInt32(timeoutMsMaybe.Value); } catch (FormatException) { throw new ArgumentException($"Configured value for {SchemaRegistryConfig.PropertyNames.SchemaRegistryRequestTimeoutMs} must be an integer."); } var identityMapCapacityMaybe = config.FirstOrDefault(prop => prop.Key.ToLower() == SchemaRegistryConfig.PropertyNames.SchemaRegistryMaxCachedSchemas); try { this.identityMapCapacity = identityMapCapacityMaybe.Value == null ? DefaultMaxCachedSchemas : Convert.ToInt32(identityMapCapacityMaybe.Value); } catch (FormatException) { throw new ArgumentException($"Configured value for {SchemaRegistryConfig.PropertyNames.SchemaRegistryMaxCachedSchemas} must be an integer."); } var basicAuthSource = config.FirstOrDefault(prop => prop.Key.ToLower() == SchemaRegistryConfig.PropertyNames.SchemaRegistryBasicAuthCredentialsSource).Value ?? ""; var basicAuthInfo = config.FirstOrDefault(prop => prop.Key.ToLower() == SchemaRegistryConfig.PropertyNames.SchemaRegistryBasicAuthUserInfo).Value ?? ""; string username = null; string password = null; if (basicAuthSource == "USER_INFO" || basicAuthSource == "") { if (basicAuthInfo != "") { var userPass = (basicAuthInfo).Split(':'); if (userPass.Length != 2) { throw new ArgumentException($"Configuration property {SchemaRegistryConfig.PropertyNames.SchemaRegistryBasicAuthUserInfo} must be of the form 'username:password'."); } username = userPass[0]; password = userPass[1]; } } else if (basicAuthSource == "SASL_INHERIT") { if (basicAuthInfo != "") { throw new ArgumentException($"{SchemaRegistryConfig.PropertyNames.SchemaRegistryBasicAuthCredentialsSource} set to 'SASL_INHERIT', but {SchemaRegistryConfig.PropertyNames.SchemaRegistryBasicAuthUserInfo} as also specified."); } var saslUsername = config.FirstOrDefault(prop => prop.Key == "sasl.username"); var saslPassword = config.FirstOrDefault(prop => prop.Key == "sasl.password"); if (saslUsername.Value == null) { throw new ArgumentException($"{SchemaRegistryConfig.PropertyNames.SchemaRegistryBasicAuthCredentialsSource} set to 'SASL_INHERIT', but 'sasl.username' property not specified."); } if (saslPassword.Value == null) { throw new ArgumentException($"{SchemaRegistryConfig.PropertyNames.SchemaRegistryBasicAuthCredentialsSource} set to 'SASL_INHERIT', but 'sasl.password' property not specified."); } username = saslUsername.Value; password = saslPassword.Value; } else { throw new ArgumentException($"Invalid value '{basicAuthSource}' specified for property '{SchemaRegistryConfig.PropertyNames.SchemaRegistryBasicAuthCredentialsSource}'"); } foreach (var property in config) { if (!property.Key.StartsWith("schema.registry.")) { continue; } if (property.Key != SchemaRegistryConfig.PropertyNames.SchemaRegistryUrl && property.Key != SchemaRegistryConfig.PropertyNames.SchemaRegistryRequestTimeoutMs && property.Key != SchemaRegistryConfig.PropertyNames.SchemaRegistryMaxCachedSchemas && property.Key != SchemaRegistryConfig.PropertyNames.SchemaRegistryBasicAuthCredentialsSource && property.Key != SchemaRegistryConfig.PropertyNames.SchemaRegistryBasicAuthUserInfo && property.Key != SchemaRegistryConfig.PropertyNames.SchemaRegistryKeySubjectNameStrategy && property.Key != SchemaRegistryConfig.PropertyNames.SchemaRegistryValueSubjectNameStrategy && property.Key != SchemaRegistryConfig.PropertyNames.SslCaLocation && property.Key != SchemaRegistryConfig.PropertyNames.SslKeystoreLocation && property.Key != SchemaRegistryConfig.PropertyNames.SslKeystorePassword && property.Key != SchemaRegistryConfig.PropertyNames.EnableSslCertificateVerification) { throw new ArgumentException($"Unknown configuration parameter {property.Key}"); } } var sslVerificationMaybe = config.FirstOrDefault(prop => prop.Key.ToLower() == SchemaRegistryConfig.PropertyNames.EnableSslCertificateVerification); bool sslVerify; try { sslVerify = sslVerificationMaybe.Value == null ? DefaultEnableSslCertificateVerification : bool.Parse(sslVerificationMaybe.Value); } catch (FormatException) { throw new ArgumentException($"Configured value for {SchemaRegistryConfig.PropertyNames.EnableSslCertificateVerification} must be a bool."); } this.restService = new RestService(schemaRegistryUris, timeoutMs, username, password, SetSslConfig(config), sslVerify); }
public SpecificSerializerImpl( ISchemaRegistryClient schemaRegistryClient, bool autoRegisterSchema, int initialBufferSize, SubjectNameStrategyDelegate subjectNameStrategy) { this.schemaRegistryClient = schemaRegistryClient; this.autoRegisterSchema = autoRegisterSchema; this.initialBufferSize = initialBufferSize; this.subjectNameStrategy = subjectNameStrategy; Type writerType = typeof(T); if (typeof(ISpecificRecord).IsAssignableFrom(writerType)) { writerSchema = (global::Avro.Schema) typeof(T).GetField("_SCHEMA", BindingFlags.Public | BindingFlags.Static).GetValue(null); } else if (writerType.Equals(typeof(int))) { writerSchema = global::Avro.Schema.Parse("int"); } else if (writerType.Equals(typeof(bool))) { writerSchema = global::Avro.Schema.Parse("boolean"); } else if (writerType.Equals(typeof(double))) { writerSchema = global::Avro.Schema.Parse("double"); } else if (writerType.Equals(typeof(string))) { // Note: It would arguably be better to make this a union with null, to // exactly match the .NET string type, however we don't for consistency // with the Java Avro serializer. writerSchema = global::Avro.Schema.Parse("string"); } else if (writerType.Equals(typeof(float))) { writerSchema = global::Avro.Schema.Parse("float"); } else if (writerType.Equals(typeof(long))) { writerSchema = global::Avro.Schema.Parse("long"); } else if (writerType.Equals(typeof(byte[]))) { // Note: It would arguably be better to make this a union with null, to // exactly match the .NET byte[] type, however we don't for consistency // with the Java Avro serializer. writerSchema = global::Avro.Schema.Parse("bytes"); } else if (writerType.Equals(typeof(Null))) { writerSchema = global::Avro.Schema.Parse("null"); } else { throw new InvalidOperationException( $"AvroSerializer only accepts type parameters of int, bool, double, string, float, " + "long, byte[], instances of ISpecificRecord and subclasses of SpecificFixed." ); } avroWriter = new SpecificWriter <T>(writerSchema); writerSchemaString = writerSchema.ToString(); }