Exemple #1
0
        /// <summary>
        ///     Initialize a new instance of the JsonSerializer class.
        /// </summary>
        public JsonSerializer(ISchemaRegistryClient schemaRegistryClient, JsonSerializerConfig config = null)
        {
            this.schemaRegistryClient = schemaRegistryClient;

            this.schema         = NJsonSchema.JsonSchema.FromType <T>();
            this.schemaFullname = schema.Title;
            this.schemaText     = schema.ToJson();

            if (config == null)
            {
                return;
            }

            var nonJsonConfig = config.Where(item => !item.Key.StartsWith("json."));

            if (nonJsonConfig.Count() > 0)
            {
                throw new ArgumentException($"JsonSerializer: unknown configuration parameter {nonJsonConfig.First().Key}");
            }

            if (config.BufferBytes != null)
            {
                this.initialBufferSize = config.BufferBytes.Value;
            }
            if (config.AutoRegisterSchemas != null)
            {
                this.autoRegisterSchema = config.AutoRegisterSchemas.Value;
            }
            if (config.SubjectNameStrategy != null)
            {
                this.subjectNameStrategy = config.SubjectNameStrategy.Value.ToDelegate();
            }
        }
Exemple #2
0
        /// <include file='../Confluent.Kafka/include_docs.xml' path='API/Member[@name="IDeserializer_Configure"]/*' />
        public IEnumerable <KeyValuePair <string, object> > Configure(IEnumerable <KeyValuePair <string, object> > config, bool isKey)
        {
            var keyOrValue = isKey ? "Key" : "Value";
            var srConfig   = config.Where(item => item.Key.StartsWith("schema.registry."));
            var avroConfig = config.Where(item => item.Key.StartsWith("avro."));

            if (avroConfig.Count() != 0)
            {
                throw new ArgumentException($"{keyOrValue} AvroDeserializer: unexpected configuration parameter {avroConfig.First().Key}");
            }

            if (srConfig.Count() != 0)
            {
                if (schemaRegistryClient != null)
                {
                    throw new ArgumentException($"{keyOrValue} AvroDeserializer schema registry client was configured via both the constructor and configuration parameters.");
                }

                schemaRegistryClient = new CachedSchemaRegistryClient(config);
            }

            if (schemaRegistryClient == null)
            {
                throw new ArgumentException($"{keyOrValue} AvroDeserializer schema registry client was not supplied or configured.");
            }

            return(config.Where(item => !item.Key.StartsWith("schema.registry.") && !item.Key.StartsWith("avro.")));
        }
Exemple #3
0
        private void SetupProducer()
        {
            if (producer != null)
            {
                return;
            }

            lock (myLock)
            {
                Dispose();
            }

            var config = new ProducerConfig {
                BootstrapServers = kafkaOptions.BootstrapServerUri
            };
            var schemaRegistryConfig = new SchemaRegistryConfig {
                Url = kafkaOptions.SchemaRegistryUri
            };
            var jsonSerializerConfig = new JsonSerializerConfig();

            schemaRegistryClient = new CachedSchemaRegistryClient(schemaRegistryConfig);
            producer             = new ProducerBuilder <Null, T>(config)
                                   .SetValueSerializer(new JsonSerializer <T>(schemaRegistryClient, jsonSerializerConfig))
                                   .Build();
        }
Exemple #4
0
 /// <summary>
 /// Set the message key deserializer.
 /// </summary>
 /// <param name="consumerBuilder">
 /// The <see cref="ConsumerBuilder{TKey, TValue}" /> instance to be configured.
 /// </param>
 /// <param name="registryClient">
 /// The client to use for Schema Registry operations. The client should only be disposed
 /// after the consumer; the deserializer will use it to request schemas as messages are
 /// being consumed.
 /// </param>
 public static ConsumerBuilder <TKey, TValue> SetAvroKeyDeserializer <TKey, TValue>(
     this ConsumerBuilder <TKey, TValue> consumerBuilder,
     ISchemaRegistryClient registryClient
     ) => consumerBuilder.SetKeyDeserializer(
     new AsyncSchemaRegistryDeserializer <TKey>(
         registryClient
         ).AsSyncOverAsync());
Exemple #5
0
        /// <summary>
        /// Creates a serializer.
        /// </summary>
        /// <param name="registryClient">
        /// The client to use for Schema Registry operations. (The client will not be disposed.)
        /// </param>
        /// <param name="registerAutomatically">
        /// Whether to automatically register schemas that match the type being serialized.
        /// </param>
        /// <param name="schemaBuilder">
        /// The schema builder to use to create a schema for a C# type when registering automatically.
        /// If none is provided, the default schema builder will be used.
        /// </param>
        /// <param name="schemaReader">
        /// The JSON schema reader to use to convert schemas received from the registry into abstract
        /// representations. If none is provided, the default schema reader will be used.
        /// </param>
        /// <param name="schemaWriter">
        /// The JSON schema writer to use to convert abstract schema representations when registering
        /// automatically. If none is provided, the default schema writer will be used.
        /// </param>
        /// <param name="serializerBuilder">
        /// The deserializer builder to use to build serialization functions for C# types. If none
        /// is provided, the default serializer builder will be used.
        /// </param>
        /// <param name="subjectNameBuilder">
        /// A function that determines the subject name given the topic name and a component type
        /// (key or value). If none is provided, the default "{topic name}-{component}" naming
        /// convention will be used.
        /// </param>
        /// <exception cref="ArgumentNullException">
        /// Thrown when the registry client is null.
        /// </exception>
        public AsyncSchemaRegistrySerializer(
            ISchemaRegistryClient registryClient,
            bool registerAutomatically                             = false,
            Abstract.ISchemaBuilder?schemaBuilder                  = null,
            IJsonSchemaReader?schemaReader                         = null,
            IJsonSchemaWriter?schemaWriter                         = null,
            IBinarySerializerBuilder?serializerBuilder             = null,
            Func <SerializationContext, string>?subjectNameBuilder = null
            )
        {
            if (registryClient == null)
            {
                throw new ArgumentNullException(nameof(registryClient));
            }

            RegisterAutomatically = registerAutomatically;
            SchemaBuilder         = schemaBuilder ?? new Abstract.SchemaBuilder();
            SchemaReader          = schemaReader ?? new JsonSchemaReader();
            SchemaWriter          = schemaWriter ?? new JsonSchemaWriter();
            SerializerBuilder     = serializerBuilder ?? new BinarySerializerBuilder();
            SubjectNameBuilder    = subjectNameBuilder ??
                                    (c => $"{c.Topic}-{(c.Component == MessageComponentType.Key ? "key" : "value")}");

            _cache    = new ConcurrentDictionary <string, Task <Func <T, byte[]> > >();
            _register = (subject, json) => registryClient.RegisterSchemaAsync(subject, json);
            _resolve  = subject => registryClient.GetLatestSchemaAsync(subject);
        }
Exemple #6
0
        private static async Task <IProducer <OrderKey, OrderEventRecord> > CreateProducer(
            ISchemaRegistryClient registryClient,
            AutomaticRegistrationBehavior automaticRegistrationBehavior)
        {
            var schemaBuilder = new SchemaBuilder(
                SchemaBuilder.CreateDefaultCaseBuilders()
                .Prepend(builder => new OrderEventUnionSchemaBuilderCase(builder)));

            using var serializerBuilder = new SchemaRegistrySerializerBuilder(
                      registryClient,
                      schemaBuilder,
                      serializerBuilder: new BinarySerializerBuilder(
                          BinarySerializerBuilder.CreateDefaultCaseBuilders()
                          .Prepend(builder => new OrderEventUnionSerializerBuilderCase(builder))));

            var producerBuilder = new ProducerBuilder <OrderKey, OrderEventRecord>(
                new ProducerConfig
            {
                BootstrapServers = BootstrapServers,
            });

            await producerBuilder.SetAvroKeySerializer(
                registryClient,
                SubjectNameStrategy.Topic.ConstructKeySubjectName(Topic),
                automaticRegistrationBehavior);

            await producerBuilder.SetAvroValueSerializer(
                serializerBuilder,
                SubjectNameStrategy.Topic.ConstructKeySubjectName(Topic),
                automaticRegistrationBehavior);

            return(producerBuilder.Build());
        }
Exemple #7
0
        public KafkaProducer(IOptions <KafkaProducerOptions> options, ISemanticLog log, IJsonSerializer jsonSerializer)
        {
            this.jsonSerializer = jsonSerializer;

            textProducer = new ProducerBuilder <string, string>(options.Value)
                           .SetErrorHandler((p, error) =>
            {
                LogError(log, error);
            })
                           .SetLogHandler((p, message) =>
            {
                LogMessage(log, message);
            })
                           .SetKeySerializer(Serializers.Utf8)
                           .SetValueSerializer(Serializers.Utf8)
                           .Build();

            if (options.Value.IsSchemaRegistryConfigured())
            {
                schemaRegistry = new CachedSchemaRegistryClient(options.Value.SchemaRegistry);

                avroProducer = new ProducerBuilder <string, GenericRecord>(options.Value)
                               .SetErrorHandler((p, error) =>
                {
                    LogError(log, error);
                })
                               .SetLogHandler((p, message) =>
                {
                    LogMessage(log, message);
                })
                               .SetKeySerializer(Serializers.Utf8)
                               .SetValueSerializer(new AvroSerializer <GenericRecord>(schemaRegistry, options.Value.AvroSerializer))
                               .Build();
            }
        }
Exemple #8
0
        /// <summary>
        /// Constructor.
        /// </summary>
        /// <param name="settings">the settings dictionary, which follows
        /// the norm defined by Kafka settings.</param>
        /// <exception cref="ArgumentNullException">if any of the method parameters is null.</exception>
        /// <exception cref="ArgumentException">if any of the mandatory Kafka parameters is not set.</exception>
        public GojulMQKafkaMessageConsumer(Dictionary <string, string> settings)
        {
            Condition.Requires(settings, "settings").IsNotNull();
            Condition.Requires((string)settings[BootstrapServers], BootstrapServers)
            .IsNotNull()
            .IsNotEmpty();
            Condition.Requires((string)settings[GroupId], GroupId)
            .IsNotNull()
            .IsNotEmpty();
            Condition.Requires((string)settings[SchemaRegistryUrl], SchemaRegistryUrl)
            .IsNotNull()
            .IsNotEmpty();

            _disposed = false;

            _schemaRegistry = new CachedSchemaRegistryClient(settings);
            _consumer       = new ConsumerBuilder <string, T>(KafkaSettingsList.SanitizeConfiguration(settings))
                              .SetKeyDeserializer(Deserializers.Utf8)
                              .SetValueDeserializer(new AvroDeserializer <T>(_schemaRegistry).AsSyncOverAsync())
                              .SetErrorHandler((_, error) =>
            {
                _log.Error(string.Format("Error while processing message %s - Skipping this message !", error));
            })
                              .Build();
        }
Exemple #9
0
        /// <summary>
        /// Initializes a new instance of the <see cref="AsyncSchemaRegistrySerializer{T}" />
        /// class with a Schema Registry client.
        /// </summary>
        /// <param name="registryClient">
        /// A Schema Registry client to use for Registry operations. (The client will not be
        /// disposed.)
        /// </param>
        /// <param name="registerAutomatically">
        /// Whether the serializer should automatically register schemas that match the type being
        /// serialized.
        /// </param>
        /// <param name="schemaBuilder">
        /// A schema builder instance that should be used to create a schema for <typeparamref name="T" />
        /// when registering automatically. If none is provided, the default <see cref="SchemaBuilder" />
        /// will be used.
        /// </param>
        /// <param name="schemaReader">
        /// A schema reader instance that should be used to convert schemas received from the
        /// Registry into abstract representations. If none is provided, the default
        /// <see cref="JsonSchemaReader" /> will be used.
        /// </param>
        /// <param name="schemaWriter">
        /// A schema writer instance that should be used to convert abstract schema representations
        /// when registering automatically. If none is provided, the default <see cref="JsonSchemaWriter" />
        /// will be used.
        /// </param>
        /// <param name="serializerBuilder">
        /// A serializer builder instance that should be used to generate serialization functions
        /// for .NET <see cref="Type" />s. If none is provided, the default <see cref="BinarySerializerBuilder" />
        /// will be used.
        /// </param>
        /// <param name="subjectNameBuilder">
        /// A function that determines a subject name given a topic name and a component type (key
        /// or value). If none is provided, the default <c>{topic name}-{component}</c> naming
        /// convention will be used.
        /// </param>
        /// <param name="tombstoneBehavior">
        /// How the serializer should handle tombstone records.
        /// </param>
        /// <exception cref="ArgumentNullException">
        /// Thrown when <paramref name="registryClient" /> is <c>null</c>.
        /// </exception>
        /// <exception cref="UnsupportedTypeException">
        /// Thrown when <paramref name="tombstoneBehavior" /> is incompatible with
        /// <typeparamref name="T" />.
        /// </exception>
        public AsyncSchemaRegistrySerializer(
            ISchemaRegistryClient registryClient,
            AutomaticRegistrationBehavior registerAutomatically = AutomaticRegistrationBehavior.Never,
            Abstract.ISchemaBuilder schemaBuilder                  = null,
            IJsonSchemaReader schemaReader                         = null,
            IJsonSchemaWriter schemaWriter                         = null,
            IBinarySerializerBuilder serializerBuilder             = null,
            Func <SerializationContext, string> subjectNameBuilder = null,
            TombstoneBehavior tombstoneBehavior                    = TombstoneBehavior.None)
        {
            if (tombstoneBehavior != TombstoneBehavior.None && default(T) != null)
            {
                throw new UnsupportedTypeException(typeof(T), $"{typeof(T)} cannot represent tombstone values.");
            }

            RegisterAutomatically = registerAutomatically;
            RegistryClient        = registryClient ?? throw new ArgumentNullException(nameof(registryClient));
            SchemaBuilder         = schemaBuilder ?? new Abstract.SchemaBuilder();
            SchemaReader          = schemaReader ?? new JsonSchemaReader();
            SchemaWriter          = schemaWriter ?? new JsonSchemaWriter();
            SerializerBuilder     = serializerBuilder ?? new BinarySerializerBuilder();
            SubjectNameBuilder    = subjectNameBuilder ??
                                    (c => $"{c.Topic}-{(c.Component == MessageComponentType.Key ? "key" : "value")}");
            TombstoneBehavior = tombstoneBehavior;

            cache = new Dictionary <string, Task <Func <T, byte[]> > >();
            disposeRegistryClient = false;
        }
Exemple #10
0
        /// <summary>
        /// Creates a deserializer.
        /// </summary>
        /// <param name="registryClient">
        /// The client to use for Schema Registry operations. (The client will not be disposed.)
        /// </param>
        /// <param name="deserializerBuilder">
        /// The deserializer builder used to generate deserialization functions for C# types. If
        /// none is provided, the default deserializer builder will be used.
        /// </param>
        /// <param name="schemaReader">
        /// The JSON schema reader used to convert schemas received from the registry into abstract
        /// representations. If none is provided, the default schema reader will be used.
        /// </param>
        /// <param name="tombstoneBehavior">
        /// The behavior of the deserializer on tombstone records.
        /// </param>
        /// <exception cref="ArgumentNullException">
        /// Thrown when the registry client is null.
        /// </exception>
        public AsyncSchemaRegistryDeserializer(
            ISchemaRegistryClient registryClient,
            IBinaryDeserializerBuilder deserializerBuilder = null,
            IJsonSchemaReader schemaReader      = null,
            TombstoneBehavior tombstoneBehavior = TombstoneBehavior.None
            )
        {
            if (registryClient == null)
            {
                throw new ArgumentNullException(nameof(registryClient));
            }

            if (tombstoneBehavior != TombstoneBehavior.None && default(T) != null)
            {
                throw new UnsupportedTypeException(typeof(T), $"{typeof(T)} cannot represent tombstone values.");
            }

            DeserializerBuilder = deserializerBuilder ?? new BinaryDeserializerBuilder();
            RegistryClient      = registryClient;
            SchemaReader        = schemaReader ?? new JsonSchemaReader();
            TombstoneBehavior   = tombstoneBehavior;

            _cache = new Dictionary <int, Task <Func <Stream, T> > >();
            _disposeRegistryClient = false;
        }
        /// <summary>
        /// Creates a serializer.
        /// </summary>
        /// <param name="registryClient">
        /// A client to use for Schema Registry operations. (The client will not be disposed.)
        /// </param>
        /// <param name="registerAutomatically">
        /// Whether to automatically register schemas that match the type being serialized.
        /// </param>
        /// <param name="schemaBuilder">
        /// A schema builder (used to build a schema for a C# type when registering automatically).
        /// If none is provided, the default schema builder will be used.
        /// </param>
        /// <param name="schemaReader">
        /// A JSON schema reader (used to convert schemas received from the registry into abstract
        /// representations). If none is provided, the default schema reader will be used.
        /// </param>
        /// <param name="schemaWriter">
        /// A JSON schema writer (used to convert abstract schema representations when registering
        /// automatically). If none is provided, the default schema writer will be used.
        /// </param>
        /// <param name="serializerBuilder">
        /// A deserializer builder (used to build serialization functions for C# types). If none is
        /// provided, the default serializer builder will be used.
        /// </param>
        /// <param name="subjectNameBuilder">
        /// A function that determines the subject name given the topic name and a component type
        /// (key or value). If none is provided, the default "{topic name}-{component}" naming
        /// convention will be used.
        /// </param>
        /// <exception cref="ArgumentNullException">
        /// Thrown when the registry client is null.
        /// </exception>
        public AsyncSchemaRegistrySerializer(
            ISchemaRegistryClient registryClient,
            bool registerAutomatically                             = false,
            Abstract.ISchemaBuilder schemaBuilder                  = null,
            IJsonSchemaReader schemaReader                         = null,
            IJsonSchemaWriter schemaWriter                         = null,
            IBinarySerializerBuilder serializerBuilder             = null,
            Func <SerializationContext, string> subjectNameBuilder = null
            ) : this(
                registerAutomatically,
                schemaBuilder,
                schemaReader,
                schemaWriter,
                serializerBuilder,
                subjectNameBuilder
                )
        {
            if (registryClient == null)
            {
                throw new ArgumentNullException(nameof(registryClient));
            }

            _register = (subject, json) => registryClient.RegisterSchemaAsync(subject, json);
            _resolve  = subject => registryClient.GetLatestSchemaAsync(subject);
        }
Exemple #12
0
        /// <summary>
        ///     Initialize a new instance of the ProtobufSerializer class.
        /// </summary>
        public ProtobufSerializer(ISchemaRegistryClient schemaRegistryClient, ProtobufSerializerConfig config = null)
        {
            this.schemaRegistryClient = schemaRegistryClient;

            if (config == null)
            {
                this.referenceSubjectNameStrategy = ReferenceSubjectNameStrategy.ReferenceName.ToDelegate();
                return;
            }

            var nonProtobufConfig = config.Where(item => !item.Key.StartsWith("protobuf."));

            if (nonProtobufConfig.Count() > 0)
            {
                throw new ArgumentException($"ProtobufSerializer: unknown configuration parameter {nonProtobufConfig.First().Key}");
            }

            if (config.BufferBytes != null)
            {
                this.initialBufferSize = config.BufferBytes.Value;
            }
            if (config.AutoRegisterSchemas != null)
            {
                this.autoRegisterSchema = config.AutoRegisterSchemas.Value;
            }
            if (config.SubjectNameStrategy != null)
            {
                this.subjectNameStrategy = config.SubjectNameStrategy.Value.ToDelegate();
            }
            this.referenceSubjectNameStrategy = config.ReferenceSubjectNameStrategy == null
                ? ReferenceSubjectNameStrategy.ReferenceName.ToDelegate()
                : config.ReferenceSubjectNameStrategy.Value.ToDelegate();
        }
Exemple #13
0
 public AvroExternalStreamDeserializer(ISchemaRegistryClient schemaRegistry)
 {
     _registry              = schemaRegistry;
     _deserializers         = new ConcurrentDictionary <Type, AvroDeserializerWrapper>();
     _getOrCreateMethodInfo = typeof(AvroExternalStreamDeserializer)
                              .GetMethod(nameof(GetOrCreate), BindingFlags.NonPublic | BindingFlags.Instance);
 }
 public AvroSubjectNameStrategySerializer
 (
     ISchemaRegistryClient schemaRegistryClient
 )
 {
     _schemaRegistryClient = schemaRegistryClient;
 }
 public KafkaProducer(ILogger logger, ISchemaRegistryClient schemaRegistryClient, ProducerConfig config)
 {
     _logger  = logger;
     Producer = new ProducerBuilder <TKey, TValue>(config)
                .SetKeySerializer(new AvroSerializer <TKey>(schemaRegistryClient))
                .SetValueSerializer(new AvroSerializer <TValue>(schemaRegistryClient))
                .Build();
 }
Exemple #16
0
        internal ProducerContext(ProducerConfig config, ISchemaRegistryClient registry)
        {
            _registry = registry;

            Producer = new ProducerBuilder <string, T>(config)
                       .SetValueSerializer(new ProtobufSerializer <T>(_registry))
                       .Build();
        }
        internal ConsumerContext(ConsumerConfig config, ISchemaRegistryClient registry)
        {
            _registry = registry;

            Consumer = new ConsumerBuilder <string, T>(config)
                       .SetValueDeserializer(new ProtobufDeserializer <T>().AsSyncOverAsync())
                       .Build();
        }
        public CachedSchemaRegistryClient(ISchemaRegistryClient innerClient, KafkaSchemaRegistryConfiguration configuration)
        {
            _innerClient   = innerClient;
            _configuration = configuration;

            _schemaVersionListCache = new ConcurrentDictionary <string, CacheItem <int[]> >();
            _schemaDetailsCache     = new ConcurrentDictionary <string, CacheItem <SchemaDetails> >();
        }
Exemple #19
0
 public AvroSubjectNameStrategyDeserializer
 (
     ISchemaRegistryClient schemaRegistryClient,
     SubjectNameSchemaCache cache
 )
 {
     _schemaRegistryClient = schemaRegistryClient;
     _cache = cache;
 }
Exemple #20
0
        /// <summary>
        /// Initializes a new instance of the <see cref="ConfluentProtobufSerializer"/> class.
        /// </summary>
        /// <param name="resolver">An instance of <see cref="IDependencyResolver"/></param>
        /// <param name="serializerConfig">An instance of <see cref="ProtobufSerializerConfig"/></param>
        public ConfluentProtobufSerializer(IDependencyResolver resolver, ProtobufSerializerConfig serializerConfig)
        {
            this.schemaRegistryClient =
                resolver.Resolve <ISchemaRegistryClient>() ??
                throw new InvalidOperationException(
                          $"No schema registry configuration was found. Set it using {nameof(ClusterConfigurationBuilderExtensions.WithSchemaRegistry)} on cluster configuration");

            this.serializerConfig = serializerConfig;
        }
Exemple #21
0
 /// <summary>
 /// Set the message value deserializer.
 /// </summary>
 /// <param name="consumerBuilder">
 /// The <see cref="ConsumerBuilder{TKey, TValue}" /> instance to be configured.
 /// </param>
 /// <param name="registryClient">
 /// The client to use for Schema Registry operations. The client should only be disposed
 /// after the consumer; the deserializer will use it to request schemas as messages are
 /// being consumed.
 /// </param>
 /// <param name="tombstoneBehavior">
 /// The behavior of the deserializer on tombstone records.
 /// </param>
 public static ConsumerBuilder <TKey, TValue> SetAvroValueDeserializer <TKey, TValue>(
     this ConsumerBuilder <TKey, TValue> consumerBuilder,
     ISchemaRegistryClient registryClient,
     TombstoneBehavior tombstoneBehavior = TombstoneBehavior.None
     ) => consumerBuilder.SetValueDeserializer(
     new AsyncSchemaRegistryDeserializer <TValue>(
         registryClient,
         tombstoneBehavior: tombstoneBehavior
         ).AsSyncOverAsync());
 public GenericSerializerImpl(
     ISchemaRegistryClient schemaRegistryClient,
     bool autoRegisterSchema,
     int initialBufferSize)
 {
     this.schemaRegistryClient = schemaRegistryClient;
     this.autoRegisterSchema   = autoRegisterSchema;
     this.initialBufferSize    = initialBufferSize;
 }
 /// <summary>
 /// Set the message key serializer.
 /// </summary>
 /// <param name="producerBuilder">
 /// The <see cref="ProducerBuilder{TKey, TValue}" /> instance to be configured.
 /// </param>
 /// <param name="registryClient">
 /// A client to use for Schema Registry operations. The client should only be disposed
 /// after the producer; the serializer will use it to request schemas as messages are being
 /// produced.
 /// </param>
 /// <param name="registerAutomatically">
 /// Whether to automatically register schemas that match the type being serialized.
 /// </param>
 /// <param name="subjectNameBuilder">
 /// A function that determines the subject name given the topic name and a component type
 /// (key or value). If none is provided, the default "{topic name}-{component}" naming
 /// convention will be used.
 /// </param>
 public static ProducerBuilder <TKey, TValue> SetAvroKeySerializer <TKey, TValue>(
     this ProducerBuilder <TKey, TValue> producerBuilder,
     ISchemaRegistryClient registryClient,
     bool registerAutomatically = false,
     Func <SerializationContext, string> subjectNameBuilder = null
     ) => producerBuilder.SetKeySerializer(new AsyncSchemaRegistrySerializer <TKey>(
                                               registryClient,
                                               registerAutomatically: registerAutomatically,
                                               subjectNameBuilder: subjectNameBuilder
                                               ));
        /// <summary>
        ///     Initialize a new DynamicSpecificRecordDeserializer instance.
        /// </summary>
        /// <param name="schemaRegistryClient">
        ///     An implementation of ISchemaRegistryClient used for
        ///     communication with Confluent Schema Registry.
        /// </param>
        /// <param name="config">
        ///     Deserializer configuration properties (refer to
        ///     <see cref="AvroDeserializerConfig" />).
        /// </param>
        public DynamicSpecificRecordDeserializer(ISchemaRegistryClient schemaRegistryClient,
                                                 IReadOnlyCollection <KeyValuePair <string, string> > config = null)
        {
            // There is some config validation that happens so make sure we can construct one
            // ReSharper disable once ObjectCreationAsStatement
            new AvroDeserializer <ISpecificRecord>(schemaRegistryClient, config);

            _schemaRegistryClient = schemaRegistryClient;
            _config = config;
        }
        /// <summary>
        ///     Initialize a new instance of the AvroSerializer class.
        ///     When passed as a parameter to the Confluent.Kafka.Producer constructor,
        ///     the following configuration properties will be extracted from the producer's
        ///     configuration property collection:
        ///
        ///     avro.serializer.buffer.bytes (default: 128) - Initial size (in bytes) of the buffer
        ///         used for message serialization. Use a value high enough to avoid resizing
        ///         the buffer, but small enough to avoid excessive memory use. Inspect the size of
        ///         the byte array returned by the Serialize method to estimate an appropriate value.
        ///         Note: each call to serialize creates a new buffer.
        ///
        ///     avro.serializer.auto.register.schemas (default: true) - true if the serializer should
        ///         attempt to auto-register unrecognized schemas with Confluent Schema Registry,
        ///         false if not.
        /// </summary>
        /// <param name="schemaRegistryClient">
        ///     An implementation of ISchemaRegistryClient used for
        ///     communication with Confluent Schema Registry.
        /// </param>
        /// <param name="config">
        ///     Serializer configuration properties (refer to
        ///     <see cref="AvroSerializerConfig" />)
        /// </param>
        public AvroSerializer(ISchemaRegistryClient schemaRegistryClient, AvroSerializerConfig config = null)
        {
            this.schemaRegistryClient = schemaRegistryClient;

            if (config == null)
            {
                return;
            }

            var nonAvroConfig = config.Where(item => !item.Key.StartsWith("avro."));

            if (nonAvroConfig.Count() > 0)
            {
                throw new ArgumentException($"AvroSerializer: unknown configuration parameter {nonAvroConfig.First().Key}");
            }

            var avroConfig = config.Where(item => item.Key.StartsWith("avro."));

            foreach (var property in avroConfig)
            {
                if (property.Key != AvroSerializerConfig.PropertyNames.AutoRegisterSchemas &&
                    property.Key != AvroSerializerConfig.PropertyNames.UseLatestVersion &&
                    property.Key != AvroSerializerConfig.PropertyNames.BufferBytes &&
                    property.Key != AvroSerializerConfig.PropertyNames.SubjectNameStrategy)
                {
                    throw new ArgumentException($"AvroSerializer: unknown configuration property {property.Key}");
                }
            }

            if (config.BufferBytes != null)
            {
                this.initialBufferSize = config.BufferBytes.Value;
            }
            if (config.AutoRegisterSchemas != null)
            {
                this.autoRegisterSchema = config.AutoRegisterSchemas.Value;
            }
            if (config.NormalizeSchemas != null)
            {
                this.normalizeSchemas = config.NormalizeSchemas.Value;
            }
            if (config.UseLatestVersion != null)
            {
                this.useLatestVersion = config.UseLatestVersion.Value;
            }
            if (config.SubjectNameStrategy != null)
            {
                this.subjectNameStrategy = config.SubjectNameStrategy.Value.ToDelegate();
            }

            if (this.useLatestVersion && this.autoRegisterSchema)
            {
                throw new ArgumentException($"AvroSerializer: cannot enable both use.latest.version and auto.register.schemas");
            }
        }
Exemple #26
0
 public GenericSerializerImpl(
     ISchemaRegistryClient schemaRegistryClient,
     bool autoRegisterSchema,
     int initialBufferSize,
     SubjectNameStrategyDelegate subjectNameStrategy)
 {
     this.schemaRegistryClient = schemaRegistryClient;
     this.autoRegisterSchema   = autoRegisterSchema;
     this.initialBufferSize    = initialBufferSize;
     this.subjectNameStrategy  = subjectNameStrategy;
 }
Exemple #27
0
 public SubjectNameSerializer
 (
     ISchemaRegistryClient schemaRegistryClient,
     bool autoRegisterSchema,
     int initialBufferSize
 )
 {
     _schemaRegistryClient = schemaRegistryClient;
     _autoRegisterSchema   = autoRegisterSchema;
     _initialBufferSize    = initialBufferSize;
 }
        /// <summary>
        /// Sets an Avro deserializer for keys.
        /// </summary>
        /// <typeparam name="TKey">
        /// The type of key to be deserialized.
        /// </typeparam>
        /// <typeparam name="TValue">
        /// The type of value to be deserialized.
        /// </typeparam>
        /// <param name="consumerBuilder">
        /// A <see cref="ConsumerBuilder{TKey, TValue}" /> instance to be configured.
        /// </param>
        /// <param name="registryClient">
        /// A Schema Registry client to use to resolve the schema. (The client will not be
        /// disposed.)
        /// </param>
        /// <param name="id">
        /// The ID of the schema that should be used to deserialize keys.
        /// </param>
        /// <returns>
        /// <paramref name="consumerBuilder" /> with an Avro deserializer configured for
        /// <typeparamref name="TKey" />.
        /// </returns>
        public static async Task <ConsumerBuilder <TKey, TValue> > SetAvroKeyDeserializer <TKey, TValue>(
            this ConsumerBuilder <TKey, TValue> consumerBuilder,
            ISchemaRegistryClient registryClient,
            int id)
        {
            using var deserializerBuilder = new SchemaRegistryDeserializerBuilder(registryClient);

            return(await consumerBuilder
                   .SetAvroKeyDeserializer(deserializerBuilder, id)
                   .ConfigureAwait(false));
        }
 /// <summary>
 /// Set the message key serializer.
 /// </summary>
 /// <param name="producerBuilder">
 /// The <see cref="ProducerBuilder{TKey, TValue}" /> instance to be configured.
 /// </param>
 /// <param name="registryClient">
 /// A client to use to resolve the schema. (The client will not be disposed.)
 /// </param>
 /// <param name="id">
 /// The ID of the schema that should be used to serialize keys.
 /// </param>
 public static async Task <ProducerBuilder <TKey, TValue> > SetAvroKeySerializer <TKey, TValue>(
     this ProducerBuilder <TKey, TValue> producerBuilder,
     ISchemaRegistryClient registryClient,
     int id
     )
 {
     using (var serializerBuilder = new SchemaRegistrySerializerBuilder(registryClient))
     {
         return(await producerBuilder.SetAvroKeySerializer(serializerBuilder, id));
     }
 }
 /// <summary>
 /// Set the message value deserializer.
 /// </summary>
 /// <param name="consumerBuilder">
 /// The <see cref="ConsumerBuilder{TKey, TValue}" /> instance to be configured.
 /// </param>
 /// <param name="registryClient">
 /// The client to use to resolve the schema. (The client will not be disposed.)
 /// </param>
 /// <param name="subject">
 /// The subject of the schema that should be used to deserialize values. The latest version
 /// of the subject will be resolved.
 /// </param>
 public static async Task <ConsumerBuilder <TKey, TValue> > SetAvroValueDeserializer <TKey, TValue>(
     this ConsumerBuilder <TKey, TValue> consumerBuilder,
     ISchemaRegistryClient registryClient,
     string subject
     )
 {
     using (var deserializerBuilder = new SchemaRegistryDeserializerBuilder(registryClient))
     {
         return(await consumerBuilder.SetAvroValueDeserializer(deserializerBuilder, subject));
     }
 }