internal AvroSerializerConfig GetSerializerConfig(ISchemaRegistryConfig config)
        {
            var c = new AvroSerializerConfig();

            if (config.AutoRegisterSchemas.HasValue)
            {
                c.AutoRegisterSchemas = config.AutoRegisterSchemas;
            }

            if (config.SubjectNameStrategy.HasValue)
            {
                c.SubjectNameStrategy = (Confluent.SchemaRegistry.SubjectNameStrategy)config.SubjectNameStrategy.Value;
            }

            if (config.UseLatestVersion.HasValue)
            {
                c.UseLatestVersion = config.UseLatestVersion.Value;
            }

            if (config.BufferBytes.HasValue)
            {
                c.BufferBytes = config.BufferBytes.Value;
            }

            return(c);
        }
        protected BaseKafkaService(IOptions <KafkaOptions> option)
        {
            Console.WriteLine(JsonConvert.SerializeObject(option));

            try
            {
                var publisher = option.Value.Publishers.First(i => i.ServiceName == this.GetType().Name);
                if (publisher != null)
                {
                    _publisher         = publisher;
                    _schemaRegistryUrl =
                        $"{option.Value.Servers.SchemaRegistry.PublicIp}:{option.Value.Servers.SchemaRegistry.Port}";
                    _avroSerializerConfig = new AvroSerializerConfig
                    {
                        BufferBytes         = 500,
                        AutoRegisterSchemas = true
                    };
                    _producerConfig = new ProducerConfig
                    {
                        BootstrapServers = GenerateKafkaBrokerString(option.Value),
                        SocketTimeoutMs  = 5000,
                        MessageTimeoutMs = 5000,
                        RequestTimeoutMs = 5000
                    };
                }
            }
            catch (Exception)
            {
                Console.WriteLine("Failed to find Publisher information for service class {0}", this.GetType().Name);
            }
        }
 /// <summary>
 /// Registers a middleware to serialize avro messages using schema registry
 /// </summary>
 /// <param name="middlewares">The middleware configuration builder</param>
 /// <param name="config">The avro serializer configuration</param>
 /// <returns></returns>
 public static IProducerMiddlewareConfigurationBuilder AddSchemaRegistryAvroSerializer(
     this IProducerMiddlewareConfigurationBuilder middlewares,
     AvroSerializerConfig config = null)
 {
     return(middlewares.AddSerializer(
                resolver => new ConfluentAvroSerializer(resolver, config),
                resolver => new SchemaRegistryTypeResolver(new ConfluentAvroTypeNameResolver(resolver.Resolve <ISchemaRegistryClient>()))));
 }
Beispiel #4
0
 public KafkaConnection(ProducerConfig producerConfig, ConsumerConfig consumerConfig,
                        SchemaRegistryConfig schemaRegistryConfig, AvroSerializerConfig avroSerializerConfig)
 {
     this._producerConfiguration       = producerConfig ?? throw new ArgumentNullException(nameof(producerConfig));
     this._consumerConfiguration       = consumerConfig ?? throw new ArgumentNullException(nameof(consumerConfig));
     this._schemaRegistryConfiguration = schemaRegistryConfig ?? throw new ArgumentNullException(nameof(schemaRegistryConfig));
     this._avroSerializerConfiguration = avroSerializerConfig ?? throw new ArgumentNullException(nameof(avroSerializerConfig));
 }
        /// <summary>
        /// </summary>
        /// <param name="resolver">The <see cref="IDependencyResolver"/> to be used by the framework</param>
        /// <param name="serializerConfig">Avro serializer configuration</param>
        public ApacheAvroMessageSerializer(
            IDependencyResolver resolver,
            AvroSerializerConfig serializerConfig)
        {
            this.schemaRegistryClient = resolver.Resolve <ISchemaRegistryClient>() ??
                                        throw new InvalidOperationException($"No schema registry configuration was found. Set it using {nameof(ClusterConfigurationBuilderExtensions.WithSchemaRegistry)} on cluster configuration");

            this.serializerConfig = serializerConfig;
        }
Beispiel #6
0
        static void Main(string[] args)
        {
            string bootstrapServers  = "127.0.0.1:9092";
            string schemaRegistryUrl = "127.0.0.1:8081";
            string topicName         = "enhanced-end-of-cure";

            var consumerConfig = new ConsumerConfig
            {
                BootstrapServers = bootstrapServers,
                GroupId          = "console-group"
            };

            var schemaRegistryConfig = new SchemaRegistryConfig
            {
                Url = schemaRegistryUrl
            };

            var avroSerializerConfig = new AvroSerializerConfig
            {
            };

            using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
                using (var consumer = new ConsumerBuilder <string, EnhancedEndOfCure>(consumerConfig)
                                      .SetKeyDeserializer(new AvroDeserializer <string>(schemaRegistry, avroSerializerConfig).AsSyncOverAsync())
                                      .SetValueDeserializer(new AvroDeserializer <EnhancedEndOfCure>(schemaRegistry, avroSerializerConfig).AsSyncOverAsync())
                                      .Build())
                {
                    consumer.Subscribe(topicName);

                    try
                    {
                        CancellationTokenSource cts = new CancellationTokenSource();
                        while (true)
                        {
                            try
                            {
                                var consumeResult = consumer.Consume(cts.Token);
                                var v             = (EnhancedEndOfCure)consumeResult.Message.Value;
                                Console.WriteLine($"End Of cure : \n\r" +
                                                  " - Date : " + v.Date +
                                                  " - ItemCode : " + v.ItemCode +
                                                  " - CureEquipmentId : " + v.CureEquipmentId +
                                                  " - ShiftCode : " + v.ShiftCode);
                            }
                            catch (ConsumeException e)
                            {
                                Console.WriteLine($"Consume error: {e.Error.Reason}");
                            }
                        }
                    }
                    catch (OperationCanceledException)
                    {
                        consumer.Close();
                    }
                }
        }
        public void SerializerThrowsOnUnrelated()
        {
            var config = new Dictionary <string, string>
            {
                { "some.random.config.param", "false" }
            };
            var config_ = new AvroSerializerConfig(config);

            Assert.Throws <ArgumentException>(() => { var avroSerializer = new AvroSerializer <int>(null, config_); });
        }
        private AvroSerializerConfig GetSerializerConfig(ISchemaRegistryConfig config)
        {
            AvroSerializerConfig c = new AvroSerializerConfig();

            if (config.AutoRegisterSchemas.HasValue)
            {
                c.AutoRegisterSchemas = config.AutoRegisterSchemas;
            }

            return(c);
        }
        public void SerializerConfigure()
        {
            var config = new AvroSerializerConfig
            {
                BufferBytes         = 42,
                AutoRegisterSchemas = false
            };

            // should not throw.
            var avroSerializer = new AvroSerializer <int>(null, config);
        }
Beispiel #10
0
        static void Main(string[] args)
        {
            var topicName = "Motion";

            var schemaRegistryConfig = new SchemaRegistryConfig
            {
                Url = "localhost:8081"
            };

            var consumerConfig = new ConsumerConfig
            {
                BootstrapServers = "localhost:9092",
                GroupId          = "comsumer-group"
            };

            var avroSerializerConfig = new AvroSerializerConfig
            {
                // optional Avro serializer properties:
                BufferBytes = 100
            };

            using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
                using (var consumer =
                           new ConsumerBuilder <string, MotionEvent>(consumerConfig)
                           .SetKeyDeserializer(new AvroDeserializer <string>(schemaRegistry).AsSyncOverAsync())
                           .SetValueDeserializer(new AvroDeserializer <MotionEvent>(schemaRegistry).AsSyncOverAsync())
                           .SetErrorHandler((_, e) => Console.WriteLine($"Error: {e.Reason}"))
                           .Build())
                {
                    CancellationTokenSource cts = new CancellationTokenSource();

                    consumer.Subscribe(topicName);

                    try
                    {
                        while (true)
                        {
                            try
                            {
                                var consumeResult = consumer.Consume(cts.Token);
                                Console.WriteLine($"Consumed id={consumeResult.Message.Value.id}, timestamp= {consumeResult.Message.Value.timestamp}, camera={consumeResult.Message.Value.camera}.");
                            }
                            catch (ConsumeException e)
                            {
                                Console.WriteLine($"Consume error: {e.Error.Reason}");
                            }
                        }
                    }
                    catch (OperationCanceledException)
                    {
                        consumer.Close();
                    }
                }
        }
        public void SerializerUnexpectedAvroConfigParam()
        {
            var config = new Dictionary <string, string>
            {
                { "avro.serializer.buffer.bytes", "42" },
                { "avro.serializer.auto.register.schemas", "false" },
                { "avro.unknown", "70" }
            };
            var config_ = new AvroSerializerConfig(config);

            Assert.Throws <ArgumentException>(() => { var avroSerializer = new AvroSerializer <int>(null, config_); });
        }
Beispiel #12
0
        static void Main(string[] args)
        {
            string bootstrapServers  = "127.0.0.1:9092";
            string schemaRegistryUrl = "127.0.0.1:8081";
            string topicName         = "end-of-cure";

            var producerConfig = new ProducerConfig
            {
                BootstrapServers = bootstrapServers
            };

            var schemaRegistryConfig = new SchemaRegistryConfig
            {
                Url = schemaRegistryUrl
            };

            var avroSerializerConfig = new AvroSerializerConfig
            {
                BufferBytes = 100
            };

            using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
                using (var producer = new ProducerBuilder <string, EndOfCure>(producerConfig)
                                      .SetKeySerializer(new AvroSerializer <string>(schemaRegistry, avroSerializerConfig))
                                      .SetValueSerializer(new AvroSerializer <EndOfCure>(schemaRegistry, avroSerializerConfig))
                                      .Build())
                {
                    while (true)
                    {
                        var evt = new EndOfCure();
                        evt.Date            = DateTime.Now.ToLongDateString();
                        evt.ItemCode        = DateTime.Now.Ticks.ToString();
                        evt.CureEquipmentId = "001";

                        producer
                        .ProduceAsync(topicName, new Message <string, EndOfCure> {
                            Key = evt.CureEquipmentId, Value = evt
                        })
                        .ContinueWith(task =>

                        {
                            if (!task.IsFaulted)
                            {
                                Console.WriteLine($"produced to: {task.Result.TopicPartitionOffset}");
                            }
                            Console.WriteLine($"error producing message: {task.Exception.InnerException}");
                        });

                        Thread.Sleep(1000);
                    }
                }
        }
        private static ProducerBuilder <PositionKey, PositionValue> CreateProducerBuilder(ISchemaRegistryClient schemaRegistryClient)
        {
            var settings = new ProducerConfig
            {
                ClientId         = "vp-producer-avro",
                BootstrapServers = "kafka:9092",
            };
            var avroSerializerConfig = new AvroSerializerConfig
            {
                AutoRegisterSchemas = true,
            };
            var producerBuilder = new ProducerBuilder <PositionKey, PositionValue>(settings)
                                  .SetKeySerializer(new AvroSerializer <PositionKey>(schemaRegistryClient, avroSerializerConfig))
                                  .SetValueSerializer(new AvroSerializer <PositionValue>(schemaRegistryClient, avroSerializerConfig));

            return(producerBuilder);
        }
Beispiel #14
0
        public void ConfigureServices(IServiceCollection services)
        {
            services.AddLogging();
            services.AddSingleton <IConfigurationRoot>(Configuration);

            var producerConfiguration = new ProducerConfig {
                BootstrapServers = "localhost:9092"
            };
            var schemaRegistryConfiguration = new SchemaRegistryConfig
            {
                SchemaRegistryUrl = "localhost:8081",
                SchemaRegistryRequestTimeoutMs = 5000,
                SchemaRegistryMaxCachedSchemas = 10
            };
            var avroSerializerConfiguration = new AvroSerializerConfig
            {
                // optional Avro serializer properties:
                // BufferBytes = 100,
                AutoRegisterSchemas = true,
            };

            var consumerConfiguration = new ConsumerConfig
            {
                BootstrapServers = "localhost:9092",
                GroupId          = Assembly.GetExecutingAssembly().GetName().Name
            };

            //Set up the event bus
            services.AddSingleton <KafkaConnection>(new KafkaConnection(
                                                        producerConfiguration
                                                        , consumerConfiguration
                                                        , schemaRegistryConfiguration
                                                        , avroSerializerConfiguration));
            services.AddSingleton <IEventBusSubscriptionManager, EventBusSubscriptionManager>();

            services.AddSingleton <IEventBus, KafkaEventBus.KafkaEventBus>(sp =>
            {
                var kafkaConnection             = sp.GetRequiredService <KafkaConnection>();
                var logger                      = sp.GetRequiredService <ILogger <KafkaEventBus.KafkaEventBus> >();
                var eventBusSubcriptionsManager = sp.GetRequiredService <IEventBusSubscriptionManager>();
                return(new KafkaEventBus.KafkaEventBus(eventBusSubcriptionsManager, logger, kafkaConnection, sp));
            });

            services.AddTransient <UserCreatedIntegrationEventHandler>();
            services.AddTransient <UserCreatedIntegrationEventHandler1>();
        }
Beispiel #15
0
        public IProducer <string, TAvro> Build()
        {
            var config = new ProducerConfig
            {
                BootstrapServers = _kafkaOptions.BootstrapServers,
                //producer mais seguro
                Acks = _kafkaOptions.Acks,
                EnableIdempotence     = _kafkaOptions.EnableIdempotence,
                MessageSendMaxRetries = _kafkaOptions.MessageSendMaxRetries,
                MaxInFlight           = _kafkaOptions.MaxInFlight,

                //melhorar taxa de transferencia
                CompressionType = _kafkaOptions.CompressionType,
                LingerMs        = _kafkaOptions.LingerMs,
                BatchSize       = _kafkaOptions.BatchSizeKB * 1024
            };

            var schemaRegistryConfig = new SchemaRegistryConfig
            {
                // Note: you can specify more than one schema registry url using the
                // schema.registry.url property for redundancy (comma separated list).
                // The property name is not plural to follow the convention set by
                // the Java implementation.
                Url = _kafkaOptions.SchemaRegistryUrl
            };

            var avroSerializerConfig = new AvroSerializerConfig
            {
                // optional Avro serializer properties:
                BufferBytes = 100
            };

            var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig);

            var producerBuilder =
                new ProducerBuilder <string, TAvro>(config)
                .SetKeySerializer(new AvroSerializer <string>(schemaRegistry, avroSerializerConfig))
                .SetValueSerializer(new AvroSerializer <TAvro>(schemaRegistry, avroSerializerConfig));



            return(producerBuilder.Build());
        }
Beispiel #16
0
        public ProducerConnectionBuilder <TKey, TValue> WithSchemaRegistry(string url)
        {
            if (typeof(TKey) == typeof(string) || typeof(TValue) == typeof(string))
            {
                return(this);
            }

            var schemaRegistryConfig = new SchemaRegistryConfig()
            {
                SchemaRegistryUrl = url
            };
            var schemaRegistryClient = new CachedSchemaRegistryClient(schemaRegistryConfig);
            var serializerConfig     = new AvroSerializerConfig();

            producerBuilder.SetKeySerializer(new AvroSerializer <TKey>(schemaRegistryClient, serializerConfig));
            producerBuilder.SetValueSerializer(new AvroSerializer <TValue>(schemaRegistryClient, serializerConfig));

            return(this);
        }
Beispiel #17
0
        public static IServiceCollection AddEventBus(this IServiceCollection services)
        {
            ProducerConfig producerConfiguration = new ProducerConfig {
                BootstrapServers = "localhost:9092"
            };
            SchemaRegistryConfig schemaRegistryConfiguration = new SchemaRegistryConfig
            {
                SchemaRegistryUrl = "localhost:8081",
                SchemaRegistryRequestTimeoutMs = 5000,
                SchemaRegistryMaxCachedSchemas = 10
            };
            AvroSerializerConfig avroSerializerConfiguration = new AvroSerializerConfig
            {
                // optional Avro serializer properties:
                // BufferBytes = 100,
                AutoRegisterSchemas = true,
            };

            ConsumerConfig consumerConfiguration = new ConsumerConfig
            {
                BootstrapServers = "localhost:9092",
                GroupId          = Assembly.GetExecutingAssembly().GetName().Name
            };

            //Set up the event bus
            services.AddSingleton <KafkaConnection>(new KafkaConnection(
                                                        producerConfiguration
                                                        , consumerConfiguration
                                                        , schemaRegistryConfiguration
                                                        , avroSerializerConfiguration));
            services.AddSingleton <IEventBusSubscriptionManager, EventBusSubscriptionManager>();

            services.AddSingleton <IEventBus, KafkaEventBus.KafkaEventBus>(sp =>
            {
                KafkaConnection kafkaConnection = sp.GetRequiredService <KafkaConnection>();
                ILogger <KafkaEventBus.KafkaEventBus> logger             = sp.GetRequiredService <ILogger <KafkaEventBus.KafkaEventBus> >();
                IEventBusSubscriptionManager eventBusSubcriptionsManager = sp.GetRequiredService <IEventBusSubscriptionManager>();
                return(new KafkaEventBus.KafkaEventBus(eventBusSubcriptionsManager, logger, kafkaConnection, sp));
            });

            return(services);
        }
        public BaseKafkaService(IOptions <KafkaOptions> option)
        {
            Console.WriteLine(JsonConvert.SerializeObject(option));

            try
            {
                _defaultBrokerString = GenerateKafkaBrokerString(option.Value);
                _schemaRegistryUrl   =
                    $"{option.Value.Servers.SchemaRegistry.PublicIp}:{option.Value.Servers.SchemaRegistry.Port}";
                _avroSerializerConfig = new AvroSerializerConfig
                {
                    BufferBytes         = 500,
                    AutoRegisterSchemas = true
                };
            }
            catch (Exception)
            {
                Console.WriteLine("Failed to find Publisher information for service class {0}", this.GetType().Name);
            }
        }
        private static void ProduceInvalid_NotRecord2(string bootstrapServers, string schemaRegistryServers)
        {
            string topic          = Guid.NewGuid().ToString();
            var    producerConfig = new ProducerConfig {
                BootstrapServers = bootstrapServers
            };

            // 2. Record naming strategy only works with avro record types (value)
            var schemaRegistryConfig2 = new SchemaRegistryConfig
            {
                Url = schemaRegistryServers
            };
            var avroSerializerConfig = new AvroSerializerConfig
            {
                SubjectNameStrategy = SubjectNameStrategy.Record
            };

            using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig2))
                using (var producer =
                           new ProducerBuilder <string, string>(producerConfig)
                           // not a record type.
                           .SetValueSerializer(new AvroSerializer <string>(schemaRegistry, avroSerializerConfig))
                           .Build())
                {
                    Exception caught = null;
                    try
                    {
                        producer.ProduceAsync(topic, new Message <string, string> {
                            Key = "hello", Value = "world"
                        }).GetAwaiter().GetResult();
                    }
                    catch (Exception e)
                    {
                        caught = e;
                    }

                    Assert.NotNull(caught);
                    Assert.IsType <ProduceException <string, string> >(caught);
                    Assert.Equal(ErrorCode.Local_ValueSerialization, ((ProduceException <string, string>)caught).Error.Code);
                }
        }
Beispiel #20
0
        public IConsumer <string, TAvro> Build()
        {
            var consumerConfig = new ConsumerConfig()
            {
                BootstrapServers = _kafkaOptions.BootstrapServers,
                GroupId          = _kafkaOptions.ConsumerGroupId,
                AutoOffsetReset  = AutoOffsetReset.Earliest
            };

            var schemaRegistryConfig = new SchemaRegistryConfig
            {
                // Note: you can specify more than one schema registry url using the
                // schema.registry.url property for redundancy (comma separated list).
                // The property name is not plural to follow the convention set by
                // the Java implementation.
                Url = _kafkaOptions.SchemaRegistryUrl
            };

            var avroSerializerConfig = new AvroSerializerConfig
            {
                // optional Avro serializer properties:
                BufferBytes = 100
            };

            var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig);

            var consumerBuilder =
                new ConsumerBuilder <string, TAvro>(consumerConfig)
                .SetKeyDeserializer(new AvroDeserializer <string>(schemaRegistry).AsSyncOverAsync())
                .SetValueDeserializer(new AvroDeserializer <TAvro>(schemaRegistry).AsSyncOverAsync())
                .SetErrorHandler((_, e) =>
            {
                _logger.LogError(e.Reason);
            });

            return(consumerBuilder.Build());
        }
Beispiel #21
0
        static void Main(string[] args)
        {
            if (args.Length != 3)
            {
                Console.WriteLine("Usage: .. bootstrapServers schemaRegistryUrl topicName");
                return;
            }

            string bootstrapServers  = args[0];
            string schemaRegistryUrl = args[1];
            string topicName         = args[2];

            var producerConfig = new ProducerConfig
            {
                BootstrapServers = bootstrapServers
            };

            var schemaRegistryConfig = new SchemaRegistryConfig
            {
                // Note: you can specify more than one schema registry url using the
                // schema.registry.url property for redundancy (comma separated list).
                // The property name is not plural to follow the convention set by
                // the Java implementation.
                Url = schemaRegistryUrl
            };

            var consumerConfig = new ConsumerConfig
            {
                BootstrapServers = bootstrapServers,
                GroupId          = "avro-specific-example-group"
            };

            var avroSerializerConfig = new AvroSerializerConfig
            {
                // optional Avro serializer properties:
                BufferBytes = 100
            };

            CancellationTokenSource cts = new CancellationTokenSource();
            var consumeTask             = Task.Run(() =>
            {
                using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
                    using (var consumer =
                               new ConsumerBuilder <string, User>(consumerConfig)
                               .SetValueDeserializer(new AvroDeserializer <User>(schemaRegistry).AsSyncOverAsync())
                               .SetErrorHandler((_, e) => Console.WriteLine($"Error: {e.Reason}"))
                               .Build())
                    {
                        consumer.Subscribe(topicName);

                        try
                        {
                            while (true)
                            {
                                try
                                {
                                    var consumeResult = consumer.Consume(cts.Token);
                                    var user          = consumeResult.Message.Value;
                                    Console.WriteLine($"key: {consumeResult.Message.Key}, user name: {user.name}, favorite number: {user.favorite_number}, favorite color: {user.favorite_color}, hourly_rate: {user.hourly_rate}");
                                }
                                catch (ConsumeException e)
                                {
                                    Console.WriteLine($"Consume error: {e.Error.Reason}");
                                }
                            }
                        }
                        catch (OperationCanceledException)
                        {
                            consumer.Close();
                        }
                    }
            });

            using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
                using (var producer =
                           new ProducerBuilder <string, User>(producerConfig)
                           .SetValueSerializer(new AvroSerializer <User>(schemaRegistry, avroSerializerConfig))
                           .Build())
                {
                    Console.WriteLine($"{producer.Name} producing on {topicName}. Enter user names, q to exit.");

                    int    i = 1;
                    string text;
                    while ((text = Console.ReadLine()) != "q")
                    {
                        User user = new User {
                            name = text, favorite_color = "green", favorite_number = ++i, hourly_rate = new Avro.AvroDecimal(67.99)
                        };
                        producer
                        .ProduceAsync(topicName, new Message <string, User> {
                            Key = text, Value = user
                        })
                        .ContinueWith(task =>
                        {
                            if (!task.IsFaulted)
                            {
                                Console.WriteLine($"produced to: {task.Result.TopicPartitionOffset}");
                                return;
                            }

                            // Task.Exception is of type AggregateException. Use the InnerException property
                            // to get the underlying ProduceException. In some cases (notably Schema Registry
                            // connectivity issues), the InnerException of the ProduceException will contain
                            // additional information pertaining to the root cause of the problem. Note: this
                            // information is automatically included in the output of the ToString() method of
                            // the ProduceException which is called implicitly in the below.
                            Console.WriteLine($"error producing message: {task.Exception.InnerException}");
                        });
                    }
                }

            cts.Cancel();
        }
Beispiel #22
0
        public void Consume(string bootstrapServers, string schemaRegistryUrl, string topicName)
        {
            var schemaRegistryRequestTimeoutMs = 30000;
            var schemaRegistryMaxCachedSchemas = 1000;
            var groupID            = Guid.NewGuid().ToString();
            var bufferBytes        = 1024;
            var autoRegisterSchema = true;

            var producerConfig = new ProducerConfig
            {
                BootstrapServers = bootstrapServers
            };

            var schemaRegistryConfig = new SchemaRegistryConfig
            {
                SchemaRegistryUrl = schemaRegistryUrl,
                // optional schema registry client properties:
                SchemaRegistryRequestTimeoutMs = schemaRegistryRequestTimeoutMs,
                SchemaRegistryMaxCachedSchemas = schemaRegistryMaxCachedSchemas
            };

            var consumerConfig = new ConsumerConfig
            {
                BootstrapServers = bootstrapServers,
                AutoOffsetReset  = AutoOffsetReset.Earliest,
                GroupId          = groupID // "Test" //Guid.NewGuid().ToString()
            };

            var avroSerializerConfig = new AvroSerializerConfig
            {
                // optional Avro serializer properties:
                BufferBytes         = bufferBytes,
                AutoRegisterSchemas = autoRegisterSchema
            };

            NewConstructionAddressEvent addr = new NewConstructionAddressEvent();

            using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
                using (var consumer =
                           new ConsumerBuilder <string, NewConstructionAddressEvent>(consumerConfig)
                           .SetKeyDeserializer(new AvroDeserializer <string>(schemaRegistry).AsSyncOverAsync())
                           .SetValueDeserializer(new AvroDeserializer <NewConstructionAddressEvent>(schemaRegistry).AsSyncOverAsync())
                           .SetErrorHandler((_, e) => logger.LogError($"Error: {e.Reason}"))
                           .Build())
                {
                    try
                    {
                        logger.LogInformation($"Starting consumer.subscribe.");

                        consumer.Subscribe(topicName);

                        while (true)
                        {
                            try
                            {
                                logger.LogInformation($"Starting: consumer.Consume");
                                var consumeResult = consumer.Consume();

                                string k = consumeResult.Key;
                                logger.LogInformation($"BusMessage: {consumeResult.Message}, constructionAddressId: {consumeResult.Value.constructionAddressId}");
                            }
                            catch (OperationCanceledException)
                            {
                                logger.LogInformation($"OperationCancelled for consumer.Consume");
                                break;
                            }
                            catch (ConsumeException e)
                            {
                                logger.LogInformation(e, $"Consume error: {e.Error.Reason}");
                                break;
                            }
                        }
                    }
                    catch (Exception ex)
                    {
                        logger.LogError(ex, $"Consume error: {ex.Message}");
                    }
                    finally
                    {
                        consumer.Close();
                    }
                }
        }
Beispiel #23
0
        static async Task Main(string[] args)
        {
            var topicName = "Motion";

            var config = new ProducerConfig
            {
                BootstrapServers = "localhost:9092"
            };

            var schemaRegistryConfig = new SchemaRegistryConfig
            {
                Url = "localhost:8081"
            };

            var avroSerializerConfig = new AvroSerializerConfig
            {
                // optional Avro serializer properties:
                BufferBytes = 100
            };

            using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
                using (var producer =
                           new ProducerBuilder <string, MotionEvent>(config)
                           .SetKeySerializer(new AvroSerializer <string>(schemaRegistry, avroSerializerConfig))
                           .SetValueSerializer(new AvroSerializer <MotionEvent>(schemaRegistry, avroSerializerConfig))
                           .Build())
                {
                    Console.WriteLine($"Starting producer {producer.Name} producing on {topicName} version {nameof(MotionEvent)}. Ctl-C to abort.");

                    var i = 0;
                    while (true)
                    {
                        var now         = DateTime.UtcNow;
                        var key         = $"prod15-{now.ToString("ddHHmmss")}";
                        var motionEvent = new MotionEvent {
                            id        = key,
                            timestamp = now,
                            camera    = "GUSTAV 1M",
                        };

                        try
                        {
                            var deliveryReport = await producer.ProduceAsync(
                                topicName, new Message <string, MotionEvent>
                            {
                                Key   = key,
                                Value = motionEvent
                            });

                            Console.WriteLine($"Prod#1 id={motionEvent.id} cam={motionEvent.camera}");

                            i++;
                        }
                        catch (ProduceException <string, MotionEvent> e)
                        {
                            Console.WriteLine($"Failed to produce event {i} {e.Message} [{e.Error.Code}]");
                        }

                        await Task.Delay(1000);
                    }
                }
        }
Beispiel #24
0
        static async Task Main(string[] args)
        {
            if (args.Length != 3)
            {
                Console.WriteLine("Usage: .. bootstrapServers schemaRegistryUrl topicName");
                return;
            }

            string bootstrapServers  = args[0];
            string schemaRegistryUrl = args[1];
            string topicName         = args[2];

            var producerConfig = new ProducerConfig
            {
                BootstrapServers = bootstrapServers
            };

            var schemaRegistryConfig = new SchemaRegistryConfig
            {
                // Note: you can specify more than one schema registry url using the
                // schema.registry.url property for redundancy (comma separated list).
                // The property name is not plural to follow the convention set by
                // the Java implementation.
                SchemaRegistryUrl = schemaRegistryUrl,
                // optional schema registry client properties:
                SchemaRegistryRequestTimeoutMs = 5000,
                SchemaRegistryMaxCachedSchemas = 10
            };

            var consumerConfig = new ConsumerConfig
            {
                BootstrapServers = bootstrapServers,
                GroupId          = "avro-specific-example-group"
            };

            var avroSerializerConfig = new AvroSerializerConfig
            {
                // optional Avro serializer properties:
                BufferBytes         = 100,
                AutoRegisterSchemas = true
            };

            // Note: The User class in this project was generated using the Confluent fork of the avrogen.exe tool
            // (avaliable from: https://github.com/confluentinc/avro/tree/confluent-fork) which includes modifications
            // that prevent namespace clashes with user namespaces that include the identifier 'Avro'. AvroSerializer
            // and AvroDeserializer are also compatible with classes generated by the official avrogen.exe tool
            // (available from: https://github.com/apache/avro), with the above limitation.

            CancellationTokenSource cts = new CancellationTokenSource();
            var consumeTask             = Task.Run(() =>
            {
                using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
                    using (var consumer =
                               new ConsumerBuilder <string, User>(consumerConfig)
                               .SetKeyDeserializer(new AvroDeserializer <string>(schemaRegistry))
                               .SetValueDeserializer(new AvroDeserializer <User>(schemaRegistry))
                               .SetErrorHandler((_, e) => Console.WriteLine($"Error: {e.Reason}"))
                               .Build())
                    {
                        consumer.Subscribe(topicName);

                        try
                        {
                            while (true)
                            {
                                try
                                {
                                    var consumeResult = consumer.Consume(cts.Token);

                                    Console.WriteLine($"user key name: {consumeResult.Message.Key}, user value favorite color: {consumeResult.Value.favorite_color}");
                                }
                                catch (ConsumeException e)
                                {
                                    Console.WriteLine($"Consume error: {e.Error.Reason}");
                                }
                            }
                        }
                        catch (OperationCanceledException)
                        {
                            consumer.Close();
                        }
                    }
            });

            using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
                using (var producer =
                           new ProducerBuilder <string, User>(producerConfig)
                           .SetKeySerializer(new AvroSerializer <string>(schemaRegistry))
                           .SetValueSerializer(new AvroSerializer <User>(schemaRegistry))
                           .Build())
                {
                    Console.WriteLine($"{producer.Name} producing on {topicName}. Enter user names, q to exit.");

                    int    i = 0;
                    string text;
                    while ((text = Console.ReadLine()) != "q")
                    {
                        User user = new User {
                            name = text, favorite_color = "green", favorite_number = i++
                        };
                        await producer
                        .ProduceAsync(topicName, new Message <string, User> {
                            Key = text, Value = user
                        })
                        .ContinueWith(task => task.IsFaulted
                            ? $"error producing message: {task.Exception.Message}"
                            : $"produced to: {task.Result.TopicPartitionOffset}");
                    }
                }

            cts.Cancel();
        }
Beispiel #25
0
        private static void BMSpecificProduceConsume(string bootstrapServers, string schemaRegistryServers)
        {
            var producerConfig = new ProducerConfig {
                BootstrapServers = bootstrapServers
            };

            var schemaRegistryConfig = new SchemaRegistryConfig
            {
                Url = schemaRegistryServers
            };

            var adminClientConfig = new AdminClientConfig
            {
                BootstrapServers = bootstrapServers
            };

            var topic = Guid.NewGuid().ToString();

            Console.Error.WriteLine($"topic: {topic}");

            using (var adminClient = new AdminClientBuilder(adminClientConfig).Build())
            {
                adminClient.CreateTopicsAsync(
                    new List <TopicSpecification> {
                    new TopicSpecification {
                        Name = topic, NumPartitions = 1, ReplicationFactor = 1
                    }
                }).Wait();
            }

            var    srClient = new CachedSchemaRegistryClient(schemaRegistryConfig);
            Schema schema1  = new Schema(EventA._SCHEMA.ToString(), SchemaType.Avro);
            Schema schema2  = new Schema(EventB._SCHEMA.ToString(), SchemaType.Avro);
            var    id1      = srClient.RegisterSchemaAsync("events-a", schema1).Result;
            var    id2      = srClient.RegisterSchemaAsync("events-b", schema2).Result;

            var             avroUnion   = @"[""Confluent.Kafka.Examples.AvroSpecific.EventA"",""Confluent.Kafka.Examples.AvroSpecific.EventB""]";
            Schema          unionSchema = new Schema(avroUnion, SchemaType.Avro);
            SchemaReference reference   = new SchemaReference(
                "Confluent.Kafka.Examples.AvroSpecific.EventA",
                "events-a",
                srClient.GetLatestSchemaAsync("events-a").Result.Version);

            unionSchema.References.Add(reference);
            reference = new SchemaReference(
                "Confluent.Kafka.Examples.AvroSpecific.EventB",
                "events-b",
                srClient.GetLatestSchemaAsync("events-b").Result.Version);
            unionSchema.References.Add(reference);

            var id3 = srClient.RegisterSchemaAsync($"{topic}-value", unionSchema).Result;

            AvroSerializerConfig avroSerializerConfig = new AvroSerializerConfig {
                AutoRegisterSchemas = false, UseLatestSchemaVersion = true
            };

            using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
                using (var producer =
                           new ProducerBuilder <string, ISpecificRecord>(producerConfig)
                           .SetKeySerializer(new AvroSerializer <string>(schemaRegistry))
                           .SetValueSerializer(new BmSpecificSerializerImpl <ISpecificRecord>(
                                                   schemaRegistry,
                                                   false,
                                                   1024,
                                                   SubjectNameStrategy.Topic.ToDelegate(),
                                                   true))
                           .Build())
                {
                    for (int i = 0; i < 3; ++i)
                    {
                        var eventA = new EventA
                        {
                            A         = "I'm event A",
                            EventId   = Guid.NewGuid().ToString(),
                            EventType = "EventType-A",
                            OccuredOn = DateTime.UtcNow.Ticks,
                        };

                        producer.ProduceAsync(topic, new Message <string, ISpecificRecord> {
                            Key = "DomainEvent", Value = eventA
                        }).Wait();
                    }

                    for (int i = 0; i < 3; ++i)
                    {
                        var eventB = new EventB
                        {
                            B         = 123456987,
                            EventId   = Guid.NewGuid().ToString(),
                            EventType = "EventType-B",
                            OccuredOn = DateTime.UtcNow.Ticks,
                        };

                        producer.ProduceAsync(topic, new Message <string, ISpecificRecord> {
                            Key = "DomainEvent", Value = eventB
                        }).Wait();
                    }

                    Assert.Equal(0, producer.Flush(TimeSpan.FromSeconds(10)));
                }

            var consumerConfig = new ConsumerConfig
            {
                BootstrapServers   = bootstrapServers,
                GroupId            = Guid.NewGuid().ToString(),
                SessionTimeoutMs   = 6000,
                AutoOffsetReset    = AutoOffsetReset.Earliest,
                EnablePartitionEof = true
            };

            using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
                using (var consumer =
                           new ConsumerBuilder <string, GenericRecord>(consumerConfig)
                           .SetKeyDeserializer(new AvroDeserializer <string>(schemaRegistry).AsSyncOverAsync())
                           .SetValueDeserializer(new BmGenericDeserializerImpl(schemaRegistry).AsSyncOverAsync())
                           .SetErrorHandler((_, e) => Assert.True(false, e.Reason))
                           .Build())
                {
                    consumer.Subscribe(topic);

                    int i = 0;
                    while (true)
                    {
                        var record = consumer.Consume(TimeSpan.FromMilliseconds(100));
                        if (record == null)
                        {
                            continue;
                        }
                        if (record.IsPartitionEOF)
                        {
                            break;
                        }

                        Console.WriteLine(record.Message.Value["EventType"]);
                        i += 1;
                    }

                    Assert.Equal(6, i);

                    consumer.Close();
                }
        }
Beispiel #26
0
        static async Task Main(string[] args)
        {
            if (args.Length != 3)
            {
                Console.WriteLine("Usage: .. bootstrapServers schemaRegistryUrl topicName");
                return;
            }

            string bootstrapServers  = args[0];
            string schemaRegistryUrl = args[1];
            string topicName         = args[2];

            var producerConfig = new ProducerConfig
            {
                BootstrapServers = bootstrapServers
            };

            var schemaRegistryConfig = new SchemaRegistryConfig
            {
                // Note: you can specify more than one schema registry url using the
                // schema.registry.url property for redundancy (comma separated list).
                // The property name is not plural to follow the convention set by
                // the Java implementation.
                Url = schemaRegistryUrl
            };

            var consumerConfig = new ConsumerConfig
            {
                BootstrapServers = bootstrapServers,
                GroupId          = "avro-specific-example-group"
            };

            var avroSerializerConfig = new AvroSerializerConfig
            {
                // optional Avro serializer properties:
                BufferBytes = 100
            };

            CancellationTokenSource cts = new CancellationTokenSource();
            var consumeTask             = Task.Run(() =>
            {
                using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
                    using (var consumer =
                               new ConsumerBuilder <string, User>(consumerConfig)
                               .SetKeyDeserializer(new AvroDeserializer <string>(schemaRegistry).AsSyncOverAsync())
                               .SetValueDeserializer(new AvroDeserializer <User>(schemaRegistry).AsSyncOverAsync())
                               .SetErrorHandler((_, e) => Console.WriteLine($"Error: {e.Reason}"))
                               .Build())
                    {
                        consumer.Subscribe(topicName);

                        try
                        {
                            while (true)
                            {
                                try
                                {
                                    var consumeResult = consumer.Consume(cts.Token);
                                    Console.WriteLine($"user name: {consumeResult.Message.Key}, favorite color: {consumeResult.Message.Value.favorite_color}, hourly_rate: {consumeResult.Message.Value.hourly_rate}");
                                }
                                catch (ConsumeException e)
                                {
                                    Console.WriteLine($"Consume error: {e.Error.Reason}");
                                }
                            }
                        }
                        catch (OperationCanceledException)
                        {
                            consumer.Close();
                        }
                    }
            });

            using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
                using (var producer =
                           new ProducerBuilder <string, User>(producerConfig)
                           .SetKeySerializer(new AvroSerializer <string>(schemaRegistry, avroSerializerConfig))
                           .SetValueSerializer(new AvroSerializer <User>(schemaRegistry, avroSerializerConfig))
                           .Build())
                {
                    Console.WriteLine($"{producer.Name} producing on {topicName}. Enter user names, q to exit.");

                    int    i = 0;
                    string text;
                    while ((text = Console.ReadLine()) != "q")
                    {
                        User user = new User {
                            name = text, favorite_color = "green", favorite_number = i++, hourly_rate = new Avro.AvroDecimal(67.99)
                        };
                        await producer
                        .ProduceAsync(topicName, new Message <string, User> {
                            Key = text, Value = user
                        })
                        .ContinueWith(task => task.IsFaulted
                            ? $"error producing message: {task.Exception.Message}"
                            : $"produced to: {task.Result.TopicPartitionOffset}");
                    }
                }

            cts.Cancel();
        }
Beispiel #27
0
        private static void ProduceConsumeNull(string bootstrapServers, string schemaRegistryServers, SubjectNameStrategy nameStrategy)
        {
            var producerConfig = new ProducerConfig
            {
                BootstrapServers = bootstrapServers
            };

            var consumerConfig = new ConsumerConfig
            {
                BootstrapServers   = bootstrapServers,
                GroupId            = Guid.NewGuid().ToString(),
                SessionTimeoutMs   = 6000,
                AutoOffsetReset    = AutoOffsetReset.Earliest,
                EnablePartitionEof = true
            };

            var schemaRegistryConfig = new SchemaRegistryConfig
            {
                Url = schemaRegistryServers,
            };

            var avroSerializerConfig = new AvroSerializerConfig
            {
                SubjectNameStrategy = nameStrategy
            };

            var adminClientConfig = new AdminClientConfig
            {
                BootstrapServers = bootstrapServers
            };

            string topic = Guid.NewGuid().ToString();

            using (var adminClient = new AdminClientBuilder(adminClientConfig).Build())
            {
                adminClient.CreateTopicsAsync(
                    new List <TopicSpecification> {
                    new TopicSpecification {
                        Name = topic, NumPartitions = 1, ReplicationFactor = 1
                    }
                }).Wait();
            }

            using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
                using (var producer =
                           new ProducerBuilder <string, ProduceConsumeUser>(producerConfig)
                           .SetKeySerializer(new AvroSerializer <string>(schemaRegistry))
                           // Test ValueSubjectNameStrategy here,
                           // and KeySubjectNameStrategy in ProduceConsumeGeneric.
                           .SetValueSerializer(new AvroSerializer <ProduceConsumeUser>(schemaRegistry, avroSerializerConfig))
                           .Build())
                {
                    for (int i = 0; i < 100; ++i)
                    {
                        producer
                        .ProduceAsync(topic, new Message <string, ProduceConsumeUser> {
                            Key = null, Value = null
                        })
                        .Wait();
                    }
                    Assert.Equal(0, producer.Flush(TimeSpan.FromSeconds(10)));
                }

            using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
                using (var consumer =
                           new ConsumerBuilder <string, ProduceConsumeUser>(consumerConfig)
                           .SetKeyDeserializer(new AvroDeserializer <string>(schemaRegistry).AsSyncOverAsync())
                           .SetValueDeserializer(new AvroDeserializer <ProduceConsumeUser>(schemaRegistry).AsSyncOverAsync())
                           .SetErrorHandler((_, e) => Assert.True(false, e.Reason))
                           .Build())
                {
                    consumer.Subscribe(topic);

                    int i = 0;
                    while (true)
                    {
                        var record = consumer.Consume(TimeSpan.FromMilliseconds(100));
                        if (record == null)
                        {
                            continue;
                        }
                        if (record.IsPartitionEOF)
                        {
                            break;
                        }

                        Assert.Null(record.Message.Key);
                        Assert.Null(record.Message.Value);
                        i += 1;
                    }

                    Assert.Equal(100, i);

                    consumer.Close();
                }
        }
Beispiel #28
0
        private static void ProduceConsumeGeneric(string bootstrapServers, string schemaRegistryServers, SubjectNameStrategy nameStrategy)
        {
            var rs = (RecordSchema)RecordSchema.Parse(
                @"{
                    ""namespace"": ""Confluent.Kafka.Examples.AvroSpecific"",
                    ""type"": ""record"",
                    ""name"": ""ProduceConsumeUser2"",
                    ""fields"": [
                        {""name"": ""name"", ""type"": ""string""},
                        {""name"": ""favorite_number"",  ""type"": [""int"", ""null""]},
                        {""name"": ""favorite_color"", ""type"": [""string"", ""null""]}
                    ]
                  }"
                );

            var config = new ProducerConfig {
                BootstrapServers = bootstrapServers
            };
            var schemaRegistryConfig = new SchemaRegistryConfig
            {
                Url = schemaRegistryServers,
            };

            var avroSerializerConfig = new AvroSerializerConfig
            {
                SubjectNameStrategy = nameStrategy
            };

            var topic = Guid.NewGuid().ToString();

            DeliveryResult <GenericRecord, Null> dr;

            using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
                using (var p =
                           new ProducerBuilder <GenericRecord, Null>(config)
                           // Test KeySubjectNameStrategy here,
                           // and ValueSubjectNameStrategy in ProduceConsume.
                           .SetKeySerializer(new AvroSerializer <GenericRecord>(schemaRegistry, avroSerializerConfig))
                           .SetValueSerializer(Serializers.Null)
                           .Build())
                {
                    var record = new GenericRecord(rs);
                    record.Add("name", "my name 2");
                    record.Add("favorite_number", 44);
                    record.Add("favorite_color", null);
                    dr = p.ProduceAsync(topic, new Message <GenericRecord, Null> {
                        Key = record
                    }).Result;
                }

            // produce a specific record (to later consume back as a generic record).
            using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
                using (var p =
                           new ProducerBuilder <ProduceConsumeUser2, Null>(config)
                           .SetKeySerializer(new AvroSerializer <ProduceConsumeUser2>(schemaRegistry, avroSerializerConfig))
                           .SetValueSerializer(Serializers.Null)
                           .Build())
                {
                    var user = new ProduceConsumeUser2
                    {
                        name            = "my name 3",
                        favorite_number = 47,
                        favorite_color  = "orange"
                    };

                    p.ProduceAsync(topic, new Message <ProduceConsumeUser2, Null> {
                        Key = user
                    }).Wait();
                }

            Assert.Null(dr.Message.Value);
            Assert.NotNull(dr.Message.Key);
            dr.Message.Key.TryGetValue("name", out object name);
            dr.Message.Key.TryGetValue("favorite_number", out object number);
            dr.Message.Key.TryGetValue("favorite_color", out object color);

            Assert.IsType <string>(name);
            Assert.IsType <int>(number);

            Assert.Equal("my name 2", name);
            Assert.Equal(44, number);
            Assert.Null(color);

            var cconfig = new ConsumerConfig {
                GroupId = Guid.NewGuid().ToString(), BootstrapServers = bootstrapServers
            };

            using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
                using (var consumer =
                           new ConsumerBuilder <GenericRecord, Null>(cconfig)
                           .SetKeyDeserializer(new AvroDeserializer <GenericRecord>(schemaRegistry).AsSyncOverAsync())
                           .SetValueDeserializer(Deserializers.Null)
                           .Build())
                {
                    // consume generic record produced as a generic record.
                    consumer.Assign(new List <TopicPartitionOffset> {
                        new TopicPartitionOffset(topic, 0, dr.Offset)
                    });
                    var record = consumer.Consume(new CancellationTokenSource(TimeSpan.FromSeconds(10)).Token);
                    record.Message.Key.TryGetValue("name", out object msgName);
                    record.Message.Key.TryGetValue("favorite_number", out object msgNumber);
                    record.Message.Key.TryGetValue("favorite_color", out object msgColor);

                    Assert.IsType <string>(msgName);
                    Assert.IsType <int>(msgNumber);

                    Assert.Equal("my name 2", msgName);
                    Assert.Equal(44, msgNumber);
                    Assert.Null(msgColor);

                    // consume generic record produced as a specific record.
                    record = consumer.Consume(new CancellationTokenSource(TimeSpan.FromSeconds(10)).Token);
                    record.Message.Key.TryGetValue("name", out msgName);
                    record.Message.Key.TryGetValue("favorite_number", out msgNumber);
                    record.Message.Key.TryGetValue("favorite_color", out msgColor);

                    Assert.IsType <string>(msgName);
                    Assert.IsType <int>(msgNumber);
                    Assert.IsType <string>(msgColor);

                    Assert.Equal("my name 3", msgName);
                    Assert.Equal(47, msgNumber);
                    Assert.Equal("orange", msgColor);
                }

            using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
                using (var consumer =
                           new ConsumerBuilder <ProduceConsumeUser2, Null>(cconfig)
                           .SetKeyDeserializer(new AvroDeserializer <ProduceConsumeUser2>(schemaRegistry).AsSyncOverAsync())
                           .SetValueDeserializer(Deserializers.Null)
                           .Build())
                {
                    consumer.Assign(new List <TopicPartitionOffset> {
                        new TopicPartitionOffset(topic, 0, dr.Offset)
                    });
                    var record = consumer.Consume(new CancellationTokenSource(TimeSpan.FromSeconds(10)).Token);
                    Assert.Equal("my name 2", record.Message.Key.name);
                    Assert.Equal(44, record.Message.Key.favorite_number);
                    Assert.Null(record.Message.Key.favorite_color);
                }

            // Check that what's in schema registry is what's expected.
            using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
            {
                var subjects = schemaRegistry.GetAllSubjectsAsync().Result;

                if (nameStrategy == SubjectNameStrategy.TopicRecord)
                {
                    Assert.Single(subjects.Where(s => s.Contains(topic)));
                    Assert.Single(subjects.Where(s => s == $"{topic}-{((Avro.RecordSchema)ProduceConsumeUser2._SCHEMA).Fullname}"));
                }

                if (nameStrategy == SubjectNameStrategy.Topic)
                {
                    Assert.Single(subjects.Where(s => s.Contains(topic)));
                    Assert.Single(subjects.Where(s => s == $"{topic}-key"));
                }

                if (nameStrategy == SubjectNameStrategy.Record)
                {
                    Assert.Single(subjects.Where(s => s.Contains(topic))); // the string key.
                    Assert.Single(subjects.Where(s => s == $"{((Avro.RecordSchema)ProduceConsumeUser2._SCHEMA).Fullname}"));
                }
            }
        }
        private static void ProduceConsume(string bootstrapServers, string schemaRegistryServers, SubjectNameStrategy nameStrategy)
        {
            var producerConfig = new ProducerConfig
            {
                BootstrapServers = bootstrapServers
            };

            var consumerConfig = new ConsumerConfig
            {
                BootstrapServers   = bootstrapServers,
                GroupId            = Guid.NewGuid().ToString(),
                SessionTimeoutMs   = 6000,
                AutoOffsetReset    = AutoOffsetReset.Earliest,
                EnablePartitionEof = true
            };

            var schemaRegistryConfig = new SchemaRegistryConfig
            {
                Url = schemaRegistryServers,
            };

            var avroSerializerConfig = new AvroSerializerConfig
            {
                SubjectNameStrategy = nameStrategy
            };

            var adminClientConfig = new AdminClientConfig
            {
                BootstrapServers = bootstrapServers
            };

            string topic = Guid.NewGuid().ToString();

            using (var adminClient = new AdminClientBuilder(adminClientConfig).Build())
            {
                adminClient.CreateTopicsAsync(
                    new List <TopicSpecification> {
                    new TopicSpecification {
                        Name = topic, NumPartitions = 1, ReplicationFactor = 1
                    }
                }).Wait();
            }

            using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
                using (var producer =
                           new ProducerBuilder <string, ProduceConsumeUser>(producerConfig)
                           .SetKeySerializer(new AvroSerializer <string>(schemaRegistry))
                           // Test ValueSubjectNameStrategy here,
                           // and KeySubjectNameStrategy in ProduceConsumeGeneric.
                           .SetValueSerializer(new AvroSerializer <ProduceConsumeUser>(schemaRegistry, avroSerializerConfig))
                           .Build())
                {
                    for (int i = 0; i < 100; ++i)
                    {
                        var user = new ProduceConsumeUser
                        {
                            name            = i.ToString(),
                            favorite_number = i,
                            favorite_color  = "blue"
                        };

                        producer
                        .ProduceAsync(topic, new Message <string, ProduceConsumeUser> {
                            Key = user.name, Value = user
                        })
                        .Wait();
                    }
                    Assert.Equal(0, producer.Flush(TimeSpan.FromSeconds(10)));
                }

            using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
                using (var consumer =
                           new ConsumerBuilder <string, ProduceConsumeUser>(consumerConfig)
                           .SetKeyDeserializer(new AvroDeserializer <string>(schemaRegistry).AsSyncOverAsync())
                           .SetValueDeserializer(new AvroDeserializer <ProduceConsumeUser>(schemaRegistry).AsSyncOverAsync())
                           .SetErrorHandler((_, e) => Assert.True(false, e.Reason))
                           .Build())
                {
                    consumer.Subscribe(topic);

                    int i = 0;
                    while (true)
                    {
                        var record = consumer.Consume(TimeSpan.FromMilliseconds(100));
                        if (record == null)
                        {
                            continue;
                        }
                        if (record.IsPartitionEOF)
                        {
                            break;
                        }

                        Assert.Equal(i.ToString(), record.Message.Key);
                        Assert.Equal(i.ToString(), record.Message.Value.name);
                        Assert.Equal(i, record.Message.Value.favorite_number);
                        Assert.Equal("blue", record.Message.Value.favorite_color);
                        i += 1;
                    }

                    Assert.Equal(100, i);

                    consumer.Close();
                }

            // Check that what's in schema registry is what's expected.
            using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
            {
                var subjects = schemaRegistry.GetAllSubjectsAsync().Result;

                if (nameStrategy == SubjectNameStrategy.TopicRecord)
                {
                    Assert.Equal(2, (int)subjects.Where(s => s.Contains(topic)).Count());
                    Assert.Single(subjects.Where(s => s == $"{topic}-key"));
                    Assert.Single(subjects.Where(s => s == $"{topic}-{((Avro.RecordSchema)ProduceConsumeUser._SCHEMA).Fullname}"));
                }

                if (nameStrategy == SubjectNameStrategy.Topic)
                {
                    Assert.Equal(2, (int)subjects.Where(s => s.Contains(topic)).Count());
                    Assert.Single(subjects.Where(s => s == $"{topic}-key"));
                    Assert.Single(subjects.Where(s => s == $"{topic}-value"));
                }

                if (nameStrategy == SubjectNameStrategy.Record)
                {
                    Assert.Single(subjects.Where(s => s.Contains(topic))); // the string key.
                    Assert.Single(subjects.Where(s => s == $"{topic}-key"));
                    Assert.Single(subjects.Where(s => s == $"{((Avro.RecordSchema)ProduceConsumeUser._SCHEMA).Fullname}"));
                }
            }
        }