private void init() { AvroSerdeProvider serdeProvider = new AvroSerdeProvider(consumerConfiguration.avroConfig); Consumer <String, DeleteClient> consumerDelete = new Consumer <String, DeleteClient>( consumerConfiguration.consumerConfig, serdeProvider.GetDeserializerGenerator <String>(), serdeProvider.GetDeserializerGenerator <DeleteClient>()); CancellationTokenSource cts = new CancellationTokenSource(); var consumeTask = Task.Run(() => { consumerDelete.OnError += (_, e) => Console.WriteLine($"Error: {e.Reason}"); consumerDelete.Subscribe("DeleteClient"); while (!cts.Token.IsCancellationRequested) { try { var consumeResult = consumerDelete.Consume(cts.Token); Console.WriteLine($"Delete Client - key: {consumeResult.Message.Key}, value: {consumeResult.Value}"); var item = context.ClientItems.Find(new Guid(consumeResult.Value.id)); if (item != null) { context.Remove(item); context.SaveChanges(); clientDeletedProducer.sendEvent(item); } } catch (ConsumeException e) { Console.WriteLine("Consume error: " + e.Error.Reason); } } consumerDelete.Close(); }, cts.Token); }
public AvroConsumerStrategy(ConsumerConfig config, AvroSerdeProviderConfig avroConfig) { _serdeProvider = new AvroSerdeProvider(avroConfig); _consumer = new Consumer <TKey, TValue>(config, _serdeProvider.GetDeserializerGenerator <TKey>(), _serdeProvider.GetDeserializerGenerator <TValue>()); }
private static Consumer <TKey, TValue> CreateConsumer <TKey, TValue>(bool enableAutoCommit) { Log.Information("Creating consumer"); var avroSerdeProvider = new AvroSerdeProvider(CreateAvroSerdeProviderConfig()); var config = CreateConsumerConfig(enableAutoCommit); var keyDeserializer = avroSerdeProvider.GetDeserializerGenerator <TKey>(); var valueDeserializer = avroSerdeProvider.GetDeserializerGenerator <TValue>(); var consumer = new Consumer <TKey, TValue>(config, keyDeserializer, valueDeserializer); consumer.OnError += (_, error) => Console.WriteLine($"[ERROR] Code: {error.Code}, Reason: {error.Reason}"); consumer.OnLog += (_, log) => Console.WriteLine($"[LOG] {log.Message}"); consumer.OnStatistics += (_, statistic) => Console.WriteLine($"[STATISTIC] {statistic}"); Log.Information("Consumer created"); consumer.Subscribe(Topic); Log.Information("Subscribed to: {0}", string.Join(',', Topic)); return(consumer); }
private void init() { AvroSerdeProvider serdeProvider = new AvroSerdeProvider(consumerConfiguration.avroConfig); Consumer <int, ClientDuplicatesFound> consumerDuplicatesFound = new Consumer <int, ClientDuplicatesFound>( consumerConfiguration.consumerConfig, serdeProvider.GetDeserializerGenerator <int>(), serdeProvider.GetDeserializerGenerator <ClientDuplicatesFound>()); CancellationTokenSource cts = new CancellationTokenSource(); var consumeTask = Task.Run(() => { consumerDuplicatesFound.OnError += (_, e) => Console.WriteLine($"Error: {e.Reason}"); consumerDuplicatesFound.Subscribe("ClientDuplicatesFound"); while (!cts.Token.IsCancellationRequested) { try { var consumeResult = consumerDuplicatesFound.Consume(cts.Token); Console.WriteLine($"Client Duplicate Found - key: {consumeResult.Message.Key}, value: {consumeResult.Value}"); var oldItem = context.ClientItems.Find(new Guid(consumeResult.Value.oldclientid)); if (oldItem != null) { context.ClientItems.Remove(oldItem); context.SaveChanges(); } } catch (ConsumeException e) { Console.WriteLine("Consume error: " + e.Error.Reason); } } consumerDuplicatesFound.Close(); }, cts.Token); }
private static void StartConsumingAvro <T>(string topic) where T : ISpecificRecord { var avroSerdeProvider = new AvroSerdeProvider(new AvroSerdeProviderConfig { SchemaRegistryUrl = Configuration.SchemaRegistryUrl }); using (var consumer = new Consumer <Ignore, T>(Configuration.ConsumerConfig, null, avroSerdeProvider.GetDeserializerGenerator <T>())) { var consumerConfig = Configuration.ConsumerConfig; Log($"Consumer for group: '{consumerConfig.GroupId}' was created. Data type: '{Configuration.Type}'."); var consuming = true; consumer.OnError += (s, e) => { consuming = !e.IsFatal; Log(e.ToString()); }; consumer.Subscribe(new[] { topic }); Log($"Subscribed to topic: '{topic}'."); while (consuming) { try { var consumeResult = consumer.Consume(); var type = typeof(T); if (type == typeof(Avro.Models.Block)) { var result = consumeResult as ConsumeResult <Ignore, Avro.Models.Block>; var block = result.Value; Log($"Block: {block.blockNumber} {block.blockHash}"); } else if (type == typeof(Avro.Models.FullTransaction)) { var result = consumeResult as ConsumeResult <Ignore, Avro.Models.FullTransaction>; var transaction = result.Value; Log($"Transaction for block: {transaction.blockNumber} {transaction.receipt.blockHash}"); } else { Log($"Unknown data type: {type.Name}"); continue; } Log($"Consumed value at '{consumeResult.TopicPartitionOffset}'."); } catch (ConsumeException exception) { Log($"Consumer error occured: {exception.Error.Reason}"); } catch (Exception exception) { Log(exception.Message); } } consumer.Close(); } }
static void Main(string[] args) { if (args.Length != 3) { Console.WriteLine("Usage: .. bootstrapServers schemaRegistryUrl topicName"); return; } string bootstrapServers = args[0]; string schemaRegistryUrl = args[1]; string topicName = args[2]; var producerConfig = new ProducerConfig { BootstrapServers = bootstrapServers }; var avroConfig = new AvroSerdeProviderConfig { // Note: you can specify more than one schema registry url using the // schema.registry.url property for redundancy (comma separated list). // The property name is not plural to follow the convention set by // the Java implementation. SchemaRegistryUrl = schemaRegistryUrl, // optional schema registry client properties: SchemaRegistryRequestTimeoutMs = 5000, SchemaRegistryMaxCachedSchemas = 10, // optional avro serializer properties: AvroSerializerBufferBytes = 50, AvroSerializerAutoRegisterSchemas = true }; var consumerConfig = new ConsumerConfig { BootstrapServers = bootstrapServers, GroupId = Guid.NewGuid().ToString() }; // Note: The User class in this project was generated using the Confluent fork of the avrogen.exe tool // (avaliable from: https://github.com/confluentinc/avro/tree/confluent-fork) which includes modifications // that prevent namespace clashes with user namespaces that include the identifier 'Avro'. AvroSerializer // and AvroDeserializer are also compatible with classes generated by the official avrogen.exe tool // (available from: https://github.com/apache/avro), with the above limitation. CancellationTokenSource cts = new CancellationTokenSource(); var consumeTask = Task.Run(() => { using (var serdeProvider = new AvroSerdeProvider(avroConfig)) using (var consumer = new Consumer <string, User>(consumerConfig, serdeProvider.GetDeserializerGenerator <string>(), serdeProvider.GetDeserializerGenerator <User>())) { consumer.OnError += (_, e) => Console.WriteLine($"Error: {e.Reason}"); consumer.Subscribe(topicName); while (!cts.Token.IsCancellationRequested) { try { var consumeResult = consumer.Consume(cts.Token); Console.WriteLine($"user key name: {consumeResult.Message.Key}, user value favorite color: {consumeResult.Value.favorite_color}"); } catch (ConsumeException e) { Console.WriteLine("Consume error: " + e.Error.Reason); } } consumer.Close(); } }, cts.Token); using (var serdeProvider = new AvroSerdeProvider(avroConfig)) using (var producer = new Producer <string, User>(producerConfig, serdeProvider.GetSerializerGenerator <string>(), serdeProvider.GetSerializerGenerator <User>())) { Console.WriteLine($"{producer.Name} producing on {topicName}. Enter user names, q to exit."); int i = 0; string text; while ((text = Console.ReadLine()) != "q") { User user = new User { name = text, favorite_color = "green", favorite_number = i++ }; producer .ProduceAsync(topicName, new Message <string, User> { Key = text, Value = user }) .ContinueWith(task => task.IsFaulted ? $"error producing message: {task.Exception.Message}" : $"produced to: {task.Result.TopicPartitionOffset}"); } } cts.Cancel(); }
public static void ProduceConsumeGeneric(string bootstrapServers, string schemaRegistryServers) { var s = (RecordSchema)Schema.Parse( @"{ ""namespace"": ""Confluent.Kafka.Examples.AvroSpecific"", ""type"": ""record"", ""name"": ""User"", ""fields"": [ {""name"": ""name"", ""type"": ""string""}, {""name"": ""favorite_number"", ""type"": [""int"", ""null""]}, {""name"": ""favorite_color"", ""type"": [""string"", ""null""]} ] }" ); var config = new ProducerConfig { BootstrapServers = bootstrapServers }; var serdeProviderConfig = new AvroSerdeProviderConfig { SchemaRegistryUrl = schemaRegistryServers }; var topic = Guid.NewGuid().ToString(); DeliveryReport <Null, GenericRecord> dr; using (var serdeProvider = new AvroSerdeProvider(serdeProviderConfig)) using (var p = new Producer <Null, GenericRecord>(config, null, serdeProvider.GetSerializerGenerator <GenericRecord>())) { var record = new GenericRecord(s); record.Add("name", "my name 2"); record.Add("favorite_number", 44); record.Add("favorite_color", null); dr = p.ProduceAsync(topic, new Message <Null, GenericRecord> { Value = record }).Result; } // produce a specific record (to later consume back as a generic record). using (var serdeProvider = new AvroSerdeProvider(serdeProviderConfig)) using (var p = new Producer <Null, User>(config, null, serdeProvider.GetSerializerGenerator <User>())) { var user = new User { name = "my name 3", favorite_number = 47, favorite_color = "orange" }; p.ProduceAsync(topic, new Message <Null, User> { Value = user }).Wait(); } Assert.Null(dr.Message.Key); Assert.NotNull(dr.Message.Value); dr.Message.Value.TryGetValue("name", out object name); dr.Message.Value.TryGetValue("favorite_number", out object number); dr.Message.Value.TryGetValue("favorite_color", out object color); Assert.IsType <string>(name); Assert.IsType <int>(number); Assert.Equal("my name 2", name); Assert.Equal(44, number); Assert.Null(color); var cconfig = new ConsumerConfig { GroupId = Guid.NewGuid().ToString(), BootstrapServers = bootstrapServers }; using (var serdeProvider = new AvroSerdeProvider(serdeProviderConfig)) using (var consumer = new Consumer <Null, GenericRecord>(cconfig, null, serdeProvider.GetDeserializerGenerator <GenericRecord>())) { // consume generic record produced as a generic record. consumer.Assign(new List <TopicPartitionOffset> { new TopicPartitionOffset(topic, 0, dr.Offset) }); var record = consumer.Consume(TimeSpan.FromSeconds(20)); record.Message.Value.TryGetValue("name", out object msgName); record.Message.Value.TryGetValue("favorite_number", out object msgNumber); record.Message.Value.TryGetValue("favorite_color", out object msgColor); Assert.IsType <string>(msgName); Assert.IsType <int>(msgNumber); Assert.Equal("my name 2", msgName); Assert.Equal(44, msgNumber); Assert.Null(msgColor); // consume generic record produced as a specific record. record = consumer.Consume(TimeSpan.FromSeconds(20)); record.Message.Value.TryGetValue("name", out msgName); record.Message.Value.TryGetValue("favorite_number", out msgNumber); record.Message.Value.TryGetValue("favorite_color", out msgColor); Assert.IsType <string>(msgName); Assert.IsType <int>(msgNumber); Assert.IsType <string>(msgColor); Assert.Equal("my name 3", msgName); Assert.Equal(47, msgNumber); Assert.Equal("orange", msgColor); } using (var serdeProvider = new AvroSerdeProvider(serdeProviderConfig)) using (var consumer = new Consumer <Null, User>(cconfig, null, serdeProvider.GetDeserializerGenerator <User>())) { consumer.Assign(new List <TopicPartitionOffset> { new TopicPartitionOffset(topic, 0, dr.Offset) }); var record = consumer.Consume(TimeSpan.FromSeconds(20)); Assert.Equal("my name 2", record.Message.Value.name); Assert.Equal(44, record.Message.Value.favorite_number); Assert.Null(record.Message.Value.favorite_color); } }
protected override async Task ExecuteAsync(CancellationToken stoppingToken) { Log.Information("server1.dbo.products démarré à : {Now}", DateTime.Now); stoppingToken.Register(() => _logger.Information("server1.dbo.products a débuté son arrêt à : {Now}", DateTime.Now)); var tacheConsommateur = Task.Run(() => { using (var serdeProvider = new AvroSerdeProvider(new AvroSerdeProviderConfig { SchemaRegistryUrl = schemaRegistryConfig.SchemaRegistryUrl })) using (var consumer = new Consumer <GenericRecord, Envelope>(consumerConfig, serdeProvider.GetDeserializerGenerator <GenericRecord>().Invoke(true), serdeProvider.GetDeserializerGenerator <Envelope>().Invoke(false))) { consumer.Subscribe(TOPIC); try { while (true) { try { var consumeResult = consumer.Consume(stoppingToken); if (consumeResult.Value.before == null && consumeResult.Value.after != null) { //Ajout product produit = new product() { idProduit = consumeResult.Value.after.id, name = consumeResult.Value.after.name, description = consumeResult.Value.after.description, weight = consumeResult.Value.after.weight }; _serviceProduct.Create(produit); } if (consumeResult.Value.before != null && consumeResult.Value.after != null) { //MAJ product produit = new product() { idProduit = consumeResult.Value.after.id, name = consumeResult.Value.after.name, description = consumeResult.Value.after.description, weight = consumeResult.Value.after.weight }; _serviceProduct.Update(produit.idProduit, produit); } if (consumeResult.Value.before != null && consumeResult.Value.after == null) { //Suppression _serviceProduct.Remove(consumeResult.Value.before.id); } _logger.Information("Key: {Key}, Value: {Value}", consumeResult.Message.Key, consumeResult.Value); } catch (ConsumeException e) { _logger.Error("Consommation erreur: {Reason}", e.Error.Reason); } } } catch (OperationCanceledException) { // commit final offsets and leave the group. consumer.Close(); } } }); //_logger.Information("service server1.dbo.products arrêté à : {Now}", DateTime.Now); }
static void Main(string[] args) { if (args.Length != 3) { Console.WriteLine("Usage: .. bootstrapServers schemaRegistryUrl topicName"); return; } string bootstrapServers = args[0]; string schemaRegistryUrl = args[1]; string topicName = args[2]; var producerConfig = new ProducerConfig { BootstrapServers = bootstrapServers }; var avroConfig = new AvroSerdeProviderConfig { // Note: you can specify more than one schema registry url using the // schemaRegistryUrl property for redundancy (comma separated list). SchemaRegistryUrl = schemaRegistryUrl, // optional schema registry client properties: // SchemaRegistryRequestTimeoutMs = 5000, // SchemaRegistryMaxCachedSchemas = 10, // optional avro serializer properties: // AvroSerializerBufferBytes = 50, // AvroSerializerAutoRegisterSchemas = true }; var consumerConfig = new ConsumerConfig { BootstrapServers = bootstrapServers, GroupId = Guid.NewGuid().ToString() }; // var s = (RecordSchema)Schema.Parse(File.ReadAllText("my-schema.json")); var s = (RecordSchema)Schema.Parse( @"{ ""namespace"": ""Confluent.Kafka.Examples.AvroSpecific"", ""type"": ""record"", ""name"": ""User"", ""fields"": [ {""name"": ""name"", ""type"": ""string""}, {""name"": ""favorite_number"", ""type"": [""int"", ""null""]}, {""name"": ""favorite_color"", ""type"": [""string"", ""null""]} ] }" ); CancellationTokenSource cts = new CancellationTokenSource(); var consumeTask = Task.Run(() => { using (var serdeProvider = new AvroSerdeProvider(avroConfig)) using (var consumer = new Consumer <string, GenericRecord>(consumerConfig, serdeProvider.GetDeserializerGenerator <string>(), serdeProvider.GetDeserializerGenerator <GenericRecord>())) { consumer.OnError += (_, e) => Console.WriteLine($"Error: {e.Reason}"); consumer.Subscribe(topicName); while (!cts.Token.IsCancellationRequested) { try { var consumeResult = consumer.Consume(cts.Token); Console.WriteLine($"Key: {consumeResult.Message.Key}\nValue: {consumeResult.Value}"); } catch (ConsumeException e) { Console.WriteLine("Consume error: " + e.Error.Reason); } } consumer.Close(); } }, cts.Token); using (var serdeProvider = new AvroSerdeProvider(avroConfig)) using (var producer = new Producer <string, GenericRecord>(producerConfig, serdeProvider.GetSerializerGenerator <string>(), serdeProvider.GetSerializerGenerator <GenericRecord>())) { Console.WriteLine($"{producer.Name} producing on {topicName}. Enter user names, q to exit."); int i = 0; string text; while ((text = Console.ReadLine()) != "q") { var record = new GenericRecord(s); record.Add("name", text); record.Add("favorite_number", i++); record.Add("favorite_color", "blue"); producer .ProduceAsync(topicName, new Message <string, GenericRecord> { Key = text, Value = record }) .ContinueWith(task => task.IsFaulted ? $"error producing message: {task.Exception.Message}" : $"produced to: {task.Result.TopicPartitionOffset}"); } } cts.Cancel(); }
public static void ProduceConsume(string bootstrapServers, string schemaRegistryServers) { string topic = Guid.NewGuid().ToString(); var producerConfig = new ProducerConfig { BootstrapServers = bootstrapServers }; var consumerConfig = new ConsumerConfig { BootstrapServers = bootstrapServers, GroupId = Guid.NewGuid().ToString(), SessionTimeoutMs = 6000, AutoOffsetReset = AutoOffsetResetType.Earliest }; var serdeProviderConfig = new AvroSerdeProviderConfig { SchemaRegistryUrl = schemaRegistryServers }; using (var serdeProvider = new AvroSerdeProvider(serdeProviderConfig)) using (var producer = new Producer <string, User>(producerConfig, serdeProvider.GetSerializerGenerator <string>(), serdeProvider.GetSerializerGenerator <User>())) { for (int i = 0; i < 100; ++i) { var user = new User { name = i.ToString(), favorite_number = i, favorite_color = "blue" }; producer.ProduceAsync(topic, new Message <string, User> { Key = user.name, Value = user }); } Assert.Equal(0, producer.Flush(TimeSpan.FromSeconds(10))); } using (var serdeProvider = new AvroSerdeProvider(serdeProviderConfig)) using (var consumer = new Consumer <string, User>(consumerConfig, serdeProvider.GetDeserializerGenerator <string>(), serdeProvider.GetDeserializerGenerator <User>())) { bool consuming = true; consumer.OnPartitionEOF += (_, topicPartitionOffset) => consuming = false; consumer.OnError += (_, e) => Assert.True(false, e.Reason); consumer.Subscribe(topic); int i = 0; while (consuming) { var record = consumer.Consume(TimeSpan.FromMilliseconds(100)); if (record != null) { Assert.Equal(i.ToString(), record.Message.Key); Assert.Equal(i.ToString(), record.Message.Value.name); Assert.Equal(i, record.Message.Value.favorite_number); Assert.Equal("blue", record.Message.Value.favorite_color); i += 1; } } Assert.Equal(100, i); consumer.Close(); } }
protected override DeserializerGenerator <TMessage> GetDeserializer <TMessage>() => _avroProvider.GetDeserializerGenerator <TMessage>();
/// <summary> /// In this example /// - offsets are manually committed. /// - no extra thread is created for the Poll (Consume) loop. /// </summary> public static void Run_Consume(string brokerList, List <string> topics, CancellationToken cancellationToken) { var config = new ConsumerConfig { BootstrapServers = brokerList, GroupId = "csharp-consumer", EnableAutoCommit = false, StatisticsIntervalMs = 5000, SessionTimeoutMs = 6000, AutoOffsetReset = AutoOffsetResetType.Earliest }; const int commitPeriod = 5; using (var serdeProvider = new AvroSerdeProvider(new AvroSerdeProviderConfig { SchemaRegistryUrl = "192.168.100.153:8081" })) using (var consumer = new Consumer <Ignore, com.landoop.social.reddit.post.reddit_post>(config, null, serdeProvider.GetDeserializerGenerator <com.landoop.social.reddit.post.reddit_post>())) // using (var consumer = new Consumer<Ingore, string>(config)) { // Note: All event handlers are called on the main .Consume thread. // Raised when the consumer has been notified of a new assignment set. // You can use this event to perform actions such as retrieving offsets // from an external source / manually setting start offsets using // the Assign method. You can even call Assign with a different set of // partitions than those in the assignment. If you do not call Assign // in a handler of this event, the consumer will be automatically // assigned to the partitions of the assignment set and consumption // will start from last committed offsets or in accordance with // the auto.offset.reset configuration parameter for partitions where // there is no committed offset. consumer.OnPartitionsAssigned += (_, partitions) => Console.WriteLine($"Assigned partitions: [{string.Join(", ", partitions)}], member id: {consumer.MemberId}"); // Raised when the consumer's current assignment set has been revoked. consumer.OnPartitionsRevoked += (_, partitions) => Console.WriteLine($"Revoked partitions: [{string.Join(", ", partitions)}]"); consumer.OnPartitionEOF += (_, tpo) => Console.WriteLine($"Reached end of topic {tpo.Topic} partition {tpo.Partition}, next message will be at offset {tpo.Offset}"); consumer.OnError += (_, e) => Console.WriteLine($"Error: {e.Reason}"); //consumer.OnStatistics += (_, json) // => Console.WriteLine($"Statistics: {json}"); consumer.Subscribe(topics); while (!cancellationToken.IsCancellationRequested) { try { var consumeResult = consumer.Consume(cancellationToken); Console.WriteLine($"Topic: {consumeResult.Topic}"); Console.WriteLine($"Partition: {consumeResult.Partition} Offset: {consumeResult.Offset}"); Console.WriteLine($"Subreddit Topic: { consumeResult.Value.subreddit }"); Console.WriteLine($"{ consumeResult.Value.body }"); Console.WriteLine("---"); Thread.Sleep(1000); if (consumeResult.Offset % commitPeriod == 0) { // The Commit method sends a "commit offsets" request to the Kafka // cluster and synchronously waits for the response. This is very // slow compared to the rate at which the consumer is capable of // consuming messages. A high performance application will typically // commit offsets relatively infrequently and be designed handle // duplicate messages in the event of failure. var committedOffsets = consumer.Commit(consumeResult); Console.WriteLine($"Committed offset: {committedOffsets}"); } } catch (ConsumeException e) { Console.WriteLine($"Consume error: {e.Error}"); } } consumer.Close(); } }
static void ConsumeSpecific(string bootstrapServers, string schemaRegistryUrl) { CancellationTokenSource cts = new CancellationTokenSource(); Console.CancelKeyPress += (_, e) => { e.Cancel = true; // prevent the process from terminating. cts.Cancel(); }; var consumerConfig = new ConsumerConfig { GroupId = Guid.NewGuid().ToString(), BootstrapServers = bootstrapServers, AutoOffsetReset = AutoOffsetResetType.Earliest }; using (var serdeProvider = new AvroSerdeProvider(new AvroSerdeProviderConfig { SchemaRegistryUrl = schemaRegistryUrl })) using (var consumer = new Consumer <Null, MessageTypes.LogMessage>(consumerConfig, null, serdeProvider.GetDeserializerGenerator <MessageTypes.LogMessage>())) { consumer.Subscribe("log-messages"); while (!cts.IsCancellationRequested) { try { var consumeResult = consumer.Consume(cts.Token); Console.WriteLine($"{consumeResult.Message.Timestamp.UtcDateTime.ToString("yyyy-MM-dd HH:mm:ss")}: [{consumeResult.Value.Severity}] {consumeResult.Value.Message}"); } catch (ConsumeException e) { Console.WriteLine($"an error occured: {e.Error.Reason}"); } } // commit final offsets and leave the group. consumer.Close(); } }
public static void ConsumeIncompatibleTypes(string bootstrapServers, string schemaRegistryServers) { string topic = Guid.NewGuid().ToString(); var producerConfig = new ProducerConfig { BootstrapServers = bootstrapServers }; var consumerConfig = new ConsumerConfig { BootstrapServers = bootstrapServers, GroupId = Guid.NewGuid().ToString(), SessionTimeoutMs = 6000, AutoOffsetReset = AutoOffsetResetType.Earliest }; var serdeProviderConfig = new AvroSerdeProviderConfig { SchemaRegistryUrl = schemaRegistryServers }; using (var serdeProvider = new AvroSerdeProvider(serdeProviderConfig)) using (var producer = new Producer <string, User>(producerConfig, serdeProvider.GetSerializerGenerator <string>(), serdeProvider.GetSerializerGenerator <User>())) { var user = new User { name = "username", favorite_number = 107, favorite_color = "orange" }; producer.ProduceAsync(topic, new Message <string, User> { Key = user.name, Value = user }); Assert.Equal(0, producer.Flush(TimeSpan.FromSeconds(10))); } using (var serdeProvider = new AvroSerdeProvider(serdeProviderConfig)) using (var consumer = new Consumer <User, User>(consumerConfig, serdeProvider.GetDeserializerGenerator <User>(), serdeProvider.GetDeserializerGenerator <User>())) { consumer.Assign(new List <TopicPartitionOffset> { new TopicPartitionOffset(topic, 0, 0) }); bool hadError = false; try { consumer.Consume(TimeSpan.FromSeconds(10)); } catch (ConsumeException e) { if (e.Error.Code == ErrorCode.Local_KeyDeserialization) { hadError = true; } } Assert.True(hadError); } using (var serdeProvider = new AvroSerdeProvider(serdeProviderConfig)) using (var consumer = new Consumer <string, string>(consumerConfig, serdeProvider.GetDeserializerGenerator <string>(), serdeProvider.GetDeserializerGenerator <string>())) { consumer.Assign(new List <TopicPartitionOffset> { new TopicPartitionOffset(topic, 0, 0) }); bool hadError = false; try { consumer.Consume(TimeSpan.FromSeconds(10)); } catch (ConsumeException e) { if (e.Error.Code == ErrorCode.Local_ValueDeserialization) { hadError = true; } } Assert.True(hadError); } }
public static void PrimitiveTypes(string bootstrapServers, string schemaRegistryServers) { var serdeProviderConfig = new AvroSerdeProviderConfig { SchemaRegistryUrl = schemaRegistryServers }; using (var serdeProvider = new AvroSerdeProvider(serdeProviderConfig)) { var producerConfig = new ProducerConfig { BootstrapServers = bootstrapServers }; var consumerConfig = new ConsumerConfig { BootstrapServers = bootstrapServers, GroupId = Guid.NewGuid().ToString(), SessionTimeoutMs = 6000, AutoOffsetReset = AutoOffsetResetType.Earliest }; var stringTopic = Guid.NewGuid().ToString(); using (var producer = new Producer <string, string>(producerConfig, serdeProvider.GetSerializerGenerator <string>(), serdeProvider.GetSerializerGenerator <string>())) { producer.ProduceAsync(stringTopic, new Message <string, string> { Key = "hello", Value = "world" }); Assert.Equal(0, producer.Flush(TimeSpan.FromSeconds(10))); } var bytesTopic = Guid.NewGuid().ToString(); using (var producer = new Producer <byte[], byte[]>(producerConfig, serdeProvider.GetSerializerGenerator <byte[]>(), serdeProvider.GetSerializerGenerator <byte[]>())) { producer.ProduceAsync(bytesTopic, new Message <byte[], byte[]> { Key = new byte[] { 1, 4, 11 }, Value = new byte[] {} }); Assert.Equal(0, producer.Flush(TimeSpan.FromSeconds(10))); } var intTopic = Guid.NewGuid().ToString(); using (var producer = new Producer <int, int>(producerConfig, serdeProvider.GetSerializerGenerator <int>(), serdeProvider.GetSerializerGenerator <int>())) { producer.ProduceAsync(intTopic, new Message <int, int> { Key = 42, Value = 43 }); Assert.Equal(0, producer.Flush(TimeSpan.FromSeconds(10))); } var longTopic = Guid.NewGuid().ToString(); using (var producer = new Producer <long, long>(producerConfig, serdeProvider.GetSerializerGenerator <long>(), serdeProvider.GetSerializerGenerator <long>())) { producer.ProduceAsync(longTopic, new Message <long, long> { Key = -32, Value = -33 }); Assert.Equal(0, producer.Flush(TimeSpan.FromSeconds(10))); } var boolTopic = Guid.NewGuid().ToString(); using (var producer = new Producer <bool, bool>(producerConfig, serdeProvider.GetSerializerGenerator <bool>(), serdeProvider.GetSerializerGenerator <bool>())) { producer.ProduceAsync(boolTopic, new Message <bool, bool> { Key = true, Value = false }); Assert.Equal(0, producer.Flush(TimeSpan.FromSeconds(10))); } var floatTopic = Guid.NewGuid().ToString(); using (var producer = new Producer <float, float>(producerConfig, serdeProvider.GetSerializerGenerator <float>(), serdeProvider.GetSerializerGenerator <float>())) { producer.ProduceAsync(floatTopic, new Message <float, float> { Key = 44.0f, Value = 45.0f }); Assert.Equal(0, producer.Flush(TimeSpan.FromSeconds(10))); } var doubleTopic = Guid.NewGuid().ToString(); using (var producer = new Producer <double, double>(producerConfig, serdeProvider.GetSerializerGenerator <double>(), serdeProvider.GetSerializerGenerator <double>())) { producer.ProduceAsync(doubleTopic, new Message <double, double> { Key = 46.0, Value = 47.0 }); Assert.Equal(0, producer.Flush(TimeSpan.FromSeconds(10))); } var nullTopic = Guid.NewGuid().ToString(); using (var producer = new Producer <Null, Null>(producerConfig, serdeProvider.GetSerializerGenerator <Null>(), serdeProvider.GetSerializerGenerator <Null>())) { producer.ProduceAsync(nullTopic, new Message <Null, Null>()); Assert.Equal(0, producer.Flush(TimeSpan.FromSeconds(10))); } using (var consumer = new Consumer <string, string>(consumerConfig, serdeProvider.GetDeserializerGenerator <string>(), serdeProvider.GetDeserializerGenerator <string>())) { consumer.Assign(new List <TopicPartitionOffset> { new TopicPartitionOffset(stringTopic, 0, 0) }); var result = consumer.Consume(TimeSpan.FromSeconds(10)); Assert.Equal("hello", result.Message.Key); Assert.Equal("world", result.Message.Value); } using (var consumer = new Consumer <byte[], byte[]>(consumerConfig, serdeProvider.GetDeserializerGenerator <byte[]>(), serdeProvider.GetDeserializerGenerator <byte[]>())) { consumer.Assign(new List <TopicPartitionOffset> { new TopicPartitionOffset(bytesTopic, 0, 0) }); var result = consumer.Consume(TimeSpan.FromSeconds(10)); Assert.Equal(new byte[] { 1, 4, 11 }, result.Message.Key); Assert.Equal(new byte[] { }, result.Message.Value); } using (var consumer = new Consumer <int, int>(consumerConfig, serdeProvider.GetDeserializerGenerator <int>(), serdeProvider.GetDeserializerGenerator <int>())) { consumer.Assign(new List <TopicPartitionOffset> { new TopicPartitionOffset(intTopic, 0, 0) }); var result = consumer.Consume(TimeSpan.FromSeconds(10)); Assert.Equal(42, result.Message.Key); Assert.Equal(43, result.Message.Value); } using (var consumer = new Consumer <long, long>(consumerConfig, serdeProvider.GetDeserializerGenerator <long>(), serdeProvider.GetDeserializerGenerator <long>())) { consumer.Assign(new List <TopicPartitionOffset> { new TopicPartitionOffset(longTopic, 0, 0) }); var result = consumer.Consume(TimeSpan.FromSeconds(10)); Assert.Equal(-32, result.Message.Key); Assert.Equal(-33, result.Message.Value); } using (var consumer = new Consumer <bool, bool>(consumerConfig, serdeProvider.GetDeserializerGenerator <bool>(), serdeProvider.GetDeserializerGenerator <bool>())) { consumer.Assign(new List <TopicPartitionOffset> { new TopicPartitionOffset(boolTopic, 0, 0) }); var result = consumer.Consume(TimeSpan.FromSeconds(10)); Assert.True(result.Message.Key); Assert.False(result.Message.Value); } using (var consumer = new Consumer <float, float>(consumerConfig, serdeProvider.GetDeserializerGenerator <float>(), serdeProvider.GetDeserializerGenerator <float>())) { consumer.Assign(new List <TopicPartitionOffset> { new TopicPartitionOffset(floatTopic, 0, 0) }); var result = consumer.Consume(TimeSpan.FromSeconds(10)); Assert.Equal(44.0f, result.Message.Key); Assert.Equal(45.0f, result.Message.Value); } using (var consumer = new Consumer <double, double>(consumerConfig, serdeProvider.GetDeserializerGenerator <double>(), serdeProvider.GetDeserializerGenerator <double>())) { consumer.Assign(new List <TopicPartitionOffset> { new TopicPartitionOffset(doubleTopic, 0, 0) }); var result = consumer.Consume(TimeSpan.FromSeconds(10)); Assert.Equal(46.0, result.Message.Key); Assert.Equal(47.0, result.Message.Value); } using (var consumer = new Consumer <Null, Null>(consumerConfig, serdeProvider.GetDeserializerGenerator <Null>(), serdeProvider.GetDeserializerGenerator <Null>())) { consumer.Assign(new List <TopicPartitionOffset> { new TopicPartitionOffset(nullTopic, 0, 0) }); var result = consumer.Consume(TimeSpan.FromSeconds(10)); Assert.Null(result.Key); Assert.Null(result.Value); } } }