private void AddToBuffer(Consumer <Null, GenericRecord> consumer, Message <Null, GenericRecord> avroMessage, BlockingCollection <MessageProxy <RowChange> > accumulatedChanges, AvroTableTypeConverter avroTableTypeConverter) { var tableChange = avroTableTypeConverter.GetRowChange(avroMessage.Value); var msg = new MessageProxy <RowChange>(consumer, avroMessage) { Payload = tableChange }; accumulatedChanges.Add(msg); }
public NonKeyedAvroProducer(string bootstrapServers, string schemaRegistryUrl, string topic, AvroTableTypeConverter avroTableTypeConverter, TableSchema tableSchema) : base(topic) { _config = new Dictionary <string, object> { { "bootstrap.servers", bootstrapServers }, { "schema.registry.url", schemaRegistryUrl }, { "socket.blocking.max.ms", "1" } // workaround for https://github.com/confluentinc/confluent-kafka-dotnet/issues/501 }; _producer = new Producer <Null, GenericRecord>(_config, null, new AvroSerializer <GenericRecord>()); _avroTypeConverter = avroTableTypeConverter; _tableSchema = tableSchema; }
public KeyedAvroProducer(string bootstrapServers, string schemaRegistryUrl, string topic, AvroTableTypeConverter avroTableTypeConverter, TableSchema tableSchema) : base(topic) { _config = new Dictionary <string, object> { { "bootstrap.servers", bootstrapServers }, { "schema.registry.url", schemaRegistryUrl } }; if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) { _config.Add("socket.blocking.max.ms", "1"); // workaround for https://github.com/confluentinc/confluent-kafka-dotnet/issues/501 } _producer = new Producer <string, GenericRecord>(_config, new StringSerializer(Encoding.UTF8), new AvroSerializer <GenericRecord>()); _avroTypeConverter = avroTableTypeConverter; _tableSchema = tableSchema; }
private void Consume(CancellationToken token, BlockingCollection <MessageProxy <RowChange> > accumulatedChanges, string topic, string table) { var conf = new Dictionary <string, object> { { "group.id", $"{table}-consumer-group" }, { "statistics.interval.ms", 60000 }, { "bootstrap.servers", _kafkaBootstrapServers }, { "schema.registry.url", _schemaRegistryUrl } }; foreach (var confPair in conf) { Console.WriteLine(topic + " - " + confPair.Key + ": " + confPair.Value); } AvroTableTypeConverter avroTableTypeConverter = null; using (var consumer = new Consumer <Null, GenericRecord>(conf, null, new AvroDeserializer <GenericRecord>())) { //consumer.OnPartitionEOF += (_, end) // => Console.WriteLine($"Reached end of topic {end.Topic} partition {end.Partition}, next message will be at offset {end.Offset}"); consumer.OnError += (_, msg) => Console.WriteLine($"{topic} - Error: {msg.Reason}"); consumer.OnConsumeError += (_, msg) => Console.WriteLine($"{topic} - Consume error: {msg.Error.Reason}"); consumer.OnPartitionsAssigned += (_, partitions) => { Console.WriteLine($"{topic} - Assigned partitions: [{string.Join(", ", partitions)}], member id: {consumer.MemberId}"); consumer.Assign(partitions); }; consumer.OnPartitionsRevoked += (_, partitions) => { Console.WriteLine($"{topic} - Revoked partitions: [{string.Join(", ", partitions)}]"); consumer.Unassign(); }; //consumer.OnStatistics += (_, json) // => Console.WriteLine($"{topic} - Statistics: {json}"); Console.WriteLine($"Subscribing to topic {topic}"); consumer.Subscribe(topic); int secondsWithoutMessage = 0; while (!token.IsCancellationRequested) { Message <Null, GenericRecord> msg = null; if (consumer.Consume(out msg, TimeSpan.FromSeconds(1))) { if (avroTableTypeConverter == null) { avroTableTypeConverter = new AvroTableTypeConverter(msg.Value.Schema); } else if (!avroTableTypeConverter.SchemaMatches(msg.Value.Schema)) { avroTableTypeConverter = new AvroTableTypeConverter(msg.Value.Schema); } AddToBuffer(consumer, msg, accumulatedChanges, avroTableTypeConverter); secondsWithoutMessage = 0; } else { secondsWithoutMessage++; if (secondsWithoutMessage % 30 == 0) { Console.WriteLine($"{topic}: No messages in last {secondsWithoutMessage} seconds"); } Task.Delay(100).Wait(); } } } accumulatedChanges.CompleteAdding(); // notifies consumers that no more messages will come }