private void AddToBuffer(Consumer <Null, GenericRecord> consumer,
Message <Null, GenericRecord> avroMessage,
BlockingCollection <MessageProxy <RowChange> > accumulatedChanges,
AvroTableTypeConverter avroTableTypeConverter)
{
var tableChange = avroTableTypeConverter.GetRowChange(avroMessage.Value);
var msg = new MessageProxy <RowChange>(consumer, avroMessage)
{
Payload = tableChange
};
accumulatedChanges.Add(msg);
}
public NonKeyedAvroProducer(string bootstrapServers, string schemaRegistryUrl, string topic, AvroTableTypeConverter avroTableTypeConverter, TableSchema tableSchema)
: base(topic)
{
_config = new Dictionary <string, object>
{
{ "bootstrap.servers", bootstrapServers },
{ "schema.registry.url", schemaRegistryUrl },
{ "socket.blocking.max.ms", "1" } // workaround for https://github.com/confluentinc/confluent-kafka-dotnet/issues/501
};
_producer = new Producer <Null, GenericRecord>(_config, null, new AvroSerializer <GenericRecord>());
_avroTypeConverter = avroTableTypeConverter;
_tableSchema = tableSchema;
}
public KeyedAvroProducer(string bootstrapServers, string schemaRegistryUrl, string topic, AvroTableTypeConverter avroTableTypeConverter, TableSchema tableSchema)
: base(topic)
{
_config = new Dictionary <string, object>
{
{ "bootstrap.servers", bootstrapServers },
{ "schema.registry.url", schemaRegistryUrl }
};
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
_config.Add("socket.blocking.max.ms", "1"); // workaround for https://github.com/confluentinc/confluent-kafka-dotnet/issues/501
}
_producer = new Producer <string, GenericRecord>(_config, new StringSerializer(Encoding.UTF8), new AvroSerializer <GenericRecord>());
_avroTypeConverter = avroTableTypeConverter;
_tableSchema = tableSchema;
}
private void Consume(CancellationToken token, BlockingCollection <MessageProxy <RowChange> > accumulatedChanges, string topic, string table)
{
var conf = new Dictionary <string, object>
{
{ "group.id", $"{table}-consumer-group" },
{ "statistics.interval.ms", 60000 },
{ "bootstrap.servers", _kafkaBootstrapServers },
{ "schema.registry.url", _schemaRegistryUrl }
};
foreach (var confPair in conf)
{
Console.WriteLine(topic + " - " + confPair.Key + ": " + confPair.Value);
}
AvroTableTypeConverter avroTableTypeConverter = null;
using (var consumer = new Consumer <Null, GenericRecord>(conf, null, new AvroDeserializer <GenericRecord>()))
{
//consumer.OnPartitionEOF += (_, end)
// => Console.WriteLine($"Reached end of topic {end.Topic} partition {end.Partition}, next message will be at offset {end.Offset}");
consumer.OnError += (_, msg)
=> Console.WriteLine($"{topic} - Error: {msg.Reason}");
consumer.OnConsumeError += (_, msg)
=> Console.WriteLine($"{topic} - Consume error: {msg.Error.Reason}");
consumer.OnPartitionsAssigned += (_, partitions) =>
{
Console.WriteLine($"{topic} - Assigned partitions: [{string.Join(", ", partitions)}], member id: {consumer.MemberId}");
consumer.Assign(partitions);
};
consumer.OnPartitionsRevoked += (_, partitions) =>
{
Console.WriteLine($"{topic} - Revoked partitions: [{string.Join(", ", partitions)}]");
consumer.Unassign();
};
//consumer.OnStatistics += (_, json)
// => Console.WriteLine($"{topic} - Statistics: {json}");
Console.WriteLine($"Subscribing to topic {topic}");
consumer.Subscribe(topic);
int secondsWithoutMessage = 0;
while (!token.IsCancellationRequested)
{
Message <Null, GenericRecord> msg = null;
if (consumer.Consume(out msg, TimeSpan.FromSeconds(1)))
{
if (avroTableTypeConverter == null)
{
avroTableTypeConverter = new AvroTableTypeConverter(msg.Value.Schema);
}
else if (!avroTableTypeConverter.SchemaMatches(msg.Value.Schema))
{
avroTableTypeConverter = new AvroTableTypeConverter(msg.Value.Schema);
}
AddToBuffer(consumer, msg, accumulatedChanges, avroTableTypeConverter);
secondsWithoutMessage = 0;
}
else
{
secondsWithoutMessage++;
if (secondsWithoutMessage % 30 == 0)
{
Console.WriteLine($"{topic}: No messages in last {secondsWithoutMessage} seconds");
}
Task.Delay(100).Wait();
}
}
}
accumulatedChanges.CompleteAdding(); // notifies consumers that no more messages will come
}
