public AvroConsumerStrategy(ConsumerConfig config, AvroSerdeProviderConfig avroConfig)
{
_serdeProvider = new AvroSerdeProvider(avroConfig);
_consumer = new Consumer <TKey, TValue>(config,
_serdeProvider.GetDeserializerGenerator <TKey>(),
_serdeProvider.GetDeserializerGenerator <TValue>());
}
private void Init(ProducerConfig config)
{
if (_logger.IsDebug)
{
_logger.Debug($"Initializing {Name} type producer for Kafka...");
}
try
{
_serdeProvider = new AvroSerdeProvider(new AvroSerdeProviderConfig
{
SchemaRegistryUrl = _schemaRegistryUrl
});
_producerBlocks =
new Producer <Null, Block>(config, null, _serdeProvider.GetSerializerGenerator <Block>());
_producerTransactions = new Producer <Null, FullTransaction>(config, null,
_serdeProvider.GetSerializerGenerator <FullTransaction>());
_producerBlocks.OnError += (s, e) => _logger.Error(e.ToString());
_producerTransactions.OnError += (s, e) => _logger.Error(e.ToString());
_initialized = true;
if (_logger.IsDebug)
{
_logger.Debug($"Initialized {Name} type producer for Kafka.");
}
}
catch (Exception e)
{
_logger.Error(e.Message, e);
}
}
private static void StartConsumingAvro <T>(string topic) where T : ISpecificRecord
{
var avroSerdeProvider = new AvroSerdeProvider(new AvroSerdeProviderConfig
{
SchemaRegistryUrl = Configuration.SchemaRegistryUrl
});
using (var consumer = new Consumer <Ignore, T>(Configuration.ConsumerConfig, null,
avroSerdeProvider.GetDeserializerGenerator <T>()))
{
var consumerConfig = Configuration.ConsumerConfig;
Log($"Consumer for group: '{consumerConfig.GroupId}' was created. Data type: '{Configuration.Type}'.");
var consuming = true;
consumer.OnError += (s, e) =>
{
consuming = !e.IsFatal;
Log(e.ToString());
};
consumer.Subscribe(new[] { topic });
Log($"Subscribed to topic: '{topic}'.");
while (consuming)
{
try
{
var consumeResult = consumer.Consume();
var type = typeof(T);
if (type == typeof(Avro.Models.Block))
{
var result = consumeResult as ConsumeResult <Ignore, Avro.Models.Block>;
var block = result.Value;
Log($"Block: {block.blockNumber} {block.blockHash}");
}
else if (type == typeof(Avro.Models.FullTransaction))
{
var result = consumeResult as ConsumeResult <Ignore, Avro.Models.FullTransaction>;
var transaction = result.Value;
Log($"Transaction for block: {transaction.blockNumber} {transaction.receipt.blockHash}");
}
else
{
Log($"Unknown data type: {type.Name}");
continue;
}
Log($"Consumed value at '{consumeResult.TopicPartitionOffset}'.");
}
catch (ConsumeException exception)
{
Log($"Consumer error occured: {exception.Error.Reason}");
}
catch (Exception exception)
{
Log(exception.Message);
}
}
consumer.Close();
}
}
private void init()
{
AvroSerdeProvider serdeProvider = new AvroSerdeProvider(consumerConfiguration.avroConfig);
Consumer <String, DeleteClient> consumerDelete =
new Consumer <String, DeleteClient>(
consumerConfiguration.consumerConfig,
serdeProvider.GetDeserializerGenerator <String>(),
serdeProvider.GetDeserializerGenerator <DeleteClient>());
CancellationTokenSource cts = new CancellationTokenSource();
var consumeTask = Task.Run(() =>
{
consumerDelete.OnError += (_, e)
=> Console.WriteLine($"Error: {e.Reason}");
consumerDelete.Subscribe("DeleteClient");
while (!cts.Token.IsCancellationRequested)
{
try {
var consumeResult = consumerDelete.Consume(cts.Token);
Console.WriteLine($"Delete Client - key: {consumeResult.Message.Key}, value: {consumeResult.Value}");
var item = context.ClientItems.Find(new Guid(consumeResult.Value.id));
if (item != null)
{
context.Remove(item);
context.SaveChanges();
clientDeletedProducer.sendEvent(item);
}
} catch (ConsumeException e) {
Console.WriteLine("Consume error: " + e.Error.Reason);
}
}
consumerDelete.Close();
}, cts.Token);
}
static void ProduceSpecific(string bootstrapServers, string schemaRegistryUrl)
{
var producerConfig = new ProducerConfig {
BootstrapServers = bootstrapServers
};
using (var serdeProvider = new AvroSerdeProvider(new AvroSerdeProviderConfig {
SchemaRegistryUrl = schemaRegistryUrl
}))
using (var producer = new Producer <Null, MessageTypes.LogMessage>(producerConfig, null, serdeProvider.GetSerializerGenerator <MessageTypes.LogMessage>()))
{
producer.ProduceAsync("log-messages",
new Message <Null, MessageTypes.LogMessage>
{
Value = new MessageTypes.LogMessage
{
IP = "192.168.0.1",
Message = "a test message 2",
Severity = MessageTypes.LogLevel.Info,
Tags = new Dictionary <string, string> {
{ "location", "CA" }
}
}
});
producer.Flush(TimeSpan.FromSeconds(30));
}
}
static void ProduceGeneric(string bootstrapServers, string schemaRegistryUrl)
{
var producerConfig = new ProducerConfig {
BootstrapServers = bootstrapServers
};
using (var serdeProvider = new AvroSerdeProvider(new AvroSerdeProviderConfig {
SchemaRegistryUrl = schemaRegistryUrl
}))
using (var producer = new Producer <Null, GenericRecord>(producerConfig, null, serdeProvider.GetSerializerGenerator <GenericRecord>()))
{
var logLevelSchema = (EnumSchema)Schema.Parse(
File.ReadAllText("LogLevel.asvc"));
var logMessageSchema = (RecordSchema)Schema
.Parse(File.ReadAllText("LogMessage.V1.asvc")
.Replace(
"MessageTypes.LogLevel",
File.ReadAllText("LogLevel.asvc")));
var record = new GenericRecord(logMessageSchema);
record.Add("IP", "127.0.0.1");
record.Add("Message", "a test log message");
record.Add("Severity", new GenericEnum(logLevelSchema, "Error"));
producer.ProduceAsync("log-messages", new Message <Null, GenericRecord> {
Value = record
})
.ContinueWith(task => Console.WriteLine(
task.IsFaulted
? $"error producing message: {task.Exception.Message}"
: $"produced to: {task.Result.TopicPartitionOffset}"));
producer.Flush(TimeSpan.FromSeconds(30));
}
}
private void init()
{
AvroSerdeProvider serdeProvider = new AvroSerdeProvider(consumerConfiguration.avroConfig);
Consumer <int, ClientDuplicatesFound> consumerDuplicatesFound =
new Consumer <int, ClientDuplicatesFound>(
consumerConfiguration.consumerConfig,
serdeProvider.GetDeserializerGenerator <int>(),
serdeProvider.GetDeserializerGenerator <ClientDuplicatesFound>());
CancellationTokenSource cts = new CancellationTokenSource();
var consumeTask = Task.Run(() =>
{
consumerDuplicatesFound.OnError += (_, e)
=> Console.WriteLine($"Error: {e.Reason}");
consumerDuplicatesFound.Subscribe("ClientDuplicatesFound");
while (!cts.Token.IsCancellationRequested)
{
try {
var consumeResult = consumerDuplicatesFound.Consume(cts.Token);
Console.WriteLine($"Client Duplicate Found - key: {consumeResult.Message.Key}, value: {consumeResult.Value}");
var oldItem = context.ClientItems.Find(new Guid(consumeResult.Value.oldclientid));
if (oldItem != null)
{
context.ClientItems.Remove(oldItem);
context.SaveChanges();
}
} catch (ConsumeException e) {
Console.WriteLine("Consume error: " + e.Error.Reason);
}
}
consumerDuplicatesFound.Close();
}, cts.Token);
}
private static Consumer <TKey, TValue> CreateConsumer <TKey, TValue>(bool enableAutoCommit)
{
Log.Information("Creating consumer");
var avroSerdeProvider = new AvroSerdeProvider(CreateAvroSerdeProviderConfig());
var config = CreateConsumerConfig(enableAutoCommit);
var keyDeserializer = avroSerdeProvider.GetDeserializerGenerator <TKey>();
var valueDeserializer = avroSerdeProvider.GetDeserializerGenerator <TValue>();
var consumer = new Consumer <TKey, TValue>(config, keyDeserializer, valueDeserializer);
consumer.OnError += (_, error) => Console.WriteLine($"[ERROR] Code: {error.Code}, Reason: {error.Reason}");
consumer.OnLog += (_, log) => Console.WriteLine($"[LOG] {log.Message}");
consumer.OnStatistics += (_, statistic) => Console.WriteLine($"[STATISTIC] {statistic}");
Log.Information("Consumer created");
consumer.Subscribe(Topic);
Log.Information("Subscribed to: {0}", string.Join(',', Topic));
return(consumer);
}
public AvroKafkaConsumer(IOptions <MessagingOptions> options,
ILogger <AvroKafkaConsumer> logger) : base(options, logger)
{
var avroConfig = new AvroSerdeProviderConfig {
SchemaRegistryUrl = options.Value.SchemeRegistry
};
_avroProvider = new AvroSerdeProvider(avroConfig);
}
public ClientDeletedProducer(ProducerConfiguration producerConfiguration)
{
this.producerConfiguration = producerConfiguration;
AvroSerdeProvider serdeProvider = new AvroSerdeProvider(producerConfiguration.avroConfig);
producerCreated = new Producer <String, ClientDeleted>(
producerConfiguration.producerConfig,
serdeProvider.GetSerializerGenerator <String>(),
serdeProvider.GetSerializerGenerator <ClientDeleted>());
}
protected override Producer <Null, TMessage> GetProducer()
{
var producerConfig = new ProducerConfig {
BootstrapServers = _options.Servers
};
var avroProvider = new AvroSerdeProvider(new AvroSerdeProviderConfig {
SchemaRegistryUrl = _options.SchemeRegistry
});
return(new Producer <Null, TMessage>(producerConfig, null, avroProvider.GetSerializerGenerator <TMessage>()));
}
public static void ProduceGenericMultipleTopics(string bootstrapServers, string schemaRegistryServers)
{
var s = (RecordSchema)Schema.Parse(
@"{
""namespace"": ""Confluent.Kafka.Examples.AvroSpecific"",
""type"": ""record"",
""name"": ""User"",
""fields"": [
{""name"": ""name"", ""type"": ""string""},
{""name"": ""favorite_number"", ""type"": [""int"", ""null""]},
{""name"": ""favorite_color"", ""type"": [""string"", ""null""]}
]
}"
);
var config = new ProducerConfig {
BootstrapServers = bootstrapServers
};
var serdeProviderConfig = new AvroSerdeProviderConfig {
SchemaRegistryUrl = schemaRegistryServers
};
var topic = Guid.NewGuid().ToString();
var topic2 = Guid.NewGuid().ToString();
DeliveryReport <Null, GenericRecord> dr;
DeliveryReport <Null, GenericRecord> dr2;
using (var serdeProvider = new AvroSerdeProvider(serdeProviderConfig))
using (var p = new Producer <Null, GenericRecord>(config, null, serdeProvider.GetSerializerGenerator <GenericRecord>()))
{
var record = new GenericRecord(s);
record.Add("name", "my name 2");
record.Add("favorite_number", 44);
record.Add("favorite_color", null);
dr = p.ProduceAsync(topic, new Message <Null, GenericRecord> {
Key = null, Value = record
}).Result;
dr2 = p.ProduceAsync(topic2, new Message <Null, GenericRecord> {
Key = null, Value = record
}).Result;
}
Assert.Null(dr.Key);
Assert.NotNull(dr.Value);
Assert.Null(dr2.Key);
Assert.NotNull(dr2.Value);
}
private static Producer <TKey, TValue> CreateProducer <TKey, TValue>()
{
Log.Information("Creating producer");
var avroSerdeProvider = new AvroSerdeProvider(CreateAvroSerdeProviderConfig());
var producer = new Producer <TKey, TValue>(CreateProducerConfig(), avroSerdeProvider.GetSerializerGenerator <TKey>(), avroSerdeProvider.GetSerializerGenerator <TValue>());
producer.OnError += (_, error) => Log.Error("{0}: {1}", error.Code, error.Reason);
producer.OnLog += (_, log) => Log.Information("Log: {0}", log.Message);
producer.OnStatistics += (_, statistic) => Log.Information("Statistics: {0}", statistic);
Log.Information("Producer created");
return(producer);
}
static void ConsumeSpecific(string bootstrapServers, string schemaRegistryUrl)
{
CancellationTokenSource cts = new CancellationTokenSource();
Console.CancelKeyPress += (_, e) => {
e.Cancel = true; // prevent the process from terminating.
cts.Cancel();
};
var consumerConfig = new ConsumerConfig
{
GroupId = Guid.NewGuid().ToString(),
BootstrapServers = bootstrapServers,
AutoOffsetReset = AutoOffsetResetType.Earliest
};
using (var serdeProvider = new AvroSerdeProvider(new AvroSerdeProviderConfig {
SchemaRegistryUrl = schemaRegistryUrl
}))
using (var consumer = new Consumer <Null, MessageTypes.LogMessage>(consumerConfig, null, serdeProvider.GetDeserializerGenerator <MessageTypes.LogMessage>()))
{
consumer.Subscribe("log-messages");
while (!cts.IsCancellationRequested)
{
try
{
var consumeResult = consumer.Consume(cts.Token);
Console.WriteLine($"{consumeResult.Message.Timestamp.UtcDateTime.ToString("yyyy-MM-dd HH:mm:ss")}: [{consumeResult.Value.Severity}] {consumeResult.Value.Message}");
}
catch (ConsumeException e)
{
Console.WriteLine($"an error occured: {e.Error.Reason}");
}
}
// commit final offsets and leave the group.
consumer.Close();
}
}
public static void PrimitiveTypes(string bootstrapServers, string schemaRegistryServers)
{
var serdeProviderConfig = new AvroSerdeProviderConfig {
SchemaRegistryUrl = schemaRegistryServers
};
using (var serdeProvider = new AvroSerdeProvider(serdeProviderConfig))
{
var producerConfig = new ProducerConfig {
BootstrapServers = bootstrapServers
};
var consumerConfig = new ConsumerConfig
{
BootstrapServers = bootstrapServers,
GroupId = Guid.NewGuid().ToString(),
SessionTimeoutMs = 6000,
AutoOffsetReset = AutoOffsetResetType.Earliest
};
var stringTopic = Guid.NewGuid().ToString();
using (var producer = new Producer <string, string>(producerConfig, serdeProvider.GetSerializerGenerator <string>(), serdeProvider.GetSerializerGenerator <string>()))
{
producer.ProduceAsync(stringTopic, new Message <string, string> {
Key = "hello", Value = "world"
});
Assert.Equal(0, producer.Flush(TimeSpan.FromSeconds(10)));
}
var bytesTopic = Guid.NewGuid().ToString();
using (var producer = new Producer <byte[], byte[]>(producerConfig, serdeProvider.GetSerializerGenerator <byte[]>(), serdeProvider.GetSerializerGenerator <byte[]>()))
{
producer.ProduceAsync(bytesTopic, new Message <byte[], byte[]> {
Key = new byte[] { 1, 4, 11 }, Value = new byte[] {}
});
Assert.Equal(0, producer.Flush(TimeSpan.FromSeconds(10)));
}
var intTopic = Guid.NewGuid().ToString();
using (var producer = new Producer <int, int>(producerConfig, serdeProvider.GetSerializerGenerator <int>(), serdeProvider.GetSerializerGenerator <int>()))
{
producer.ProduceAsync(intTopic, new Message <int, int> {
Key = 42, Value = 43
});
Assert.Equal(0, producer.Flush(TimeSpan.FromSeconds(10)));
}
var longTopic = Guid.NewGuid().ToString();
using (var producer = new Producer <long, long>(producerConfig, serdeProvider.GetSerializerGenerator <long>(), serdeProvider.GetSerializerGenerator <long>()))
{
producer.ProduceAsync(longTopic, new Message <long, long> {
Key = -32, Value = -33
});
Assert.Equal(0, producer.Flush(TimeSpan.FromSeconds(10)));
}
var boolTopic = Guid.NewGuid().ToString();
using (var producer = new Producer <bool, bool>(producerConfig, serdeProvider.GetSerializerGenerator <bool>(), serdeProvider.GetSerializerGenerator <bool>()))
{
producer.ProduceAsync(boolTopic, new Message <bool, bool> {
Key = true, Value = false
});
Assert.Equal(0, producer.Flush(TimeSpan.FromSeconds(10)));
}
var floatTopic = Guid.NewGuid().ToString();
using (var producer = new Producer <float, float>(producerConfig, serdeProvider.GetSerializerGenerator <float>(), serdeProvider.GetSerializerGenerator <float>()))
{
producer.ProduceAsync(floatTopic, new Message <float, float> {
Key = 44.0f, Value = 45.0f
});
Assert.Equal(0, producer.Flush(TimeSpan.FromSeconds(10)));
}
var doubleTopic = Guid.NewGuid().ToString();
using (var producer = new Producer <double, double>(producerConfig, serdeProvider.GetSerializerGenerator <double>(), serdeProvider.GetSerializerGenerator <double>()))
{
producer.ProduceAsync(doubleTopic, new Message <double, double> {
Key = 46.0, Value = 47.0
});
Assert.Equal(0, producer.Flush(TimeSpan.FromSeconds(10)));
}
var nullTopic = Guid.NewGuid().ToString();
using (var producer = new Producer <Null, Null>(producerConfig, serdeProvider.GetSerializerGenerator <Null>(), serdeProvider.GetSerializerGenerator <Null>()))
{
producer.ProduceAsync(nullTopic, new Message <Null, Null>());
Assert.Equal(0, producer.Flush(TimeSpan.FromSeconds(10)));
}
using (var consumer = new Consumer <string, string>(consumerConfig, serdeProvider.GetDeserializerGenerator <string>(), serdeProvider.GetDeserializerGenerator <string>()))
{
consumer.Assign(new List <TopicPartitionOffset> {
new TopicPartitionOffset(stringTopic, 0, 0)
});
var result = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.Equal("hello", result.Message.Key);
Assert.Equal("world", result.Message.Value);
}
using (var consumer = new Consumer <byte[], byte[]>(consumerConfig, serdeProvider.GetDeserializerGenerator <byte[]>(), serdeProvider.GetDeserializerGenerator <byte[]>()))
{
consumer.Assign(new List <TopicPartitionOffset> {
new TopicPartitionOffset(bytesTopic, 0, 0)
});
var result = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.Equal(new byte[] { 1, 4, 11 }, result.Message.Key);
Assert.Equal(new byte[] { }, result.Message.Value);
}
using (var consumer = new Consumer <int, int>(consumerConfig, serdeProvider.GetDeserializerGenerator <int>(), serdeProvider.GetDeserializerGenerator <int>()))
{
consumer.Assign(new List <TopicPartitionOffset> {
new TopicPartitionOffset(intTopic, 0, 0)
});
var result = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.Equal(42, result.Message.Key);
Assert.Equal(43, result.Message.Value);
}
using (var consumer = new Consumer <long, long>(consumerConfig, serdeProvider.GetDeserializerGenerator <long>(), serdeProvider.GetDeserializerGenerator <long>()))
{
consumer.Assign(new List <TopicPartitionOffset> {
new TopicPartitionOffset(longTopic, 0, 0)
});
var result = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.Equal(-32, result.Message.Key);
Assert.Equal(-33, result.Message.Value);
}
using (var consumer = new Consumer <bool, bool>(consumerConfig, serdeProvider.GetDeserializerGenerator <bool>(), serdeProvider.GetDeserializerGenerator <bool>()))
{
consumer.Assign(new List <TopicPartitionOffset> {
new TopicPartitionOffset(boolTopic, 0, 0)
});
var result = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.True(result.Message.Key);
Assert.False(result.Message.Value);
}
using (var consumer = new Consumer <float, float>(consumerConfig, serdeProvider.GetDeserializerGenerator <float>(), serdeProvider.GetDeserializerGenerator <float>()))
{
consumer.Assign(new List <TopicPartitionOffset> {
new TopicPartitionOffset(floatTopic, 0, 0)
});
var result = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.Equal(44.0f, result.Message.Key);
Assert.Equal(45.0f, result.Message.Value);
}
using (var consumer = new Consumer <double, double>(consumerConfig, serdeProvider.GetDeserializerGenerator <double>(), serdeProvider.GetDeserializerGenerator <double>()))
{
consumer.Assign(new List <TopicPartitionOffset> {
new TopicPartitionOffset(doubleTopic, 0, 0)
});
var result = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.Equal(46.0, result.Message.Key);
Assert.Equal(47.0, result.Message.Value);
}
using (var consumer = new Consumer <Null, Null>(consumerConfig, serdeProvider.GetDeserializerGenerator <Null>(), serdeProvider.GetDeserializerGenerator <Null>()))
{
consumer.Assign(new List <TopicPartitionOffset> {
new TopicPartitionOffset(nullTopic, 0, 0)
});
var result = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.Null(result.Key);
Assert.Null(result.Value);
}
}
}
static void Main(string[] args)
{
if (args.Length != 3)
{
Console.WriteLine("Usage: .. bootstrapServers schemaRegistryUrl topicName");
return;
}
string bootstrapServers = args[0];
string schemaRegistryUrl = args[1];
string topicName = args[2];
var producerConfig = new ProducerConfig {
BootstrapServers = bootstrapServers
};
var avroConfig = new AvroSerdeProviderConfig
{
// Note: you can specify more than one schema registry url using the
// schema.registry.url property for redundancy (comma separated list).
// The property name is not plural to follow the convention set by
// the Java implementation.
SchemaRegistryUrl = schemaRegistryUrl,
// optional schema registry client properties:
SchemaRegistryRequestTimeoutMs = 5000,
SchemaRegistryMaxCachedSchemas = 10,
// optional avro serializer properties:
AvroSerializerBufferBytes = 50,
AvroSerializerAutoRegisterSchemas = true
};
var consumerConfig = new ConsumerConfig
{
BootstrapServers = bootstrapServers,
GroupId = Guid.NewGuid().ToString()
};
// Note: The User class in this project was generated using the Confluent fork of the avrogen.exe tool
// (avaliable from: https://github.com/confluentinc/avro/tree/confluent-fork) which includes modifications
// that prevent namespace clashes with user namespaces that include the identifier 'Avro'. AvroSerializer
// and AvroDeserializer are also compatible with classes generated by the official avrogen.exe tool
// (available from: https://github.com/apache/avro), with the above limitation.
CancellationTokenSource cts = new CancellationTokenSource();
var consumeTask = Task.Run(() =>
{
using (var serdeProvider = new AvroSerdeProvider(avroConfig))
using (var consumer = new Consumer <string, User>(consumerConfig, serdeProvider.GetDeserializerGenerator <string>(), serdeProvider.GetDeserializerGenerator <User>()))
{
consumer.OnError += (_, e)
=> Console.WriteLine($"Error: {e.Reason}");
consumer.Subscribe(topicName);
while (!cts.Token.IsCancellationRequested)
{
try
{
var consumeResult = consumer.Consume(cts.Token);
Console.WriteLine($"user key name: {consumeResult.Message.Key}, user value favorite color: {consumeResult.Value.favorite_color}");
}
catch (ConsumeException e)
{
Console.WriteLine("Consume error: " + e.Error.Reason);
}
}
consumer.Close();
}
}, cts.Token);
using (var serdeProvider = new AvroSerdeProvider(avroConfig))
using (var producer = new Producer <string, User>(producerConfig, serdeProvider.GetSerializerGenerator <string>(), serdeProvider.GetSerializerGenerator <User>()))
{
Console.WriteLine($"{producer.Name} producing on {topicName}. Enter user names, q to exit.");
int i = 0;
string text;
while ((text = Console.ReadLine()) != "q")
{
User user = new User {
name = text, favorite_color = "green", favorite_number = i++
};
producer
.ProduceAsync(topicName, new Message <string, User> {
Key = text, Value = user
})
.ContinueWith(task => task.IsFaulted
? $"error producing message: {task.Exception.Message}"
: $"produced to: {task.Result.TopicPartitionOffset}");
}
}
cts.Cancel();
}
public static void ProduceConsumeGeneric(string bootstrapServers, string schemaRegistryServers)
{
var s = (RecordSchema)Schema.Parse(
@"{
""namespace"": ""Confluent.Kafka.Examples.AvroSpecific"",
""type"": ""record"",
""name"": ""User"",
""fields"": [
{""name"": ""name"", ""type"": ""string""},
{""name"": ""favorite_number"", ""type"": [""int"", ""null""]},
{""name"": ""favorite_color"", ""type"": [""string"", ""null""]}
]
}"
);
var config = new ProducerConfig {
BootstrapServers = bootstrapServers
};
var serdeProviderConfig = new AvroSerdeProviderConfig {
SchemaRegistryUrl = schemaRegistryServers
};
var topic = Guid.NewGuid().ToString();
DeliveryReport <Null, GenericRecord> dr;
using (var serdeProvider = new AvroSerdeProvider(serdeProviderConfig))
using (var p = new Producer <Null, GenericRecord>(config, null, serdeProvider.GetSerializerGenerator <GenericRecord>()))
{
var record = new GenericRecord(s);
record.Add("name", "my name 2");
record.Add("favorite_number", 44);
record.Add("favorite_color", null);
dr = p.ProduceAsync(topic, new Message <Null, GenericRecord> {
Value = record
}).Result;
}
// produce a specific record (to later consume back as a generic record).
using (var serdeProvider = new AvroSerdeProvider(serdeProviderConfig))
using (var p = new Producer <Null, User>(config, null, serdeProvider.GetSerializerGenerator <User>()))
{
var user = new User
{
name = "my name 3",
favorite_number = 47,
favorite_color = "orange"
};
p.ProduceAsync(topic, new Message <Null, User> {
Value = user
}).Wait();
}
Assert.Null(dr.Message.Key);
Assert.NotNull(dr.Message.Value);
dr.Message.Value.TryGetValue("name", out object name);
dr.Message.Value.TryGetValue("favorite_number", out object number);
dr.Message.Value.TryGetValue("favorite_color", out object color);
Assert.IsType <string>(name);
Assert.IsType <int>(number);
Assert.Equal("my name 2", name);
Assert.Equal(44, number);
Assert.Null(color);
var cconfig = new ConsumerConfig {
GroupId = Guid.NewGuid().ToString(), BootstrapServers = bootstrapServers
};
using (var serdeProvider = new AvroSerdeProvider(serdeProviderConfig))
using (var consumer = new Consumer <Null, GenericRecord>(cconfig, null, serdeProvider.GetDeserializerGenerator <GenericRecord>()))
{
// consume generic record produced as a generic record.
consumer.Assign(new List <TopicPartitionOffset> {
new TopicPartitionOffset(topic, 0, dr.Offset)
});
var record = consumer.Consume(TimeSpan.FromSeconds(20));
record.Message.Value.TryGetValue("name", out object msgName);
record.Message.Value.TryGetValue("favorite_number", out object msgNumber);
record.Message.Value.TryGetValue("favorite_color", out object msgColor);
Assert.IsType <string>(msgName);
Assert.IsType <int>(msgNumber);
Assert.Equal("my name 2", msgName);
Assert.Equal(44, msgNumber);
Assert.Null(msgColor);
// consume generic record produced as a specific record.
record = consumer.Consume(TimeSpan.FromSeconds(20));
record.Message.Value.TryGetValue("name", out msgName);
record.Message.Value.TryGetValue("favorite_number", out msgNumber);
record.Message.Value.TryGetValue("favorite_color", out msgColor);
Assert.IsType <string>(msgName);
Assert.IsType <int>(msgNumber);
Assert.IsType <string>(msgColor);
Assert.Equal("my name 3", msgName);
Assert.Equal(47, msgNumber);
Assert.Equal("orange", msgColor);
}
using (var serdeProvider = new AvroSerdeProvider(serdeProviderConfig))
using (var consumer = new Consumer <Null, User>(cconfig, null, serdeProvider.GetDeserializerGenerator <User>()))
{
consumer.Assign(new List <TopicPartitionOffset> {
new TopicPartitionOffset(topic, 0, dr.Offset)
});
var record = consumer.Consume(TimeSpan.FromSeconds(20));
Assert.Equal("my name 2", record.Message.Value.name);
Assert.Equal(44, record.Message.Value.favorite_number);
Assert.Null(record.Message.Value.favorite_color);
}
}
public static void ProduceIncompatibleTypes(string bootstrapServers, string schemaRegistryServers)
{
var producerConfig = new ProducerConfig {
BootstrapServers = bootstrapServers
};
var consumerConfig = new ConsumerConfig
{
BootstrapServers = bootstrapServers,
GroupId = Guid.NewGuid().ToString(),
SessionTimeoutMs = 6000,
AutoOffsetReset = AutoOffsetResetType.Earliest,
};
var serdeProviderConfig = new AvroSerdeProviderConfig {
SchemaRegistryUrl = schemaRegistryServers
};
var topic = Guid.NewGuid().ToString();
using (var serdeProvider = new AvroSerdeProvider(serdeProviderConfig))
using (var producer = new Producer <string, string>(producerConfig, serdeProvider.GetSerializerGenerator <string>(), serdeProvider.GetSerializerGenerator <string>()))
{
producer.ProduceAsync(topic, new Message <string, string> {
Key = "hello", Value = "world"
});
Assert.Equal(0, producer.Flush(TimeSpan.FromSeconds(10)));
}
using (var serdeProvider = new AvroSerdeProvider(serdeProviderConfig))
using (var producer = new Producer <int, string>(producerConfig, serdeProvider.GetSerializerGenerator <int>(), serdeProvider.GetSerializerGenerator <string>()))
{
Assert.Throws <SchemaRegistryException>(() =>
{
try
{
producer.ProduceAsync(topic, new Message <int, string> {
Key = 42, Value = "world"
}).Wait();
}
catch (AggregateException e)
{
throw e.InnerException;
}
});
}
using (var serdeProvider = new AvroSerdeProvider(serdeProviderConfig))
using (var producer = new Producer <string, int>(producerConfig, serdeProvider.GetSerializerGenerator <string>(), serdeProvider.GetSerializerGenerator <int>()))
{
Assert.Throws <SchemaRegistryException>(() =>
{
try
{
producer.ProduceAsync(topic, new Message <string, int> {
Key = "world", Value = 42
}).Wait();
}
catch (AggregateException e)
{
throw e.InnerException;
}
});
}
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
Log.Information("server1.dbo.products démarré à : {Now}", DateTime.Now);
stoppingToken.Register(() =>
_logger.Information("server1.dbo.products a débuté son arrêt à : {Now}", DateTime.Now));
var tacheConsommateur = Task.Run(() =>
{
using (var serdeProvider = new AvroSerdeProvider(new AvroSerdeProviderConfig {
SchemaRegistryUrl = schemaRegistryConfig.SchemaRegistryUrl
}))
using (var consumer = new Consumer <GenericRecord, Envelope>(consumerConfig,
serdeProvider.GetDeserializerGenerator <GenericRecord>().Invoke(true),
serdeProvider.GetDeserializerGenerator <Envelope>().Invoke(false)))
{
consumer.Subscribe(TOPIC);
try
{
while (true)
{
try
{
var consumeResult = consumer.Consume(stoppingToken);
if (consumeResult.Value.before == null && consumeResult.Value.after != null)
{
//Ajout
product produit = new product()
{
idProduit = consumeResult.Value.after.id,
name = consumeResult.Value.after.name,
description = consumeResult.Value.after.description,
weight = consumeResult.Value.after.weight
};
_serviceProduct.Create(produit);
}
if (consumeResult.Value.before != null && consumeResult.Value.after != null)
{
//MAJ
product produit = new product()
{
idProduit = consumeResult.Value.after.id,
name = consumeResult.Value.after.name,
description = consumeResult.Value.after.description,
weight = consumeResult.Value.after.weight
};
_serviceProduct.Update(produit.idProduit, produit);
}
if (consumeResult.Value.before != null && consumeResult.Value.after == null)
{
//Suppression
_serviceProduct.Remove(consumeResult.Value.before.id);
}
_logger.Information("Key: {Key}, Value: {Value}",
consumeResult.Message.Key,
consumeResult.Value);
}
catch (ConsumeException e)
{
_logger.Error("Consommation erreur: {Reason}", e.Error.Reason);
}
}
}
catch (OperationCanceledException)
{
// commit final offsets and leave the group.
consumer.Close();
}
}
});
//_logger.Information("service server1.dbo.products arrêté à : {Now}", DateTime.Now);
}
static void Main(string[] args)
{
if (args.Length != 3)
{
Console.WriteLine("Usage: .. bootstrapServers schemaRegistryUrl topicName");
return;
}
string bootstrapServers = args[0];
string schemaRegistryUrl = args[1];
string topicName = args[2];
var producerConfig = new ProducerConfig {
BootstrapServers = bootstrapServers
};
var avroConfig = new AvroSerdeProviderConfig
{
// Note: you can specify more than one schema registry url using the
// schemaRegistryUrl property for redundancy (comma separated list).
SchemaRegistryUrl = schemaRegistryUrl,
// optional schema registry client properties:
// SchemaRegistryRequestTimeoutMs = 5000,
// SchemaRegistryMaxCachedSchemas = 10,
// optional avro serializer properties:
// AvroSerializerBufferBytes = 50,
// AvroSerializerAutoRegisterSchemas = true
};
var consumerConfig = new ConsumerConfig
{
BootstrapServers = bootstrapServers,
GroupId = Guid.NewGuid().ToString()
};
// var s = (RecordSchema)Schema.Parse(File.ReadAllText("my-schema.json"));
var s = (RecordSchema)Schema.Parse(
@"{
""namespace"": ""Confluent.Kafka.Examples.AvroSpecific"",
""type"": ""record"",
""name"": ""User"",
""fields"": [
{""name"": ""name"", ""type"": ""string""},
{""name"": ""favorite_number"", ""type"": [""int"", ""null""]},
{""name"": ""favorite_color"", ""type"": [""string"", ""null""]}
]
}"
);
CancellationTokenSource cts = new CancellationTokenSource();
var consumeTask = Task.Run(() =>
{
using (var serdeProvider = new AvroSerdeProvider(avroConfig))
using (var consumer = new Consumer <string, GenericRecord>(consumerConfig, serdeProvider.GetDeserializerGenerator <string>(), serdeProvider.GetDeserializerGenerator <GenericRecord>()))
{
consumer.OnError += (_, e)
=> Console.WriteLine($"Error: {e.Reason}");
consumer.Subscribe(topicName);
while (!cts.Token.IsCancellationRequested)
{
try
{
var consumeResult = consumer.Consume(cts.Token);
Console.WriteLine($"Key: {consumeResult.Message.Key}\nValue: {consumeResult.Value}");
}
catch (ConsumeException e)
{
Console.WriteLine("Consume error: " + e.Error.Reason);
}
}
consumer.Close();
}
}, cts.Token);
using (var serdeProvider = new AvroSerdeProvider(avroConfig))
using (var producer = new Producer <string, GenericRecord>(producerConfig, serdeProvider.GetSerializerGenerator <string>(), serdeProvider.GetSerializerGenerator <GenericRecord>()))
{
Console.WriteLine($"{producer.Name} producing on {topicName}. Enter user names, q to exit.");
int i = 0;
string text;
while ((text = Console.ReadLine()) != "q")
{
var record = new GenericRecord(s);
record.Add("name", text);
record.Add("favorite_number", i++);
record.Add("favorite_color", "blue");
producer
.ProduceAsync(topicName, new Message <string, GenericRecord> {
Key = text, Value = record
})
.ContinueWith(task => task.IsFaulted
? $"error producing message: {task.Exception.Message}"
: $"produced to: {task.Result.TopicPartitionOffset}");
}
}
cts.Cancel();
}
public static void ProduceConsume(string bootstrapServers, string schemaRegistryServers)
{
string topic = Guid.NewGuid().ToString();
var producerConfig = new ProducerConfig {
BootstrapServers = bootstrapServers
};
var consumerConfig = new ConsumerConfig
{
BootstrapServers = bootstrapServers,
GroupId = Guid.NewGuid().ToString(),
SessionTimeoutMs = 6000,
AutoOffsetReset = AutoOffsetResetType.Earliest
};
var serdeProviderConfig = new AvroSerdeProviderConfig {
SchemaRegistryUrl = schemaRegistryServers
};
using (var serdeProvider = new AvroSerdeProvider(serdeProviderConfig))
using (var producer = new Producer <string, User>(producerConfig, serdeProvider.GetSerializerGenerator <string>(), serdeProvider.GetSerializerGenerator <User>()))
{
for (int i = 0; i < 100; ++i)
{
var user = new User
{
name = i.ToString(),
favorite_number = i,
favorite_color = "blue"
};
producer.ProduceAsync(topic, new Message <string, User> {
Key = user.name, Value = user
});
}
Assert.Equal(0, producer.Flush(TimeSpan.FromSeconds(10)));
}
using (var serdeProvider = new AvroSerdeProvider(serdeProviderConfig))
using (var consumer = new Consumer <string, User>(consumerConfig, serdeProvider.GetDeserializerGenerator <string>(), serdeProvider.GetDeserializerGenerator <User>()))
{
bool consuming = true;
consumer.OnPartitionEOF += (_, topicPartitionOffset)
=> consuming = false;
consumer.OnError += (_, e)
=> Assert.True(false, e.Reason);
consumer.Subscribe(topic);
int i = 0;
while (consuming)
{
var record = consumer.Consume(TimeSpan.FromMilliseconds(100));
if (record != null)
{
Assert.Equal(i.ToString(), record.Message.Key);
Assert.Equal(i.ToString(), record.Message.Value.name);
Assert.Equal(i, record.Message.Value.favorite_number);
Assert.Equal("blue", record.Message.Value.favorite_color);
i += 1;
}
}
Assert.Equal(100, i);
consumer.Close();
}
}
static void Main(string[] args)
{
int messages = 1;
if (args.Length > 0)
{
messages = int.Parse(args[0]);
}
var config = new ProducerConfig
{
BootstrapServers = "broker1:9092",
};
var avroConfig = new AvroSerdeProviderConfig
{
// Note: you can specify more than one schema registry url using the
// schema.registry.url property for redundancy (comma separated list).
// The property name is not plural to follow the convention set by
// the Java implementation.
SchemaRegistryUrl = "http://*****:*****@somewhere.io",
Value = httpRequest
});
}
producer.Flush(TimeSpan.FromSeconds(30));
}
}
public static void ConsumeIncompatibleTypes(string bootstrapServers, string schemaRegistryServers)
{
string topic = Guid.NewGuid().ToString();
var producerConfig = new ProducerConfig {
BootstrapServers = bootstrapServers
};
var consumerConfig = new ConsumerConfig
{
BootstrapServers = bootstrapServers,
GroupId = Guid.NewGuid().ToString(),
SessionTimeoutMs = 6000,
AutoOffsetReset = AutoOffsetResetType.Earliest
};
var serdeProviderConfig = new AvroSerdeProviderConfig {
SchemaRegistryUrl = schemaRegistryServers
};
using (var serdeProvider = new AvroSerdeProvider(serdeProviderConfig))
using (var producer = new Producer <string, User>(producerConfig, serdeProvider.GetSerializerGenerator <string>(), serdeProvider.GetSerializerGenerator <User>()))
{
var user = new User
{
name = "username",
favorite_number = 107,
favorite_color = "orange"
};
producer.ProduceAsync(topic, new Message <string, User> {
Key = user.name, Value = user
});
Assert.Equal(0, producer.Flush(TimeSpan.FromSeconds(10)));
}
using (var serdeProvider = new AvroSerdeProvider(serdeProviderConfig))
using (var consumer = new Consumer <User, User>(consumerConfig, serdeProvider.GetDeserializerGenerator <User>(), serdeProvider.GetDeserializerGenerator <User>()))
{
consumer.Assign(new List <TopicPartitionOffset> {
new TopicPartitionOffset(topic, 0, 0)
});
bool hadError = false;
try
{
consumer.Consume(TimeSpan.FromSeconds(10));
}
catch (ConsumeException e)
{
if (e.Error.Code == ErrorCode.Local_KeyDeserialization)
{
hadError = true;
}
}
Assert.True(hadError);
}
using (var serdeProvider = new AvroSerdeProvider(serdeProviderConfig))
using (var consumer = new Consumer <string, string>(consumerConfig, serdeProvider.GetDeserializerGenerator <string>(), serdeProvider.GetDeserializerGenerator <string>()))
{
consumer.Assign(new List <TopicPartitionOffset> {
new TopicPartitionOffset(topic, 0, 0)
});
bool hadError = false;
try
{
consumer.Consume(TimeSpan.FromSeconds(10));
}
catch (ConsumeException e)
{
if (e.Error.Code == ErrorCode.Local_ValueDeserialization)
{
hadError = true;
}
}
Assert.True(hadError);
}
}
/// <summary>
/// In this example
/// - offsets are manually committed.
/// - no extra thread is created for the Poll (Consume) loop.
/// </summary>
public static void Run_Consume(string brokerList, List <string> topics, CancellationToken cancellationToken)
{
var config = new ConsumerConfig
{
BootstrapServers = brokerList,
GroupId = "csharp-consumer",
EnableAutoCommit = false,
StatisticsIntervalMs = 5000,
SessionTimeoutMs = 6000,
AutoOffsetReset = AutoOffsetResetType.Earliest
};
const int commitPeriod = 5;
using (var serdeProvider = new AvroSerdeProvider(new AvroSerdeProviderConfig {
SchemaRegistryUrl = "192.168.100.153:8081"
}))
using (var consumer = new Consumer <Ignore, com.landoop.social.reddit.post.reddit_post>(config, null, serdeProvider.GetDeserializerGenerator <com.landoop.social.reddit.post.reddit_post>()))
// using (var consumer = new Consumer<Ingore, string>(config))
{
// Note: All event handlers are called on the main .Consume thread.
// Raised when the consumer has been notified of a new assignment set.
// You can use this event to perform actions such as retrieving offsets
// from an external source / manually setting start offsets using
// the Assign method. You can even call Assign with a different set of
// partitions than those in the assignment. If you do not call Assign
// in a handler of this event, the consumer will be automatically
// assigned to the partitions of the assignment set and consumption
// will start from last committed offsets or in accordance with
// the auto.offset.reset configuration parameter for partitions where
// there is no committed offset.
consumer.OnPartitionsAssigned += (_, partitions)
=> Console.WriteLine($"Assigned partitions: [{string.Join(", ", partitions)}], member id: {consumer.MemberId}");
// Raised when the consumer's current assignment set has been revoked.
consumer.OnPartitionsRevoked += (_, partitions)
=> Console.WriteLine($"Revoked partitions: [{string.Join(", ", partitions)}]");
consumer.OnPartitionEOF += (_, tpo)
=> Console.WriteLine($"Reached end of topic {tpo.Topic} partition {tpo.Partition}, next message will be at offset {tpo.Offset}");
consumer.OnError += (_, e)
=> Console.WriteLine($"Error: {e.Reason}");
//consumer.OnStatistics += (_, json)
// => Console.WriteLine($"Statistics: {json}");
consumer.Subscribe(topics);
while (!cancellationToken.IsCancellationRequested)
{
try
{
var consumeResult = consumer.Consume(cancellationToken);
Console.WriteLine($"Topic: {consumeResult.Topic}");
Console.WriteLine($"Partition: {consumeResult.Partition} Offset: {consumeResult.Offset}");
Console.WriteLine($"Subreddit Topic: { consumeResult.Value.subreddit }");
Console.WriteLine($"{ consumeResult.Value.body }");
Console.WriteLine("---");
Thread.Sleep(1000);
if (consumeResult.Offset % commitPeriod == 0)
{
// The Commit method sends a "commit offsets" request to the Kafka
// cluster and synchronously waits for the response. This is very
// slow compared to the rate at which the consumer is capable of
// consuming messages. A high performance application will typically
// commit offsets relatively infrequently and be designed handle
// duplicate messages in the event of failure.
var committedOffsets = consumer.Commit(consumeResult);
Console.WriteLine($"Committed offset: {committedOffsets}");
}
}
catch (ConsumeException e)
{
Console.WriteLine($"Consume error: {e.Error}");
}
}
consumer.Close();
}
}
public static void AutoRegisterSchemaDisabled(string bootstrapServers, string schemaRegistryServers)
{
string topic = Guid.NewGuid().ToString();
var producerConfig = new ProducerConfig {
BootstrapServers = bootstrapServers
};
var avroConfig = new AvroSerdeProviderConfig {
SchemaRegistryUrl = schemaRegistryServers, AvroSerializerAutoRegisterSchemas = false
};
var consumerConfig = new ConsumerConfig
{
BootstrapServers = bootstrapServers,
GroupId = Guid.NewGuid().ToString(),
SessionTimeoutMs = 6000,
AutoOffsetReset = AutoOffsetResetType.Earliest
};
using (var serdeProvider = new AvroSerdeProvider(avroConfig))
using (var producer = new Producer <string, int>(producerConfig, serdeProvider.GetSerializerGenerator <string>(), serdeProvider.GetSerializerGenerator <int>()))
{
Assert.Throws <SchemaRegistryException>(() =>
{
try
{
producer.ProduceAsync(topic, new Message <string, int> {
Key = "test", Value = 112
}).Wait();
}
catch (AggregateException e)
{
throw e.InnerException;
}
});
}
var producerConfig2 = new ProducerConfig {
BootstrapServers = bootstrapServers
};
var avroConfig2 = new AvroSerdeProviderConfig {
SchemaRegistryUrl = schemaRegistryServers
};
using (var serdeProvider = new AvroSerdeProvider(avroConfig2))
using (var producer = new Producer <string, int>(producerConfig2, serdeProvider.GetSerializerGenerator <string>(), serdeProvider.GetSerializerGenerator <int>()))
{
producer.ProduceAsync(topic, new Message <string, int> {
Key = "test", Value = 112
}).Wait();
}
var producerConfig3 = new ProducerConfig {
BootstrapServers = bootstrapServers
};
var avroConfig3 = new AvroSerdeProviderConfig {
SchemaRegistryUrl = schemaRegistryServers, AvroSerializerAutoRegisterSchemas = false
};
// config with avro.serializer.auto.register.schemas == false should work now.
using (var serdeProvider = new AvroSerdeProvider(avroConfig3))
using (var producer = new Producer <string, int>(producerConfig3, serdeProvider.GetSerializerGenerator <string>(), serdeProvider.GetSerializerGenerator <int>()))
{
producer.ProduceAsync(topic, new Message <string, int> {
Key = "test", Value = 112
}).Wait();
}
}
