public AvroConsumerStrategy(ConsumerConfig config, AvroSerdeProviderConfig avroConfig)
{
_serdeProvider = new AvroSerdeProvider(avroConfig);
_consumer = new Consumer <TKey, TValue>(config,
_serdeProvider.GetDeserializerGenerator <TKey>(),
_serdeProvider.GetDeserializerGenerator <TValue>());
}
public AvroKafkaConsumer(IOptions <MessagingOptions> options,
ILogger <AvroKafkaConsumer> logger) : base(options, logger)
{
var avroConfig = new AvroSerdeProviderConfig {
SchemaRegistryUrl = options.Value.SchemeRegistry
};
_avroProvider = new AvroSerdeProvider(avroConfig);
}
private static AvroSerdeProviderConfig CreateAvroSerdeProviderConfig()
{
var config = new AvroSerdeProviderConfig();
if (!File.Exists("beta.producer.schemaregistry.json"))
{
return(config);
}
var content = File.ReadAllText("beta.producer.schemaregistry.json");
var json = JsonConvert.DeserializeObject <Dictionary <string, string> >(content);
if (json.ContainsKey(AvroSerdeProviderConfig.PropertyNames.AvroSerializerAutoRegisterSchemas))
{
if (bool.TryParse(json[AvroSerdeProviderConfig.PropertyNames.AvroSerializerAutoRegisterSchemas], out var value))
{
config.AvroSerializerAutoRegisterSchemas = value;
}
}
if (json.ContainsKey(AvroSerdeProviderConfig.PropertyNames.AvroSerializerBufferBytes))
{
if (int.TryParse(json[AvroSerdeProviderConfig.PropertyNames.AvroSerializerBufferBytes], out var value))
{
config.AvroSerializerBufferBytes = value;
}
}
if (json.ContainsKey(SchemaRegistryConfig.PropertyNames.SchemaRegistryMaxCachedSchemas))
{
if (int.TryParse(json[SchemaRegistryConfig.PropertyNames.SchemaRegistryMaxCachedSchemas], out var value))
{
config.SchemaRegistryMaxCachedSchemas = value;
}
}
if (json.ContainsKey(SchemaRegistryConfig.PropertyNames.SchemaRegistryRequestTimeoutMs))
{
if (int.TryParse(json[SchemaRegistryConfig.PropertyNames.SchemaRegistryRequestTimeoutMs], out var value))
{
config.SchemaRegistryRequestTimeoutMs = value;
}
}
if (json.ContainsKey(SchemaRegistryConfig.PropertyNames.SchemaRegistryUrl))
{
config.SchemaRegistryUrl = json[SchemaRegistryConfig.PropertyNames.SchemaRegistryUrl];
}
config.ToList().ForEach(i => Log.Information("{0}[{1}]: {2}", nameof(AvroSerdeProviderConfig), i.Key, i.Value));
return(config);
}
public static void ProduceGenericMultipleTopics(string bootstrapServers, string schemaRegistryServers)
{
var s = (RecordSchema)Schema.Parse(
@"{
""namespace"": ""Confluent.Kafka.Examples.AvroSpecific"",
""type"": ""record"",
""name"": ""User"",
""fields"": [
{""name"": ""name"", ""type"": ""string""},
{""name"": ""favorite_number"", ""type"": [""int"", ""null""]},
{""name"": ""favorite_color"", ""type"": [""string"", ""null""]}
]
}"
);
var config = new ProducerConfig {
BootstrapServers = bootstrapServers
};
var serdeProviderConfig = new AvroSerdeProviderConfig {
SchemaRegistryUrl = schemaRegistryServers
};
var topic = Guid.NewGuid().ToString();
var topic2 = Guid.NewGuid().ToString();
DeliveryReport <Null, GenericRecord> dr;
DeliveryReport <Null, GenericRecord> dr2;
using (var serdeProvider = new AvroSerdeProvider(serdeProviderConfig))
using (var p = new Producer <Null, GenericRecord>(config, null, serdeProvider.GetSerializerGenerator <GenericRecord>()))
{
var record = new GenericRecord(s);
record.Add("name", "my name 2");
record.Add("favorite_number", 44);
record.Add("favorite_color", null);
dr = p.ProduceAsync(topic, new Message <Null, GenericRecord> {
Key = null, Value = record
}).Result;
dr2 = p.ProduceAsync(topic2, new Message <Null, GenericRecord> {
Key = null, Value = record
}).Result;
}
Assert.Null(dr.Key);
Assert.NotNull(dr.Value);
Assert.Null(dr2.Key);
Assert.NotNull(dr2.Value);
}
public override IConsumerStrategy <TKey, TValue> CreateConsumerStrategy()
{
var config = new ConsumerConfig
{
BootstrapServers = BrokerUrls,
GroupId = ConsumerGroupId,
EnableAutoCommit = true,
AutoOffsetReset = AutoOffsetResetType.Latest
};
var avroConfig = new AvroSerdeProviderConfig
{
// Note: you can specify more than one schema registry url using the
// schema.registry.url property for redundancy (comma separated list).
// The property name is not plural to follow the convention set by
// the Java implementation.
SchemaRegistryUrl = SchemaRegistryUrls,
// optional schema registry client properties:
SchemaRegistryRequestTimeoutMs = 5000,
SchemaRegistryMaxCachedSchemas = 10
};
return(new AvroConsumerStrategy <TKey, TValue>(config, avroConfig));
}
static void Main(string[] args)
{
if (args.Length != 3)
{
Console.WriteLine("Usage: .. bootstrapServers schemaRegistryUrl topicName");
return;
}
string bootstrapServers = args[0];
string schemaRegistryUrl = args[1];
string topicName = args[2];
var producerConfig = new ProducerConfig {
BootstrapServers = bootstrapServers
};
var avroConfig = new AvroSerdeProviderConfig
{
// Note: you can specify more than one schema registry url using the
// schema.registry.url property for redundancy (comma separated list).
// The property name is not plural to follow the convention set by
// the Java implementation.
SchemaRegistryUrl = schemaRegistryUrl,
// optional schema registry client properties:
SchemaRegistryRequestTimeoutMs = 5000,
SchemaRegistryMaxCachedSchemas = 10,
// optional avro serializer properties:
AvroSerializerBufferBytes = 50,
AvroSerializerAutoRegisterSchemas = true
};
var consumerConfig = new ConsumerConfig
{
BootstrapServers = bootstrapServers,
GroupId = Guid.NewGuid().ToString()
};
// Note: The User class in this project was generated using the Confluent fork of the avrogen.exe tool
// (avaliable from: https://github.com/confluentinc/avro/tree/confluent-fork) which includes modifications
// that prevent namespace clashes with user namespaces that include the identifier 'Avro'. AvroSerializer
// and AvroDeserializer are also compatible with classes generated by the official avrogen.exe tool
// (available from: https://github.com/apache/avro), with the above limitation.
CancellationTokenSource cts = new CancellationTokenSource();
var consumeTask = Task.Run(() =>
{
using (var serdeProvider = new AvroSerdeProvider(avroConfig))
using (var consumer = new Consumer <string, User>(consumerConfig, serdeProvider.GetDeserializerGenerator <string>(), serdeProvider.GetDeserializerGenerator <User>()))
{
consumer.OnError += (_, e)
=> Console.WriteLine($"Error: {e.Reason}");
consumer.Subscribe(topicName);
while (!cts.Token.IsCancellationRequested)
{
try
{
var consumeResult = consumer.Consume(cts.Token);
Console.WriteLine($"user key name: {consumeResult.Message.Key}, user value favorite color: {consumeResult.Value.favorite_color}");
}
catch (ConsumeException e)
{
Console.WriteLine("Consume error: " + e.Error.Reason);
}
}
consumer.Close();
}
}, cts.Token);
using (var serdeProvider = new AvroSerdeProvider(avroConfig))
using (var producer = new Producer <string, User>(producerConfig, serdeProvider.GetSerializerGenerator <string>(), serdeProvider.GetSerializerGenerator <User>()))
{
Console.WriteLine($"{producer.Name} producing on {topicName}. Enter user names, q to exit.");
int i = 0;
string text;
while ((text = Console.ReadLine()) != "q")
{
User user = new User {
name = text, favorite_color = "green", favorite_number = i++
};
producer
.ProduceAsync(topicName, new Message <string, User> {
Key = text, Value = user
})
.ContinueWith(task => task.IsFaulted
? $"error producing message: {task.Exception.Message}"
: $"produced to: {task.Result.TopicPartitionOffset}");
}
}
cts.Cancel();
}
public static void ProduceConsumeGeneric(string bootstrapServers, string schemaRegistryServers)
{
var s = (RecordSchema)Schema.Parse(
@"{
""namespace"": ""Confluent.Kafka.Examples.AvroSpecific"",
""type"": ""record"",
""name"": ""User"",
""fields"": [
{""name"": ""name"", ""type"": ""string""},
{""name"": ""favorite_number"", ""type"": [""int"", ""null""]},
{""name"": ""favorite_color"", ""type"": [""string"", ""null""]}
]
}"
);
var config = new ProducerConfig {
BootstrapServers = bootstrapServers
};
var serdeProviderConfig = new AvroSerdeProviderConfig {
SchemaRegistryUrl = schemaRegistryServers
};
var topic = Guid.NewGuid().ToString();
DeliveryReport <Null, GenericRecord> dr;
using (var serdeProvider = new AvroSerdeProvider(serdeProviderConfig))
using (var p = new Producer <Null, GenericRecord>(config, null, serdeProvider.GetSerializerGenerator <GenericRecord>()))
{
var record = new GenericRecord(s);
record.Add("name", "my name 2");
record.Add("favorite_number", 44);
record.Add("favorite_color", null);
dr = p.ProduceAsync(topic, new Message <Null, GenericRecord> {
Value = record
}).Result;
}
// produce a specific record (to later consume back as a generic record).
using (var serdeProvider = new AvroSerdeProvider(serdeProviderConfig))
using (var p = new Producer <Null, User>(config, null, serdeProvider.GetSerializerGenerator <User>()))
{
var user = new User
{
name = "my name 3",
favorite_number = 47,
favorite_color = "orange"
};
p.ProduceAsync(topic, new Message <Null, User> {
Value = user
}).Wait();
}
Assert.Null(dr.Message.Key);
Assert.NotNull(dr.Message.Value);
dr.Message.Value.TryGetValue("name", out object name);
dr.Message.Value.TryGetValue("favorite_number", out object number);
dr.Message.Value.TryGetValue("favorite_color", out object color);
Assert.IsType <string>(name);
Assert.IsType <int>(number);
Assert.Equal("my name 2", name);
Assert.Equal(44, number);
Assert.Null(color);
var cconfig = new ConsumerConfig {
GroupId = Guid.NewGuid().ToString(), BootstrapServers = bootstrapServers
};
using (var serdeProvider = new AvroSerdeProvider(serdeProviderConfig))
using (var consumer = new Consumer <Null, GenericRecord>(cconfig, null, serdeProvider.GetDeserializerGenerator <GenericRecord>()))
{
// consume generic record produced as a generic record.
consumer.Assign(new List <TopicPartitionOffset> {
new TopicPartitionOffset(topic, 0, dr.Offset)
});
var record = consumer.Consume(TimeSpan.FromSeconds(20));
record.Message.Value.TryGetValue("name", out object msgName);
record.Message.Value.TryGetValue("favorite_number", out object msgNumber);
record.Message.Value.TryGetValue("favorite_color", out object msgColor);
Assert.IsType <string>(msgName);
Assert.IsType <int>(msgNumber);
Assert.Equal("my name 2", msgName);
Assert.Equal(44, msgNumber);
Assert.Null(msgColor);
// consume generic record produced as a specific record.
record = consumer.Consume(TimeSpan.FromSeconds(20));
record.Message.Value.TryGetValue("name", out msgName);
record.Message.Value.TryGetValue("favorite_number", out msgNumber);
record.Message.Value.TryGetValue("favorite_color", out msgColor);
Assert.IsType <string>(msgName);
Assert.IsType <int>(msgNumber);
Assert.IsType <string>(msgColor);
Assert.Equal("my name 3", msgName);
Assert.Equal(47, msgNumber);
Assert.Equal("orange", msgColor);
}
using (var serdeProvider = new AvroSerdeProvider(serdeProviderConfig))
using (var consumer = new Consumer <Null, User>(cconfig, null, serdeProvider.GetDeserializerGenerator <User>()))
{
consumer.Assign(new List <TopicPartitionOffset> {
new TopicPartitionOffset(topic, 0, dr.Offset)
});
var record = consumer.Consume(TimeSpan.FromSeconds(20));
Assert.Equal("my name 2", record.Message.Value.name);
Assert.Equal(44, record.Message.Value.favorite_number);
Assert.Null(record.Message.Value.favorite_color);
}
}
static void Main(string[] args)
{
if (args.Length != 3)
{
Console.WriteLine("Usage: .. bootstrapServers schemaRegistryUrl topicName");
return;
}
string bootstrapServers = args[0];
string schemaRegistryUrl = args[1];
string topicName = args[2];
var producerConfig = new ProducerConfig {
BootstrapServers = bootstrapServers
};
var avroConfig = new AvroSerdeProviderConfig
{
// Note: you can specify more than one schema registry url using the
// schemaRegistryUrl property for redundancy (comma separated list).
SchemaRegistryUrl = schemaRegistryUrl,
// optional schema registry client properties:
// SchemaRegistryRequestTimeoutMs = 5000,
// SchemaRegistryMaxCachedSchemas = 10,
// optional avro serializer properties:
// AvroSerializerBufferBytes = 50,
// AvroSerializerAutoRegisterSchemas = true
};
var consumerConfig = new ConsumerConfig
{
BootstrapServers = bootstrapServers,
GroupId = Guid.NewGuid().ToString()
};
// var s = (RecordSchema)Schema.Parse(File.ReadAllText("my-schema.json"));
var s = (RecordSchema)Schema.Parse(
@"{
""namespace"": ""Confluent.Kafka.Examples.AvroSpecific"",
""type"": ""record"",
""name"": ""User"",
""fields"": [
{""name"": ""name"", ""type"": ""string""},
{""name"": ""favorite_number"", ""type"": [""int"", ""null""]},
{""name"": ""favorite_color"", ""type"": [""string"", ""null""]}
]
}"
);
CancellationTokenSource cts = new CancellationTokenSource();
var consumeTask = Task.Run(() =>
{
using (var serdeProvider = new AvroSerdeProvider(avroConfig))
using (var consumer = new Consumer <string, GenericRecord>(consumerConfig, serdeProvider.GetDeserializerGenerator <string>(), serdeProvider.GetDeserializerGenerator <GenericRecord>()))
{
consumer.OnError += (_, e)
=> Console.WriteLine($"Error: {e.Reason}");
consumer.Subscribe(topicName);
while (!cts.Token.IsCancellationRequested)
{
try
{
var consumeResult = consumer.Consume(cts.Token);
Console.WriteLine($"Key: {consumeResult.Message.Key}\nValue: {consumeResult.Value}");
}
catch (ConsumeException e)
{
Console.WriteLine("Consume error: " + e.Error.Reason);
}
}
consumer.Close();
}
}, cts.Token);
using (var serdeProvider = new AvroSerdeProvider(avroConfig))
using (var producer = new Producer <string, GenericRecord>(producerConfig, serdeProvider.GetSerializerGenerator <string>(), serdeProvider.GetSerializerGenerator <GenericRecord>()))
{
Console.WriteLine($"{producer.Name} producing on {topicName}. Enter user names, q to exit.");
int i = 0;
string text;
while ((text = Console.ReadLine()) != "q")
{
var record = new GenericRecord(s);
record.Add("name", text);
record.Add("favorite_number", i++);
record.Add("favorite_color", "blue");
producer
.ProduceAsync(topicName, new Message <string, GenericRecord> {
Key = text, Value = record
})
.ContinueWith(task => task.IsFaulted
? $"error producing message: {task.Exception.Message}"
: $"produced to: {task.Result.TopicPartitionOffset}");
}
}
cts.Cancel();
}
public static void ProduceConsume(string bootstrapServers, string schemaRegistryServers)
{
string topic = Guid.NewGuid().ToString();
var producerConfig = new ProducerConfig {
BootstrapServers = bootstrapServers
};
var consumerConfig = new ConsumerConfig
{
BootstrapServers = bootstrapServers,
GroupId = Guid.NewGuid().ToString(),
SessionTimeoutMs = 6000,
AutoOffsetReset = AutoOffsetResetType.Earliest
};
var serdeProviderConfig = new AvroSerdeProviderConfig {
SchemaRegistryUrl = schemaRegistryServers
};
using (var serdeProvider = new AvroSerdeProvider(serdeProviderConfig))
using (var producer = new Producer <string, User>(producerConfig, serdeProvider.GetSerializerGenerator <string>(), serdeProvider.GetSerializerGenerator <User>()))
{
for (int i = 0; i < 100; ++i)
{
var user = new User
{
name = i.ToString(),
favorite_number = i,
favorite_color = "blue"
};
producer.ProduceAsync(topic, new Message <string, User> {
Key = user.name, Value = user
});
}
Assert.Equal(0, producer.Flush(TimeSpan.FromSeconds(10)));
}
using (var serdeProvider = new AvroSerdeProvider(serdeProviderConfig))
using (var consumer = new Consumer <string, User>(consumerConfig, serdeProvider.GetDeserializerGenerator <string>(), serdeProvider.GetDeserializerGenerator <User>()))
{
bool consuming = true;
consumer.OnPartitionEOF += (_, topicPartitionOffset)
=> consuming = false;
consumer.OnError += (_, e)
=> Assert.True(false, e.Reason);
consumer.Subscribe(topic);
int i = 0;
while (consuming)
{
var record = consumer.Consume(TimeSpan.FromMilliseconds(100));
if (record != null)
{
Assert.Equal(i.ToString(), record.Message.Key);
Assert.Equal(i.ToString(), record.Message.Value.name);
Assert.Equal(i, record.Message.Value.favorite_number);
Assert.Equal("blue", record.Message.Value.favorite_color);
i += 1;
}
}
Assert.Equal(100, i);
consumer.Close();
}
}
public static void AutoRegisterSchemaDisabled(string bootstrapServers, string schemaRegistryServers)
{
string topic = Guid.NewGuid().ToString();
var producerConfig = new ProducerConfig {
BootstrapServers = bootstrapServers
};
var avroConfig = new AvroSerdeProviderConfig {
SchemaRegistryUrl = schemaRegistryServers, AvroSerializerAutoRegisterSchemas = false
};
var consumerConfig = new ConsumerConfig
{
BootstrapServers = bootstrapServers,
GroupId = Guid.NewGuid().ToString(),
SessionTimeoutMs = 6000,
AutoOffsetReset = AutoOffsetResetType.Earliest
};
using (var serdeProvider = new AvroSerdeProvider(avroConfig))
using (var producer = new Producer <string, int>(producerConfig, serdeProvider.GetSerializerGenerator <string>(), serdeProvider.GetSerializerGenerator <int>()))
{
Assert.Throws <SchemaRegistryException>(() =>
{
try
{
producer.ProduceAsync(topic, new Message <string, int> {
Key = "test", Value = 112
}).Wait();
}
catch (AggregateException e)
{
throw e.InnerException;
}
});
}
var producerConfig2 = new ProducerConfig {
BootstrapServers = bootstrapServers
};
var avroConfig2 = new AvroSerdeProviderConfig {
SchemaRegistryUrl = schemaRegistryServers
};
using (var serdeProvider = new AvroSerdeProvider(avroConfig2))
using (var producer = new Producer <string, int>(producerConfig2, serdeProvider.GetSerializerGenerator <string>(), serdeProvider.GetSerializerGenerator <int>()))
{
producer.ProduceAsync(topic, new Message <string, int> {
Key = "test", Value = 112
}).Wait();
}
var producerConfig3 = new ProducerConfig {
BootstrapServers = bootstrapServers
};
var avroConfig3 = new AvroSerdeProviderConfig {
SchemaRegistryUrl = schemaRegistryServers, AvroSerializerAutoRegisterSchemas = false
};
// config with avro.serializer.auto.register.schemas == false should work now.
using (var serdeProvider = new AvroSerdeProvider(avroConfig3))
using (var producer = new Producer <string, int>(producerConfig3, serdeProvider.GetSerializerGenerator <string>(), serdeProvider.GetSerializerGenerator <int>()))
{
producer.ProduceAsync(topic, new Message <string, int> {
Key = "test", Value = 112
}).Wait();
}
}
static void Main(string[] args)
{
int messages = 1;
if (args.Length > 0)
{
messages = int.Parse(args[0]);
}
var config = new ProducerConfig
{
BootstrapServers = "broker1:9092",
};
var avroConfig = new AvroSerdeProviderConfig
{
// Note: you can specify more than one schema registry url using the
// schema.registry.url property for redundancy (comma separated list).
// The property name is not plural to follow the convention set by
// the Java implementation.
SchemaRegistryUrl = "http://*****:*****@somewhere.io",
Value = httpRequest
});
}
producer.Flush(TimeSpan.FromSeconds(30));
}
}
public static void ProduceIncompatibleTypes(string bootstrapServers, string schemaRegistryServers)
{
var producerConfig = new ProducerConfig {
BootstrapServers = bootstrapServers
};
var consumerConfig = new ConsumerConfig
{
BootstrapServers = bootstrapServers,
GroupId = Guid.NewGuid().ToString(),
SessionTimeoutMs = 6000,
AutoOffsetReset = AutoOffsetResetType.Earliest,
};
var serdeProviderConfig = new AvroSerdeProviderConfig {
SchemaRegistryUrl = schemaRegistryServers
};
var topic = Guid.NewGuid().ToString();
using (var serdeProvider = new AvroSerdeProvider(serdeProviderConfig))
using (var producer = new Producer <string, string>(producerConfig, serdeProvider.GetSerializerGenerator <string>(), serdeProvider.GetSerializerGenerator <string>()))
{
producer.ProduceAsync(topic, new Message <string, string> {
Key = "hello", Value = "world"
});
Assert.Equal(0, producer.Flush(TimeSpan.FromSeconds(10)));
}
using (var serdeProvider = new AvroSerdeProvider(serdeProviderConfig))
using (var producer = new Producer <int, string>(producerConfig, serdeProvider.GetSerializerGenerator <int>(), serdeProvider.GetSerializerGenerator <string>()))
{
Assert.Throws <SchemaRegistryException>(() =>
{
try
{
producer.ProduceAsync(topic, new Message <int, string> {
Key = 42, Value = "world"
}).Wait();
}
catch (AggregateException e)
{
throw e.InnerException;
}
});
}
using (var serdeProvider = new AvroSerdeProvider(serdeProviderConfig))
using (var producer = new Producer <string, int>(producerConfig, serdeProvider.GetSerializerGenerator <string>(), serdeProvider.GetSerializerGenerator <int>()))
{
Assert.Throws <SchemaRegistryException>(() =>
{
try
{
producer.ProduceAsync(topic, new Message <string, int> {
Key = "world", Value = 42
}).Wait();
}
catch (AggregateException e)
{
throw e.InnerException;
}
});
}
}
public static void ConsumeIncompatibleTypes(string bootstrapServers, string schemaRegistryServers)
{
string topic = Guid.NewGuid().ToString();
var producerConfig = new ProducerConfig {
BootstrapServers = bootstrapServers
};
var consumerConfig = new ConsumerConfig
{
BootstrapServers = bootstrapServers,
GroupId = Guid.NewGuid().ToString(),
SessionTimeoutMs = 6000,
AutoOffsetReset = AutoOffsetResetType.Earliest
};
var serdeProviderConfig = new AvroSerdeProviderConfig {
SchemaRegistryUrl = schemaRegistryServers
};
using (var serdeProvider = new AvroSerdeProvider(serdeProviderConfig))
using (var producer = new Producer <string, User>(producerConfig, serdeProvider.GetSerializerGenerator <string>(), serdeProvider.GetSerializerGenerator <User>()))
{
var user = new User
{
name = "username",
favorite_number = 107,
favorite_color = "orange"
};
producer.ProduceAsync(topic, new Message <string, User> {
Key = user.name, Value = user
});
Assert.Equal(0, producer.Flush(TimeSpan.FromSeconds(10)));
}
using (var serdeProvider = new AvroSerdeProvider(serdeProviderConfig))
using (var consumer = new Consumer <User, User>(consumerConfig, serdeProvider.GetDeserializerGenerator <User>(), serdeProvider.GetDeserializerGenerator <User>()))
{
consumer.Assign(new List <TopicPartitionOffset> {
new TopicPartitionOffset(topic, 0, 0)
});
bool hadError = false;
try
{
consumer.Consume(TimeSpan.FromSeconds(10));
}
catch (ConsumeException e)
{
if (e.Error.Code == ErrorCode.Local_KeyDeserialization)
{
hadError = true;
}
}
Assert.True(hadError);
}
using (var serdeProvider = new AvroSerdeProvider(serdeProviderConfig))
using (var consumer = new Consumer <string, string>(consumerConfig, serdeProvider.GetDeserializerGenerator <string>(), serdeProvider.GetDeserializerGenerator <string>()))
{
consumer.Assign(new List <TopicPartitionOffset> {
new TopicPartitionOffset(topic, 0, 0)
});
bool hadError = false;
try
{
consumer.Consume(TimeSpan.FromSeconds(10));
}
catch (ConsumeException e)
{
if (e.Error.Code == ErrorCode.Local_ValueDeserialization)
{
hadError = true;
}
}
Assert.True(hadError);
}
}
public static void PrimitiveTypes(string bootstrapServers, string schemaRegistryServers)
{
var serdeProviderConfig = new AvroSerdeProviderConfig {
SchemaRegistryUrl = schemaRegistryServers
};
using (var serdeProvider = new AvroSerdeProvider(serdeProviderConfig))
{
var producerConfig = new ProducerConfig {
BootstrapServers = bootstrapServers
};
var consumerConfig = new ConsumerConfig
{
BootstrapServers = bootstrapServers,
GroupId = Guid.NewGuid().ToString(),
SessionTimeoutMs = 6000,
AutoOffsetReset = AutoOffsetResetType.Earliest
};
var stringTopic = Guid.NewGuid().ToString();
using (var producer = new Producer <string, string>(producerConfig, serdeProvider.GetSerializerGenerator <string>(), serdeProvider.GetSerializerGenerator <string>()))
{
producer.ProduceAsync(stringTopic, new Message <string, string> {
Key = "hello", Value = "world"
});
Assert.Equal(0, producer.Flush(TimeSpan.FromSeconds(10)));
}
var bytesTopic = Guid.NewGuid().ToString();
using (var producer = new Producer <byte[], byte[]>(producerConfig, serdeProvider.GetSerializerGenerator <byte[]>(), serdeProvider.GetSerializerGenerator <byte[]>()))
{
producer.ProduceAsync(bytesTopic, new Message <byte[], byte[]> {
Key = new byte[] { 1, 4, 11 }, Value = new byte[] {}
});
Assert.Equal(0, producer.Flush(TimeSpan.FromSeconds(10)));
}
var intTopic = Guid.NewGuid().ToString();
using (var producer = new Producer <int, int>(producerConfig, serdeProvider.GetSerializerGenerator <int>(), serdeProvider.GetSerializerGenerator <int>()))
{
producer.ProduceAsync(intTopic, new Message <int, int> {
Key = 42, Value = 43
});
Assert.Equal(0, producer.Flush(TimeSpan.FromSeconds(10)));
}
var longTopic = Guid.NewGuid().ToString();
using (var producer = new Producer <long, long>(producerConfig, serdeProvider.GetSerializerGenerator <long>(), serdeProvider.GetSerializerGenerator <long>()))
{
producer.ProduceAsync(longTopic, new Message <long, long> {
Key = -32, Value = -33
});
Assert.Equal(0, producer.Flush(TimeSpan.FromSeconds(10)));
}
var boolTopic = Guid.NewGuid().ToString();
using (var producer = new Producer <bool, bool>(producerConfig, serdeProvider.GetSerializerGenerator <bool>(), serdeProvider.GetSerializerGenerator <bool>()))
{
producer.ProduceAsync(boolTopic, new Message <bool, bool> {
Key = true, Value = false
});
Assert.Equal(0, producer.Flush(TimeSpan.FromSeconds(10)));
}
var floatTopic = Guid.NewGuid().ToString();
using (var producer = new Producer <float, float>(producerConfig, serdeProvider.GetSerializerGenerator <float>(), serdeProvider.GetSerializerGenerator <float>()))
{
producer.ProduceAsync(floatTopic, new Message <float, float> {
Key = 44.0f, Value = 45.0f
});
Assert.Equal(0, producer.Flush(TimeSpan.FromSeconds(10)));
}
var doubleTopic = Guid.NewGuid().ToString();
using (var producer = new Producer <double, double>(producerConfig, serdeProvider.GetSerializerGenerator <double>(), serdeProvider.GetSerializerGenerator <double>()))
{
producer.ProduceAsync(doubleTopic, new Message <double, double> {
Key = 46.0, Value = 47.0
});
Assert.Equal(0, producer.Flush(TimeSpan.FromSeconds(10)));
}
var nullTopic = Guid.NewGuid().ToString();
using (var producer = new Producer <Null, Null>(producerConfig, serdeProvider.GetSerializerGenerator <Null>(), serdeProvider.GetSerializerGenerator <Null>()))
{
producer.ProduceAsync(nullTopic, new Message <Null, Null>());
Assert.Equal(0, producer.Flush(TimeSpan.FromSeconds(10)));
}
using (var consumer = new Consumer <string, string>(consumerConfig, serdeProvider.GetDeserializerGenerator <string>(), serdeProvider.GetDeserializerGenerator <string>()))
{
consumer.Assign(new List <TopicPartitionOffset> {
new TopicPartitionOffset(stringTopic, 0, 0)
});
var result = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.Equal("hello", result.Message.Key);
Assert.Equal("world", result.Message.Value);
}
using (var consumer = new Consumer <byte[], byte[]>(consumerConfig, serdeProvider.GetDeserializerGenerator <byte[]>(), serdeProvider.GetDeserializerGenerator <byte[]>()))
{
consumer.Assign(new List <TopicPartitionOffset> {
new TopicPartitionOffset(bytesTopic, 0, 0)
});
var result = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.Equal(new byte[] { 1, 4, 11 }, result.Message.Key);
Assert.Equal(new byte[] { }, result.Message.Value);
}
using (var consumer = new Consumer <int, int>(consumerConfig, serdeProvider.GetDeserializerGenerator <int>(), serdeProvider.GetDeserializerGenerator <int>()))
{
consumer.Assign(new List <TopicPartitionOffset> {
new TopicPartitionOffset(intTopic, 0, 0)
});
var result = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.Equal(42, result.Message.Key);
Assert.Equal(43, result.Message.Value);
}
using (var consumer = new Consumer <long, long>(consumerConfig, serdeProvider.GetDeserializerGenerator <long>(), serdeProvider.GetDeserializerGenerator <long>()))
{
consumer.Assign(new List <TopicPartitionOffset> {
new TopicPartitionOffset(longTopic, 0, 0)
});
var result = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.Equal(-32, result.Message.Key);
Assert.Equal(-33, result.Message.Value);
}
using (var consumer = new Consumer <bool, bool>(consumerConfig, serdeProvider.GetDeserializerGenerator <bool>(), serdeProvider.GetDeserializerGenerator <bool>()))
{
consumer.Assign(new List <TopicPartitionOffset> {
new TopicPartitionOffset(boolTopic, 0, 0)
});
var result = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.True(result.Message.Key);
Assert.False(result.Message.Value);
}
using (var consumer = new Consumer <float, float>(consumerConfig, serdeProvider.GetDeserializerGenerator <float>(), serdeProvider.GetDeserializerGenerator <float>()))
{
consumer.Assign(new List <TopicPartitionOffset> {
new TopicPartitionOffset(floatTopic, 0, 0)
});
var result = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.Equal(44.0f, result.Message.Key);
Assert.Equal(45.0f, result.Message.Value);
}
using (var consumer = new Consumer <double, double>(consumerConfig, serdeProvider.GetDeserializerGenerator <double>(), serdeProvider.GetDeserializerGenerator <double>()))
{
consumer.Assign(new List <TopicPartitionOffset> {
new TopicPartitionOffset(doubleTopic, 0, 0)
});
var result = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.Equal(46.0, result.Message.Key);
Assert.Equal(47.0, result.Message.Value);
}
using (var consumer = new Consumer <Null, Null>(consumerConfig, serdeProvider.GetDeserializerGenerator <Null>(), serdeProvider.GetDeserializerGenerator <Null>()))
{
consumer.Assign(new List <TopicPartitionOffset> {
new TopicPartitionOffset(nullTopic, 0, 0)
});
var result = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.Null(result.Key);
Assert.Null(result.Value);
}
}
}
