public static async Task PublishEvent(DataObjectMapping dataObjectMapping, SchemaRegistryConfig schemaRegistryConfig, ClientConfig kafkaConfig, string[] topics)
{
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
{
//using (var producer = new Producer<string, string>(config, new StringSerializer(Encoding.UTF8), new StringSerializer(Encoding.UTF8)))
// Produce events to the topic
var producer = new ProducerBuilder <string, DataObjectMapping>(kafkaConfig)
.SetValueSerializer(new JsonSerializer <DataObjectMapping>(schemaRegistry, new JsonSerializerConfig {
BufferBytes = 100
}))
.Build();
var localMessage = new Message <string, DataObjectMapping>();
localMessage.Key = "DataObjectMapping";
localMessage.Value = dataObjectMapping;
// Synchronous producer, does not work with Json serialisation
//producer.Produce(topics[0], localMessage, SyncHandler);
// Create asynchronous task (and wait for it)
var delivery = producer.ProduceAsync(topics[0], localMessage);
await delivery.ContinueWith(AsyncHandlerDataObjectMapping);
producer.Flush(TimeSpan.FromSeconds(10));
}
return;
}
public TopicConsumer(IServiceProvider serviceProvider, IServiceCollection services)
{
_serviceProvider = serviceProvider;
_services = services;
_consumerConfig = serviceProvider.GetRequiredService <ConsumerConfig>();
_schemaRegistryConfig = serviceProvider.GetRequiredService <SchemaRegistryConfig>();
}
public static void FillTheCache(Config config)
{
const int capacity = 16;
const string testSchema =
"{\"type\":\"record\",\"name\":\"User\",\"namespace\":\"Confluent.Kafka.Examples.AvroSpecific" +
"\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"favorite_number\",\"type\":[\"i" +
"nt\",\"null\"]},{\"name\":\"favorite_color\",\"type\":[\"string\",\"null\"]}]}";
var srConfig = new SchemaRegistryConfig
{
Url = config.Server,
RequestTimeoutMs = 3000,
MaxCachedSchemas = capacity
};
var sr = new CachedSchemaRegistryClient(srConfig);
var registerCount = capacity + 10;
var subjects = new List <string>();
var ids = new List <int>();
// test is that this does not throw. Also, inspect in debugger.
for (int i = 0; i < registerCount; ++i)
{
var topicName = Guid.NewGuid().ToString();
var subject = sr.ConstructValueSubjectName(topicName);
subjects.Add(subject);
var id = sr.RegisterSchemaAsync(subject, testSchema).Result;
ids.Add(id);
}
}
private static async Task Run_Producer <TKey, TValue>(string brokerList, string topicName,
SecurityProtocol securityProtocol, SaslMechanism saslMechanism, string saslUsername, string saslPassword,
string schemaRegistryUrl, string basicAuthUserInfo, Message <Ignore, TValue> consumerMessage)
where TKey : class, IMessage <TKey>, new()
where TValue : class, IMessage <TValue>, new()
{
var message = CreateMessage <TKey, TValue>(consumerMessage.Value);
var config = new ProducerConfig
{
BootstrapServers = brokerList,
};
var schemaRegistryConfig = new SchemaRegistryConfig
{
Url = schemaRegistryUrl,
};
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
using (var producer = new ProducerBuilder <TKey, TValue>(config)
.SetKeySerializer(new ProtobufSerializer <TKey>(schemaRegistry))
.SetValueSerializer(new ProtobufSerializer <TValue>(schemaRegistry))
.Build())
{
try
{
var deliveryReport = await producer.ProduceAsync(topicName, message);
Console.WriteLine($"delivered to: {deliveryReport.TopicPartitionOffset} for producer: {producer.Name}");
}
catch (ProduceException <int, TValue> e)
{
Console.WriteLine($"failed to deliver message: {e.Message} [{e.Error.Code}]");
}
}
}
public static IProducer <TKey, TValue> CreateProducer <TKey, TValue>(
BrokerOptions brokerOptions, ILogger logger)
where TKey : class, IMessage <TKey>, new()
where TValue : class, IMessage <TValue>, new()
{
var config = new ProducerConfig
{
BootstrapServers = brokerOptions.Brokers,
// EnableIdempotence = true,
// MessageSendMaxRetries = 10000000,
// SecurityProtocol = brokerOptions.SecurityProtocol,
// SaslMechanism = brokerOptions.SaslMechanism,
// SaslUsername = brokerOptions.SaslUsername,
// SaslPassword = brokerOptions.SaslPassword,
};
var schemaRegistryConfig = new SchemaRegistryConfig
{
Url = brokerOptions.SchemaRegistryUrl,
// BasicAuthCredentialsSource = AuthCredentialsSource.UserInfo,
// BasicAuthUserInfo = brokerOptions.SchemaRegistryAuth,
};
var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig);
var producer = new ProducerBuilder <TKey, TValue>(config)
.SetKeySerializer(new ProtobufSerializer <TKey>(schemaRegistry))
.SetValueSerializer(new ProtobufSerializer <TValue>(schemaRegistry))
.Build();
return(producer);
}
private void SetupProducer()
{
if (producer != null)
{
return;
}
lock (myLock)
{
Dispose();
}
var config = new ProducerConfig {
BootstrapServers = kafkaOptions.BootstrapServerUri
};
var schemaRegistryConfig = new SchemaRegistryConfig {
Url = kafkaOptions.SchemaRegistryUri
};
var jsonSerializerConfig = new JsonSerializerConfig();
schemaRegistryClient = new CachedSchemaRegistryClient(schemaRegistryConfig);
producer = new ProducerBuilder <Null, T>(config)
.SetValueSerializer(new JsonSerializer <T>(schemaRegistryClient, jsonSerializerConfig))
.Build();
}
static void Main(string[] args)
{
var(logger, config) = Bootstrap();
var schemaRegistryConfig = new SchemaRegistryConfig
{
Url = config["SCHEMA_REGISTRY_URL"],
};
StartProducing(logger, config, schemaRegistryConfig);
var consumerFactory = new ConsumerFactory <string, Record>(
logger,
new ProtobufDeserializer <Record>(),
KafkaOptions.ForConsumer(config));
var consumer = consumerFactory.CreateConsumer("Group-1");
var consumer2 = consumerFactory.CreateConsumer("Group-2");
var eventListener = new EventListener(consumer, PERSON_TOPIC, logger);
var eventListener2 = new EventListener(consumer2, PERSON_TOPIC, logger);
var personEventHandler = new PersonEventHandler();
var addressEventHandler = new AddressEventHandler();
var handlers = new List <IEventHandler> {
personEventHandler, addressEventHandler
};
var eventProcessor = new EventProcessor(eventListener, handlers, logger);
var eventProcessor2 = new EventProcessor(eventListener2, handlers, logger);
eventProcessor.StartProcessing(CreateCancellation().Token).GetAwaiter().GetResult();
eventProcessor2.StartProcessing(CreateCancellation().Token).GetAwaiter().GetResult();
}
internal SchemaRegistryConfig GetConfig(ISchemaRegistryConfig config)
{
var c = new SchemaRegistryConfig
{
Url = config.SchemaRegistryUrl
};
if (config.SchemaRegistryMaxCachedSchemas.HasValue)
{
c.MaxCachedSchemas = config.SchemaRegistryMaxCachedSchemas;
}
if (config.SchemaRegistryRequestTimeoutMs.HasValue)
{
c.RequestTimeoutMs = config.SchemaRegistryRequestTimeoutMs;
}
if (!string.IsNullOrEmpty(config.BasicAuthUserInfo))
{
c.BasicAuthUserInfo = config.BasicAuthUserInfo;
}
if (config.BasicAuthCredentialsSource.HasValue)
{
c.BasicAuthCredentialsSource = (AuthCredentialsSource)config.BasicAuthCredentialsSource.Value;
}
return(c);
}
public static void ProduceConsumeSchemaManyMessagesProtobuf(string bootstrapServers, string schemaRegistryServers)
{
var producerConfig = new ProducerConfig {
BootstrapServers = bootstrapServers
};
var schemaRegistryConfig = new SchemaRegistryConfig {
Url = schemaRegistryServers
};
using (var topic = new TemporaryTopic(bootstrapServers, 1))
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
using (var producer =
new ProducerBuilder <string, Msg230>(producerConfig)
.SetValueSerializer(new ProtobufSerializer <Msg230>(schemaRegistry))
.Build())
{
var u = new Msg230();
u.Value = 41;
producer.ProduceAsync(topic.Name, new Message <string, Msg230> {
Key = "test1", Value = u
}).Wait();
var consumerConfig = new ConsumerConfig
{
BootstrapServers = bootstrapServers,
GroupId = Guid.NewGuid().ToString(),
AutoOffsetReset = AutoOffsetReset.Earliest
};
// Test the protobuf deserializer can read this message
using (var consumer =
new ConsumerBuilder <string, UInt32Value>(consumerConfig)
.SetValueDeserializer(new ProtobufDeserializer <UInt32Value>().AsSyncOverAsync())
.Build())
{
consumer.Subscribe(topic.Name);
var cr = consumer.Consume();
Assert.Equal(u.Value, cr.Message.Value.Value);
}
// Check the pre-data bytes are as expected.
using (var consumer = new ConsumerBuilder <string, byte[]>(consumerConfig).Build())
{
consumer.Subscribe(topic.Name);
var cr = consumer.Consume();
// magic byte + schema id + expected array index length + at least one data byte.
Assert.True(cr.Message.Value.Length >= 1 + 4 + 1 + 2 + 1);
// magic byte
Assert.Equal(0, cr.Message.Value[0]);
// index array length
Assert.Equal(1, cr.Message.Value[5]);
// there are 231 messages in the schema. message 230 has index 230. varint is 2 bytes:
// in binary: 11100110.
// -> &7f |80 -> 11100110 = 230
Assert.Equal(230, cr.Message.Value[6]);
// >>7 -> 00000001
Assert.Equal(1, cr.Message.Value[7]);
}
}
}
private static IProducer <string, Payment> AddProducerPayment(IConfiguration configuration)
{
var config = new ProducerConfig
{
BootstrapServers = configuration["Kafka:Servers"],
ClientId = configuration["Kafka:ClientId"] + "-" + Dns.GetHostName(),
Acks = Acks.All,
SecurityProtocol = SecurityProtocol.SaslSsl,
SaslMechanism = SaslMechanism.Plain,
SaslUsername = configuration["Kafka:Username"],
SaslPassword = configuration["Kafka:Password"]
};
var schemaConfig = new SchemaRegistryConfig
{
Url = configuration["SchemaRegistry:Url"],
BasicAuthUserInfo = configuration["SchemaRegistry:UsernamePassword"]
};
var schemaRegistry = new CachedSchemaRegistryClient(schemaConfig);
return(new ProducerBuilder <string, Payment>(config)
//.SetValueSerializer(new AnimaJsonSerializer<Payment>())
.SetValueSerializer(new AvroSerializer <Payment>(schemaRegistry).AsSyncOverAsync())
.Build());
}
public KafkaBackgroundReceiver(Microsoft.AspNetCore.SignalR.IHubContext <ImageMarshallingHub> imageHub)
{
// build the kafka consumer
_imageHub = imageHub;
var conf = new ConsumerConfig
{
GroupId = "test-consumer-group",
BootstrapServers = "kafka-server1:9092",
// Note: The AutoOffsetReset property detemines the start offset in the event
// there are not yet any committed offsets for the consumer group for the
// topic/partitions of interest. By default, offsets are committed
// automatically, so in this example, consumption will only start from the
// earliest message in the topic 'my-topic' the first time you run the program.
AutoOffsetReset = AutoOffsetReset.Earliest
};
var schemaRegistryConfig = new SchemaRegistryConfig
{
Url = "kafka-schema-registry:8081"
};
var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig);
//_consumer = new ConsumerBuilder<Ignore, string>(conf).Build();
_consumer = new ConsumerBuilder <Ignore, imageResponse>(conf).SetValueDeserializer(new AvroDeserializer <imageResponse>(schemaRegistry).AsSyncOverAsync <imageResponse>()).Build();
_consumer.Subscribe("imageResponse");
}
public KafkaConnection(ProducerConfig producerConfig, ConsumerConfig consumerConfig,
SchemaRegistryConfig schemaRegistryConfig, AvroSerializerConfig avroSerializerConfig)
{
this._producerConfiguration = producerConfig ?? throw new ArgumentNullException(nameof(producerConfig));
this._consumerConfiguration = consumerConfig ?? throw new ArgumentNullException(nameof(consumerConfig));
this._schemaRegistryConfiguration = schemaRegistryConfig ?? throw new ArgumentNullException(nameof(schemaRegistryConfig));
this._avroSerializerConfiguration = avroSerializerConfig ?? throw new ArgumentNullException(nameof(avroSerializerConfig));
}
static async Task Main(string[] args)
{
var config = new ProducerConfig()
{
BootstrapServers = "192.168.99.109:9092"
};
var schemaRegistryConfig = new SchemaRegistryConfig
{
Url = "192.168.99.109:8081",
// Note: you can specify more than one schema registry url using the
// schema.registry.url property for redundancy (comma separated list).
// The property name is not plural to follow the convention set by
// the Java implementation.
// optional schema registry client properties:
RequestTimeoutMs = 5000,
MaxCachedSchemas = 10
};
// If serializers are not specified, default serializers from
// `Confluent.Kafka.Serializers` will be automatically used where
// available. Note: by default strings are encoded as UTF8.
using var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig);
using var p = new ProducerBuilder <MessageHeader, NewCustomerTopic>(config)
.SetKeySerializer(new AvroSerializer <MessageHeader>(schemaRegistry))
.SetValueSerializer(new AvroSerializer <NewCustomerTopic>(schemaRegistry))
.Build();
try
{
Parallel.ForEach(Enumerable.Range(1, 100), i => {
var dr = p.ProduceAsync("new-customer-topic2", new Message <MessageHeader, NewCustomerTopic>
{
Key = new MessageHeader
{
AppOrigin = "DemoKafkaClient",
CurrentUser = "******",
Timestamp = DateTime.Now.ToString("s"),
Token = Guid.NewGuid().ToString()
},
Value = new NewCustomerTopic
{
FirstName = "Customer " + i,
LastName = "Asyik"
}
}).Result;
Console.WriteLine($"Delivered '{dr.Value}' to '{dr.TopicPartitionOffset}'");
});
p.Flush();
}
catch (ProduceException <Null, string> e)
{
Console.WriteLine($"Delivery failed: {e.Error.Reason}");
}
}
public static void ProduceConsumeNestedProtobuf(string bootstrapServers, string schemaRegistryServers)
{
var producerConfig = new ProducerConfig {
BootstrapServers = bootstrapServers
};
var schemaRegistryConfig = new SchemaRegistryConfig {
Url = schemaRegistryServers
};
using (var topic = new TemporaryTopic(bootstrapServers, 1))
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
using (var producer =
new ProducerBuilder <string, NestedOuter.Types.NestedMid2.Types.NestedLower>(producerConfig)
.SetValueSerializer(new ProtobufSerializer <NestedOuter.Types.NestedMid2.Types.NestedLower>(schemaRegistry))
.Build())
{
var u = new NestedOuter.Types.NestedMid2.Types.NestedLower();
u.Field2 = "field_2_value";
producer.ProduceAsync(topic.Name, new Message <string, NestedOuter.Types.NestedMid2.Types.NestedLower> {
Key = "test1", Value = u
}).Wait();
var consumerConfig = new ConsumerConfig
{
BootstrapServers = bootstrapServers,
GroupId = Guid.NewGuid().ToString(),
AutoOffsetReset = AutoOffsetReset.Earliest
};
// Test the protobuf deserializer can read this message
using (var consumer =
new ConsumerBuilder <string, NestedOuter.Types.NestedMid2.Types.NestedLower>(consumerConfig)
.SetValueDeserializer(new ProtobufDeserializer <NestedOuter.Types.NestedMid2.Types.NestedLower>()
.AsSyncOverAsync())
.Build())
{
consumer.Subscribe(topic.Name);
var cr = consumer.Consume();
Assert.Equal(u.Field2, cr.Message.Value.Field2);
}
// Check the pre-data bytes are as expected.
using (var consumer = new ConsumerBuilder <string, byte[]>(consumerConfig).Build())
{
consumer.Subscribe(topic.Name);
var cr = consumer.Consume();
// magic byte + schema id + expected array index length + at least one data byte.
Assert.True(cr.Message.Value.Length > 1 + 4 + 4 + 1);
// magic byte
Assert.Equal(0, cr.Message.Value[0]);
// index array.
Assert.Equal(3, cr.Message.Value[5]);
Assert.Equal(2, cr.Message.Value[6]);
Assert.Equal(1, cr.Message.Value[7]);
Assert.Equal(0, cr.Message.Value[8]);
}
}
}
public static ISchemaRegistryClient Create()
{
var schemaConfig = new SchemaRegistryConfig()
{
Url = "http://localhost:8081"
};
return(new CachedSchemaRegistryClient(schemaConfig));
}
private static ISchemaRegistryClient CreateSchemaRegistryClient()
{
var schemaRegistryConfig = new SchemaRegistryConfig
{
Url = "http://schema-registry:8081/",
};
return(new CachedSchemaRegistryClient(schemaRegistryConfig));
}
public EventFactory(IOptions <KafkaProducerConfiguration> kafkaConfigOptions)
{
var kafkaConfiguration = kafkaConfigOptions.Value;
_schemaRegistryConfig = new SchemaRegistryConfig
{
Url = kafkaConfiguration.AvroSchemaRegistryUrls
};
}
public void InvalidSubjectNameStrategy()
{
var config = new SchemaRegistryConfig {
Url = "irrelevanthost:8081"
};
config.Set(SchemaRegistryConfig.PropertyNames.SchemaRegistryKeySubjectNameStrategy, "bad_value");
Assert.Throws <ArgumentException>(() => new CachedSchemaRegistryClient(config));
}
public void ConstructKeySubjectName_Topic1()
{
var config = new SchemaRegistryConfig {
Url = "irrelevanthost:8081"
};
var src = new CachedSchemaRegistryClient(config);
Assert.Equal("mytopic-key", src.ConstructKeySubjectName("mytopic", "myschemaname"));
}
static void Main(string[] args)
{
string bootstrapServers = "127.0.0.1:9092";
string schemaRegistryUrl = "127.0.0.1:8081";
string topicName = "enhanced-end-of-cure";
var consumerConfig = new ConsumerConfig
{
BootstrapServers = bootstrapServers,
GroupId = "console-group"
};
var schemaRegistryConfig = new SchemaRegistryConfig
{
Url = schemaRegistryUrl
};
var avroSerializerConfig = new AvroSerializerConfig
{
};
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
using (var consumer = new ConsumerBuilder <string, EnhancedEndOfCure>(consumerConfig)
.SetKeyDeserializer(new AvroDeserializer <string>(schemaRegistry, avroSerializerConfig).AsSyncOverAsync())
.SetValueDeserializer(new AvroDeserializer <EnhancedEndOfCure>(schemaRegistry, avroSerializerConfig).AsSyncOverAsync())
.Build())
{
consumer.Subscribe(topicName);
try
{
CancellationTokenSource cts = new CancellationTokenSource();
while (true)
{
try
{
var consumeResult = consumer.Consume(cts.Token);
var v = (EnhancedEndOfCure)consumeResult.Message.Value;
Console.WriteLine($"End Of cure : \n\r" +
" - Date : " + v.Date +
" - ItemCode : " + v.ItemCode +
" - CureEquipmentId : " + v.CureEquipmentId +
" - ShiftCode : " + v.ShiftCode);
}
catch (ConsumeException e)
{
Console.WriteLine($"Consume error: {e.Error.Reason}");
}
}
}
catch (OperationCanceledException)
{
consumer.Close();
}
}
}
public void SSLWithNoCertificateProvided()
{
var config = new SchemaRegistryConfig
{
Url = "irrelevanthost:8081",
SchemaRegistrySecurityProtocol = "SSL",
};
Assert.Throws <ArgumentException>(() => new CachedSchemaRegistryClient(config));
}
public void ConstructValueSubjectName()
{
var config = new SchemaRegistryConfig {
SchemaRegistryUrl = "irrelevanthost:8081"
};
CachedSchemaRegistryClient src = new CachedSchemaRegistryClient(config);
Assert.Equal("mytopic-value", src.ConstructValueSubjectName("mytopic"));
}
/// <summary>
/// Configures schema registry to the cluster
/// </summary>
/// <param name="cluster">Instance of <see cref="IClusterConfigurationBuilder"/></param>
/// <param name="handler">A handler to set the configuration values</param>
/// <returns></returns>
public static IClusterConfigurationBuilder WithSchemaRegistry(
this IClusterConfigurationBuilder cluster,
Action <SchemaRegistryConfig> handler)
{
var config = new SchemaRegistryConfig();
handler(config);
cluster.DependencyConfigurator.AddSingleton <ISchemaRegistryClient>(_ => new CachedSchemaRegistryClient(config));
return(cluster);
}
static void Main(string[] args)
{
Action <DeliveryReport <Null, Cliente> > handler =
r => Console.WriteLine(!r.Error.IsError ? $"Delivered message to {r.TopicPartitionOffset}" : $"Delivery Error: {r.Error.Reason}");
var schemaRegistryConfig = new SchemaRegistryConfig
{
Url = "https://schema-registry:8181",
RequestTimeoutMs = 5000,
MaxCachedSchemas = 10,
SslCaLocation = "C:/Users/raul/workspace/kafka-ssl-compose/secrets/CAroot.pem",
SslKeystoreLocation = "C:/Users/raul/workspace/kafka-ssl-compose/secrets/schema-registry.keystore.jks",
SslKeystorePassword = "******"
};
var config = new ProducerConfig
{
BootstrapServers = "kafka-ssl:9092",
ClientId = "my_producer",
SecurityProtocol = SecurityProtocol.Ssl,
SslCaLocation = "C:/Users/raul/workspace/kafka-ssl-compose/secrets/CAroot.pem",
SslKeystoreLocation = "C:/Users/raul/workspace/kafka-ssl-compose/secrets/producer.keystore.jks",
SslKeystorePassword = "******",
Debug = "security",
CompressionType = CompressionType.Gzip,
EnableIdempotence = true,
BatchNumMessages = 16000,
LingerMs = 5,
Acks = Acks.All,
};
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
using (var producer = new ProducerBuilder <Null, Cliente>(config)
.SetValueSerializer(new AvroSerializer <Cliente>(schemaRegistry, new AvroSerializerConfig {
AutoRegisterSchemas = false
}).AsSyncOverAsync())
.Build())
{
for (int i = 0; i < 5; ++i)
{
Cliente cliente = new Cliente {
clienteId = i + "", name = "John"
};
producer.Produce("client", new Message <Null, Cliente> {
Value = cliente
}, handler);
Console.WriteLine("Sending message");
producer.Flush(TimeSpan.FromSeconds(10));
}
}
}
/// <summary>
/// Main method for console app.
/// </summary>
/// <param name="args">No arguments used.</param>
public static void Main(string[] args)
{
Console.WriteLine("Starting .net Avro consumer.");
var consumerConfig = new ConsumerConfig
{
BootstrapServers = "kafka:9092",
GroupId = "csharp-consumer-avro",
AutoOffsetReset = AutoOffsetReset.Earliest,
PluginLibraryPaths = "monitoring-interceptor",
};
// Configure schema registry location
var schemaRegistryConfig = new SchemaRegistryConfig {
Url = "http://schema-registry:8081"
};
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
using (var consumer = new ConsumerBuilder <string, PositionValue>(consumerConfig)
.SetValueDeserializer(new AvroDeserializer <PositionValue>(schemaRegistry).AsSyncOverAsync())
.Build())
{
// Subscribe to our topic
consumer.Subscribe(KafkaTopic);
CancellationTokenSource cts = new CancellationTokenSource();
Console.CancelKeyPress += (_, e) =>
{
e.Cancel = true; // prevent the process from terminating.
cts.Cancel();
};
try
{
while (true)
{
try
{
// Poll for available records
var cr = consumer.Consume(cts.Token);
Console.WriteLine($"Key:{cr.Message.Key} Latitude:{cr.Message.Value.latitude} Longitude:{cr.Message.Value.longitude} [partition {cr.Partition.Value}]");
}
catch (ConsumeException e)
{
Console.WriteLine($"Error occured: {e.Error.Reason}");
}
}
}
catch (OperationCanceledException)
{
// Clean up when the application exits
Console.WriteLine("Closing consumer.");
consumer.Close();
}
}
}
static void Main(string[] args)
{
var topicName = "Motion";
var schemaRegistryConfig = new SchemaRegistryConfig
{
Url = "localhost:8081"
};
var consumerConfig = new ConsumerConfig
{
BootstrapServers = "localhost:9092",
GroupId = "comsumer-group"
};
var avroSerializerConfig = new AvroSerializerConfig
{
// optional Avro serializer properties:
BufferBytes = 100
};
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
using (var consumer =
new ConsumerBuilder <string, MotionEvent>(consumerConfig)
.SetKeyDeserializer(new AvroDeserializer <string>(schemaRegistry).AsSyncOverAsync())
.SetValueDeserializer(new AvroDeserializer <MotionEvent>(schemaRegistry).AsSyncOverAsync())
.SetErrorHandler((_, e) => Console.WriteLine($"Error: {e.Reason}"))
.Build())
{
CancellationTokenSource cts = new CancellationTokenSource();
consumer.Subscribe(topicName);
try
{
while (true)
{
try
{
var consumeResult = consumer.Consume(cts.Token);
Console.WriteLine($"Consumed id={consumeResult.Message.Value.id}, timestamp= {consumeResult.Message.Value.timestamp}, camera={consumeResult.Message.Value.camera}.");
}
catch (ConsumeException e)
{
Console.WriteLine($"Consume error: {e.Error.Reason}");
}
}
}
catch (OperationCanceledException)
{
consumer.Close();
}
}
}
public static ISerializer <AuditEvent> GetAuditSerializer()
{
var schemaRegistryConfig = new SchemaRegistryConfig
{
Url = schemaRegistryURL,
BasicAuthUserInfo = "CUXIJ3GC5X4TPREW:aX7pqhnvDl8heSJRD1tDXMJAxVjTzdoGoXYe8a4tBCW6hqbwLDCxL54xCpNGfJEU"
};
var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig);
return(new AvroSerializer <AuditEvent>(schemaRegistry).AsSyncOverAsync());
}
public void SSLWithCertificateNotFound()
{
var config = new SchemaRegistryConfig
{
Url = "irrelevanthost:8081",
SchemaRegistrySecurityProtocol = "SSL",
SchemaRegistryPfx = "pfxCertificate-notfound.pfx"
};
Assert.Throws <ArgumentException>(() => new CachedSchemaRegistryClient(config));
}
public void ConstructKeySubjectName_TopicRecord()
{
var config = new SchemaRegistryConfig
{
Url = "irrelevanthost:8081",
KeySubjectNameStrategy = SubjectNameStrategy.TopicRecord
};
var src = new CachedSchemaRegistryClient(config);
Assert.Equal("mytopic-myschemaname", src.ConstructKeySubjectName("mytopic", "myschemaname"));
}
public static void ProduceConsumeBasicProtobuf(string bootstrapServers, string schemaRegistryServers)
{
var producerConfig = new ProducerConfig {
BootstrapServers = bootstrapServers
};
var schemaRegistryConfig = new SchemaRegistryConfig {
Url = schemaRegistryServers
};
using (var topic = new TemporaryTopic(bootstrapServers, 1))
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
using (var producer =
new ProducerBuilder <string, UInt32Value>(producerConfig)
.SetValueSerializer(new ProtobufSerializer <UInt32Value>(schemaRegistry))
.Build())
{
var u = new UInt32Value();
u.Value = 42;
producer.ProduceAsync(topic.Name, new Message <string, UInt32Value> {
Key = "test1", Value = u
}).Wait();
var consumerConfig = new ConsumerConfig
{
BootstrapServers = bootstrapServers,
GroupId = Guid.NewGuid().ToString(),
AutoOffsetReset = AutoOffsetReset.Earliest
};
// Test the protobuf deserializer can read this message
using (var consumer =
new ConsumerBuilder <string, UInt32Value>(consumerConfig)
.SetValueDeserializer(new ProtobufDeserializer <UInt32Value>().AsSyncOverAsync())
.Build())
{
consumer.Subscribe(topic.Name);
var cr = consumer.Consume();
Assert.Equal(u.Value, cr.Message.Value.Value);
}
// Check the pre-data bytes are as expected.
using (var consumer = new ConsumerBuilder <string, byte[]>(consumerConfig).Build())
{
consumer.Subscribe(topic.Name);
var cr = consumer.Consume();
// magic byte + schema id + expected array index length + at least one data byte.
Assert.True(cr.Message.Value.Length >= 1 + 4 + 1 + 1);
// magic byte
Assert.Equal(0, cr.Message.Value[0]);
// array index (special value as an optimization)
Assert.Equal(0, cr.Message.Value[5]);
}
}
}
