public static void ProduceConsumeSchemaManyMessagesProtobuf(string bootstrapServers, string schemaRegistryServers)
{
var producerConfig = new ProducerConfig {
BootstrapServers = bootstrapServers
};
var schemaRegistryConfig = new SchemaRegistryConfig {
Url = schemaRegistryServers
};
using (var topic = new TemporaryTopic(bootstrapServers, 1))
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
using (var producer =
new ProducerBuilder <string, Msg230>(producerConfig)
.SetValueSerializer(new ProtobufSerializer <Msg230>(schemaRegistry))
.Build())
{
var u = new Msg230();
u.Value = 41;
producer.ProduceAsync(topic.Name, new Message <string, Msg230> {
Key = "test1", Value = u
}).Wait();
var consumerConfig = new ConsumerConfig
{
BootstrapServers = bootstrapServers,
GroupId = Guid.NewGuid().ToString(),
AutoOffsetReset = AutoOffsetReset.Earliest
};
// Test the protobuf deserializer can read this message
using (var consumer =
new ConsumerBuilder <string, UInt32Value>(consumerConfig)
.SetValueDeserializer(new ProtobufDeserializer <UInt32Value>().AsSyncOverAsync())
.Build())
{
consumer.Subscribe(topic.Name);
var cr = consumer.Consume();
Assert.Equal(u.Value, cr.Message.Value.Value);
}
// Check the pre-data bytes are as expected.
using (var consumer = new ConsumerBuilder <string, byte[]>(consumerConfig).Build())
{
consumer.Subscribe(topic.Name);
var cr = consumer.Consume();
// magic byte + schema id + expected array index length + at least one data byte.
Assert.True(cr.Message.Value.Length >= 1 + 4 + 1 + 2 + 1);
// magic byte
Assert.Equal(0, cr.Message.Value[0]);
// index array length
Assert.Equal(1, cr.Message.Value[5]);
// there are 231 messages in the schema. message 230 has index 230. varint is 2 bytes:
// in binary: 11100110.
// -> &7f |80 -> 11100110 = 230
Assert.Equal(230, cr.Message.Value[6]);
// >>7 -> 00000001
Assert.Equal(1, cr.Message.Value[7]);
}
}
}
public static void ProduceConsumeNestedProtobuf(string bootstrapServers, string schemaRegistryServers)
{
var producerConfig = new ProducerConfig {
BootstrapServers = bootstrapServers
};
var schemaRegistryConfig = new SchemaRegistryConfig {
Url = schemaRegistryServers
};
using (var topic = new TemporaryTopic(bootstrapServers, 1))
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
using (var producer =
new ProducerBuilder <string, NestedOuter.Types.NestedMid2.Types.NestedLower>(producerConfig)
.SetValueSerializer(new ProtobufSerializer <NestedOuter.Types.NestedMid2.Types.NestedLower>(schemaRegistry))
.Build())
{
var u = new NestedOuter.Types.NestedMid2.Types.NestedLower();
u.Field2 = "field_2_value";
producer.ProduceAsync(topic.Name, new Message <string, NestedOuter.Types.NestedMid2.Types.NestedLower> {
Key = "test1", Value = u
}).Wait();
var consumerConfig = new ConsumerConfig
{
BootstrapServers = bootstrapServers,
GroupId = Guid.NewGuid().ToString(),
AutoOffsetReset = AutoOffsetReset.Earliest
};
// Test the protobuf deserializer can read this message
using (var consumer =
new ConsumerBuilder <string, NestedOuter.Types.NestedMid2.Types.NestedLower>(consumerConfig)
.SetValueDeserializer(new ProtobufDeserializer <NestedOuter.Types.NestedMid2.Types.NestedLower>()
.AsSyncOverAsync())
.Build())
{
consumer.Subscribe(topic.Name);
var cr = consumer.Consume();
Assert.Equal(u.Field2, cr.Message.Value.Field2);
}
// Check the pre-data bytes are as expected.
using (var consumer = new ConsumerBuilder <string, byte[]>(consumerConfig).Build())
{
consumer.Subscribe(topic.Name);
var cr = consumer.Consume();
// magic byte + schema id + expected array index length + at least one data byte.
Assert.True(cr.Message.Value.Length > 1 + 4 + 4 + 1);
// magic byte
Assert.Equal(0, cr.Message.Value[0]);
// index array.
Assert.Equal(3, cr.Message.Value[5]);
Assert.Equal(2, cr.Message.Value[6]);
Assert.Equal(1, cr.Message.Value[7]);
Assert.Equal(0, cr.Message.Value[8]);
}
}
}
public static void ProduceConsumeBasicProtobuf(string bootstrapServers, string schemaRegistryServers)
{
var producerConfig = new ProducerConfig {
BootstrapServers = bootstrapServers
};
var schemaRegistryConfig = new SchemaRegistryConfig {
Url = schemaRegistryServers
};
using (var topic = new TemporaryTopic(bootstrapServers, 1))
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
using (var producer =
new ProducerBuilder <string, UInt32Value>(producerConfig)
.SetValueSerializer(new ProtobufSerializer <UInt32Value>(schemaRegistry))
.Build())
{
var u = new UInt32Value();
u.Value = 42;
producer.ProduceAsync(topic.Name, new Message <string, UInt32Value> {
Key = "test1", Value = u
}).Wait();
var consumerConfig = new ConsumerConfig
{
BootstrapServers = bootstrapServers,
GroupId = Guid.NewGuid().ToString(),
AutoOffsetReset = AutoOffsetReset.Earliest
};
// Test the protobuf deserializer can read this message
using (var consumer =
new ConsumerBuilder <string, UInt32Value>(consumerConfig)
.SetValueDeserializer(new ProtobufDeserializer <UInt32Value>().AsSyncOverAsync())
.Build())
{
consumer.Subscribe(topic.Name);
var cr = consumer.Consume();
Assert.Equal(u.Value, cr.Message.Value.Value);
}
// Check the pre-data bytes are as expected.
using (var consumer = new ConsumerBuilder <string, byte[]>(consumerConfig).Build())
{
consumer.Subscribe(topic.Name);
var cr = consumer.Consume();
// magic byte + schema id + expected array index length + at least one data byte.
Assert.True(cr.Message.Value.Length >= 1 + 4 + 1 + 1);
// magic byte
Assert.Equal(0, cr.Message.Value[0]);
// array index (special value as an optimization)
Assert.Equal(0, cr.Message.Value[5]);
}
}
}
public static void ProduceConsumeNestedProtobuf(string bootstrapServers, string schemaRegistryServers)
{
var producerConfig = new ProducerConfig {
BootstrapServers = bootstrapServers
};
var schemaRegistryConfig = new SchemaRegistryConfig {
Url = schemaRegistryServers
};
using (var topic = new TemporaryTopic(bootstrapServers, 1))
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
using (var producer =
new ProducerBuilder <string, NestedOuter.Types.NestedMid2.Types.NestedLower>(producerConfig)
.SetValueSerializer(new ProtobufSerializer <NestedOuter.Types.NestedMid2.Types.NestedLower>(schemaRegistry))
.Build())
{
var u = new NestedOuter.Types.NestedMid2.Types.NestedLower();
u.Field2 = "field_2_value";
producer.ProduceAsync(topic.Name, new Message <string, NestedOuter.Types.NestedMid2.Types.NestedLower> {
Key = "test1", Value = u
}).Wait();
var consumerConfig = new ConsumerConfig
{
BootstrapServers = bootstrapServers,
GroupId = Guid.NewGuid().ToString(),
AutoOffsetReset = AutoOffsetReset.Earliest
};
// Test the protobuf deserializer can read this message
using (var consumer =
new ConsumerBuilder <string, NestedOuter.Types.NestedMid2.Types.NestedLower>(consumerConfig)
.SetValueDeserializer(new ProtobufDeserializer <NestedOuter.Types.NestedMid2.Types.NestedLower>()
.AsSyncOverAsync())
.Build())
{
consumer.Subscribe(topic.Name);
var cr = consumer.Consume();
Assert.Equal(u.Field2, cr.Message.Value.Field2);
}
}
}
public static void ProduceConsumeSchemaManyMessagesProtobuf(string bootstrapServers, string schemaRegistryServers)
{
var producerConfig = new ProducerConfig {
BootstrapServers = bootstrapServers
};
var schemaRegistryConfig = new SchemaRegistryConfig {
Url = schemaRegistryServers
};
using (var topic = new TemporaryTopic(bootstrapServers, 1))
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
using (var producer =
new ProducerBuilder <string, Msg230>(producerConfig)
.SetValueSerializer(new ProtobufSerializer <Msg230>(schemaRegistry))
.Build())
{
var u = new Msg230();
u.Value = 41;
producer.ProduceAsync(topic.Name, new Message <string, Msg230> {
Key = "test1", Value = u
}).Wait();
var consumerConfig = new ConsumerConfig
{
BootstrapServers = bootstrapServers,
GroupId = Guid.NewGuid().ToString(),
AutoOffsetReset = AutoOffsetReset.Earliest
};
// Test the protobuf deserializer can read this message
using (var consumer =
new ConsumerBuilder <string, UInt32Value>(consumerConfig)
.SetValueDeserializer(new ProtobufDeserializer <UInt32Value>().AsSyncOverAsync())
.Build())
{
consumer.Subscribe(topic.Name);
var cr = consumer.Consume();
Assert.Equal(u.Value, cr.Message.Value.Value);
}
}
}
public static void SubjectNameStrategiesProtobuf(string bootstrapServers, string schemaRegistryServers)
{
var producerConfig = new ProducerConfig {
BootstrapServers = bootstrapServers
};
var schemaRegistryConfig = new SchemaRegistryConfig {
Url = schemaRegistryServers
};
using (var topic = new TemporaryTopic(bootstrapServers, 1))
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
{
using (var producer =
new ProducerBuilder <string, UInt32Value>(producerConfig)
.SetValueSerializer(new ProtobufSerializer <UInt32Value>(schemaRegistry, new ProtobufSerializerConfig {
SubjectNameStrategy = SubjectNameStrategy.TopicRecord
}))
.Build())
{
var u = new UInt32Value();
u.Value = 42;
producer.ProduceAsync(topic.Name, new Message <string, UInt32Value> {
Key = "test1", Value = u
}).Wait();
var subjects = schemaRegistry.GetAllSubjectsAsync().Result;
Assert.Contains(topic.Name + "-UInt32Value", subjects);
Assert.DoesNotContain(topic.Name + "-value", subjects);
// May contain the record name subject from a previous test.
}
using (var producer =
new ProducerBuilder <string, UInt32Value>(producerConfig)
.SetValueSerializer(new ProtobufSerializer <UInt32Value>(schemaRegistry, new ProtobufSerializerConfig {
SubjectNameStrategy = SubjectNameStrategy.Record
}))
.Build())
{
var u = new UInt32Value();
u.Value = 42;
producer.ProduceAsync(topic.Name, new Message <string, UInt32Value> {
Key = "test1", Value = u
}).Wait();
var subjects = schemaRegistry.GetAllSubjectsAsync().Result;
// Note: If this value is in SR by any means (even if not via this test),
// it implies what is being tested here is functional.
Assert.Contains("UInt32Value", subjects);
Assert.DoesNotContain(topic.Name + "-value", subjects);
}
using (var producer =
new ProducerBuilder <string, UInt32Value>(producerConfig)
.SetValueSerializer(new ProtobufSerializer <UInt32Value>(schemaRegistry))
.Build())
{
var u = new UInt32Value();
u.Value = 42;
producer.ProduceAsync(topic.Name, new Message <string, UInt32Value> {
Key = "test1", Value = u
}).Wait();
var subjects = schemaRegistry.GetAllSubjectsAsync().Result;
Assert.Contains(topic.Name + "-value", subjects);
}
}
}
public static void AutoRegisterSchemaDisabled(string bootstrapServers, string schemaRegistryServers)
{
using (var topic = new TemporaryTopic(bootstrapServers, 1))
{
var producerConfig = new ProducerConfig
{
BootstrapServers = bootstrapServers
};
var consumerConfig = new ConsumerConfig
{
BootstrapServers = bootstrapServers,
GroupId = Guid.NewGuid().ToString(),
SessionTimeoutMs = 6000,
AutoOffsetReset = AutoOffsetReset.Earliest
};
var schemaRegistryConfig = new SchemaRegistryConfig
{
SchemaRegistryUrl = schemaRegistryServers
};
// first a quick check the value case fails.
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
using (var producer =
new ProducerBuilder <string, int>(producerConfig)
.SetKeySerializer(new AsyncAvroSerializer <string>(schemaRegistry))
.SetValueSerializer(new AsyncAvroSerializer <int>(schemaRegistry, new AvroSerializerConfig {
AutoRegisterSchemas = false
}))
.Build())
{
Assert.Throws <SchemaRegistryException>(() =>
{
string guidTopic = new Guid().ToString();
try
{
producer
.ProduceAsync(guidTopic, new Message <string, int> {
Key = "test", Value = 112
})
.GetAwaiter()
.GetResult();
}
catch (Exception e)
{
Assert.True(e is ProduceException <string, int>);
Assert.Equal(ErrorCode.Local_ValueSerialization, ((ProduceException <string, int>)e).Error.Code);
// Test message fields are appropriately set in the case of a serialization error.
Assert.Equal("test", ((ProduceException <string, int>)e).DeliveryResult.Key);
Assert.Equal(112, ((ProduceException <string, int>)e).DeliveryResult.Value);
Assert.Equal(Offset.Unset, ((ProduceException <string, int>)e).DeliveryResult.Offset);
Assert.Equal(Partition.Any, ((ProduceException <string, int>)e).DeliveryResult.Partition);
Assert.Equal(guidTopic, ((ProduceException <string, int>)e).DeliveryResult.Topic);
Assert.Equal(PersistenceStatus.NotPersisted, ((ProduceException <string, int>)e).DeliveryResult.Status);
Assert.Equal(Timestamp.Default, ((ProduceException <string, int>)e).DeliveryResult.Timestamp);
Assert.Null(((ProduceException <string, int>)e).DeliveryResult.Headers);
// should be SerializationException.
throw e.InnerException;
}
});
}
// the following tests all check behavior in the key case.
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
using (var producer =
new ProducerBuilder <string, int>(producerConfig)
.SetKeySerializer(new AsyncAvroSerializer <string>(schemaRegistry, new AvroSerializerConfig {
AutoRegisterSchemas = false
}))
.SetValueSerializer(new AsyncAvroSerializer <int>(schemaRegistry))
.Build())
{
Assert.Throws <SchemaRegistryException>(() =>
{
try
{
producer.ProduceAsync(topic.Name, new Message <string, int> {
Key = "test", Value = 112
})
.GetAwaiter()
.GetResult();
}
catch (Exception e)
{
Assert.True(e is ProduceException <string, int>);
throw e.InnerException;
}
});
}
// allow auto register..
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
using (var producer =
new ProducerBuilder <string, int>(producerConfig)
.SetKeySerializer(new AsyncAvroSerializer <string>(schemaRegistry))
.SetValueSerializer(new AsyncAvroSerializer <int>(schemaRegistry))
.Build())
{
producer.ProduceAsync(topic.Name, new Message <string, int> {
Key = "test", Value = 112
}).Wait();
}
// config with avro.serializer.auto.register.schemas == false should work now.
using (var schemaRegistry = new CachedSchemaRegistryClient(new SchemaRegistryConfig {
SchemaRegistryUrl = schemaRegistryServers
}))
using (var producer =
new ProducerBuilder <string, int>(producerConfig)
.SetKeySerializer(new AsyncAvroSerializer <string>(schemaRegistry, new AvroSerializerConfig {
AutoRegisterSchemas = false
}))
.SetValueSerializer(new AsyncAvroSerializer <int>(schemaRegistry))
.Build())
{
producer.ProduceAsync(topic.Name, new Message <string, int> {
Key = "test", Value = 112
}).Wait();
}
}
}
public static void ConsumePartitionEOF(string bootstrapServers, string schemaRegistryServers)
{
var producerConfig = new ProducerConfig
{
BootstrapServers = bootstrapServers
};
var schemaRegistryConfig = new SchemaRegistryConfig
{
SchemaRegistryUrl = schemaRegistryServers
};
using (var topic = new TemporaryTopic(bootstrapServers, 1))
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
using (var producer =
new ProducerBuilder <Null, User>(producerConfig)
.SetKeySerializer(Serializers.Null)
.SetValueSerializer(new AvroSerializer <User>(schemaRegistry))
.Build())
{
producer.ProduceAsync(topic.Name, new Message <Null, User> {
Value = new User {
name = "test"
}
});
var consumerConfig = new ConsumerConfig
{
BootstrapServers = bootstrapServers,
GroupId = Guid.NewGuid().ToString(),
SessionTimeoutMs = 6000,
AutoOffsetReset = AutoOffsetReset.Earliest,
EnablePartitionEof = true
};
using (var consumer =
new ConsumerBuilder <Null, User>(consumerConfig)
.SetKeyDeserializer(Deserializers.Null)
.SetValueDeserializer(new AvroDeserializer <User>(schemaRegistry))
.SetRebalanceHandler((c, e) =>
{
if (e.IsAssignment)
{
c.Assign(e.Partitions.Select(tp => new TopicPartitionOffset(tp, Offset.Beginning)));
}
})
.Build())
{
consumer.Subscribe(topic.Name);
var cr1 = consumer.Consume();
Assert.NotNull(cr1);
Assert.NotNull(cr1.Message);
Assert.False(cr1.IsPartitionEOF);
var cr2 = consumer.Consume();
Assert.NotNull(cr2);
Assert.Null(cr2.Message);
Assert.True(cr2.IsPartitionEOF);
}
consumerConfig = new ConsumerConfig
{
BootstrapServers = bootstrapServers,
GroupId = Guid.NewGuid().ToString(),
SessionTimeoutMs = 6000,
AutoOffsetReset = AutoOffsetReset.Earliest,
EnablePartitionEof = false
};
using (var consumer =
new ConsumerBuilder <Null, User>(consumerConfig)
.SetKeyDeserializer(Deserializers.Null)
.SetValueDeserializer(new AvroDeserializer <User>(schemaRegistry))
.SetRebalanceHandler((c, e) =>
{
if (e.IsAssignment)
{
c.Assign(e.Partitions.Select(tp => new TopicPartitionOffset(tp, Offset.Beginning)));
}
})
.Build())
{
consumer.Subscribe(topic.Name);
var cr1 = consumer.Consume();
Assert.NotNull(cr1);
Assert.NotNull(cr1.Message);
Assert.False(cr1.IsPartitionEOF);
var cr2 = consumer.Consume(TimeSpan.FromSeconds(2));
Assert.Null(cr2);
}
}
}
public static void RegularAndAvro(string bootstrapServers, string schemaRegistryServers)
{
using (var topic1 = new TemporaryTopic(bootstrapServers, 1))
using (var topic2 = new TemporaryTopic(bootstrapServers, 1))
{
var producerConfig = new ProducerConfig
{
BootstrapServers = bootstrapServers
};
var consumerConfig = new ConsumerConfig
{
BootstrapServers = bootstrapServers,
GroupId = Guid.NewGuid().ToString(),
SessionTimeoutMs = 6000,
AutoOffsetReset = AutoOffsetReset.Earliest
};
var schemaRegistryConfig = new SchemaRegistryConfig
{
SchemaRegistryUrl = schemaRegistryServers
};
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
using (var producer =
new ProducerBuilder <string, string>(producerConfig)
.SetKeySerializer(Serializers.Utf8)
.SetValueSerializer(new AvroSerializer <string>(schemaRegistry))
.Build())
{
// implicit check that this does not fail.
producer.ProduceAsync(topic1.Name, new Message <string, string> {
Key = "hello", Value = "world"
}).Wait();
// check that the value type was registered with SR, and the key was not.
Assert.Throws <SchemaRegistryException>(() =>
{
try
{
schemaRegistry.GetLatestSchemaAsync(schemaRegistry.ConstructKeySubjectName(topic1.Name)).Wait();
}
catch (AggregateException e)
{
throw e.InnerException;
}
});
var s2 = schemaRegistry.GetLatestSchemaAsync(schemaRegistry.ConstructValueSubjectName(topic1.Name)).Result;
}
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
using (var producer =
new ProducerBuilder <string, string>(producerConfig)
.SetKeySerializer(new AvroSerializer <string>(schemaRegistry))
.SetValueSerializer(Serializers.Utf8)
.Build())
{
// implicit check that this does not fail.
producer.ProduceAsync(topic2.Name, new Message <string, string> {
Key = "hello", Value = "world"
}).Wait();
// check that the key type was registered with SR, and the value was not.
Assert.Throws <SchemaRegistryException>(() =>
{
try
{
schemaRegistry.GetLatestSchemaAsync(schemaRegistry.ConstructValueSubjectName(topic2.Name)).Wait();
}
catch (AggregateException e)
{
throw e.InnerException;
}
});
var s2 = schemaRegistry.GetLatestSchemaAsync(schemaRegistry.ConstructKeySubjectName(topic2.Name)).Result;
}
// check the above can be consumed (using regular / Avro serializers as appropriate)
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
{
using (var consumer =
new ConsumerBuilder <string, string>(consumerConfig)
.SetKeyDeserializer(Deserializers.Utf8)
.SetValueDeserializer(new AvroDeserializer <string>(schemaRegistry))
.Build())
{
consumer.Assign(new TopicPartitionOffset(topic1.Name, 0, 0));
var cr = consumer.Consume();
Assert.Equal("hello", cr.Key);
Assert.Equal("world", cr.Value);
}
using (var consumer =
new ConsumerBuilder <string, string>(consumerConfig)
.SetKeyDeserializer(new AvroDeserializer <string>(schemaRegistry))
.SetValueDeserializer(Deserializers.Utf8).Build())
{
consumer.Assign(new TopicPartitionOffset(topic2.Name, 0, 0));
var cr = consumer.Consume();
Assert.Equal("hello", cr.Key);
Assert.Equal("world", cr.Value);
}
using (var consumer =
new ConsumerBuilder <string, string>(consumerConfig)
.SetKeyDeserializer(Deserializers.Utf8)
.SetValueDeserializer(new AvroDeserializer <string>(schemaRegistry))
.Build())
{
consumer.Assign(new TopicPartitionOffset(topic2.Name, 0, 0));
Assert.ThrowsAny <ConsumeException>(() =>
{
try
{
consumer.Consume();
}
catch (AggregateException e)
{
throw e.InnerException;
}
});
}
using (var consumer =
new ConsumerBuilder <string, string>(consumerConfig)
.SetKeyDeserializer(new AvroDeserializer <string>(schemaRegistry))
.SetValueDeserializer(Deserializers.Utf8)
.Build())
{
consumer.Assign(new TopicPartitionOffset(topic1.Name, 0, 0));
Assert.ThrowsAny <ConsumeException>(() =>
{
try
{
consumer.Consume();
}
catch (AggregateException e)
{
throw e.InnerException;
}
});
}
}
}
}
public static void ProduceConsumeMixedJson(string bootstrapServers, string schemaRegistryServers)
{
var producerConfig = new ProducerConfig {
BootstrapServers = bootstrapServers
};
var schemaRegistryConfig = new SchemaRegistryConfig {
Url = schemaRegistryServers
};
using (var topic = new TemporaryTopic(bootstrapServers, 1))
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
using (var producer =
new ProducerBuilder <string, PersonPoco>(producerConfig)
.SetValueSerializer(new JsonSerializer <PersonPoco>(schemaRegistry))
.Build())
{
// test that a System.ComponentModel.DataAnnotations constraint is effective.
{
var p = new PersonPoco
{
FirstName = "Test",
LastName = "User",
NumberWithRange = 7 // range should be between 2 and 5.
};
Assert.Throws <ProduceException <string, PersonPoco> >(() => {
try
{
producer.ProduceAsync(topic.Name, new Message <string, PersonPoco> {
Key = "test1", Value = p
}).Wait();
}
catch (AggregateException ax)
{
// what would be thrown if the call was awaited.
throw ax.InnerException;
}
});
}
// test that a Newtonsoft.Json constraint is effective.
{
var p = new PersonPoco
{
FirstName = "Test",
// Omit LastName
NumberWithRange = 3
};
Assert.Throws <AggregateException>(() => {
producer.ProduceAsync(topic.Name, new Message <string, PersonPoco> {
Key = "test1", Value = p
}).Wait();
});
}
// test all fields valid.
{
var p = new PersonPoco
{
FirstName = "A",
MiddleName = "Test",
LastName = "User",
Gender = GenderEnum.Male,
NumberWithRange = 3,
Birthday = new DateTime(2010, 11, 1),
Company = new CompanyPoco
{
Name = "Confluent"
},
Cars = new List <CarPoco>
{
new CarPoco
{
Name = "Volvo",
Manufacturer = new CompanyPoco
{
Name = "Confluent"
}
}
}
};
producer.ProduceAsync(topic.Name, new Message <string, PersonPoco> {
Key = "test1", Value = p
}).Wait();
var schema = schemaRegistry.GetLatestSchemaAsync(SubjectNameStrategy.Topic.ConstructValueSubjectName(topic.Name, null)).Result.SchemaString;
var consumerConfig = new ConsumerConfig
{
BootstrapServers = bootstrapServers,
GroupId = Guid.NewGuid().ToString(),
AutoOffsetReset = AutoOffsetReset.Earliest
};
using (var consumer =
new ConsumerBuilder <string, PersonPoco>(consumerConfig)
.SetValueDeserializer(new JsonDeserializer <PersonPoco>().AsSyncOverAsync())
.Build())
{
consumer.Subscribe(topic.Name);
var cr = consumer.Consume();
Assert.Equal(p.FirstName, cr.Message.Value.FirstName);
Assert.Equal(p.MiddleName, cr.Message.Value.MiddleName);
Assert.Equal(p.LastName, cr.Message.Value.LastName);
Assert.Equal(p.Gender, cr.Message.Value.Gender);
Assert.Equal(p.NumberWithRange, cr.Message.Value.NumberWithRange);
Assert.Equal(p.Birthday, cr.Message.Value.Birthday);
Assert.Equal(p.Company.Name, cr.Message.Value.Company.Name);
Assert.Single(cr.Message.Value.Cars);
Assert.Equal(cr.Message.Value.Cars[0].Manufacturer.Name, p.Cars[0].Manufacturer.Name);
Assert.Equal(cr.Message.Value.Cars[0].Name, p.Cars[0].Name);
}
}
}
}
public static void AutoRegisterSchemaDisabled(string bootstrapServers, string schemaRegistryServers)
{
using (var topic = new TemporaryTopic(bootstrapServers, 1))
{
var producerConfig = new ProducerConfig
{
BootstrapServers = bootstrapServers
};
var consumerConfig = new ConsumerConfig
{
BootstrapServers = bootstrapServers,
GroupId = Guid.NewGuid().ToString(),
SessionTimeoutMs = 6000,
AutoOffsetReset = AutoOffsetResetType.Earliest
};
var schemaRegistryConfig = new SchemaRegistryConfig
{
SchemaRegistryUrl = schemaRegistryServers
};
// first a quick check the value case fails.
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
using (var producer = new Producer <string, int>(
producerConfig,
new AvroSerializer <string>(schemaRegistry),
new AvroSerializer <int>(schemaRegistry, new AvroSerializerConfig {
AutoRegisterSchemas = false
})))
{
Assert.Throws <SerializationException>(() =>
{
try
{
producer
.ProduceAsync(new Guid().ToString(), new Message <string, int> {
Key = "test", Value = 112
})
.Wait();
}
catch (AggregateException e)
{
throw e.InnerException;
}
});
}
// the following tests all check behavior in the key case.
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
using (var producer = new Producer <string, int>(
producerConfig,
new AvroSerializer <string>(schemaRegistry, new AvroSerializerConfig {
AutoRegisterSchemas = false
}),
new AvroSerializer <int>(schemaRegistry)))
{
Assert.Throws <SerializationException>(() =>
{
try
{
producer.ProduceAsync(topic.Name, new Message <string, int> {
Key = "test", Value = 112
}).Wait();
}
catch (AggregateException e)
{
throw e.InnerException;
}
});
}
// allow auto register..
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
using (var producer = new Producer <string, int>(producerConfig,
new AvroSerializer <string>(schemaRegistry), new AvroSerializer <int>(schemaRegistry)))
{
producer.ProduceAsync(topic.Name, new Message <string, int> {
Key = "test", Value = 112
}).Wait();
}
// config with avro.serializer.auto.register.schemas == false should work now.
using (var schemaRegistry = new CachedSchemaRegistryClient(new SchemaRegistryConfig {
SchemaRegistryUrl = schemaRegistryServers
}))
using (var producer = new Producer <string, int>(
producerConfig,
new AvroSerializer <string>(schemaRegistry, new AvroSerializerConfig {
AutoRegisterSchemas = false
}),
new AvroSerializer <int>(schemaRegistry)))
{
producer.ProduceAsync(topic.Name, new Message <string, int> {
Key = "test", Value = 112
}).Wait();
}
}
}
public static void ProduceConsumeGoogleRefProtobuf(string bootstrapServers, string schemaRegistryServers)
{
var producerConfig = new ProducerConfig {
BootstrapServers = bootstrapServers
};
var schemaRegistryConfig = new SchemaRegistryConfig {
Url = schemaRegistryServers
};
using (var topic = new TemporaryTopic(bootstrapServers, 1))
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
using (var producer =
new ProducerBuilder <string, WithGoogleRefs.TheRecord>(producerConfig)
.SetValueSerializer(new ProtobufSerializer <WithGoogleRefs.TheRecord>(schemaRegistry))
.Build())
{
var u = new WithGoogleRefs.TheRecord();
u.ListType = 41;
u.ReceivedTime = new Google.Protobuf.WellKnownTypes.Timestamp();
u.ReceivedTime.Seconds = 1591364591;
producer.ProduceAsync(topic.Name, new Message <string, WithGoogleRefs.TheRecord> {
Key = "test1", Value = u
}).Wait();
var consumerConfig = new ConsumerConfig
{
BootstrapServers = bootstrapServers,
GroupId = Guid.NewGuid().ToString(),
AutoOffsetReset = AutoOffsetReset.Earliest
};
// Test the protobuf deserializer can read this message
using (var consumer =
new ConsumerBuilder <string, WithGoogleRefs.TheRecord>(consumerConfig)
.SetValueDeserializer(new ProtobufDeserializer <WithGoogleRefs.TheRecord>().AsSyncOverAsync())
.Build())
{
consumer.Subscribe(topic.Name);
var cr = consumer.Consume();
Assert.Equal(u.ListType.Value, cr.Message.Value.ListType.Value);
Assert.Equal(u.ReceivedTime.Seconds, cr.Message.Value.ReceivedTime.Seconds);
}
// Check the pre-data bytes are as expected.
using (var consumer = new ConsumerBuilder <string, byte[]>(consumerConfig).Build())
{
consumer.Subscribe(topic.Name);
var cr = consumer.Consume();
// magic byte + schema id + expected array index length + at least one data byte.
Assert.True(cr.Message.Value.Length >= 1 + 4 + 1 + 1);
// magic byte
Assert.Equal(0, cr.Message.Value[0]);
// array index (special value as an optimization)
Assert.Equal(0, cr.Message.Value[5]);
}
// Check the referenced schemas are in schema registry.
var subjects = schemaRegistry.GetAllSubjectsAsync().Result;
Assert.Contains("google/protobuf/timestamp.proto", subjects);
Assert.Contains("google/protobuf/wrappers.proto", subjects);
}
}
