public static void ProduceConsumeSchemaManyMessagesProtobuf(string bootstrapServers, string schemaRegistryServers) { var producerConfig = new ProducerConfig { BootstrapServers = bootstrapServers }; var schemaRegistryConfig = new SchemaRegistryConfig { Url = schemaRegistryServers }; using (var topic = new TemporaryTopic(bootstrapServers, 1)) using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig)) using (var producer = new ProducerBuilder <string, Msg230>(producerConfig) .SetValueSerializer(new ProtobufSerializer <Msg230>(schemaRegistry)) .Build()) { var u = new Msg230(); u.Value = 41; producer.ProduceAsync(topic.Name, new Message <string, Msg230> { Key = "test1", Value = u }).Wait(); var consumerConfig = new ConsumerConfig { BootstrapServers = bootstrapServers, GroupId = Guid.NewGuid().ToString(), AutoOffsetReset = AutoOffsetReset.Earliest }; // Test the protobuf deserializer can read this message using (var consumer = new ConsumerBuilder <string, UInt32Value>(consumerConfig) .SetValueDeserializer(new ProtobufDeserializer <UInt32Value>().AsSyncOverAsync()) .Build()) { consumer.Subscribe(topic.Name); var cr = consumer.Consume(); Assert.Equal(u.Value, cr.Message.Value.Value); } // Check the pre-data bytes are as expected. using (var consumer = new ConsumerBuilder <string, byte[]>(consumerConfig).Build()) { consumer.Subscribe(topic.Name); var cr = consumer.Consume(); // magic byte + schema id + expected array index length + at least one data byte. Assert.True(cr.Message.Value.Length >= 1 + 4 + 1 + 2 + 1); // magic byte Assert.Equal(0, cr.Message.Value[0]); // index array length Assert.Equal(1, cr.Message.Value[5]); // there are 231 messages in the schema. message 230 has index 230. varint is 2 bytes: // in binary: 11100110. // -> &7f |80 -> 11100110 = 230 Assert.Equal(230, cr.Message.Value[6]); // >>7 -> 00000001 Assert.Equal(1, cr.Message.Value[7]); } } }
public static void ProduceConsumeNestedProtobuf(string bootstrapServers, string schemaRegistryServers) { var producerConfig = new ProducerConfig { BootstrapServers = bootstrapServers }; var schemaRegistryConfig = new SchemaRegistryConfig { Url = schemaRegistryServers }; using (var topic = new TemporaryTopic(bootstrapServers, 1)) using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig)) using (var producer = new ProducerBuilder <string, NestedOuter.Types.NestedMid2.Types.NestedLower>(producerConfig) .SetValueSerializer(new ProtobufSerializer <NestedOuter.Types.NestedMid2.Types.NestedLower>(schemaRegistry)) .Build()) { var u = new NestedOuter.Types.NestedMid2.Types.NestedLower(); u.Field2 = "field_2_value"; producer.ProduceAsync(topic.Name, new Message <string, NestedOuter.Types.NestedMid2.Types.NestedLower> { Key = "test1", Value = u }).Wait(); var consumerConfig = new ConsumerConfig { BootstrapServers = bootstrapServers, GroupId = Guid.NewGuid().ToString(), AutoOffsetReset = AutoOffsetReset.Earliest }; // Test the protobuf deserializer can read this message using (var consumer = new ConsumerBuilder <string, NestedOuter.Types.NestedMid2.Types.NestedLower>(consumerConfig) .SetValueDeserializer(new ProtobufDeserializer <NestedOuter.Types.NestedMid2.Types.NestedLower>() .AsSyncOverAsync()) .Build()) { consumer.Subscribe(topic.Name); var cr = consumer.Consume(); Assert.Equal(u.Field2, cr.Message.Value.Field2); } // Check the pre-data bytes are as expected. using (var consumer = new ConsumerBuilder <string, byte[]>(consumerConfig).Build()) { consumer.Subscribe(topic.Name); var cr = consumer.Consume(); // magic byte + schema id + expected array index length + at least one data byte. Assert.True(cr.Message.Value.Length > 1 + 4 + 4 + 1); // magic byte Assert.Equal(0, cr.Message.Value[0]); // index array. Assert.Equal(3, cr.Message.Value[5]); Assert.Equal(2, cr.Message.Value[6]); Assert.Equal(1, cr.Message.Value[7]); Assert.Equal(0, cr.Message.Value[8]); } } }
public static void ProduceConsumeBasicProtobuf(string bootstrapServers, string schemaRegistryServers) { var producerConfig = new ProducerConfig { BootstrapServers = bootstrapServers }; var schemaRegistryConfig = new SchemaRegistryConfig { Url = schemaRegistryServers }; using (var topic = new TemporaryTopic(bootstrapServers, 1)) using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig)) using (var producer = new ProducerBuilder <string, UInt32Value>(producerConfig) .SetValueSerializer(new ProtobufSerializer <UInt32Value>(schemaRegistry)) .Build()) { var u = new UInt32Value(); u.Value = 42; producer.ProduceAsync(topic.Name, new Message <string, UInt32Value> { Key = "test1", Value = u }).Wait(); var consumerConfig = new ConsumerConfig { BootstrapServers = bootstrapServers, GroupId = Guid.NewGuid().ToString(), AutoOffsetReset = AutoOffsetReset.Earliest }; // Test the protobuf deserializer can read this message using (var consumer = new ConsumerBuilder <string, UInt32Value>(consumerConfig) .SetValueDeserializer(new ProtobufDeserializer <UInt32Value>().AsSyncOverAsync()) .Build()) { consumer.Subscribe(topic.Name); var cr = consumer.Consume(); Assert.Equal(u.Value, cr.Message.Value.Value); } // Check the pre-data bytes are as expected. using (var consumer = new ConsumerBuilder <string, byte[]>(consumerConfig).Build()) { consumer.Subscribe(topic.Name); var cr = consumer.Consume(); // magic byte + schema id + expected array index length + at least one data byte. Assert.True(cr.Message.Value.Length >= 1 + 4 + 1 + 1); // magic byte Assert.Equal(0, cr.Message.Value[0]); // array index (special value as an optimization) Assert.Equal(0, cr.Message.Value[5]); } } }
public static void ProduceConsumeNestedProtobuf(string bootstrapServers, string schemaRegistryServers) { var producerConfig = new ProducerConfig { BootstrapServers = bootstrapServers }; var schemaRegistryConfig = new SchemaRegistryConfig { Url = schemaRegistryServers }; using (var topic = new TemporaryTopic(bootstrapServers, 1)) using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig)) using (var producer = new ProducerBuilder <string, NestedOuter.Types.NestedMid2.Types.NestedLower>(producerConfig) .SetValueSerializer(new ProtobufSerializer <NestedOuter.Types.NestedMid2.Types.NestedLower>(schemaRegistry)) .Build()) { var u = new NestedOuter.Types.NestedMid2.Types.NestedLower(); u.Field2 = "field_2_value"; producer.ProduceAsync(topic.Name, new Message <string, NestedOuter.Types.NestedMid2.Types.NestedLower> { Key = "test1", Value = u }).Wait(); var consumerConfig = new ConsumerConfig { BootstrapServers = bootstrapServers, GroupId = Guid.NewGuid().ToString(), AutoOffsetReset = AutoOffsetReset.Earliest }; // Test the protobuf deserializer can read this message using (var consumer = new ConsumerBuilder <string, NestedOuter.Types.NestedMid2.Types.NestedLower>(consumerConfig) .SetValueDeserializer(new ProtobufDeserializer <NestedOuter.Types.NestedMid2.Types.NestedLower>() .AsSyncOverAsync()) .Build()) { consumer.Subscribe(topic.Name); var cr = consumer.Consume(); Assert.Equal(u.Field2, cr.Message.Value.Field2); } } }
public static void ProduceConsumeSchemaManyMessagesProtobuf(string bootstrapServers, string schemaRegistryServers) { var producerConfig = new ProducerConfig { BootstrapServers = bootstrapServers }; var schemaRegistryConfig = new SchemaRegistryConfig { Url = schemaRegistryServers }; using (var topic = new TemporaryTopic(bootstrapServers, 1)) using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig)) using (var producer = new ProducerBuilder <string, Msg230>(producerConfig) .SetValueSerializer(new ProtobufSerializer <Msg230>(schemaRegistry)) .Build()) { var u = new Msg230(); u.Value = 41; producer.ProduceAsync(topic.Name, new Message <string, Msg230> { Key = "test1", Value = u }).Wait(); var consumerConfig = new ConsumerConfig { BootstrapServers = bootstrapServers, GroupId = Guid.NewGuid().ToString(), AutoOffsetReset = AutoOffsetReset.Earliest }; // Test the protobuf deserializer can read this message using (var consumer = new ConsumerBuilder <string, UInt32Value>(consumerConfig) .SetValueDeserializer(new ProtobufDeserializer <UInt32Value>().AsSyncOverAsync()) .Build()) { consumer.Subscribe(topic.Name); var cr = consumer.Consume(); Assert.Equal(u.Value, cr.Message.Value.Value); } } }
public static void SubjectNameStrategiesProtobuf(string bootstrapServers, string schemaRegistryServers) { var producerConfig = new ProducerConfig { BootstrapServers = bootstrapServers }; var schemaRegistryConfig = new SchemaRegistryConfig { Url = schemaRegistryServers }; using (var topic = new TemporaryTopic(bootstrapServers, 1)) using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig)) { using (var producer = new ProducerBuilder <string, UInt32Value>(producerConfig) .SetValueSerializer(new ProtobufSerializer <UInt32Value>(schemaRegistry, new ProtobufSerializerConfig { SubjectNameStrategy = SubjectNameStrategy.TopicRecord })) .Build()) { var u = new UInt32Value(); u.Value = 42; producer.ProduceAsync(topic.Name, new Message <string, UInt32Value> { Key = "test1", Value = u }).Wait(); var subjects = schemaRegistry.GetAllSubjectsAsync().Result; Assert.Contains(topic.Name + "-UInt32Value", subjects); Assert.DoesNotContain(topic.Name + "-value", subjects); // May contain the record name subject from a previous test. } using (var producer = new ProducerBuilder <string, UInt32Value>(producerConfig) .SetValueSerializer(new ProtobufSerializer <UInt32Value>(schemaRegistry, new ProtobufSerializerConfig { SubjectNameStrategy = SubjectNameStrategy.Record })) .Build()) { var u = new UInt32Value(); u.Value = 42; producer.ProduceAsync(topic.Name, new Message <string, UInt32Value> { Key = "test1", Value = u }).Wait(); var subjects = schemaRegistry.GetAllSubjectsAsync().Result; // Note: If this value is in SR by any means (even if not via this test), // it implies what is being tested here is functional. Assert.Contains("UInt32Value", subjects); Assert.DoesNotContain(topic.Name + "-value", subjects); } using (var producer = new ProducerBuilder <string, UInt32Value>(producerConfig) .SetValueSerializer(new ProtobufSerializer <UInt32Value>(schemaRegistry)) .Build()) { var u = new UInt32Value(); u.Value = 42; producer.ProduceAsync(topic.Name, new Message <string, UInt32Value> { Key = "test1", Value = u }).Wait(); var subjects = schemaRegistry.GetAllSubjectsAsync().Result; Assert.Contains(topic.Name + "-value", subjects); } } }
public static void AutoRegisterSchemaDisabled(string bootstrapServers, string schemaRegistryServers) { using (var topic = new TemporaryTopic(bootstrapServers, 1)) { var producerConfig = new ProducerConfig { BootstrapServers = bootstrapServers }; var consumerConfig = new ConsumerConfig { BootstrapServers = bootstrapServers, GroupId = Guid.NewGuid().ToString(), SessionTimeoutMs = 6000, AutoOffsetReset = AutoOffsetReset.Earliest }; var schemaRegistryConfig = new SchemaRegistryConfig { SchemaRegistryUrl = schemaRegistryServers }; // first a quick check the value case fails. using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig)) using (var producer = new ProducerBuilder <string, int>(producerConfig) .SetKeySerializer(new AsyncAvroSerializer <string>(schemaRegistry)) .SetValueSerializer(new AsyncAvroSerializer <int>(schemaRegistry, new AvroSerializerConfig { AutoRegisterSchemas = false })) .Build()) { Assert.Throws <SchemaRegistryException>(() => { string guidTopic = new Guid().ToString(); try { producer .ProduceAsync(guidTopic, new Message <string, int> { Key = "test", Value = 112 }) .GetAwaiter() .GetResult(); } catch (Exception e) { Assert.True(e is ProduceException <string, int>); Assert.Equal(ErrorCode.Local_ValueSerialization, ((ProduceException <string, int>)e).Error.Code); // Test message fields are appropriately set in the case of a serialization error. Assert.Equal("test", ((ProduceException <string, int>)e).DeliveryResult.Key); Assert.Equal(112, ((ProduceException <string, int>)e).DeliveryResult.Value); Assert.Equal(Offset.Unset, ((ProduceException <string, int>)e).DeliveryResult.Offset); Assert.Equal(Partition.Any, ((ProduceException <string, int>)e).DeliveryResult.Partition); Assert.Equal(guidTopic, ((ProduceException <string, int>)e).DeliveryResult.Topic); Assert.Equal(PersistenceStatus.NotPersisted, ((ProduceException <string, int>)e).DeliveryResult.Status); Assert.Equal(Timestamp.Default, ((ProduceException <string, int>)e).DeliveryResult.Timestamp); Assert.Null(((ProduceException <string, int>)e).DeliveryResult.Headers); // should be SerializationException. throw e.InnerException; } }); } // the following tests all check behavior in the key case. using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig)) using (var producer = new ProducerBuilder <string, int>(producerConfig) .SetKeySerializer(new AsyncAvroSerializer <string>(schemaRegistry, new AvroSerializerConfig { AutoRegisterSchemas = false })) .SetValueSerializer(new AsyncAvroSerializer <int>(schemaRegistry)) .Build()) { Assert.Throws <SchemaRegistryException>(() => { try { producer.ProduceAsync(topic.Name, new Message <string, int> { Key = "test", Value = 112 }) .GetAwaiter() .GetResult(); } catch (Exception e) { Assert.True(e is ProduceException <string, int>); throw e.InnerException; } }); } // allow auto register.. using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig)) using (var producer = new ProducerBuilder <string, int>(producerConfig) .SetKeySerializer(new AsyncAvroSerializer <string>(schemaRegistry)) .SetValueSerializer(new AsyncAvroSerializer <int>(schemaRegistry)) .Build()) { producer.ProduceAsync(topic.Name, new Message <string, int> { Key = "test", Value = 112 }).Wait(); } // config with avro.serializer.auto.register.schemas == false should work now. using (var schemaRegistry = new CachedSchemaRegistryClient(new SchemaRegistryConfig { SchemaRegistryUrl = schemaRegistryServers })) using (var producer = new ProducerBuilder <string, int>(producerConfig) .SetKeySerializer(new AsyncAvroSerializer <string>(schemaRegistry, new AvroSerializerConfig { AutoRegisterSchemas = false })) .SetValueSerializer(new AsyncAvroSerializer <int>(schemaRegistry)) .Build()) { producer.ProduceAsync(topic.Name, new Message <string, int> { Key = "test", Value = 112 }).Wait(); } } }
public static void ConsumePartitionEOF(string bootstrapServers, string schemaRegistryServers) { var producerConfig = new ProducerConfig { BootstrapServers = bootstrapServers }; var schemaRegistryConfig = new SchemaRegistryConfig { SchemaRegistryUrl = schemaRegistryServers }; using (var topic = new TemporaryTopic(bootstrapServers, 1)) using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig)) using (var producer = new ProducerBuilder <Null, User>(producerConfig) .SetKeySerializer(Serializers.Null) .SetValueSerializer(new AvroSerializer <User>(schemaRegistry)) .Build()) { producer.ProduceAsync(topic.Name, new Message <Null, User> { Value = new User { name = "test" } }); var consumerConfig = new ConsumerConfig { BootstrapServers = bootstrapServers, GroupId = Guid.NewGuid().ToString(), SessionTimeoutMs = 6000, AutoOffsetReset = AutoOffsetReset.Earliest, EnablePartitionEof = true }; using (var consumer = new ConsumerBuilder <Null, User>(consumerConfig) .SetKeyDeserializer(Deserializers.Null) .SetValueDeserializer(new AvroDeserializer <User>(schemaRegistry)) .SetRebalanceHandler((c, e) => { if (e.IsAssignment) { c.Assign(e.Partitions.Select(tp => new TopicPartitionOffset(tp, Offset.Beginning))); } }) .Build()) { consumer.Subscribe(topic.Name); var cr1 = consumer.Consume(); Assert.NotNull(cr1); Assert.NotNull(cr1.Message); Assert.False(cr1.IsPartitionEOF); var cr2 = consumer.Consume(); Assert.NotNull(cr2); Assert.Null(cr2.Message); Assert.True(cr2.IsPartitionEOF); } consumerConfig = new ConsumerConfig { BootstrapServers = bootstrapServers, GroupId = Guid.NewGuid().ToString(), SessionTimeoutMs = 6000, AutoOffsetReset = AutoOffsetReset.Earliest, EnablePartitionEof = false }; using (var consumer = new ConsumerBuilder <Null, User>(consumerConfig) .SetKeyDeserializer(Deserializers.Null) .SetValueDeserializer(new AvroDeserializer <User>(schemaRegistry)) .SetRebalanceHandler((c, e) => { if (e.IsAssignment) { c.Assign(e.Partitions.Select(tp => new TopicPartitionOffset(tp, Offset.Beginning))); } }) .Build()) { consumer.Subscribe(topic.Name); var cr1 = consumer.Consume(); Assert.NotNull(cr1); Assert.NotNull(cr1.Message); Assert.False(cr1.IsPartitionEOF); var cr2 = consumer.Consume(TimeSpan.FromSeconds(2)); Assert.Null(cr2); } } }
public static void RegularAndAvro(string bootstrapServers, string schemaRegistryServers) { using (var topic1 = new TemporaryTopic(bootstrapServers, 1)) using (var topic2 = new TemporaryTopic(bootstrapServers, 1)) { var producerConfig = new ProducerConfig { BootstrapServers = bootstrapServers }; var consumerConfig = new ConsumerConfig { BootstrapServers = bootstrapServers, GroupId = Guid.NewGuid().ToString(), SessionTimeoutMs = 6000, AutoOffsetReset = AutoOffsetReset.Earliest }; var schemaRegistryConfig = new SchemaRegistryConfig { SchemaRegistryUrl = schemaRegistryServers }; using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig)) using (var producer = new ProducerBuilder <string, string>(producerConfig) .SetKeySerializer(Serializers.Utf8) .SetValueSerializer(new AvroSerializer <string>(schemaRegistry)) .Build()) { // implicit check that this does not fail. producer.ProduceAsync(topic1.Name, new Message <string, string> { Key = "hello", Value = "world" }).Wait(); // check that the value type was registered with SR, and the key was not. Assert.Throws <SchemaRegistryException>(() => { try { schemaRegistry.GetLatestSchemaAsync(schemaRegistry.ConstructKeySubjectName(topic1.Name)).Wait(); } catch (AggregateException e) { throw e.InnerException; } }); var s2 = schemaRegistry.GetLatestSchemaAsync(schemaRegistry.ConstructValueSubjectName(topic1.Name)).Result; } using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig)) using (var producer = new ProducerBuilder <string, string>(producerConfig) .SetKeySerializer(new AvroSerializer <string>(schemaRegistry)) .SetValueSerializer(Serializers.Utf8) .Build()) { // implicit check that this does not fail. producer.ProduceAsync(topic2.Name, new Message <string, string> { Key = "hello", Value = "world" }).Wait(); // check that the key type was registered with SR, and the value was not. Assert.Throws <SchemaRegistryException>(() => { try { schemaRegistry.GetLatestSchemaAsync(schemaRegistry.ConstructValueSubjectName(topic2.Name)).Wait(); } catch (AggregateException e) { throw e.InnerException; } }); var s2 = schemaRegistry.GetLatestSchemaAsync(schemaRegistry.ConstructKeySubjectName(topic2.Name)).Result; } // check the above can be consumed (using regular / Avro serializers as appropriate) using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig)) { using (var consumer = new ConsumerBuilder <string, string>(consumerConfig) .SetKeyDeserializer(Deserializers.Utf8) .SetValueDeserializer(new AvroDeserializer <string>(schemaRegistry)) .Build()) { consumer.Assign(new TopicPartitionOffset(topic1.Name, 0, 0)); var cr = consumer.Consume(); Assert.Equal("hello", cr.Key); Assert.Equal("world", cr.Value); } using (var consumer = new ConsumerBuilder <string, string>(consumerConfig) .SetKeyDeserializer(new AvroDeserializer <string>(schemaRegistry)) .SetValueDeserializer(Deserializers.Utf8).Build()) { consumer.Assign(new TopicPartitionOffset(topic2.Name, 0, 0)); var cr = consumer.Consume(); Assert.Equal("hello", cr.Key); Assert.Equal("world", cr.Value); } using (var consumer = new ConsumerBuilder <string, string>(consumerConfig) .SetKeyDeserializer(Deserializers.Utf8) .SetValueDeserializer(new AvroDeserializer <string>(schemaRegistry)) .Build()) { consumer.Assign(new TopicPartitionOffset(topic2.Name, 0, 0)); Assert.ThrowsAny <ConsumeException>(() => { try { consumer.Consume(); } catch (AggregateException e) { throw e.InnerException; } }); } using (var consumer = new ConsumerBuilder <string, string>(consumerConfig) .SetKeyDeserializer(new AvroDeserializer <string>(schemaRegistry)) .SetValueDeserializer(Deserializers.Utf8) .Build()) { consumer.Assign(new TopicPartitionOffset(topic1.Name, 0, 0)); Assert.ThrowsAny <ConsumeException>(() => { try { consumer.Consume(); } catch (AggregateException e) { throw e.InnerException; } }); } } } }
public static void ProduceConsumeMixedJson(string bootstrapServers, string schemaRegistryServers) { var producerConfig = new ProducerConfig { BootstrapServers = bootstrapServers }; var schemaRegistryConfig = new SchemaRegistryConfig { Url = schemaRegistryServers }; using (var topic = new TemporaryTopic(bootstrapServers, 1)) using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig)) using (var producer = new ProducerBuilder <string, PersonPoco>(producerConfig) .SetValueSerializer(new JsonSerializer <PersonPoco>(schemaRegistry)) .Build()) { // test that a System.ComponentModel.DataAnnotations constraint is effective. { var p = new PersonPoco { FirstName = "Test", LastName = "User", NumberWithRange = 7 // range should be between 2 and 5. }; Assert.Throws <ProduceException <string, PersonPoco> >(() => { try { producer.ProduceAsync(topic.Name, new Message <string, PersonPoco> { Key = "test1", Value = p }).Wait(); } catch (AggregateException ax) { // what would be thrown if the call was awaited. throw ax.InnerException; } }); } // test that a Newtonsoft.Json constraint is effective. { var p = new PersonPoco { FirstName = "Test", // Omit LastName NumberWithRange = 3 }; Assert.Throws <AggregateException>(() => { producer.ProduceAsync(topic.Name, new Message <string, PersonPoco> { Key = "test1", Value = p }).Wait(); }); } // test all fields valid. { var p = new PersonPoco { FirstName = "A", MiddleName = "Test", LastName = "User", Gender = GenderEnum.Male, NumberWithRange = 3, Birthday = new DateTime(2010, 11, 1), Company = new CompanyPoco { Name = "Confluent" }, Cars = new List <CarPoco> { new CarPoco { Name = "Volvo", Manufacturer = new CompanyPoco { Name = "Confluent" } } } }; producer.ProduceAsync(topic.Name, new Message <string, PersonPoco> { Key = "test1", Value = p }).Wait(); var schema = schemaRegistry.GetLatestSchemaAsync(SubjectNameStrategy.Topic.ConstructValueSubjectName(topic.Name, null)).Result.SchemaString; var consumerConfig = new ConsumerConfig { BootstrapServers = bootstrapServers, GroupId = Guid.NewGuid().ToString(), AutoOffsetReset = AutoOffsetReset.Earliest }; using (var consumer = new ConsumerBuilder <string, PersonPoco>(consumerConfig) .SetValueDeserializer(new JsonDeserializer <PersonPoco>().AsSyncOverAsync()) .Build()) { consumer.Subscribe(topic.Name); var cr = consumer.Consume(); Assert.Equal(p.FirstName, cr.Message.Value.FirstName); Assert.Equal(p.MiddleName, cr.Message.Value.MiddleName); Assert.Equal(p.LastName, cr.Message.Value.LastName); Assert.Equal(p.Gender, cr.Message.Value.Gender); Assert.Equal(p.NumberWithRange, cr.Message.Value.NumberWithRange); Assert.Equal(p.Birthday, cr.Message.Value.Birthday); Assert.Equal(p.Company.Name, cr.Message.Value.Company.Name); Assert.Single(cr.Message.Value.Cars); Assert.Equal(cr.Message.Value.Cars[0].Manufacturer.Name, p.Cars[0].Manufacturer.Name); Assert.Equal(cr.Message.Value.Cars[0].Name, p.Cars[0].Name); } } } }
public static void AutoRegisterSchemaDisabled(string bootstrapServers, string schemaRegistryServers) { using (var topic = new TemporaryTopic(bootstrapServers, 1)) { var producerConfig = new ProducerConfig { BootstrapServers = bootstrapServers }; var consumerConfig = new ConsumerConfig { BootstrapServers = bootstrapServers, GroupId = Guid.NewGuid().ToString(), SessionTimeoutMs = 6000, AutoOffsetReset = AutoOffsetResetType.Earliest }; var schemaRegistryConfig = new SchemaRegistryConfig { SchemaRegistryUrl = schemaRegistryServers }; // first a quick check the value case fails. using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig)) using (var producer = new Producer <string, int>( producerConfig, new AvroSerializer <string>(schemaRegistry), new AvroSerializer <int>(schemaRegistry, new AvroSerializerConfig { AutoRegisterSchemas = false }))) { Assert.Throws <SerializationException>(() => { try { producer .ProduceAsync(new Guid().ToString(), new Message <string, int> { Key = "test", Value = 112 }) .Wait(); } catch (AggregateException e) { throw e.InnerException; } }); } // the following tests all check behavior in the key case. using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig)) using (var producer = new Producer <string, int>( producerConfig, new AvroSerializer <string>(schemaRegistry, new AvroSerializerConfig { AutoRegisterSchemas = false }), new AvroSerializer <int>(schemaRegistry))) { Assert.Throws <SerializationException>(() => { try { producer.ProduceAsync(topic.Name, new Message <string, int> { Key = "test", Value = 112 }).Wait(); } catch (AggregateException e) { throw e.InnerException; } }); } // allow auto register.. using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig)) using (var producer = new Producer <string, int>(producerConfig, new AvroSerializer <string>(schemaRegistry), new AvroSerializer <int>(schemaRegistry))) { producer.ProduceAsync(topic.Name, new Message <string, int> { Key = "test", Value = 112 }).Wait(); } // config with avro.serializer.auto.register.schemas == false should work now. using (var schemaRegistry = new CachedSchemaRegistryClient(new SchemaRegistryConfig { SchemaRegistryUrl = schemaRegistryServers })) using (var producer = new Producer <string, int>( producerConfig, new AvroSerializer <string>(schemaRegistry, new AvroSerializerConfig { AutoRegisterSchemas = false }), new AvroSerializer <int>(schemaRegistry))) { producer.ProduceAsync(topic.Name, new Message <string, int> { Key = "test", Value = 112 }).Wait(); } } }
public static void ProduceConsumeGoogleRefProtobuf(string bootstrapServers, string schemaRegistryServers) { var producerConfig = new ProducerConfig { BootstrapServers = bootstrapServers }; var schemaRegistryConfig = new SchemaRegistryConfig { Url = schemaRegistryServers }; using (var topic = new TemporaryTopic(bootstrapServers, 1)) using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig)) using (var producer = new ProducerBuilder <string, WithGoogleRefs.TheRecord>(producerConfig) .SetValueSerializer(new ProtobufSerializer <WithGoogleRefs.TheRecord>(schemaRegistry)) .Build()) { var u = new WithGoogleRefs.TheRecord(); u.ListType = 41; u.ReceivedTime = new Google.Protobuf.WellKnownTypes.Timestamp(); u.ReceivedTime.Seconds = 1591364591; producer.ProduceAsync(topic.Name, new Message <string, WithGoogleRefs.TheRecord> { Key = "test1", Value = u }).Wait(); var consumerConfig = new ConsumerConfig { BootstrapServers = bootstrapServers, GroupId = Guid.NewGuid().ToString(), AutoOffsetReset = AutoOffsetReset.Earliest }; // Test the protobuf deserializer can read this message using (var consumer = new ConsumerBuilder <string, WithGoogleRefs.TheRecord>(consumerConfig) .SetValueDeserializer(new ProtobufDeserializer <WithGoogleRefs.TheRecord>().AsSyncOverAsync()) .Build()) { consumer.Subscribe(topic.Name); var cr = consumer.Consume(); Assert.Equal(u.ListType.Value, cr.Message.Value.ListType.Value); Assert.Equal(u.ReceivedTime.Seconds, cr.Message.Value.ReceivedTime.Seconds); } // Check the pre-data bytes are as expected. using (var consumer = new ConsumerBuilder <string, byte[]>(consumerConfig).Build()) { consumer.Subscribe(topic.Name); var cr = consumer.Consume(); // magic byte + schema id + expected array index length + at least one data byte. Assert.True(cr.Message.Value.Length >= 1 + 4 + 1 + 1); // magic byte Assert.Equal(0, cr.Message.Value[0]); // array index (special value as an optimization) Assert.Equal(0, cr.Message.Value[5]); } // Check the referenced schemas are in schema registry. var subjects = schemaRegistry.GetAllSubjectsAsync().Result; Assert.Contains("google/protobuf/timestamp.proto", subjects); Assert.Contains("google/protobuf/wrappers.proto", subjects); } }