public virtual void TestSchemaDefinition() { var schema1 = typeof(DefaultStruct).GetSchema(); var schema2 = AvroSchema <StructWithAnnotations> .Of(typeof(StructWithAnnotations)); var schemaDef1 = schema1.ToString(); var schemaDef2 = Encoding.UTF8.GetString(schema2.SchemaInfo.Schema); Assert.NotEqual(schemaDef1, schemaDef2); var schema3 = AvroSchema <StructWithAnnotations> .Of(ISchemaDefinition <StructWithAnnotations> .Builder().WithJsonDef(schemaDef1).Build()); var schemaDef3 = Encoding.UTF8.GetString(schema3.SchemaInfo.Schema); Assert.True(schemaDef1.Contains("DefaultStruct") && schemaDef3.Contains("DefaultStruct")); Assert.NotEqual(schemaDef2, schemaDef3); var @struct = new StructWithAnnotations { Field1 = 5678 }; // schema2 is using the schema generated from POJO, // it allows field2 to be nullable, and field3 has default value. schema2.Encode(@struct); // schema3 is using the schema passed in, which doesn't allow nullable schema3.Encode(@struct); }
public virtual void TestDefaultGetProducerDataAssigned() { var fooSchema = AvroSchema <SchemaTestUtils.Foo> .Of(ISchemaDefinition <SchemaTestUtils.Foo> .Builder().WithPojo(typeof(SchemaTestUtils.Foo)).Build()); var barSchema = AvroSchema <SchemaTestUtils.Bar> .Of(ISchemaDefinition <SchemaTestUtils.Bar> .Builder().WithPojo(typeof(SchemaTestUtils.Bar)).Build()); var keyValueSchema = ISchema <object> .KeyValue(fooSchema, barSchema); var foo = new SchemaTestUtils.Foo { Field1 = "field1", Field2 = "field2", Field3 = 3 }; var bar = new SchemaTestUtils.Bar { Field1 = true }; // // Check kv.encoding.type default, not set value var encodeBytes = keyValueSchema.Encode(new KeyValue <SchemaTestUtils.Foo, SchemaTestUtils.Bar>(foo, bar)); var builder = new MessageMetadata(); builder.ProducerName = "default"; var msg = Message <KeyValue <SchemaTestUtils.Foo, SchemaTestUtils.Bar> > .Create(builder, new ReadOnlySequence <byte>(encodeBytes), keyValueSchema); var keyValue = msg.Value; Assert.Equal(keyValue.Key, foo); Assert.Equal(keyValue.Value, bar); Assert.False(builder.ShouldSerializePartitionKey()); }
public virtual async Task ReaderSourceTaggedTest() { var topic = $"reader-topics-{Guid.NewGuid()}"; var ids = PublishMessages(topic, 50); var start = MessageIdUtils.GetOffset(ids.First()); var end = MessageIdUtils.GetOffset(ids.Last()); var conf = new ReaderConfigBuilder <DataOp>().Topic(topic); var reader = _pulsarSystem.EventSource("public", "default", topic, start, end, "http://127.0.0.1:8080") .Reader(_clientConfigurationData, conf, AvroSchema <DataOp> .Of(typeof(DataOp))) .SourceMethod() .CurrentTaggedEvents(new Messages.Consumer.Tag("twitter", "mestical")); //let leave some time to wire everything up await Task.Delay(TimeSpan.FromSeconds(20)); var receivedCount = 0; while (receivedCount == 0) { await foreach (var response in reader.CurrentEvents(TimeSpan.FromSeconds(5))) { receivedCount++; _output.WriteLine(JsonSerializer.Serialize(response, new JsonSerializerOptions { WriteIndented = true })); } await Task.Delay(TimeSpan.FromSeconds(1)); } Assert.True(receivedCount > 0); }
public virtual void TestInlineKeyValueEncodingTypeSchemaEncodeAndDecode() { var fooSchema = AvroSchema <Foo> .Of(ISchemaDefinition <Foo> .Builder().WithPojo(typeof(Foo)).Build()); var barSchema = AvroSchema <Bar> .Of(ISchemaDefinition <Bar> .Builder().WithPojo(typeof(Bar)).Build()); var keyValueSchema = ISchema <object> .KeyValue(fooSchema, barSchema, KeyValueEncodingType.INLINE); var bar = new Bar { Field1 = true }; var foo = new Foo { Field1 = "field1", Field2 = "field2", Field3 = "3", Field4 = bar, Color = Color.RED }; // Check kv.encoding.type INLINE var encodeBytes = keyValueSchema.Encode(new KeyValue <Foo, Bar>(foo, bar)); Assert.True(encodeBytes.Length > 0); var keyValue = keyValueSchema.Decode(encodeBytes); var fooBack = keyValue.Key; var barBack = keyValue.Value; Assert.True(foo.Equals(fooBack)); Assert.True(bar.Equals(barBack)); }
public virtual void TestAllowNullBytesSchemaEncodeAndDecode() { var fooAvroSchema = AvroSchema <Foo> .Of(ISchemaDefinition <Foo> .Builder().WithPojo(typeof(Foo)).Build()); var barAvroSchema = AvroSchema <Bar> .Of(ISchemaDefinition <Bar> .Builder().WithPojo(typeof(Bar)).Build()); var bar = new Bar { Field1 = true }; var foo = new Foo { Field1 = "field1", Field2 = "field2", Field3 = "3", Field4 = bar, Color = Color.RED }; var fooBytes = fooAvroSchema.Encode(foo); var barBytes = barAvroSchema.Encode(bar); var encodeBytes = ISchema <object> .KvBytes().Encode(new KeyValue <byte[], byte[]>(fooBytes, barBytes)); var decodeKV = ISchema <object> .KvBytes().Decode(encodeBytes); var fooBack = fooAvroSchema.Decode(decodeKV.Key); var barBack = barAvroSchema.Decode(decodeKV.Value); Assert.True(foo.Equals(fooBack)); Assert.True(bar.Equals(barBack)); }
public void TypeLookupAdvancedTest(Type expectedType, AvroSchema schema) { var assembly = Assembly.GetExecutingAssembly(); var actualType = SchemaResolver.GetTypeFromSchema(schema, assembly); Assert.AreEqual(expectedType, actualType); }
private ISet <MessageId> PublishMessages(string topic, int count) { var ids = new HashSet <MessageId>(); var builder = new ProducerConfigBuilder <DataOp>() .Topic(topic); var producer = _client.NewProducer(AvroSchema <DataOp> .Of(typeof(DataOp)), builder); for (var i = 0; i < count; i++) { var key = "key" + i; MessageId id = null; if (i % 2 == 0) { id = producer.NewMessage().Key(key).Property("twitter", "mestical").Value(new DataOp { Text = "my-event-message-" + i, EventTime = DateTimeHelper.CurrentUnixTimeMillis() }).Send(); } else { id = producer.NewMessage().Key(key).Value(new DataOp { Text = "my-event-message-" + i, EventTime = DateTimeHelper.CurrentUnixTimeMillis() }).Send(); } ids.Add(id); } return(ids); }
public GenericAvroReaderTest() { fooSchema = AvroSchema <SchemaTestUtils.Foo> .Of(typeof(SchemaTestUtils.Foo)); fooV2Schema = AvroSchema <SchemaTestUtils.FooV2> .Of(typeof(SchemaTestUtils.FooV2)); fooSchemaNotNull = AvroSchema <SchemaTestUtils.Foo> .Of(ISchemaDefinition <SchemaTestUtils.Foo> .Builder().WithAlwaysAllowNull(false).WithPojo(typeof(SchemaTestUtils.Foo)).Build()); fooOffsetSchema = AvroSchema <SchemaTestUtils.Foo> .Of(typeof(SchemaTestUtils.Foo)); fooOffsetSchema.SchemaInfo.Properties.Add(GenericAvroSchema.OFFSET_PROP, "5"); var prop = fooOffsetSchema.AvroSchema.GetProperty(GenericAvroSchema.OFFSET_PROP); foo = new SchemaTestUtils.Foo { Field1 = "foo1", Field2 = "bar1", Field4 = new SchemaTestUtils.Bar(), FieldUnableNull = "notNull" }; fooV2 = new SchemaTestUtils.FooV2 { Field1 = ("foo1"), Field3 = (10) }; }
public static Writer[] GetWriteActions(AvroSchema schema) { var actions = new List <Expression <Writer> >(); var writerParameter = Expression.Parameter( typeof(JsonTextWriter), "w" ); var quoteParameter = Expression.Parameter( typeof(bool), "q" ); var valueParameter = Expression.Parameter( typeof(string), "v" ); AppendWriteAction( actions, schema, null, writerParameter, quoteParameter, valueParameter, null, null ); return(actions.Select(r => r.Compile() as Writer).ToArray()); }
public virtual void TestSeparatedGetProducerDataAssigned() { var fooSchema = AvroSchema <SchemaTestUtils.Foo> .Of(ISchemaDefinition <SchemaTestUtils.Foo> .Builder().WithPojo(typeof(SchemaTestUtils.Foo)).Build()); var barSchema = AvroSchema <SchemaTestUtils.Bar> .Of(ISchemaDefinition <SchemaTestUtils.Bar> .Builder().WithPojo(typeof(SchemaTestUtils.Bar)).Build()); var keyValueSchema = ISchema <int> .KeyValue(fooSchema, barSchema, KeyValueEncodingType.SEPARATED); var foo = new SchemaTestUtils.Foo { Field1 = "field1", Field2 = "field2", Field3 = 3 }; var bar = new SchemaTestUtils.Bar { Field1 = true }; // Check kv.encoding.type SPRAERATE var encodeBytes = keyValueSchema.Encode(new KeyValue <SchemaTestUtils.Foo, SchemaTestUtils.Bar>(foo, bar)); var builder = new MessageMetadata(); builder.ProducerName = "separated"; builder.PartitionKey = Convert.ToBase64String(fooSchema.Encode(foo)); builder.PartitionKeyB64Encoded = true; var msg = Message <KeyValue <SchemaTestUtils.Foo, SchemaTestUtils.Bar> > .Create(builder, new ReadOnlySequence <byte>(encodeBytes), keyValueSchema); var keyValue = msg.Value; Assert.Equal(keyValue.Key, foo); Assert.Equal(keyValue.Value, bar); Assert.True(builder.ShouldSerializePartitionKey()); }
public JsonEncoder(TextWriter stream, AvroSchema schema, string delimiter = null) { _stream = stream; _writer = new JsonTextWriter(_stream); _delimiter = delimiter; _actions = AvroJsonFormatter.GetWriteActions(schema); }
public static Action <IAvroEncoder, T> ResolveWriter <T>(AvroSchema writerSchema) { var type = typeof(T); var assembly = type.Assembly; var attributes = GetSerializationTypes(writerSchema); var streamParameter = Expression.Parameter(typeof(IAvroEncoder), "s"); var valueParameter = Expression.Parameter(type, "v"); if (type.Equals(typeof(GenericRecord)) || type.Equals(typeof(IAvroRecord)) || type.Equals(typeof(object))) { assembly = null; } var writeExpression = ResolveWriter(assembly, type, writerSchema, streamParameter, valueParameter, null); if (writeExpression == null) { throw new AvroException($"Unable to resolve writer: '{writerSchema}' for type: '{type}'"); } return(Expression.Lambda <Action <IAvroEncoder, T> >( writeExpression, streamParameter, valueParameter ).Compile()); }
public static Expression CreateReadExpression(AvroSchema schema, Type type) { var expr = default(Expression); var types = SerializationMap.GetTypes(schema); return(expr); }
public void SchemaWrite(AvroSchema schema, string expectedCanonicalAvro, string expectedDefaultAvro, string expectedFullAvro) { var canonicalAvro = new StringBuilder(); using (var writer = new StringWriter(canonicalAvro)) AvroParser.WriteAvroCanonical(writer, schema); var actualCanonicalAvro = canonicalAvro.ToString(); var defaultAvro = new StringBuilder(); using (var writer = new StringWriter(defaultAvro)) AvroParser.WriteAvro(writer, schema); var actualDefaultAvro = defaultAvro.ToString(); var fullAvro = new StringBuilder(); using (var writer = new StringWriter(fullAvro)) AvroParser.WriteAvroFull(writer, schema); var actualFullAvro = fullAvro.ToString(); Assert.AreEqual(expectedCanonicalAvro, actualCanonicalAvro, "Canonical form mismatch"); Assert.AreEqual(expectedDefaultAvro, actualDefaultAvro, "Default form mismatch"); Assert.AreEqual(expectedFullAvro, actualFullAvro, "Full form mismatch"); actualCanonicalAvro = schema.ToAvroCanonical(); actualDefaultAvro = schema.ToAvro(); actualFullAvro = schema.ToAvroFull(); Assert.AreEqual(expectedCanonicalAvro, actualCanonicalAvro, "Extension - Canonical form mismatch"); Assert.AreEqual(expectedDefaultAvro, actualDefaultAvro, "Extension - Default form mismatch"); Assert.AreEqual(expectedFullAvro, actualFullAvro, "Extension - Full form mismatch"); }
public void AddSchema(AvroSchema schema) { if (schema is UnionSchema) { AddSchemas(schema as UnionSchema); } if (schema is NamedSchema) { var namedSchema = schema as NamedSchema; var schemaName = namedSchema.FullName; if (!_code.ContainsKey(namedSchema.FullName)) { _code.Add(schemaName, CreateCode(namedSchema, _nsMap)); } if (schema is RecordSchema) { var recordSchema = schema as RecordSchema; foreach (var fieldSchema in recordSchema) { AddSchema(fieldSchema.Type); } } } }
public override bool Equals(AvroSchema other) { return(base.Equals(other) && (other is FixedSchema) && (other as FixedSchema).Size == Size ); }
public virtual void TestFillParametersToSchemainfo() { var fooSchema = AvroSchema <Foo> .Of(ISchemaDefinition <Foo> .Builder().WithPojo(typeof(Foo)).Build()); var barSchema = AvroSchema <Bar> .Of(ISchemaDefinition <Bar> .Builder().WithPojo(typeof(Bar)).Build()); fooSchema.SchemaInfo.Name = "foo"; fooSchema.SchemaInfo.Type = SchemaType.AVRO; IDictionary <string, string> keyProperties = new Dictionary <string, string> { ["foo.key1"] = "value", ["foo.key2"] = "value" }; fooSchema.SchemaInfo.Properties = keyProperties; barSchema.SchemaInfo.Name = "bar"; barSchema.SchemaInfo.Type = SchemaType.AVRO; IDictionary <string, string> valueProperties = new Dictionary <string, string> { ["bar.key"] = "key" }; barSchema.SchemaInfo.Properties = valueProperties; var keyValueSchema1 = ISchema <object> .KeyValue(fooSchema, barSchema); Assert.Equal("foo", keyValueSchema1.SchemaInfo.Properties["key.schema.name"]); Assert.Equal(SchemaType.AVRO.ToString(), keyValueSchema1.SchemaInfo.Properties["key.schema.type"]); Assert.Equal(@"{""foo.key1"":""value"",""foo.key2"":""value""}", keyValueSchema1.SchemaInfo.Properties["key.schema.properties"]); Assert.Equal("bar", keyValueSchema1.SchemaInfo.Properties["value.schema.name"]); Assert.Equal(SchemaType.AVRO.ToString(), keyValueSchema1.SchemaInfo.Properties["value.schema.type"]); Assert.Equal(@"{""bar.key"":""key""}", keyValueSchema1.SchemaInfo.Properties["value.schema.properties"]); }
public static Reader[] GetReadActions(AvroSchema schema) { var actions = new List <Expression <Reader> >(); var readerParameter = Expression.Parameter( typeof(JsonTextReader), "w" ); var quoteParameter = Expression.Parameter( typeof(bool), "q" ); var valueParameter = Expression.Parameter( typeof(string).MakeByRefType(), "v" ); AppendReadAction( actions, schema, null, readerParameter, quoteParameter, valueParameter, null, null ); return(actions.Select(r => r.Compile() as Reader).ToArray()); }
public void ProduceAndConsume() { var record1 = AvroSchema <SimpleRecord> .Of(typeof(SimpleRecord)); var consumerBuilder = new ConsumerConfigBuilder <SimpleRecord>() .Topic(_topic) .ConsumerName("avroUpgradeSchema1") .SubscriptionName("test-sub"); var consumer = _client.NewConsumer(record1, consumerBuilder); var producerBuilder = new ProducerConfigBuilder <SimpleRecord>() .Topic(_topic) .ProducerName("avroUpgradeSchema1"); var producer = _client.NewProducer(record1, producerBuilder); producer.NewMessage() .Value(new SimpleRecord { Name = "Ebere Abanonu", Age = int.MaxValue }) .Send(); Thread.Sleep(TimeSpan.FromSeconds(10)); var message = consumer.Receive(); Assert.NotNull(message); consumer.Acknowledge(message); consumer.Unsubscribe(); var record2 = AvroSchema <SimpleRecord2> .Of(typeof(SimpleRecord2)); var consumerBuilder2 = new ConsumerConfigBuilder <SimpleRecord2>() .Topic(_topic) .ConsumerName("avroUpgradeSchema2") .SubscriptionName("test-sub"); var consumer1 = _client.NewConsumer(record2, consumerBuilder2); var producerBuilder2 = new ProducerConfigBuilder <SimpleRecord2>() .Topic(_topic) .ProducerName("avroUpgradeSchema2"); var producer2 = _client.NewProducer(record2, producerBuilder2); producer2.NewMessage() .Value(new SimpleRecord2 { Name = "Ebere", Age = int.MaxValue, Surname = "Abanonu" }) .Send(); Thread.Sleep(TimeSpan.FromSeconds(10)); var msg = consumer1.Receive(); Assert.NotNull(msg); consumer1.Acknowledge(msg); consumer1.Unsubscribe(); producer.Close(); producer2.Close(); consumer.Close(); consumer1.Close(); }
public UuidSchema(AvroSchema type) : base(type, "uuid") { if (!(type is StringSchema)) { throw new AvroParseException("Expected 'string' as type for logical type 'uuid'"); } }
public TimeMillisSchema(AvroSchema type) : base(type, "time-millis") { if (!(type is IntSchema)) { throw new AvroParseException("Expected 'int' as type for logical type 'time-millis'"); } }
public override bool Equals(AvroSchema obj) { return(base.Equals(obj) && (obj is DecimalSchema) && (obj as DecimalSchema).Precision == Precision && (obj as DecimalSchema).Scale == Scale ); }
public TimestampMicrosSchema(AvroSchema type) : base(type, "timestamp-micros") { if (!(type is LongSchema)) { throw new AvroParseException("Expected 'long' as type for logical type 'timestamp-micros'"); } }
public Field(string name, AvroSchema type) : base() { Name = name; Type = type; Aliases = new List <string>(); Default = null; }
public DurationSchema(AvroSchema type) : base(type, "duration") { if (!(type is FixedSchema) || (type as FixedSchema).Size != 12) { throw new AvroParseException("Expected 'fixed' with size '12' as type for logical type 'duration'"); } }
public GenericAvroSchemaTest() { var avroFooV2Schema = AvroSchema <SchemaTestUtils.FooV2> .Of(ISchemaDefinition <SchemaTestUtils.FooV2> .Builder().WithAlwaysAllowNull(false).WithPojo(typeof(SchemaTestUtils.FooV2)).Build()); //var avroFooSchema = AvroSchema<SchemaTestUtils.Foo>.Of(ISchemaDefinition<SchemaTestUtils.Foo>.Builder().WithAlwaysAllowNull(false).WithPojo(typeof(SchemaTestUtils.Foo)).Build()); writerSchema = new GenericAvroSchema(avroFooV2Schema.SchemaInfo); readerSchema = new GenericAvroSchema(avroFooV2Schema.SchemaInfo); }
public static string PopulateAvroJson(AvroSchema schema) { var obj = Type.GetType(schema.Type).CreateTypeInstance("Schema"); var jsonObject = JObject.Parse(JsonConvert.SerializeObject(obj)); jsonObject.RemoveProperties("Schema"); return(jsonObject.ToString()); }
public DatumReader(AvroSchema readerSchema, AvroSchema writerSchema) { var methods = SchemaResolver.ResolveReader <T>(readerSchema, writerSchema); _reader = methods.Item1; _skipper = methods.Item2; ReaderSchema = readerSchema; WriterSchema = writerSchema; }
public static void ValidateString(AvroSchema schema, string value) { var jToken = default(JToken); if (value != null) { jToken = JToken.Parse(value); } ValidateJson(schema, jToken); }
public JsonDecoder(TextReader stream, AvroSchema schema, string delimiter = null) { _stream = stream; _reader = new JsonTextReader(_stream) { SupportMultipleContent = true }; _delimiter = delimiter; _actions = AvroJsonFormatter.GetReadActions(schema); _reader.Read(); }