public virtual void TestSchemaDefinition()
{
var schema1 = typeof(DefaultStruct).GetSchema();
var schema2 = AvroSchema <StructWithAnnotations> .Of(typeof(StructWithAnnotations));
var schemaDef1 = schema1.ToString();
var schemaDef2 = Encoding.UTF8.GetString(schema2.SchemaInfo.Schema);
Assert.NotEqual(schemaDef1, schemaDef2);
var schema3 = AvroSchema <StructWithAnnotations> .Of(ISchemaDefinition <StructWithAnnotations> .Builder().WithJsonDef(schemaDef1).Build());
var schemaDef3 = Encoding.UTF8.GetString(schema3.SchemaInfo.Schema);
Assert.True(schemaDef1.Contains("DefaultStruct") && schemaDef3.Contains("DefaultStruct"));
Assert.NotEqual(schemaDef2, schemaDef3);
var @struct = new StructWithAnnotations
{
Field1 = 5678
};
// schema2 is using the schema generated from POJO,
// it allows field2 to be nullable, and field3 has default value.
schema2.Encode(@struct);
// schema3 is using the schema passed in, which doesn't allow nullable
schema3.Encode(@struct);
}
public virtual void TestDefaultGetProducerDataAssigned()
{
var fooSchema = AvroSchema <SchemaTestUtils.Foo> .Of(ISchemaDefinition <SchemaTestUtils.Foo> .Builder().WithPojo(typeof(SchemaTestUtils.Foo)).Build());
var barSchema = AvroSchema <SchemaTestUtils.Bar> .Of(ISchemaDefinition <SchemaTestUtils.Bar> .Builder().WithPojo(typeof(SchemaTestUtils.Bar)).Build());
var keyValueSchema = ISchema <object> .KeyValue(fooSchema, barSchema);
var foo = new SchemaTestUtils.Foo
{
Field1 = "field1",
Field2 = "field2",
Field3 = 3
};
var bar = new SchemaTestUtils.Bar
{
Field1 = true
};
// // Check kv.encoding.type default, not set value
var encodeBytes = keyValueSchema.Encode(new KeyValue <SchemaTestUtils.Foo, SchemaTestUtils.Bar>(foo, bar));
var builder = new MessageMetadata();
builder.ProducerName = "default";
var msg = Message <KeyValue <SchemaTestUtils.Foo, SchemaTestUtils.Bar> > .Create(builder, new ReadOnlySequence <byte>(encodeBytes), keyValueSchema);
var keyValue = msg.Value;
Assert.Equal(keyValue.Key, foo);
Assert.Equal(keyValue.Value, bar);
Assert.False(builder.ShouldSerializePartitionKey());
}
public virtual async Task ReaderSourceTaggedTest()
{
var topic = $"reader-topics-{Guid.NewGuid()}";
var ids = PublishMessages(topic, 50);
var start = MessageIdUtils.GetOffset(ids.First());
var end = MessageIdUtils.GetOffset(ids.Last());
var conf = new ReaderConfigBuilder <DataOp>().Topic(topic);
var reader = _pulsarSystem.EventSource("public", "default", topic, start, end, "http://127.0.0.1:8080")
.Reader(_clientConfigurationData, conf, AvroSchema <DataOp> .Of(typeof(DataOp)))
.SourceMethod()
.CurrentTaggedEvents(new Messages.Consumer.Tag("twitter", "mestical"));
//let leave some time to wire everything up
await Task.Delay(TimeSpan.FromSeconds(20));
var receivedCount = 0;
while (receivedCount == 0)
{
await foreach (var response in reader.CurrentEvents(TimeSpan.FromSeconds(5)))
{
receivedCount++;
_output.WriteLine(JsonSerializer.Serialize(response, new JsonSerializerOptions {
WriteIndented = true
}));
}
await Task.Delay(TimeSpan.FromSeconds(1));
}
Assert.True(receivedCount > 0);
}
public virtual void TestInlineKeyValueEncodingTypeSchemaEncodeAndDecode()
{
var fooSchema = AvroSchema <Foo> .Of(ISchemaDefinition <Foo> .Builder().WithPojo(typeof(Foo)).Build());
var barSchema = AvroSchema <Bar> .Of(ISchemaDefinition <Bar> .Builder().WithPojo(typeof(Bar)).Build());
var keyValueSchema = ISchema <object> .KeyValue(fooSchema, barSchema, KeyValueEncodingType.INLINE);
var bar = new Bar
{
Field1 = true
};
var foo = new Foo
{
Field1 = "field1",
Field2 = "field2",
Field3 = "3",
Field4 = bar,
Color = Color.RED
};
// Check kv.encoding.type INLINE
var encodeBytes = keyValueSchema.Encode(new KeyValue <Foo, Bar>(foo, bar));
Assert.True(encodeBytes.Length > 0);
var keyValue = keyValueSchema.Decode(encodeBytes);
var fooBack = keyValue.Key;
var barBack = keyValue.Value;
Assert.True(foo.Equals(fooBack));
Assert.True(bar.Equals(barBack));
}
public virtual void TestAllowNullBytesSchemaEncodeAndDecode()
{
var fooAvroSchema = AvroSchema <Foo> .Of(ISchemaDefinition <Foo> .Builder().WithPojo(typeof(Foo)).Build());
var barAvroSchema = AvroSchema <Bar> .Of(ISchemaDefinition <Bar> .Builder().WithPojo(typeof(Bar)).Build());
var bar = new Bar
{
Field1 = true
};
var foo = new Foo
{
Field1 = "field1",
Field2 = "field2",
Field3 = "3",
Field4 = bar,
Color = Color.RED
};
var fooBytes = fooAvroSchema.Encode(foo);
var barBytes = barAvroSchema.Encode(bar);
var encodeBytes = ISchema <object> .KvBytes().Encode(new KeyValue <byte[], byte[]>(fooBytes, barBytes));
var decodeKV = ISchema <object> .KvBytes().Decode(encodeBytes);
var fooBack = fooAvroSchema.Decode(decodeKV.Key);
var barBack = barAvroSchema.Decode(decodeKV.Value);
Assert.True(foo.Equals(fooBack));
Assert.True(bar.Equals(barBack));
}
public void TypeLookupAdvancedTest(Type expectedType, AvroSchema schema)
{
var assembly = Assembly.GetExecutingAssembly();
var actualType = SchemaResolver.GetTypeFromSchema(schema, assembly);
Assert.AreEqual(expectedType, actualType);
}
private ISet <MessageId> PublishMessages(string topic, int count)
{
var ids = new HashSet <MessageId>();
var builder = new ProducerConfigBuilder <DataOp>()
.Topic(topic);
var producer = _client.NewProducer(AvroSchema <DataOp> .Of(typeof(DataOp)), builder);
for (var i = 0; i < count; i++)
{
var key = "key" + i;
MessageId id = null;
if (i % 2 == 0)
{
id = producer.NewMessage().Key(key).Property("twitter", "mestical").Value(new DataOp {
Text = "my-event-message-" + i, EventTime = DateTimeHelper.CurrentUnixTimeMillis()
}).Send();
}
else
{
id = producer.NewMessage().Key(key).Value(new DataOp {
Text = "my-event-message-" + i, EventTime = DateTimeHelper.CurrentUnixTimeMillis()
}).Send();
}
ids.Add(id);
}
return(ids);
}
public GenericAvroReaderTest()
{
fooSchema = AvroSchema <SchemaTestUtils.Foo> .Of(typeof(SchemaTestUtils.Foo));
fooV2Schema = AvroSchema <SchemaTestUtils.FooV2> .Of(typeof(SchemaTestUtils.FooV2));
fooSchemaNotNull = AvroSchema <SchemaTestUtils.Foo> .Of(ISchemaDefinition <SchemaTestUtils.Foo> .Builder().WithAlwaysAllowNull(false).WithPojo(typeof(SchemaTestUtils.Foo)).Build());
fooOffsetSchema = AvroSchema <SchemaTestUtils.Foo> .Of(typeof(SchemaTestUtils.Foo));
fooOffsetSchema.SchemaInfo.Properties.Add(GenericAvroSchema.OFFSET_PROP, "5");
var prop = fooOffsetSchema.AvroSchema.GetProperty(GenericAvroSchema.OFFSET_PROP);
foo = new SchemaTestUtils.Foo
{
Field1 = "foo1",
Field2 = "bar1",
Field4 = new SchemaTestUtils.Bar(),
FieldUnableNull = "notNull"
};
fooV2 = new SchemaTestUtils.FooV2
{
Field1 = ("foo1"),
Field3 = (10)
};
}
public static Writer[] GetWriteActions(AvroSchema schema)
{
var actions = new List <Expression <Writer> >();
var writerParameter =
Expression.Parameter(
typeof(JsonTextWriter),
"w"
);
var quoteParameter =
Expression.Parameter(
typeof(bool),
"q"
);
var valueParameter =
Expression.Parameter(
typeof(string),
"v"
);
AppendWriteAction(
actions,
schema,
null,
writerParameter,
quoteParameter,
valueParameter,
null,
null
);
return(actions.Select(r => r.Compile() as Writer).ToArray());
}
public virtual void TestSeparatedGetProducerDataAssigned()
{
var fooSchema = AvroSchema <SchemaTestUtils.Foo> .Of(ISchemaDefinition <SchemaTestUtils.Foo> .Builder().WithPojo(typeof(SchemaTestUtils.Foo)).Build());
var barSchema = AvroSchema <SchemaTestUtils.Bar> .Of(ISchemaDefinition <SchemaTestUtils.Bar> .Builder().WithPojo(typeof(SchemaTestUtils.Bar)).Build());
var keyValueSchema = ISchema <int> .KeyValue(fooSchema, barSchema, KeyValueEncodingType.SEPARATED);
var foo = new SchemaTestUtils.Foo
{
Field1 = "field1",
Field2 = "field2",
Field3 = 3
};
var bar = new SchemaTestUtils.Bar
{
Field1 = true
};
// Check kv.encoding.type SPRAERATE
var encodeBytes = keyValueSchema.Encode(new KeyValue <SchemaTestUtils.Foo, SchemaTestUtils.Bar>(foo, bar));
var builder = new MessageMetadata();
builder.ProducerName = "separated";
builder.PartitionKey = Convert.ToBase64String(fooSchema.Encode(foo));
builder.PartitionKeyB64Encoded = true;
var msg = Message <KeyValue <SchemaTestUtils.Foo, SchemaTestUtils.Bar> > .Create(builder, new ReadOnlySequence <byte>(encodeBytes), keyValueSchema);
var keyValue = msg.Value;
Assert.Equal(keyValue.Key, foo);
Assert.Equal(keyValue.Value, bar);
Assert.True(builder.ShouldSerializePartitionKey());
}
public JsonEncoder(TextWriter stream, AvroSchema schema, string delimiter = null)
{
_stream = stream;
_writer = new JsonTextWriter(_stream);
_delimiter = delimiter;
_actions = AvroJsonFormatter.GetWriteActions(schema);
}
public static Action <IAvroEncoder, T> ResolveWriter <T>(AvroSchema writerSchema)
{
var type = typeof(T);
var assembly = type.Assembly;
var attributes = GetSerializationTypes(writerSchema);
var streamParameter = Expression.Parameter(typeof(IAvroEncoder), "s");
var valueParameter = Expression.Parameter(type, "v");
if (type.Equals(typeof(GenericRecord)) || type.Equals(typeof(IAvroRecord)) || type.Equals(typeof(object)))
{
assembly = null;
}
var writeExpression = ResolveWriter(assembly, type, writerSchema, streamParameter, valueParameter, null);
if (writeExpression == null)
{
throw new AvroException($"Unable to resolve writer: '{writerSchema}' for type: '{type}'");
}
return(Expression.Lambda <Action <IAvroEncoder, T> >(
writeExpression,
streamParameter,
valueParameter
).Compile());
}
public static Expression CreateReadExpression(AvroSchema schema, Type type)
{
var expr = default(Expression);
var types = SerializationMap.GetTypes(schema);
return(expr);
}
public void SchemaWrite(AvroSchema schema, string expectedCanonicalAvro, string expectedDefaultAvro, string expectedFullAvro)
{
var canonicalAvro = new StringBuilder();
using (var writer = new StringWriter(canonicalAvro))
AvroParser.WriteAvroCanonical(writer, schema);
var actualCanonicalAvro = canonicalAvro.ToString();
var defaultAvro = new StringBuilder();
using (var writer = new StringWriter(defaultAvro))
AvroParser.WriteAvro(writer, schema);
var actualDefaultAvro = defaultAvro.ToString();
var fullAvro = new StringBuilder();
using (var writer = new StringWriter(fullAvro))
AvroParser.WriteAvroFull(writer, schema);
var actualFullAvro = fullAvro.ToString();
Assert.AreEqual(expectedCanonicalAvro, actualCanonicalAvro, "Canonical form mismatch");
Assert.AreEqual(expectedDefaultAvro, actualDefaultAvro, "Default form mismatch");
Assert.AreEqual(expectedFullAvro, actualFullAvro, "Full form mismatch");
actualCanonicalAvro = schema.ToAvroCanonical();
actualDefaultAvro = schema.ToAvro();
actualFullAvro = schema.ToAvroFull();
Assert.AreEqual(expectedCanonicalAvro, actualCanonicalAvro, "Extension - Canonical form mismatch");
Assert.AreEqual(expectedDefaultAvro, actualDefaultAvro, "Extension - Default form mismatch");
Assert.AreEqual(expectedFullAvro, actualFullAvro, "Extension - Full form mismatch");
}
public void AddSchema(AvroSchema schema)
{
if (schema is UnionSchema)
{
AddSchemas(schema as UnionSchema);
}
if (schema is NamedSchema)
{
var namedSchema = schema as NamedSchema;
var schemaName = namedSchema.FullName;
if (!_code.ContainsKey(namedSchema.FullName))
{
_code.Add(schemaName, CreateCode(namedSchema, _nsMap));
}
if (schema is RecordSchema)
{
var recordSchema = schema as RecordSchema;
foreach (var fieldSchema in recordSchema)
{
AddSchema(fieldSchema.Type);
}
}
}
}
public override bool Equals(AvroSchema other)
{
return(base.Equals(other) &&
(other is FixedSchema) &&
(other as FixedSchema).Size == Size
);
}
public virtual void TestFillParametersToSchemainfo()
{
var fooSchema = AvroSchema <Foo> .Of(ISchemaDefinition <Foo> .Builder().WithPojo(typeof(Foo)).Build());
var barSchema = AvroSchema <Bar> .Of(ISchemaDefinition <Bar> .Builder().WithPojo(typeof(Bar)).Build());
fooSchema.SchemaInfo.Name = "foo";
fooSchema.SchemaInfo.Type = SchemaType.AVRO;
IDictionary <string, string> keyProperties = new Dictionary <string, string>
{
["foo.key1"] = "value",
["foo.key2"] = "value"
};
fooSchema.SchemaInfo.Properties = keyProperties;
barSchema.SchemaInfo.Name = "bar";
barSchema.SchemaInfo.Type = SchemaType.AVRO;
IDictionary <string, string> valueProperties = new Dictionary <string, string>
{
["bar.key"] = "key"
};
barSchema.SchemaInfo.Properties = valueProperties;
var keyValueSchema1 = ISchema <object> .KeyValue(fooSchema, barSchema);
Assert.Equal("foo", keyValueSchema1.SchemaInfo.Properties["key.schema.name"]);
Assert.Equal(SchemaType.AVRO.ToString(), keyValueSchema1.SchemaInfo.Properties["key.schema.type"]);
Assert.Equal(@"{""foo.key1"":""value"",""foo.key2"":""value""}", keyValueSchema1.SchemaInfo.Properties["key.schema.properties"]);
Assert.Equal("bar", keyValueSchema1.SchemaInfo.Properties["value.schema.name"]);
Assert.Equal(SchemaType.AVRO.ToString(), keyValueSchema1.SchemaInfo.Properties["value.schema.type"]);
Assert.Equal(@"{""bar.key"":""key""}", keyValueSchema1.SchemaInfo.Properties["value.schema.properties"]);
}
public static Reader[] GetReadActions(AvroSchema schema)
{
var actions = new List <Expression <Reader> >();
var readerParameter =
Expression.Parameter(
typeof(JsonTextReader),
"w"
);
var quoteParameter =
Expression.Parameter(
typeof(bool),
"q"
);
var valueParameter =
Expression.Parameter(
typeof(string).MakeByRefType(),
"v"
);
AppendReadAction(
actions,
schema,
null,
readerParameter,
quoteParameter,
valueParameter,
null,
null
);
return(actions.Select(r => r.Compile() as Reader).ToArray());
}
public void ProduceAndConsume()
{
var record1 = AvroSchema <SimpleRecord> .Of(typeof(SimpleRecord));
var consumerBuilder = new ConsumerConfigBuilder <SimpleRecord>()
.Topic(_topic)
.ConsumerName("avroUpgradeSchema1")
.SubscriptionName("test-sub");
var consumer = _client.NewConsumer(record1, consumerBuilder);
var producerBuilder = new ProducerConfigBuilder <SimpleRecord>()
.Topic(_topic)
.ProducerName("avroUpgradeSchema1");
var producer = _client.NewProducer(record1, producerBuilder);
producer.NewMessage()
.Value(new SimpleRecord {
Name = "Ebere Abanonu", Age = int.MaxValue
})
.Send();
Thread.Sleep(TimeSpan.FromSeconds(10));
var message = consumer.Receive();
Assert.NotNull(message);
consumer.Acknowledge(message);
consumer.Unsubscribe();
var record2 = AvroSchema <SimpleRecord2> .Of(typeof(SimpleRecord2));
var consumerBuilder2 = new ConsumerConfigBuilder <SimpleRecord2>()
.Topic(_topic)
.ConsumerName("avroUpgradeSchema2")
.SubscriptionName("test-sub");
var consumer1 = _client.NewConsumer(record2, consumerBuilder2);
var producerBuilder2 = new ProducerConfigBuilder <SimpleRecord2>()
.Topic(_topic)
.ProducerName("avroUpgradeSchema2");
var producer2 = _client.NewProducer(record2, producerBuilder2);
producer2.NewMessage()
.Value(new SimpleRecord2 {
Name = "Ebere", Age = int.MaxValue, Surname = "Abanonu"
})
.Send();
Thread.Sleep(TimeSpan.FromSeconds(10));
var msg = consumer1.Receive();
Assert.NotNull(msg);
consumer1.Acknowledge(msg);
consumer1.Unsubscribe();
producer.Close();
producer2.Close();
consumer.Close();
consumer1.Close();
}
public UuidSchema(AvroSchema type)
: base(type, "uuid")
{
if (!(type is StringSchema))
{
throw new AvroParseException("Expected 'string' as type for logical type 'uuid'");
}
}
public TimeMillisSchema(AvroSchema type)
: base(type, "time-millis")
{
if (!(type is IntSchema))
{
throw new AvroParseException("Expected 'int' as type for logical type 'time-millis'");
}
}
public override bool Equals(AvroSchema obj)
{
return(base.Equals(obj) &&
(obj is DecimalSchema) &&
(obj as DecimalSchema).Precision == Precision &&
(obj as DecimalSchema).Scale == Scale
);
}
public TimestampMicrosSchema(AvroSchema type)
: base(type, "timestamp-micros")
{
if (!(type is LongSchema))
{
throw new AvroParseException("Expected 'long' as type for logical type 'timestamp-micros'");
}
}
public Field(string name, AvroSchema type)
: base()
{
Name = name;
Type = type;
Aliases = new List <string>();
Default = null;
}
public DurationSchema(AvroSchema type)
: base(type, "duration")
{
if (!(type is FixedSchema) || (type as FixedSchema).Size != 12)
{
throw new AvroParseException("Expected 'fixed' with size '12' as type for logical type 'duration'");
}
}
public GenericAvroSchemaTest()
{
var avroFooV2Schema = AvroSchema <SchemaTestUtils.FooV2> .Of(ISchemaDefinition <SchemaTestUtils.FooV2> .Builder().WithAlwaysAllowNull(false).WithPojo(typeof(SchemaTestUtils.FooV2)).Build());
//var avroFooSchema = AvroSchema<SchemaTestUtils.Foo>.Of(ISchemaDefinition<SchemaTestUtils.Foo>.Builder().WithAlwaysAllowNull(false).WithPojo(typeof(SchemaTestUtils.Foo)).Build());
writerSchema = new GenericAvroSchema(avroFooV2Schema.SchemaInfo);
readerSchema = new GenericAvroSchema(avroFooV2Schema.SchemaInfo);
}
public static string PopulateAvroJson(AvroSchema schema)
{
var obj = Type.GetType(schema.Type).CreateTypeInstance("Schema");
var jsonObject = JObject.Parse(JsonConvert.SerializeObject(obj));
jsonObject.RemoveProperties("Schema");
return(jsonObject.ToString());
}
public DatumReader(AvroSchema readerSchema, AvroSchema writerSchema)
{
var methods = SchemaResolver.ResolveReader <T>(readerSchema, writerSchema);
_reader = methods.Item1;
_skipper = methods.Item2;
ReaderSchema = readerSchema;
WriterSchema = writerSchema;
}
public static void ValidateString(AvroSchema schema, string value)
{
var jToken = default(JToken);
if (value != null)
{
jToken = JToken.Parse(value);
}
ValidateJson(schema, jToken);
}
public JsonDecoder(TextReader stream, AvroSchema schema, string delimiter = null)
{
_stream = stream;
_reader = new JsonTextReader(_stream)
{
SupportMultipleContent = true
};
_delimiter = delimiter;
_actions = AvroJsonFormatter.GetReadActions(schema);
_reader.Read();
}
