public void IImmutableSetValues(ISet <string> value)
{
var schema = new ArraySchema(new StringSchema());
var deserialize = deserializerBuilder.BuildDelegate <IImmutableSet <string> >(schema);
var serialize = serializerBuilder.BuildDelegate <IImmutableSet <string> >(schema);
using (stream)
{
serialize(value.ToImmutableHashSet(), new BinaryWriter(stream));
}
var reader = new BinaryReader(stream.ToArray());
Assert.Equal(value, deserialize(ref reader).OrderBy(v => v));
}
public void DynamicArrayValues(long[] value)
{
var schema = new ArraySchema(new LongSchema());
var deserialize = deserializerBuilder.BuildDelegate <dynamic>(schema);
var serialize = serializerBuilder.BuildDelegate <dynamic>(schema);
using (stream)
{
serialize(value, new BinaryWriter(stream));
}
var reader = new BinaryReader(stream.ToArray());
Assert.Equal(value.Cast <object>(), deserialize(ref reader));
}
public void HashSetValues(HashSet <string> value)
{
var schema = new ArraySchema(new StringSchema());
var deserialize = deserializerBuilder.BuildDelegate <HashSet <string> >(schema);
var serialize = serializerBuilder.BuildDelegate <HashSet <string> >(schema);
using (stream)
{
serialize(value, new BinaryWriter(stream));
}
var reader = new BinaryReader(stream.ToArray());
Assert.Equal(value, deserialize(ref reader));
}
public void ImmutableSortedSetValues(HashSet <string> value)
{
var schema = new ArraySchema(new StringSchema());
var deserialize = deserializerBuilder.BuildDelegate <ImmutableSortedSet <string> >(schema);
var serialize = serializerBuilder.BuildDelegate <ImmutableSortedSet <string> >(schema);
using (stream)
{
serialize(value.ToImmutableSortedSet(), new Utf8JsonWriter(stream));
}
var reader = new Utf8JsonReader(stream.ToArray());
Assert.Equal(value, deserialize(ref reader));
}
public void AddInvalidItems()
{
var unionSchema = new UnionSchema();
var arraySchema = new ArraySchema(new IntSchema());
var mapSchema = new MapSchema(new IntSchema());
var enumSchema = new EnumSchema("X.Y.Z", null, new string[] { "A", "B", "C" });
Assert.DoesNotThrow(() => unionSchema.Add(arraySchema));
Assert.DoesNotThrow(() => unionSchema.Add(mapSchema));
Assert.DoesNotThrow(() => unionSchema.Add(enumSchema));
Assert.Throws(typeof(AvroParseException), () => unionSchema.Add(unionSchema));
Assert.Throws(typeof(AvroParseException), () => unionSchema.Add(arraySchema));
Assert.Throws(typeof(AvroParseException), () => unionSchema.Add(mapSchema));
Assert.Throws(typeof(AvroParseException), () => unionSchema.Add(enumSchema));
}
public void CollectionValues(long[] value)
{
var schema = new ArraySchema(new LongSchema());
var deserialize = deserializerBuilder.BuildDelegate <Collection <long> >(schema);
var serialize = serializerBuilder.BuildDelegate <Collection <long> >(schema);
using (stream)
{
serialize(new Collection <long>(value), new Utf8JsonWriter(stream));
}
var reader = new Utf8JsonReader(stream.ToArray());
Assert.Equal(value, deserialize(ref reader));
}
public void JaggedArrayValues(string[][] value)
{
var schema = new ArraySchema(new ArraySchema(new StringSchema()));
var deserialize = deserializerBuilder.BuildDelegate <string[][]>(schema);
var serialize = serializerBuilder.BuildDelegate <string[][]>(schema);
using (stream)
{
serialize(value, new Utf8JsonWriter(stream));
}
var reader = new Utf8JsonReader(stream.ToArray());
Assert.Equal(value, deserialize(ref reader));
}
public void ImmutableStackValues(long[] value)
{
var schema = new ArraySchema(new LongSchema());
var deserialize = deserializerBuilder.BuildDelegate <ImmutableStack <long> >(schema);
var serialize = serializerBuilder.BuildDelegate <ImmutableStack <long> >(schema);
using (stream)
{
serialize(ImmutableStack.CreateRange(value), new Utf8JsonWriter(stream));
}
var reader = new Utf8JsonReader(stream.ToArray());
Assert.Equal(value, deserialize(ref reader));
}
public void IListValues(long[] value)
{
var schema = new ArraySchema(new LongSchema());
var deserialize = deserializerBuilder.BuildDelegate <IList <long> >(schema);
var serialize = serializerBuilder.BuildDelegate <IList <long> >(schema);
using (stream)
{
serialize(value, new BinaryWriter(stream));
}
var reader = new BinaryReader(stream.ToArray());
Assert.Equal(value, deserialize(ref reader));
}
public void ISetValues(ISet <string> value)
{
var schema = new ArraySchema(new StringSchema());
var deserialize = deserializerBuilder.BuildDelegate <ISet <string> >(schema);
var serialize = serializerBuilder.BuildDelegate <ISet <string> >(schema);
using (stream)
{
serialize(value, new Utf8JsonWriter(stream));
}
var reader = new Utf8JsonReader(stream.ToArray());
Assert.Equal(value, deserialize(ref reader).OrderBy(v => v));
}
/// <summary>
/// Serialized an array. The default implementation calls EnsureArrayObject() to ascertain that the
/// given value is an array. It then calls GetArrayLength() and GetArrayElement()
/// to access the members of the array and then serialize them.
/// </summary>
/// <param name="schema">The ArraySchema for serialization</param>
/// <param name="value">The value being serialized</param>
/// <param name="encoder">The encoder for serialization</param>
protected override void WriteArray(ArraySchema schema, object value, Encoder encoder)
{
var arr = value as System.Collections.IList;
if (arr == null)
{
throw new AvroTypeException("Array does not implement non-generic IList");
}
long l = arr.Count;
encoder.WriteArrayStart();
encoder.SetItemCount(l);
for (int i = 0; i < l; i++)
{
encoder.StartItem();
Write(schema.ItemSchema, arr[i], encoder);
}
encoder.WriteArrayEnd();
}
private void WriteArray(object source, JsonWriter jsonWriter, ArraySchema arraySchema)
{
if (source == null)
{
return;
}
WriteItem writeArrayItem = ResolveItemWriter(arraySchema.ItemSchema);
jsonWriter.WriteStartArray();
var list = (IList)source;
for (int i = 0; i < list.Count; i++)
{
if (list[i] == null)
{
continue;
}
writeArrayItem(list[i], jsonWriter);
}
jsonWriter.WriteEndArray();
}
public static Array DecodeArray(this ArraySchema schema, JToken value)
{
var jarray = value as JArray;
if (jarray != null)
{
var itemType = GetNativeType(schema.ItemSchema);
var itemArray = Array.CreateInstance(itemType, jarray.Count);
var itemIndex = 0;
foreach (var item in jarray.Values())
{
itemArray.SetValue(DecodeAny(schema.ItemSchema, item), itemIndex);
itemIndex++;
}
return(itemArray);
}
throw new ArgumentException("invalid value type: " + value.GetType().FullName);
}
public void ArrayTest()
{
var writerSchema = new ArraySchema(new IntSchema());
var readerSchema = new ArraySchema(new LongSchema());
var writer = new DatumWriter <IList <int> >(writerSchema);
var reader = new DatumReader <IList <long> >(readerSchema, writerSchema);
var expectedArray = new List <int> {
10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0
};
using (var stream = new MemoryStream())
using (var encoder = new BinaryEncoder(stream))
using (var decoder = new BinaryDecoder(stream))
{
stream.Seek(0, SeekOrigin.Begin);
writer.Write(encoder, expectedArray);
stream.Seek(0, SeekOrigin.Begin);
Assert.AreEqual(expectedArray, reader.Read(decoder));
}
}
protected virtual object ReadArray(ArraySchema writerSchema, Schema readerSchema, IReader d)
{
ArraySchema rs = (ArraySchema)readerSchema;
object[] result = new object[0];
int i = 0;
for (int n = (int)d.ReadArrayStart(); n != 0; n = (int)d.ReadArrayNext())
{
if (result.Length < i + n)
{
Array.Resize(ref result, i + n);
}
for (int j = 0; j < n; j++, i++)
{
result[i] = Read(writerSchema.ItemSchema, rs.ItemSchema, d);
}
}
if (result[0] is IDictionary)
{
Dictionary <object, object> resultDictionary = new Dictionary <object, object>();
foreach (Dictionary <string, object> keyValue in result)
{
if (!keyValue.ContainsKey("Key") || !keyValue.ContainsKey("Value"))
{
//HACK for reading c# dictionaries, which are not avro maps
return(result);
}
resultDictionary.Add(keyValue["Key"], keyValue["Value"]);
}
return(resultDictionary);
}
return(result);
}
/// <summary>
/// Deserializes an array and returns an array object. It uses CreateArray() and works on it before returning it.
/// It also uses GetArraySize(), ResizeArray(), SetArrayElement() and GetArrayElement() methods. Derived classes can
/// override these methods to customize their behavior.
/// </summary>
/// <param name="reuse">If appropriate, uses this instead of creating a new array object.</param>
/// <param name="writerSchema">The schema used by the writer.</param>
/// <param name="readerSchema">The schema that the reader uses.</param>
/// <param name="d">The decoder for deserialization.</param>
/// <returns>The deserialized array object.</returns>
protected virtual object ReadArray(object reuse, ArraySchema writerSchema, Schema readerSchema, Decoder d)
{
ArraySchema rs = (ArraySchema)readerSchema;
object result = CreateArray(reuse, rs);
int i = 0;
for (int n = (int)d.ReadArrayStart(); n != 0; n = (int)d.ReadArrayNext())
{
if (GetArraySize(result) < (i + n))
{
ResizeArray(ref result, i + n);
}
for (int j = 0; j < n; j++, i++)
{
SetArrayElement(result, i, Read(GetArrayElement(result, i), writerSchema.ItemSchema, rs.ItemSchema, d));
}
}
if (GetArraySize(result) != i)
{
ResizeArray(ref result, i);
}
return(result);
}
private IList GetSplitList(IList list, ArraySchema schema)
{
if (list.Count == 0)
{
return(list);
}
var typeToCheck = list.GetType().GetProperties()[2].PropertyType;
if (typeToCheck.GetTypeInfo().IsValueType ||
typeToCheck == typeof(string))
{
return(list);
}
List <object> result = new List <object>();
foreach (var item in list)
{
result.Add(item != null ? SplitKeyValues(item, (RecordSchema)schema.ItemSchema) : null);
}
return(result);
}
protected override ArrayAccess GetArrayAccess(ArraySchema schema)
{
return(new GenericArrayAccess());
}
protected override ArrayAccess GetArrayAccess(ArraySchema readerSchema)
{
return(new SpecificArrayAccess(readerSchema));
}
/// <summary>
/// Serialized an array. The default implementation calls EnsureArrayObject() to ascertain that the
/// given value is an array. It then calls GetArrayLength() and GetArrayElement()
/// to access the members of the array and then serialize them.
/// </summary>
/// <param name="schema">The ArraySchema for serialization</param>
/// <param name="value">The value being serialized</param>
/// <param name="encoder">The encoder for serialization</param>
protected WriteItem ResolveArray(ArraySchema schema)
{
var itemWriter = ResolveWriter(schema.ItemSchema);
return((d, e) => WriteArray(itemWriter, d, e));
}
/// <summary>
/// Creates a new array object. The initial size of the object could be anything. The users
/// should use GetArraySize() to determine the size. The default implementation creates an <c>object[]</c>.
/// </summary>
/// <param name="reuse">If appropriate use this instead of creating a new one.</param>
/// <returns>An object suitable to deserialize an avro array</returns>
protected virtual object CreateArray(object reuse, ArraySchema rs)
{
return((reuse != null && reuse is object[]) ? (object[])reuse : new object[0]);
}
/// <summary>
/// Gets the type for the specified schema
/// </summary>
/// <param name="schema"></param>
/// <returns></returns>
/// <exception cref="AvroException">
/// No type found matching the given name.
/// </exception>
public Type GetType(Schema schema)
{
switch (schema.Tag)
{
case Schema.Type.Null:
break;
case Schema.Type.Boolean:
return(typeof(bool));
case Schema.Type.Int:
return(typeof(int));
case Schema.Type.Long:
return(typeof(long));
case Schema.Type.Float:
return(typeof(float));
case Schema.Type.Double:
return(typeof(double));
case Schema.Type.Bytes:
return(typeof(byte[]));
case Schema.Type.String:
return(typeof(string));
case Schema.Type.Union:
{
if (schema is UnionSchema unSchema && unSchema.Count == 2)
{
Schema s1 = unSchema.Schemas[0];
Schema s2 = unSchema.Schemas[1];
// Nullable ?
Type itemType = null;
if (s1.Tag == Schema.Type.Null)
{
itemType = GetType(s2);
}
else if (s2.Tag == Schema.Type.Null)
{
itemType = GetType(s1);
}
if (itemType != null)
{
if (itemType.IsValueType && !itemType.IsEnum)
{
try
{
return(GenericNullableType.MakeGenericType(itemType));
}
catch
{
}
}
return(itemType);
}
}
return(typeof(object));
}
case Schema.Type.Array:
{
ArraySchema arrSchema = schema as ArraySchema;
Type itemSchema = GetType(arrSchema.ItemSchema);
return(GenericListType.MakeGenericType(itemSchema));
}
case Schema.Type.Map:
{
MapSchema mapSchema = schema as MapSchema;
Type itemSchema = GetType(mapSchema.ValueSchema);
return(GenericMapType.MakeGenericType(typeof(string), itemSchema));
}
case Schema.Type.Enumeration:
case Schema.Type.Record:
case Schema.Type.Fixed:
case Schema.Type.Error:
{
// Should all be named types
if (schema is NamedSchema named)
{
return(FindType(named.Fullname));
}
break;
}
}
// Fallback
return(FindType(schema.Name));
}
/// <summary>
/// Gets the type for the specified schema
/// </summary>
/// <param name="schema"></param>
/// <returns></returns>
public Type GetType(Schema.Schema schema)
{
switch (schema.Type)
{
case SchemaType.Null:
break;
case SchemaType.Boolean:
return(typeof(bool));
case SchemaType.Int:
return(typeof(int));
case SchemaType.Long:
return(typeof(long));
case SchemaType.Float:
return(typeof(float));
case SchemaType.Short:
return(typeof(short));
case SchemaType.Double:
return(typeof(double));
case SchemaType.Decimal:
return(typeof(decimal));
case SchemaType.Bytes:
return(typeof(byte[]));
case SchemaType.Byte:
return(typeof(byte));
case SchemaType.String:
return(typeof(string));
case SchemaType.DateTime:
return(typeof(DateTime));
case SchemaType.Array:
{
ArraySchema arrSchema = schema as ArraySchema;
Type itemSchema = GetType(arrSchema.ItemSchema);
return(GenericListType.MakeGenericType(new[] { itemSchema }));
}
case SchemaType.Map:
{
MapSchema mapSchema = schema as MapSchema;
Type itemSchema = GetType(mapSchema.ValueSchema);
return(GenericMapType.MakeGenericType(new[] { typeof(string), itemSchema }));
}
case SchemaType.Enumeration:
case SchemaType.Record:
{
// Should all be named types
var named = schema as NamedSchema;
if (null != named)
{
return(FindType(named.Fullname, true));
}
break;
}
case SchemaType.Union:
{
UnionSchema unSchema = schema as UnionSchema;
if (null != unSchema && unSchema.Count == 2)
{
var s1 = unSchema.Schemas[0];
var s2 = unSchema.Schemas[1];
// Nullable ?
Type itemType = null;
if (s1.Type == SchemaType.Null)
{
itemType = GetType(s2);
}
else if (s2.Type == SchemaType.Null)
{
itemType = GetType(s1);
}
if (null != itemType)
{
if (itemType.IsValueType && !itemType.IsEnum)
{
try
{
return(GenericNullableType.MakeGenericType(new[] { itemType }));
}
catch
{
}
}
return(itemType);
}
}
return(typeof(object));
}
}
// Fallback
return(FindType(schema.Name, true));
}
protected abstract ArrayAccess GetArrayAccess(ArraySchema readerSchema);
public void RecordWithMissingFields()
{
var boolean = new BooleanSchema();
var array = new ArraySchema(boolean);
var map = new MapSchema(new IntSchema());
var @enum = new EnumSchema("Ordinal", new[] { "None", "First", "Second", "Third", "Fourth" });
var union = new UnionSchema(new Schema[]
{
new NullSchema(),
array,
});
var schema = new RecordSchema("AllFields")
{
Fields = new[]
{
new RecordField("First", union),
new RecordField("Second", union),
new RecordField("Third", array),
new RecordField("Fourth", array),
new RecordField("Fifth", map),
new RecordField("Sixth", map),
new RecordField("Seventh", @enum),
new RecordField("Eighth", @enum),
},
};
var deserialize = deserializerBuilder.BuildDelegate <WithoutEvenFields>(schema);
var serialize = serializerBuilder.BuildDelegate <WithEvenFields>(schema);
var value = new WithEvenFields()
{
First = new List <bool>()
{
false
},
Second = new List <bool>()
{
false, false
},
Third = new List <bool>()
{
false, false, false
},
Fourth = new List <bool>()
{
false
},
Fifth = new Dictionary <string, int>()
{
{ "first", 1 }
},
Sixth = new Dictionary <string, int>()
{
{ "first", 1 }, { "second", 2 }
},
Seventh = ImplicitEnum.First,
Eighth = ImplicitEnum.None,
};
using (stream)
{
serialize(value, new Utf8JsonWriter(stream));
}
var reader = new Utf8JsonReader(stream.ToArray());
Assert.Equal(value.Seventh, deserialize(ref reader).Seventh);
}
internal Encoder.WriteItem Resolve(ArraySchema schema)
{
var itemWriter = Resolver.ResolveWriter(schema.ItemSchema);
return((d, e) => WriteArray(itemWriter, d, e));
}
protected abstract ArrayAccess GetArrayAccess(ArraySchema schema);
