public void RecordTest()
{
var expectedValue = new TestRecord()
{
FieldA = 123, FieldB = "Test", FieldC = new TestSubRecord()
{
FieldD = false
}, FieldX = TestEnum.B, TestFixed = new TestFixed()
};
var writer = new DatumWriter <TestRecord>(expectedValue.Schema);
var reader = new DatumReader <TestRecord>(expectedValue.Schema, expectedValue.Schema);
using (var stream = new MemoryStream())
using (var encoder = new BinaryEncoder(stream))
using (var decoder = new BinaryDecoder(stream))
{
writer.Write(encoder, expectedValue);
stream.Seek(0, SeekOrigin.Begin);
var actualValue = reader.Read(decoder);
for (int i = 0; i < expectedValue.FieldCount; i++)
{
Assert.AreEqual(expectedValue.Get(i), actualValue.Get(i));
}
}
}
public void RecordTestWithOverlap()
{
var expectedValue = new TestRecordWithDefault()
{
Name = "Test"
};
var writeValue = new TestRecordWithExtraField()
{
Name = "Test", Desc = "Description"
};
var writer = new DatumWriter <TestRecordWithExtraField>(writeValue.Schema);
var reader = new DatumReader <TestRecordWithDefault>(expectedValue.Schema, writeValue.Schema);
using (var stream = new MemoryStream())
using (var encoder = new BinaryEncoder(stream))
using (var decoder = new BinaryDecoder(stream))
{
writer.Write(encoder, writeValue);
stream.Seek(0, SeekOrigin.Begin);
var actualValue = reader.Read(decoder);
for (int i = 0; i < expectedValue.FieldCount; i++)
{
Assert.AreEqual(expectedValue.Get(i), actualValue.Get(i));
}
}
}
/// <exception cref="System.IO.IOException"/>
internal virtual void Write(HistoryEvent @event)
{
lock (this)
{
Event wrapper = new Event();
wrapper.type = @event.GetEventType();
wrapper.@event = @event.GetDatum();
writer.Write(wrapper, encoder);
encoder.Flush();
@out.WriteBytes("\n");
}
}
public byte[] Write(T message)
{
var ms = new MemoryStream();
Encoder e = new BinaryEncoder(ms);
_writer.Write(message, e);
ms.Flush();
ms.Position = 0;
var b = ms.ToArray();
return(b);
}
public void NullableArbitraryUnionTest()
{
var writerSchema = new UnionSchema(new StringSchema(), new DoubleSchema(), new UuidSchema(), new NullSchema());
var readerSchema = new UnionSchema(new StringSchema(), new NullSchema());
var writer = new DatumWriter <object>(writerSchema);
var reader = new DatumReader <string>(readerSchema, writerSchema);
var expectedValueNotNull = "Some String";
var expectedValueNull = null as string;
using (var stream = new MemoryStream())
using (var encoder = new BinaryEncoder(stream))
using (var decoder = new BinaryDecoder(stream))
{
writer.Write(encoder, expectedValueNotNull);
stream.Seek(0, SeekOrigin.Begin);
Assert.AreEqual(expectedValueNotNull, reader.Read(decoder));
stream.Seek(0, SeekOrigin.Begin);
writer.Write(encoder, expectedValueNull);
stream.Seek(0, SeekOrigin.Begin);
Assert.AreEqual(expectedValueNull, reader.Read(decoder));
}
}
public void NullableValueTest()
{
var writerSchema = new UnionSchema(new FloatSchema(), new NullSchema());
var readerSchema = new UnionSchema(new NullSchema(), new FloatSchema());
var writer = new DatumWriter <float?>(writerSchema);
var reader = new DatumReader <float?>(readerSchema, writerSchema);
var expectedValueNotNull = new float?(56.45F);
var expectedValueNull = new float?();
using (var stream = new MemoryStream())
using (var encoder = new BinaryEncoder(stream))
using (var decoder = new BinaryDecoder(stream))
{
writer.Write(encoder, expectedValueNotNull);
stream.Seek(0, SeekOrigin.Begin);
Assert.AreEqual(expectedValueNotNull, reader.Read(decoder));
stream.Seek(0, SeekOrigin.Begin);
writer.Write(encoder, expectedValueNull);
stream.Seek(0, SeekOrigin.Begin);
Assert.AreEqual(expectedValueNull, reader.Read(decoder));
}
}
public void UnionToUnionMixedTest()
{
var writerSchema = new UnionSchema(new BooleanSchema(), new LongSchema(), new DoubleSchema(), new StringSchema());
var readerSchema = new UnionSchema(new NullSchema(), new IntSchema(), new FloatSchema(), new BytesSchema());
var writer = new DatumWriter <object>(writerSchema);
var reader = new DatumReader <object>(readerSchema, writerSchema);
var expectedValue = 56723234L;
using (var stream = new MemoryStream())
using (var encoder = new BinaryEncoder(stream))
using (var decoder = new BinaryDecoder(stream))
{
writer.Write(encoder, expectedValue);
stream.Seek(0, SeekOrigin.Begin);
Assert.AreEqual(expectedValue, reader.Read(decoder));
}
}
public void NonUnionValueToUnionTest()
{
var writerSchema = new FloatSchema();
var readerSchema = new UnionSchema(new FloatSchema(), new StringSchema(), new BytesSchema());
var writer = new DatumWriter <float>(writerSchema);
var reader = new DatumReader <object>(readerSchema, writerSchema);
var expectedValue = 123.456F;
using (var stream = new MemoryStream())
using (var encoder = new BinaryEncoder(stream))
using (var decoder = new BinaryDecoder(stream))
{
writer.Write(encoder, expectedValue);
stream.Seek(0, SeekOrigin.Begin);
Assert.AreEqual(expectedValue, reader.Read(decoder));
}
}
public void EnumTest()
{
var enumSchema = new EnumSchema(nameof(TestEnum), typeof(TestEnum).Namespace, Enum.GetNames(typeof(TestEnum)));
var writer = new DatumWriter <TestEnum>(enumSchema);
var reader = new DatumReader <TestEnum>(enumSchema, enumSchema);
var expectedValue = TestEnum.A;
using (var stream = new MemoryStream())
using (var encoder = new BinaryEncoder(stream))
using (var decoder = new BinaryDecoder(stream))
{
writer.Write(encoder, expectedValue);
stream.Seek(0, SeekOrigin.Begin);
var actualValue = reader.Read(decoder);
Assert.AreEqual(expectedValue, actualValue);
}
}
public void Append(T datum)
{
AssertOpen();
EnsureHeader();
long usedBuffer = _blockStream.Position;
try
{
_writer.Write(datum, _blockEncoder);
}
catch (Exception e)
{
_blockStream.Position = usedBuffer;
throw new AvroRuntimeException("Error appending datum to writer", e);
}
_blockCount++;
WriteIfBlockFull();
}
public void TestReadWrite()
{
using (var stream = new MemoryStream())
using (var encoder = new BinaryEncoder(stream))
using (var decoder = new BinaryDecoder(stream))
{
_specificWriter.Write(encoder, _writeValue);
var expectedPosition = stream.Position;
stream.Seek(0, SeekOrigin.Begin);
var actualValue = _specificReader.Read(decoder);
Assert.AreEqual(_readValue, actualValue);
stream.Seek(0, SeekOrigin.Begin);
actualValue = _specificReader.Read(decoder, ref actualValue);
Assert.AreEqual(_readValue, actualValue);
stream.Seek(0, SeekOrigin.Begin);
_specificReader.Skip(decoder);
Assert.AreEqual(stream.Position, expectedPosition);
}
}
public void MapTest()
{
var writerSchema = new MapSchema(new FloatSchema());
var readerSchema = new MapSchema(new DoubleSchema());
var writer = new DatumWriter <IDictionary <string, float> >(writerSchema);
var reader = new DatumReader <IDictionary <string, double> >(readerSchema, writerSchema);
var expectedMap = new Dictionary <string, float> {
{ "Key1", 1.1F }, { "Key2", 2.2F }, { "Key3", 3.3F }, { "Key4", 4.4F }, { "Key5", 5.5F }
};
using (var stream = new MemoryStream())
using (var encoder = new BinaryEncoder(stream))
using (var decoder = new BinaryDecoder(stream))
{
stream.Seek(0, SeekOrigin.Begin);
writer.Write(encoder, expectedMap);
stream.Seek(0, SeekOrigin.Begin);
Assert.AreEqual(expectedMap, reader.Read(decoder));
}
}
public void ArrayTest()
{
var writerSchema = new ArraySchema(new IntSchema());
var readerSchema = new ArraySchema(new LongSchema());
var writer = new DatumWriter <IList <int> >(writerSchema);
var reader = new DatumReader <IList <long> >(readerSchema, writerSchema);
var expectedArray = new List <int> {
10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0
};
using (var stream = new MemoryStream())
using (var encoder = new BinaryEncoder(stream))
using (var decoder = new BinaryDecoder(stream))
{
stream.Seek(0, SeekOrigin.Begin);
writer.Write(encoder, expectedArray);
stream.Seek(0, SeekOrigin.Begin);
Assert.AreEqual(expectedArray, reader.Read(decoder));
}
}
public void RecordGenericTest()
{
var enumSchema = AvroParser.ReadSchema <EnumSchema>(@"{""name"":""Avro.Test.Generic.TestEnum"",""type"":""enum"",""symbols"":[""A"",""B"",""C""]}");
var fixedSchema = AvroParser.ReadSchema <FixedSchema>(@"{""name"":""Avro.Test.Generic.TestFixed"",""type"":""fixed"",""size"":40}");
var subRecordSchema = AvroParser.ReadSchema <RecordSchema>(@"{""name"":""Avro.Test.Generic.TestSubRecord"",""type"":""record"",""fields"":[{""name"":""FieldD"",""type"":""boolean""}]}");
var recordSchema = AvroParser.ReadSchema <RecordSchema>(@"{""name"":""Avro.Test.Generic.TestRecord"",""type"":""record"",""fields"":[{""name"":""FieldA"",""type"":""int""},{""name"":""FieldB"",""type"":""string""},{""name"":""FieldC"",""type"":{""name"":""Avro.Test.Generic.TestSubRecord"",""type"":""record"",""fields"":[{""name"":""FieldD"",""type"":""boolean""}]}},{""name"":""FieldX"",""type"":{""name"":""Avro.Test.Generic.TestEnum"",""type"":""enum"",""symbols"":[""A"",""B"",""C""]}},{""name"":""TestFixed"",""type"":{""name"":""Avro.Test.Generic.TestFixed"",""type"":""fixed"",""size"":40}}]}");
var subRecordInstance = new GenericRecord(subRecordSchema);
var fixedInstance = new GenericFixed(fixedSchema);
var enumInstance = new GenericEnum(enumSchema, "B");
subRecordInstance["FieldD"] = false;
var expectedValue = new GenericRecord(recordSchema);
expectedValue["FieldA"] = 123;
expectedValue["FieldB"] = "Test";
expectedValue["FieldC"] = subRecordInstance;
expectedValue["FieldX"] = enumInstance;
expectedValue["TestFixed"] = fixedInstance;
var writer = new DatumWriter <GenericRecord>(recordSchema);
var reader = new DatumReader <GenericRecord>(recordSchema, recordSchema);
using (var stream = new MemoryStream())
using (var encoder = new BinaryEncoder(stream))
using (var decoder = new BinaryDecoder(stream))
{
writer.Write(encoder, expectedValue);
stream.Seek(0, SeekOrigin.Begin);
var actualValue = reader.Read(decoder);
for (int i = 0; i < expectedValue.FieldCount; i++)
{
Assert.AreEqual(expectedValue[i], actualValue[i]);
}
}
}
/// <summary>Read a single byte from the stream.</summary>
/// <exception cref="System.IO.IOException"/>
public override int Read()
{
if (pos < buffer.Length)
{
return(buffer[pos++]);
}
if (!fileReader.HasNext())
{
return(-1);
}
writer.Write(fileReader.Next(), encoder);
encoder.Flush();
if (!fileReader.HasNext())
{
// Write a new line after the last Avro record.
output.Write(Runtime.GetBytesForString(Runtime.GetProperty("line.separator"
), Charsets.Utf8));
output.Flush();
}
pos = 0;
buffer = output.ToByteArray();
output.Reset();
return(Read());
}
public void FixedTest()
{
var expectedValue = new TestFixed();
expectedValue[1] = 1;
for (int i = 2; i < expectedValue.Size; i++)
{
expectedValue[i] = (byte)((expectedValue[i - 2] + expectedValue[i - 1]) % byte.MaxValue);
}
var writer = new DatumWriter <TestFixed>(expectedValue.Schema);
var reader = new DatumReader <TestFixed>(expectedValue.Schema, expectedValue.Schema);
using (var stream = new MemoryStream())
using (var encoder = new BinaryEncoder(stream))
using (var decoder = new BinaryDecoder(stream))
{
writer.Write(encoder, expectedValue);
stream.Seek(0, SeekOrigin.Begin);
var actualValue = reader.Read(decoder);
Assert.AreEqual(expectedValue, actualValue);
}
}
public void EnumGenericTest()
{
var enumSchema = AvroParser.ReadSchema <EnumSchema>(@"{""name"":""Avro.Test.Generic.TestEnum"",""type"":""enum"",""symbols"":[""A"",""B"",""C""]}");
var writer = new DatumWriter <GenericEnum>(enumSchema);
var reader = new DatumReader <GenericEnum>(enumSchema, enumSchema);
var expectedValue = new GenericEnum(enumSchema, "B");
using (var stream = new MemoryStream())
using (var encoder = new BinaryEncoder(stream))
using (var decoder = new BinaryDecoder(stream))
{
writer.Write(encoder, expectedValue);
stream.Seek(0, SeekOrigin.Begin);
var actualValue = reader.Read(decoder);
Assert.AreEqual(expectedValue, actualValue);
var expectedPosition = stream.Position;
stream.Seek(0, SeekOrigin.Begin);
reader.Skip(decoder);
Assert.AreEqual(expectedPosition, stream.Position);
}
}
public void TestSimple()
{
var schema = AvroParser.ReadSchema(@"{
""name"":""Test.Foobar.Record.Thing"",
""type"":""record"",
""fields"":[
{""name"":""ID"",""type"":""int""},
{""name"":""Name"",""type"":""string""},
{""name"":""Nuller"",""type"":""null""},
{""name"":""family"",""type"":{
""type"":""map"",""values"":""string""
}},
{""name"":""stuff"",""type"":{
""name"":""stuffs"",
""type"":""record"",
""fields"":[
{""name"":""key"",""type"":""bytes""},
{""name"":""keytype"",""type"":{
""name"":""keyenum"",
""type"":""enum"",
""symbols"":[""ANALOG"",""DIGITAL""]
}}
]
}},
{""name"":""NullableThing"",""type"":[""null"",""string""]},
{""name"":""Tags"",""type"":{
""type"":""array"",""items"":""string""
}}
]
}") as RecordSchema;
var stuffSchema = schema.First(r => r.Name == "stuff").Type as RecordSchema;
var enumSchema = stuffSchema.First(r => r.Name == "keytype").Type as EnumSchema;
var genericRecord = new GenericRecord(schema);
var reader = new DatumReader <GenericRecord>(schema);
var writer = new DatumWriter <GenericRecord>(schema);
var delimiter = Environment.NewLine;
var stringBuilder = new StringBuilder();
using (var stream = new StringWriter(stringBuilder))
using (var encoder = new JsonEncoder(stream, schema, delimiter))
{
for (int i = 0; i < 10; i++)
{
var record = new GenericRecord(genericRecord);
record[0] = i;
record[1] = $"foo{i}";
record[2] = null;
record[3] = new Dictionary <string, string>()
{
{ "Brother", "John" }
};
var stuffRecord = new GenericRecord(stuffSchema);
var keyEnum = new GenericEnum(enumSchema, i % enumSchema.Symbols.Count);
stuffRecord[0] = Guid.NewGuid().ToByteArray();
stuffRecord[1] = keyEnum;
record[4] = stuffRecord;
record[5] = (i % 2) == 0 ? "ToNullOrNotToNull" : null;
record[6] = new List <string> {
};
writer.Write(encoder, record);
var singleJson = stringBuilder.ToString();
}
}
var allJson = stringBuilder.ToString();
using (var stream = new StringReader(allJson))
using (var decoder = new JsonDecoder(stream, schema, delimiter))
{
for (int i = 0; i < 10; i++)
{
var record = reader.Read(decoder);
Debug.WriteLine(record.ToString());
}
}
}
