public void RecordTestWithOverlap() { var expectedValue = new TestRecordWithDefault() { Name = "Test" }; var writeValue = new TestRecordWithExtraField() { Name = "Test", Desc = "Description" }; var writer = new DatumWriter <TestRecordWithExtraField>(writeValue.Schema); var reader = new DatumReader <TestRecordWithDefault>(expectedValue.Schema, writeValue.Schema); using (var stream = new MemoryStream()) using (var encoder = new BinaryEncoder(stream)) using (var decoder = new BinaryDecoder(stream)) { writer.Write(encoder, writeValue); stream.Seek(0, SeekOrigin.Begin); var actualValue = reader.Read(decoder); for (int i = 0; i < expectedValue.FieldCount; i++) { Assert.AreEqual(expectedValue.Get(i), actualValue.Get(i)); } } }
public void RecordTest() { var expectedValue = new TestRecord() { FieldA = 123, FieldB = "Test", FieldC = new TestSubRecord() { FieldD = false }, FieldX = TestEnum.B, TestFixed = new TestFixed() }; var writer = new DatumWriter <TestRecord>(expectedValue.Schema); var reader = new DatumReader <TestRecord>(expectedValue.Schema, expectedValue.Schema); using (var stream = new MemoryStream()) using (var encoder = new BinaryEncoder(stream)) using (var decoder = new BinaryDecoder(stream)) { writer.Write(encoder, expectedValue); stream.Seek(0, SeekOrigin.Begin); var actualValue = reader.Read(decoder); for (int i = 0; i < expectedValue.FieldCount; i++) { Assert.AreEqual(expectedValue.Get(i), actualValue.Get(i)); } } }
public Promoted(TWriter writeValue, TReader readValue, object writeInstance, object readInstance) { _writeValue = writeValue; _readValue = readValue; _writeSchema = writeInstance as SWriter; _readSchema = readInstance as SReader; _specificWriter = new DatumWriter <TWriter>(_writeSchema); _specificReader = new DatumReader <TReader>(_readSchema, _writeSchema); }
public UnionUpscaleTest(TWriter writeValue, object readValue) { _writeValue = writeValue; _readValue = readValue; _writeSchema = new SWriter(); _readSchema = new UnionSchema(new WUnionSchemaOne(), new WUnionSchemaTwo()); _specificWriter = new DatumWriter <TWriter>(_writeSchema); _specificReader = new DatumReader <object>(_readSchema, _writeSchema); }
/// <summary> /// Open a new writer instance to append to an output stream. /// Both in and out streams must point to the same file. /// </summary> /// <param name="writer">Datum writer to use.</param> /// <param name="inStream">reading the existing file.</param> /// <param name="outStream">stream to write to, positioned at the end of the existing file.</param> /// <returns> /// A new file writer. /// </returns> /// <exception cref="AvroRuntimeException"> /// {nameof(inStream)} must have Read access /// or /// {nameof(outStream)} must have Write access /// </exception> public static IFileWriter <T> OpenAppendWriter(DatumWriter <T> writer, Stream inStream, Stream outStream) { if (!inStream.CanRead) { throw new AvroRuntimeException($"{nameof(inStream)} must have Read access"); } if (!outStream.CanWrite) { throw new AvroRuntimeException($"{nameof(outStream)} must have Write access"); } return(new DataFileWriter <T>(writer).AppendTo(inStream, outStream)); }
public AvroWriter(Avro.Schema avroSchema) { _schema = avroSchema; var type = typeof(T); if (typeof(ISpecificRecord).IsAssignableFrom(type)) { _writer = new SpecificDatumWriter <T>(avroSchema); } else { _writer = new ReflectWriter <T>(_schema); } }
public void RecordTestWithoutOverlap() { var expectedValue = new TestRecordWithoutDefault() { ID = 123, Name = "Test" }; var writeValue = new TestRecordWithExtraField() { Name = "Test", Desc = "Description" }; var writer = new DatumWriter <TestRecordWithExtraField>(writeValue.Schema); Assert.Throws <AvroException>(() => new DatumReader <TestRecordWithDefault>(expectedValue.Schema, writeValue.Schema)); }
public void UnionToUnionMixedTest() { var writerSchema = new UnionSchema(new BooleanSchema(), new LongSchema(), new DoubleSchema(), new StringSchema()); var readerSchema = new UnionSchema(new NullSchema(), new IntSchema(), new FloatSchema(), new BytesSchema()); var writer = new DatumWriter <object>(writerSchema); var reader = new DatumReader <object>(readerSchema, writerSchema); var expectedValue = 56723234L; using (var stream = new MemoryStream()) using (var encoder = new BinaryEncoder(stream)) using (var decoder = new BinaryDecoder(stream)) { writer.Write(encoder, expectedValue); stream.Seek(0, SeekOrigin.Begin); Assert.AreEqual(expectedValue, reader.Read(decoder)); } }
public void NonUnionValueToUnionTest() { var writerSchema = new FloatSchema(); var readerSchema = new UnionSchema(new FloatSchema(), new StringSchema(), new BytesSchema()); var writer = new DatumWriter <float>(writerSchema); var reader = new DatumReader <object>(readerSchema, writerSchema); var expectedValue = 123.456F; using (var stream = new MemoryStream()) using (var encoder = new BinaryEncoder(stream)) using (var decoder = new BinaryDecoder(stream)) { writer.Write(encoder, expectedValue); stream.Seek(0, SeekOrigin.Begin); Assert.AreEqual(expectedValue, reader.Read(decoder)); } }
public void EnumTest() { var enumSchema = new EnumSchema(nameof(TestEnum), typeof(TestEnum).Namespace, Enum.GetNames(typeof(TestEnum))); var writer = new DatumWriter <TestEnum>(enumSchema); var reader = new DatumReader <TestEnum>(enumSchema, enumSchema); var expectedValue = TestEnum.A; using (var stream = new MemoryStream()) using (var encoder = new BinaryEncoder(stream)) using (var decoder = new BinaryDecoder(stream)) { writer.Write(encoder, expectedValue); stream.Seek(0, SeekOrigin.Begin); var actualValue = reader.Read(decoder); Assert.AreEqual(expectedValue, actualValue); } }
public void NullableNotNullWriterTest() { var writerSchema = new IntSchema(); var readerSchema = new UnionSchema(new NullSchema(), new IntSchema()); var writer = new DatumWriter <int>(writerSchema); var reader = new DatumReader <int?>(readerSchema, writerSchema); var expectedValueNotNull = 123; using (var stream = new MemoryStream()) using (var encoder = new BinaryEncoder(stream)) using (var decoder = new BinaryDecoder(stream)) { writer.Write(encoder, expectedValueNotNull); stream.Seek(0, SeekOrigin.Begin); Assert.AreEqual(expectedValueNotNull, reader.Read(decoder)); stream.Seek(0, SeekOrigin.Begin); } }
public void MapTest() { var writerSchema = new MapSchema(new FloatSchema()); var readerSchema = new MapSchema(new DoubleSchema()); var writer = new DatumWriter <IDictionary <string, float> >(writerSchema); var reader = new DatumReader <IDictionary <string, double> >(readerSchema, writerSchema); var expectedMap = new Dictionary <string, float> { { "Key1", 1.1F }, { "Key2", 2.2F }, { "Key3", 3.3F }, { "Key4", 4.4F }, { "Key5", 5.5F } }; using (var stream = new MemoryStream()) using (var encoder = new BinaryEncoder(stream)) using (var decoder = new BinaryDecoder(stream)) { stream.Seek(0, SeekOrigin.Begin); writer.Write(encoder, expectedMap); stream.Seek(0, SeekOrigin.Begin); Assert.AreEqual(expectedMap, reader.Read(decoder)); } }
public void ArrayTest() { var writerSchema = new ArraySchema(new IntSchema()); var readerSchema = new ArraySchema(new LongSchema()); var writer = new DatumWriter <IList <int> >(writerSchema); var reader = new DatumReader <IList <long> >(readerSchema, writerSchema); var expectedArray = new List <int> { 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0 }; using (var stream = new MemoryStream()) using (var encoder = new BinaryEncoder(stream)) using (var decoder = new BinaryDecoder(stream)) { stream.Seek(0, SeekOrigin.Begin); writer.Write(encoder, expectedArray); stream.Seek(0, SeekOrigin.Begin); Assert.AreEqual(expectedArray, reader.Read(decoder)); } }
/// <exception cref="System.IO.IOException"/> public AvroFileInputStream(FileStatus status) { pos = 0; buffer = new byte[0]; GenericDatumReader <object> reader = new GenericDatumReader <object>(); FileContext fc = FileContext.GetFileContext(new Configuration()); fileReader = DataFileReader.OpenReader(new AvroFSInput(fc, status.GetPath()), reader ); Schema schema = fileReader.GetSchema(); writer = new GenericDatumWriter <object>(schema); output = new ByteArrayOutputStream(); JsonGenerator generator = new JsonFactory().CreateJsonGenerator(output, JsonEncoding .Utf8); MinimalPrettyPrinter prettyPrinter = new MinimalPrettyPrinter(); prettyPrinter.SetRootValueSeparator(Runtime.GetProperty("line.separator")); generator.SetPrettyPrinter(prettyPrinter); encoder = EncoderFactory.Get().JsonEncoder(schema, generator); }
public void NullableArbitraryUnionTest() { var writerSchema = new UnionSchema(new StringSchema(), new DoubleSchema(), new UuidSchema(), new NullSchema()); var readerSchema = new UnionSchema(new StringSchema(), new NullSchema()); var writer = new DatumWriter <object>(writerSchema); var reader = new DatumReader <string>(readerSchema, writerSchema); var expectedValueNotNull = "Some String"; var expectedValueNull = null as string; using (var stream = new MemoryStream()) using (var encoder = new BinaryEncoder(stream)) using (var decoder = new BinaryDecoder(stream)) { writer.Write(encoder, expectedValueNotNull); stream.Seek(0, SeekOrigin.Begin); Assert.AreEqual(expectedValueNotNull, reader.Read(decoder)); stream.Seek(0, SeekOrigin.Begin); writer.Write(encoder, expectedValueNull); stream.Seek(0, SeekOrigin.Begin); Assert.AreEqual(expectedValueNull, reader.Read(decoder)); } }
public void RecordGenericTest() { var enumSchema = AvroParser.ReadSchema <EnumSchema>(@"{""name"":""Avro.Test.Generic.TestEnum"",""type"":""enum"",""symbols"":[""A"",""B"",""C""]}"); var fixedSchema = AvroParser.ReadSchema <FixedSchema>(@"{""name"":""Avro.Test.Generic.TestFixed"",""type"":""fixed"",""size"":40}"); var subRecordSchema = AvroParser.ReadSchema <RecordSchema>(@"{""name"":""Avro.Test.Generic.TestSubRecord"",""type"":""record"",""fields"":[{""name"":""FieldD"",""type"":""boolean""}]}"); var recordSchema = AvroParser.ReadSchema <RecordSchema>(@"{""name"":""Avro.Test.Generic.TestRecord"",""type"":""record"",""fields"":[{""name"":""FieldA"",""type"":""int""},{""name"":""FieldB"",""type"":""string""},{""name"":""FieldC"",""type"":{""name"":""Avro.Test.Generic.TestSubRecord"",""type"":""record"",""fields"":[{""name"":""FieldD"",""type"":""boolean""}]}},{""name"":""FieldX"",""type"":{""name"":""Avro.Test.Generic.TestEnum"",""type"":""enum"",""symbols"":[""A"",""B"",""C""]}},{""name"":""TestFixed"",""type"":{""name"":""Avro.Test.Generic.TestFixed"",""type"":""fixed"",""size"":40}}]}"); var subRecordInstance = new GenericRecord(subRecordSchema); var fixedInstance = new GenericFixed(fixedSchema); var enumInstance = new GenericEnum(enumSchema, "B"); subRecordInstance["FieldD"] = false; var expectedValue = new GenericRecord(recordSchema); expectedValue["FieldA"] = 123; expectedValue["FieldB"] = "Test"; expectedValue["FieldC"] = subRecordInstance; expectedValue["FieldX"] = enumInstance; expectedValue["TestFixed"] = fixedInstance; var writer = new DatumWriter <GenericRecord>(recordSchema); var reader = new DatumReader <GenericRecord>(recordSchema, recordSchema); using (var stream = new MemoryStream()) using (var encoder = new BinaryEncoder(stream)) using (var decoder = new BinaryDecoder(stream)) { writer.Write(encoder, expectedValue); stream.Seek(0, SeekOrigin.Begin); var actualValue = reader.Read(decoder); for (int i = 0; i < expectedValue.FieldCount; i++) { Assert.AreEqual(expectedValue[i], actualValue[i]); } } }
public void FixedTest() { var expectedValue = new TestFixed(); expectedValue[1] = 1; for (int i = 2; i < expectedValue.Size; i++) { expectedValue[i] = (byte)((expectedValue[i - 2] + expectedValue[i - 1]) % byte.MaxValue); } var writer = new DatumWriter <TestFixed>(expectedValue.Schema); var reader = new DatumReader <TestFixed>(expectedValue.Schema, expectedValue.Schema); using (var stream = new MemoryStream()) using (var encoder = new BinaryEncoder(stream)) using (var decoder = new BinaryDecoder(stream)) { writer.Write(encoder, expectedValue); stream.Seek(0, SeekOrigin.Begin); var actualValue = reader.Read(decoder); Assert.AreEqual(expectedValue, actualValue); } }
public void EnumGenericTest() { var enumSchema = AvroParser.ReadSchema <EnumSchema>(@"{""name"":""Avro.Test.Generic.TestEnum"",""type"":""enum"",""symbols"":[""A"",""B"",""C""]}"); var writer = new DatumWriter <GenericEnum>(enumSchema); var reader = new DatumReader <GenericEnum>(enumSchema, enumSchema); var expectedValue = new GenericEnum(enumSchema, "B"); using (var stream = new MemoryStream()) using (var encoder = new BinaryEncoder(stream)) using (var decoder = new BinaryDecoder(stream)) { writer.Write(encoder, expectedValue); stream.Seek(0, SeekOrigin.Begin); var actualValue = reader.Read(decoder); Assert.AreEqual(expectedValue, actualValue); var expectedPosition = stream.Position; stream.Seek(0, SeekOrigin.Begin); reader.Skip(decoder); Assert.AreEqual(expectedPosition, stream.Position); } }
internal AvroSerializer(AvroSerialization <T> _enclosing, Type clazz) { this._enclosing = _enclosing; this.writer = this._enclosing.GetWriter(clazz); }
DataFileWriter(DatumWriter <T> writer) { _writer = writer; _syncInterval = DataFileConstants.DefaultSyncInterval; }
/// <summary> /// Open a new writer instance to write /// to an output stream with a specified codec /// </summary> /// <param name="writer"></param> /// <param name="outStream"></param> /// <param name="codec"></param> /// <returns></returns> public static IFileWriter <T> OpenWriter(DatumWriter <T> writer, Stream outStream, Codec codec) { return(new DataFileWriter <T>(writer).Create(writer.Schema, outStream, codec)); }
/// <summary> /// Open a new writer instance to write /// to a file path with a specified codec /// </summary> /// <param name="writer"></param> /// <param name="path"></param> /// <param name="codec"></param> /// <returns></returns> public static IFileWriter <T> OpenWriter(DatumWriter <T> writer, string path, Codec codec) { return(OpenWriter(writer, new FileStream(path, FileMode.Create), codec)); }
/// <summary> /// Open a new writer instance to write /// to an output stream, using a Null codec /// </summary> /// <param name="writer"></param> /// <param name="outStream"></param> /// <returns></returns> public static IFileWriter <T> OpenWriter(DatumWriter <T> writer, Stream outStream) { return(OpenWriter(writer, outStream, Codec.CreateCodec(Codec.Type.Null))); }
/// <summary> /// Open a new writer instance to write /// to a file path, using a Null codec /// </summary> /// <param name="writer"></param> /// <param name="path"></param> /// <returns></returns> public static IFileWriter <T> OpenWriter(DatumWriter <T> writer, string path) { return(OpenWriter(writer, new FileStream(path, FileMode.Create), Codec.CreateCodec(Codec.Type.Null))); }
private static void checkJson(Schema schema, Object datum, DatumWriter writer, DatumReader reader) { throw new NotImplementedException(); //IPC.ByteBufferOutputStream iostr = new Avro.IPC.ByteBufferOutputStream(); //Encoder encoder = new JsonEncoder(schema, iostr); //writer.Schema = schema; //writer.write(datum, encoder); ////writer.write(datum, encoder); //encoder.Flush(); //byte[] data = iostr.ToArray(); //System.IO.File.WriteAllBytes("C:\\temp\\test.js", data); //reader.Schema = schema; //Decoder decoder = new JsonDecoder(schema, new MemoryStream(data)); //Object decoded = reader.Read(null, decoder); //Assert.AreEqual(datum, decoded, "Decoded data does not match."); //decoded = reader.Read(decoded, decoder); //Assert.AreEqual(datum, decoded, "Decoded data does not match."); }
private static void checkBinary(Schema schema, Object datum, DatumWriter writer, DatumReader reader) { throw new NotImplementedException(); //Avro.IPC.ByteBufferOutputStream iostr = new Avro.IPC.ByteBufferOutputStream(); //writer.Schema = schema; //writer.write(datum, new BinaryEncoder(iostr)); //byte[] data = iostr.ToArray(); //reader.Schema = schema; //BinaryDecoder decoder = BinaryDecoder.CreateBinaryDecoder(data, null); //Assert.IsNotNull(decoder, "decoder is null"); //Object decoded = reader.Read(null, decoder); //Assert.AreEqual(datum, decoded, "Decoded data does not match."); }
public void testEnumMismatch() { Schema actual = Schema.Parse ("{\"type\":\"enum\",\"name\":\"E\",\"symbols\":[\"X\",\"Y\"]}"); Schema expected = Schema.Parse ("{\"type\":\"enum\",\"name\":\"E\",\"symbols\":[\"Y\",\"Z\"]}"); MemoryStream iostr = new MemoryStream(); DatumWriter writer = new DatumWriter(actual); BinaryEncoder encoder = new BinaryEncoder(iostr); writer.write("Y", encoder); writer.write("X", encoder); byte[] data = iostr.ToArray(); throw new NotImplementedException(); //BinaryDecoder decoder = new BinaryDecoder( //Decoder decoder = BinaryDecoder.CreateBinaryDecoder( // data, null); //DatumReader<String> input = new GenericDatumReader<String>(actual, expected); //Assert.AreEqual(input.Read(null, decoder), "Wrong value", "Y"); //try //{ // input.Read(null, decoder); // Assert.Fail("Should have thrown exception."); //} //catch (AvroTypeException e) //{ // // expected //} }
private GenericProtocolPair(AvroProtocol local, AvroProtocol remote) { LocalHash = local.MD5; RemoteHash = remote.MD5; var requestReaders = new Dictionary <string, IAvroReader <IAvroRecord> >(); var requestWriters = new Dictionary <string, IAvroWriter <IAvroRecord> >(); var responseReaders = new Dictionary <string, IAvroReader <object> >(); var responseWriters = new Dictionary <string, IAvroWriter <object> >(); var errorReaders = new Dictionary <string, IAvroReader <object> >(); var errorWriters = new Dictionary <string, IAvroWriter <object> >(); var messagePairs = from lm in local.Messages join rm in remote.Messages on lm.Name equals rm.Name select new { MessageName = lm.Name, LocalMessage = lm, RemoteMessage = rm }; foreach (var messagePair in messagePairs) { var localRequestParameters = from p in messagePair.LocalMessage.RequestParameters join t in local.Types on p.Type.FullName equals t.FullName select new RecordSchema.Field(p.Name, t) ; var remoteRequestParameters = from p in messagePair.RemoteMessage.RequestParameters join t in remote.Types on p.Type.FullName equals t.FullName select new RecordSchema.Field(p.Name, t) ; var localRequest = new RecordSchema($"{local.FullName}.messages.{messagePair.MessageName}", localRequestParameters); var remoteRequest = new RecordSchema($"{remote.FullName}.messages.{messagePair.MessageName}", remoteRequestParameters); var requestReader = new DatumReader <IAvroRecord>(localRequest, remoteRequest); var requestWriter = new DatumWriter <IAvroRecord>(localRequest); requestReaders.Add(messagePair.MessageName, requestReader); requestWriters.Add(messagePair.MessageName, requestWriter); var responseReader = new DatumReader <object>(messagePair.LocalMessage.Response, messagePair.RemoteMessage.Response); var responseWriter = new DatumWriter <object>(messagePair.LocalMessage.Response); responseReaders.Add(messagePair.MessageName, responseReader); responseWriters.Add(messagePair.MessageName, responseWriter); var errorReader = new DatumReader <object>(messagePair.LocalMessage.Error, messagePair.RemoteMessage.Error); var errorWriter = new DatumWriter <object>(messagePair.LocalMessage.Error); errorReaders.Add(messagePair.MessageName, errorReader); errorWriters.Add(messagePair.MessageName, errorWriter); } RequestReaders = requestReaders; RequestWriters = requestWriters; ResponseReaders = responseReaders; ResponseWriters = responseWriters; ErrorReaders = errorReaders; ErrorWriters = errorWriters; }
public void TestSimple() { var schema = AvroParser.ReadSchema(@"{ ""name"":""Test.Foobar.Record.Thing"", ""type"":""record"", ""fields"":[ {""name"":""ID"",""type"":""int""}, {""name"":""Name"",""type"":""string""}, {""name"":""Nuller"",""type"":""null""}, {""name"":""family"",""type"":{ ""type"":""map"",""values"":""string"" }}, {""name"":""stuff"",""type"":{ ""name"":""stuffs"", ""type"":""record"", ""fields"":[ {""name"":""key"",""type"":""bytes""}, {""name"":""keytype"",""type"":{ ""name"":""keyenum"", ""type"":""enum"", ""symbols"":[""ANALOG"",""DIGITAL""] }} ] }}, {""name"":""NullableThing"",""type"":[""null"",""string""]}, {""name"":""Tags"",""type"":{ ""type"":""array"",""items"":""string"" }} ] }") as RecordSchema; var stuffSchema = schema.First(r => r.Name == "stuff").Type as RecordSchema; var enumSchema = stuffSchema.First(r => r.Name == "keytype").Type as EnumSchema; var genericRecord = new GenericRecord(schema); var reader = new DatumReader <GenericRecord>(schema); var writer = new DatumWriter <GenericRecord>(schema); var delimiter = Environment.NewLine; var stringBuilder = new StringBuilder(); using (var stream = new StringWriter(stringBuilder)) using (var encoder = new JsonEncoder(stream, schema, delimiter)) { for (int i = 0; i < 10; i++) { var record = new GenericRecord(genericRecord); record[0] = i; record[1] = $"foo{i}"; record[2] = null; record[3] = new Dictionary <string, string>() { { "Brother", "John" } }; var stuffRecord = new GenericRecord(stuffSchema); var keyEnum = new GenericEnum(enumSchema, i % enumSchema.Symbols.Count); stuffRecord[0] = Guid.NewGuid().ToByteArray(); stuffRecord[1] = keyEnum; record[4] = stuffRecord; record[5] = (i % 2) == 0 ? "ToNullOrNotToNull" : null; record[6] = new List <string> { }; writer.Write(encoder, record); var singleJson = stringBuilder.ToString(); } } var allJson = stringBuilder.ToString(); using (var stream = new StringReader(allJson)) using (var decoder = new JsonDecoder(stream, schema, delimiter)) { for (int i = 0; i < 10; i++) { var record = reader.Read(decoder); Debug.WriteLine(record.ToString()); } } }
/// <summary> /// Open a new writer instance to append to a file path. /// </summary> /// <param name="writer">Datum writer to use.</param> /// <param name="path">Path to the file.</param> /// <returns> /// A new file writer. /// </returns> public static IFileWriter <T> OpenAppendWriter(DatumWriter <T> writer, string path) { return(new DataFileWriter <T>(writer).AppendTo(path)); }