Ejemplo n.º 1
0
        public void RecordTestWithOverlap()
        {
            var expectedValue = new TestRecordWithDefault()
            {
                Name = "Test"
            };
            var writeValue = new TestRecordWithExtraField()
            {
                Name = "Test", Desc = "Description"
            };
            var writer = new DatumWriter <TestRecordWithExtraField>(writeValue.Schema);
            var reader = new DatumReader <TestRecordWithDefault>(expectedValue.Schema, writeValue.Schema);

            using (var stream = new MemoryStream())
                using (var encoder = new BinaryEncoder(stream))
                    using (var decoder = new BinaryDecoder(stream))
                    {
                        writer.Write(encoder, writeValue);
                        stream.Seek(0, SeekOrigin.Begin);
                        var actualValue = reader.Read(decoder);
                        for (int i = 0; i < expectedValue.FieldCount; i++)
                        {
                            Assert.AreEqual(expectedValue.Get(i), actualValue.Get(i));
                        }
                    }
        }
Ejemplo n.º 2
0
        public void RecordTest()
        {
            var expectedValue = new TestRecord()
            {
                FieldA = 123, FieldB = "Test", FieldC = new TestSubRecord()
                {
                    FieldD = false
                }, FieldX = TestEnum.B, TestFixed = new TestFixed()
            };
            var writer = new DatumWriter <TestRecord>(expectedValue.Schema);
            var reader = new DatumReader <TestRecord>(expectedValue.Schema, expectedValue.Schema);

            using (var stream = new MemoryStream())
                using (var encoder = new BinaryEncoder(stream))
                    using (var decoder = new BinaryDecoder(stream))
                    {
                        writer.Write(encoder, expectedValue);
                        stream.Seek(0, SeekOrigin.Begin);
                        var actualValue = reader.Read(decoder);
                        for (int i = 0; i < expectedValue.FieldCount; i++)
                        {
                            Assert.AreEqual(expectedValue.Get(i), actualValue.Get(i));
                        }
                    }
        }
Ejemplo n.º 3
0
 public Promoted(TWriter writeValue, TReader readValue, object writeInstance, object readInstance)
 {
     _writeValue     = writeValue;
     _readValue      = readValue;
     _writeSchema    = writeInstance as SWriter;
     _readSchema     = readInstance as SReader;
     _specificWriter = new DatumWriter <TWriter>(_writeSchema);
     _specificReader = new DatumReader <TReader>(_readSchema, _writeSchema);
 }
Ejemplo n.º 4
0
 public UnionUpscaleTest(TWriter writeValue, object readValue)
 {
     _writeValue     = writeValue;
     _readValue      = readValue;
     _writeSchema    = new SWriter();
     _readSchema     = new UnionSchema(new WUnionSchemaOne(), new WUnionSchemaTwo());
     _specificWriter = new DatumWriter <TWriter>(_writeSchema);
     _specificReader = new DatumReader <object>(_readSchema, _writeSchema);
 }
Ejemplo n.º 5
0
        /// <summary>
        /// Open a new writer instance to append to an output stream.
        /// Both in and out streams must point to the same file.
        /// </summary>
        /// <param name="writer">Datum writer to use.</param>
        /// <param name="inStream">reading the existing file.</param>
        /// <param name="outStream">stream to write to, positioned at the end of the existing file.</param>
        /// <returns>
        /// A new file writer.
        /// </returns>
        /// <exception cref="AvroRuntimeException">
        /// {nameof(inStream)} must have Read access
        /// or
        /// {nameof(outStream)} must have Write access
        /// </exception>
        public static IFileWriter <T> OpenAppendWriter(DatumWriter <T> writer, Stream inStream, Stream outStream)
        {
            if (!inStream.CanRead)
            {
                throw new AvroRuntimeException($"{nameof(inStream)} must have Read access");
            }

            if (!outStream.CanWrite)
            {
                throw new AvroRuntimeException($"{nameof(outStream)} must have Write access");
            }

            return(new DataFileWriter <T>(writer).AppendTo(inStream, outStream));
        }
Ejemplo n.º 6
0
        public AvroWriter(Avro.Schema avroSchema)
        {
            _schema = avroSchema;
            var type = typeof(T);

            if (typeof(ISpecificRecord).IsAssignableFrom(type))
            {
                _writer = new SpecificDatumWriter <T>(avroSchema);
            }
            else
            {
                _writer = new ReflectWriter <T>(_schema);
            }
        }
Ejemplo n.º 7
0
        public void RecordTestWithoutOverlap()
        {
            var expectedValue = new TestRecordWithoutDefault()
            {
                ID = 123, Name = "Test"
            };
            var writeValue = new TestRecordWithExtraField()
            {
                Name = "Test", Desc = "Description"
            };
            var writer = new DatumWriter <TestRecordWithExtraField>(writeValue.Schema);

            Assert.Throws <AvroException>(() => new DatumReader <TestRecordWithDefault>(expectedValue.Schema, writeValue.Schema));
        }
Ejemplo n.º 8
0
        public void UnionToUnionMixedTest()
        {
            var writerSchema = new UnionSchema(new BooleanSchema(), new LongSchema(), new DoubleSchema(), new StringSchema());
            var readerSchema = new UnionSchema(new NullSchema(), new IntSchema(), new FloatSchema(), new BytesSchema());
            var writer       = new DatumWriter <object>(writerSchema);
            var reader       = new DatumReader <object>(readerSchema, writerSchema);

            var expectedValue = 56723234L;

            using (var stream = new MemoryStream())
                using (var encoder = new BinaryEncoder(stream))
                    using (var decoder = new BinaryDecoder(stream))
                    {
                        writer.Write(encoder, expectedValue);
                        stream.Seek(0, SeekOrigin.Begin);
                        Assert.AreEqual(expectedValue, reader.Read(decoder));
                    }
        }
Ejemplo n.º 9
0
        public void NonUnionValueToUnionTest()
        {
            var writerSchema = new FloatSchema();
            var readerSchema = new UnionSchema(new FloatSchema(), new StringSchema(), new BytesSchema());
            var writer       = new DatumWriter <float>(writerSchema);
            var reader       = new DatumReader <object>(readerSchema, writerSchema);

            var expectedValue = 123.456F;

            using (var stream = new MemoryStream())
                using (var encoder = new BinaryEncoder(stream))
                    using (var decoder = new BinaryDecoder(stream))
                    {
                        writer.Write(encoder, expectedValue);
                        stream.Seek(0, SeekOrigin.Begin);
                        Assert.AreEqual(expectedValue, reader.Read(decoder));
                    }
        }
Ejemplo n.º 10
0
        public void EnumTest()
        {
            var enumSchema = new EnumSchema(nameof(TestEnum), typeof(TestEnum).Namespace, Enum.GetNames(typeof(TestEnum)));
            var writer     = new DatumWriter <TestEnum>(enumSchema);
            var reader     = new DatumReader <TestEnum>(enumSchema, enumSchema);

            var expectedValue = TestEnum.A;

            using (var stream = new MemoryStream())
                using (var encoder = new BinaryEncoder(stream))
                    using (var decoder = new BinaryDecoder(stream))
                    {
                        writer.Write(encoder, expectedValue);
                        stream.Seek(0, SeekOrigin.Begin);
                        var actualValue = reader.Read(decoder);
                        Assert.AreEqual(expectedValue, actualValue);
                    }
        }
Ejemplo n.º 11
0
        public void NullableNotNullWriterTest()
        {
            var writerSchema = new IntSchema();
            var readerSchema = new UnionSchema(new NullSchema(), new IntSchema());
            var writer       = new DatumWriter <int>(writerSchema);
            var reader       = new DatumReader <int?>(readerSchema, writerSchema);

            var expectedValueNotNull = 123;

            using (var stream = new MemoryStream())
                using (var encoder = new BinaryEncoder(stream))
                    using (var decoder = new BinaryDecoder(stream))
                    {
                        writer.Write(encoder, expectedValueNotNull);
                        stream.Seek(0, SeekOrigin.Begin);
                        Assert.AreEqual(expectedValueNotNull, reader.Read(decoder));
                        stream.Seek(0, SeekOrigin.Begin);
                    }
        }
Ejemplo n.º 12
0
        public void MapTest()
        {
            var writerSchema = new MapSchema(new FloatSchema());
            var readerSchema = new MapSchema(new DoubleSchema());
            var writer       = new DatumWriter <IDictionary <string, float> >(writerSchema);
            var reader       = new DatumReader <IDictionary <string, double> >(readerSchema, writerSchema);

            var expectedMap = new Dictionary <string, float> {
                { "Key1", 1.1F }, { "Key2", 2.2F }, { "Key3", 3.3F }, { "Key4", 4.4F }, { "Key5", 5.5F }
            };

            using (var stream = new MemoryStream())
                using (var encoder = new BinaryEncoder(stream))
                    using (var decoder = new BinaryDecoder(stream))
                    {
                        stream.Seek(0, SeekOrigin.Begin);
                        writer.Write(encoder, expectedMap);
                        stream.Seek(0, SeekOrigin.Begin);
                        Assert.AreEqual(expectedMap, reader.Read(decoder));
                    }
        }
Ejemplo n.º 13
0
        public void ArrayTest()
        {
            var writerSchema = new ArraySchema(new IntSchema());
            var readerSchema = new ArraySchema(new LongSchema());
            var writer       = new DatumWriter <IList <int> >(writerSchema);
            var reader       = new DatumReader <IList <long> >(readerSchema, writerSchema);

            var expectedArray = new List <int> {
                10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0
            };

            using (var stream = new MemoryStream())
                using (var encoder = new BinaryEncoder(stream))
                    using (var decoder = new BinaryDecoder(stream))
                    {
                        stream.Seek(0, SeekOrigin.Begin);
                        writer.Write(encoder, expectedArray);
                        stream.Seek(0, SeekOrigin.Begin);
                        Assert.AreEqual(expectedArray, reader.Read(decoder));
                    }
        }
Ejemplo n.º 14
0
            /// <exception cref="System.IO.IOException"/>
            public AvroFileInputStream(FileStatus status)
            {
                pos    = 0;
                buffer = new byte[0];
                GenericDatumReader <object> reader = new GenericDatumReader <object>();
                FileContext fc = FileContext.GetFileContext(new Configuration());

                fileReader = DataFileReader.OpenReader(new AvroFSInput(fc, status.GetPath()), reader
                                                       );
                Schema schema = fileReader.GetSchema();

                writer = new GenericDatumWriter <object>(schema);
                output = new ByteArrayOutputStream();
                JsonGenerator generator = new JsonFactory().CreateJsonGenerator(output, JsonEncoding
                                                                                .Utf8);
                MinimalPrettyPrinter prettyPrinter = new MinimalPrettyPrinter();

                prettyPrinter.SetRootValueSeparator(Runtime.GetProperty("line.separator"));
                generator.SetPrettyPrinter(prettyPrinter);
                encoder = EncoderFactory.Get().JsonEncoder(schema, generator);
            }
Ejemplo n.º 15
0
        public void NullableArbitraryUnionTest()
        {
            var writerSchema = new UnionSchema(new StringSchema(), new DoubleSchema(), new UuidSchema(), new NullSchema());
            var readerSchema = new UnionSchema(new StringSchema(), new NullSchema());
            var writer       = new DatumWriter <object>(writerSchema);
            var reader       = new DatumReader <string>(readerSchema, writerSchema);

            var expectedValueNotNull = "Some String";
            var expectedValueNull    = null as string;

            using (var stream = new MemoryStream())
                using (var encoder = new BinaryEncoder(stream))
                    using (var decoder = new BinaryDecoder(stream))
                    {
                        writer.Write(encoder, expectedValueNotNull);
                        stream.Seek(0, SeekOrigin.Begin);
                        Assert.AreEqual(expectedValueNotNull, reader.Read(decoder));
                        stream.Seek(0, SeekOrigin.Begin);
                        writer.Write(encoder, expectedValueNull);
                        stream.Seek(0, SeekOrigin.Begin);
                        Assert.AreEqual(expectedValueNull, reader.Read(decoder));
                    }
        }
Ejemplo n.º 16
0
        public void RecordGenericTest()
        {
            var enumSchema      = AvroParser.ReadSchema <EnumSchema>(@"{""name"":""Avro.Test.Generic.TestEnum"",""type"":""enum"",""symbols"":[""A"",""B"",""C""]}");
            var fixedSchema     = AvroParser.ReadSchema <FixedSchema>(@"{""name"":""Avro.Test.Generic.TestFixed"",""type"":""fixed"",""size"":40}");
            var subRecordSchema = AvroParser.ReadSchema <RecordSchema>(@"{""name"":""Avro.Test.Generic.TestSubRecord"",""type"":""record"",""fields"":[{""name"":""FieldD"",""type"":""boolean""}]}");
            var recordSchema    = AvroParser.ReadSchema <RecordSchema>(@"{""name"":""Avro.Test.Generic.TestRecord"",""type"":""record"",""fields"":[{""name"":""FieldA"",""type"":""int""},{""name"":""FieldB"",""type"":""string""},{""name"":""FieldC"",""type"":{""name"":""Avro.Test.Generic.TestSubRecord"",""type"":""record"",""fields"":[{""name"":""FieldD"",""type"":""boolean""}]}},{""name"":""FieldX"",""type"":{""name"":""Avro.Test.Generic.TestEnum"",""type"":""enum"",""symbols"":[""A"",""B"",""C""]}},{""name"":""TestFixed"",""type"":{""name"":""Avro.Test.Generic.TestFixed"",""type"":""fixed"",""size"":40}}]}");

            var subRecordInstance = new GenericRecord(subRecordSchema);
            var fixedInstance     = new GenericFixed(fixedSchema);
            var enumInstance      = new GenericEnum(enumSchema, "B");

            subRecordInstance["FieldD"] = false;

            var expectedValue = new GenericRecord(recordSchema);

            expectedValue["FieldA"]    = 123;
            expectedValue["FieldB"]    = "Test";
            expectedValue["FieldC"]    = subRecordInstance;
            expectedValue["FieldX"]    = enumInstance;
            expectedValue["TestFixed"] = fixedInstance;

            var writer = new DatumWriter <GenericRecord>(recordSchema);
            var reader = new DatumReader <GenericRecord>(recordSchema, recordSchema);

            using (var stream = new MemoryStream())
                using (var encoder = new BinaryEncoder(stream))
                    using (var decoder = new BinaryDecoder(stream))
                    {
                        writer.Write(encoder, expectedValue);
                        stream.Seek(0, SeekOrigin.Begin);
                        var actualValue = reader.Read(decoder);
                        for (int i = 0; i < expectedValue.FieldCount; i++)
                        {
                            Assert.AreEqual(expectedValue[i], actualValue[i]);
                        }
                    }
        }
Ejemplo n.º 17
0
        public void FixedTest()
        {
            var expectedValue = new TestFixed();

            expectedValue[1] = 1;

            for (int i = 2; i < expectedValue.Size; i++)
            {
                expectedValue[i] = (byte)((expectedValue[i - 2] + expectedValue[i - 1]) % byte.MaxValue);
            }

            var writer = new DatumWriter <TestFixed>(expectedValue.Schema);
            var reader = new DatumReader <TestFixed>(expectedValue.Schema, expectedValue.Schema);

            using (var stream = new MemoryStream())
                using (var encoder = new BinaryEncoder(stream))
                    using (var decoder = new BinaryDecoder(stream))
                    {
                        writer.Write(encoder, expectedValue);
                        stream.Seek(0, SeekOrigin.Begin);
                        var actualValue = reader.Read(decoder);
                        Assert.AreEqual(expectedValue, actualValue);
                    }
        }
Ejemplo n.º 18
0
        public void EnumGenericTest()
        {
            var enumSchema = AvroParser.ReadSchema <EnumSchema>(@"{""name"":""Avro.Test.Generic.TestEnum"",""type"":""enum"",""symbols"":[""A"",""B"",""C""]}");

            var writer = new DatumWriter <GenericEnum>(enumSchema);
            var reader = new DatumReader <GenericEnum>(enumSchema, enumSchema);

            var expectedValue = new GenericEnum(enumSchema, "B");

            using (var stream = new MemoryStream())
                using (var encoder = new BinaryEncoder(stream))
                    using (var decoder = new BinaryDecoder(stream))
                    {
                        writer.Write(encoder, expectedValue);
                        stream.Seek(0, SeekOrigin.Begin);
                        var actualValue = reader.Read(decoder);
                        Assert.AreEqual(expectedValue, actualValue);

                        var expectedPosition = stream.Position;
                        stream.Seek(0, SeekOrigin.Begin);
                        reader.Skip(decoder);
                        Assert.AreEqual(expectedPosition, stream.Position);
                    }
        }
Ejemplo n.º 19
0
 internal AvroSerializer(AvroSerialization <T> _enclosing, Type clazz)
 {
     this._enclosing = _enclosing;
     this.writer     = this._enclosing.GetWriter(clazz);
 }
 DataFileWriter(DatumWriter <T> writer)
 {
     _writer       = writer;
     _syncInterval = DataFileConstants.DefaultSyncInterval;
 }
 /// <summary>
 /// Open a new writer instance to write
 /// to an output stream with a specified codec
 /// </summary>
 /// <param name="writer"></param>
 /// <param name="outStream"></param>
 /// <param name="codec"></param>
 /// <returns></returns>
 public static IFileWriter <T> OpenWriter(DatumWriter <T> writer, Stream outStream, Codec codec)
 {
     return(new DataFileWriter <T>(writer).Create(writer.Schema, outStream, codec));
 }
 /// <summary>
 /// Open a new writer instance to write
 /// to a file path with a specified codec
 /// </summary>
 /// <param name="writer"></param>
 /// <param name="path"></param>
 /// <param name="codec"></param>
 /// <returns></returns>
 public static IFileWriter <T> OpenWriter(DatumWriter <T> writer, string path, Codec codec)
 {
     return(OpenWriter(writer, new FileStream(path, FileMode.Create), codec));
 }
 /// <summary>
 /// Open a new writer instance to write
 /// to an output stream, using a Null codec
 /// </summary>
 /// <param name="writer"></param>
 /// <param name="outStream"></param>
 /// <returns></returns>
 public static IFileWriter <T> OpenWriter(DatumWriter <T> writer, Stream outStream)
 {
     return(OpenWriter(writer, outStream, Codec.CreateCodec(Codec.Type.Null)));
 }
 /// <summary>
 /// Open a new writer instance to write
 /// to a file path, using a Null codec
 /// </summary>
 /// <param name="writer"></param>
 /// <param name="path"></param>
 /// <returns></returns>
 public static IFileWriter <T> OpenWriter(DatumWriter <T> writer, string path)
 {
     return(OpenWriter(writer, new FileStream(path, FileMode.Create), Codec.CreateCodec(Codec.Type.Null)));
 }
Ejemplo n.º 25
0
        private static void checkJson(Schema schema, Object datum,
                                DatumWriter writer,
                                DatumReader reader)
        {
            throw new NotImplementedException();
            //IPC.ByteBufferOutputStream iostr = new Avro.IPC.ByteBufferOutputStream();
            //Encoder encoder = new JsonEncoder(schema, iostr);
            //writer.Schema = schema;
            //writer.write(datum, encoder);
            ////writer.write(datum, encoder);
            //encoder.Flush();
            //byte[] data = iostr.ToArray();
            //System.IO.File.WriteAllBytes("C:\\temp\\test.js", data);
            //reader.Schema = schema;
            //Decoder decoder = new JsonDecoder(schema, new MemoryStream(data));
            //Object decoded = reader.Read(null, decoder);
            //Assert.AreEqual(datum, decoded, "Decoded data does not match.");

            //decoded = reader.Read(decoded, decoder);
            //Assert.AreEqual(datum, decoded, "Decoded data does not match.");
        }
Ejemplo n.º 26
0
        private static void checkBinary(Schema schema, Object datum,
                                  DatumWriter writer,
                                  DatumReader reader)
        {
            throw new NotImplementedException();
            //Avro.IPC.ByteBufferOutputStream iostr = new Avro.IPC.ByteBufferOutputStream();
            //writer.Schema = schema;
            //writer.write(datum, new BinaryEncoder(iostr));
            //byte[] data = iostr.ToArray();

            //reader.Schema = schema;
            
            //BinaryDecoder decoder = BinaryDecoder.CreateBinaryDecoder(data, null);
            //Assert.IsNotNull(decoder, "decoder is null");
            //Object decoded = reader.Read(null, decoder);
            //Assert.AreEqual(datum, decoded, "Decoded data does not match.");
        }
Ejemplo n.º 27
0
        public void testEnumMismatch()
        {
            Schema actual = Schema.Parse
              ("{\"type\":\"enum\",\"name\":\"E\",\"symbols\":[\"X\",\"Y\"]}");
            Schema expected = Schema.Parse
              ("{\"type\":\"enum\",\"name\":\"E\",\"symbols\":[\"Y\",\"Z\"]}");
            MemoryStream iostr = new MemoryStream();
            DatumWriter writer = new DatumWriter(actual);
            BinaryEncoder encoder = new BinaryEncoder(iostr);
            writer.write("Y", encoder);
            writer.write("X", encoder);
            byte[] data = iostr.ToArray();
            throw new NotImplementedException();

            //BinaryDecoder decoder = new BinaryDecoder(
            //Decoder decoder = BinaryDecoder.CreateBinaryDecoder(
            //    data, null);
            //DatumReader<String> input = new GenericDatumReader<String>(actual, expected);
            //Assert.AreEqual(input.Read(null, decoder), "Wrong value", "Y");
            //try
            //{
            //    input.Read(null, decoder);
            //    Assert.Fail("Should have thrown exception.");
            //}
            //catch (AvroTypeException e)
            //{
            //    // expected
            //}
        }
        private GenericProtocolPair(AvroProtocol local, AvroProtocol remote)
        {
            LocalHash  = local.MD5;
            RemoteHash = remote.MD5;

            var requestReaders  = new Dictionary <string, IAvroReader <IAvroRecord> >();
            var requestWriters  = new Dictionary <string, IAvroWriter <IAvroRecord> >();
            var responseReaders = new Dictionary <string, IAvroReader <object> >();
            var responseWriters = new Dictionary <string, IAvroWriter <object> >();
            var errorReaders    = new Dictionary <string, IAvroReader <object> >();
            var errorWriters    = new Dictionary <string, IAvroWriter <object> >();

            var messagePairs =
                from lm in local.Messages
                join rm in remote.Messages on lm.Name equals rm.Name
                select new
            {
                MessageName   = lm.Name,
                LocalMessage  = lm,
                RemoteMessage = rm
            };

            foreach (var messagePair in messagePairs)
            {
                var localRequestParameters =
                    from p in messagePair.LocalMessage.RequestParameters
                    join t in local.Types on p.Type.FullName equals t.FullName
                    select new RecordSchema.Field(p.Name, t)
                ;

                var remoteRequestParameters =
                    from p in messagePair.RemoteMessage.RequestParameters
                    join t in remote.Types on p.Type.FullName equals t.FullName
                    select new RecordSchema.Field(p.Name, t)
                ;

                var localRequest  = new RecordSchema($"{local.FullName}.messages.{messagePair.MessageName}", localRequestParameters);
                var remoteRequest = new RecordSchema($"{remote.FullName}.messages.{messagePair.MessageName}", remoteRequestParameters);

                var requestReader = new DatumReader <IAvroRecord>(localRequest, remoteRequest);
                var requestWriter = new DatumWriter <IAvroRecord>(localRequest);

                requestReaders.Add(messagePair.MessageName, requestReader);
                requestWriters.Add(messagePair.MessageName, requestWriter);

                var responseReader = new DatumReader <object>(messagePair.LocalMessage.Response, messagePair.RemoteMessage.Response);
                var responseWriter = new DatumWriter <object>(messagePair.LocalMessage.Response);

                responseReaders.Add(messagePair.MessageName, responseReader);
                responseWriters.Add(messagePair.MessageName, responseWriter);

                var errorReader = new DatumReader <object>(messagePair.LocalMessage.Error, messagePair.RemoteMessage.Error);
                var errorWriter = new DatumWriter <object>(messagePair.LocalMessage.Error);

                errorReaders.Add(messagePair.MessageName, errorReader);
                errorWriters.Add(messagePair.MessageName, errorWriter);
            }

            RequestReaders  = requestReaders;
            RequestWriters  = requestWriters;
            ResponseReaders = responseReaders;
            ResponseWriters = responseWriters;
            ErrorReaders    = errorReaders;
            ErrorWriters    = errorWriters;
        }
Ejemplo n.º 29
0
        public void TestSimple()
        {
            var schema = AvroParser.ReadSchema(@"{
                ""name"":""Test.Foobar.Record.Thing"",
                ""type"":""record"",
                ""fields"":[
                    {""name"":""ID"",""type"":""int""},
                    {""name"":""Name"",""type"":""string""},
                    {""name"":""Nuller"",""type"":""null""},
                    {""name"":""family"",""type"":{
                        ""type"":""map"",""values"":""string""
                    }},
                    {""name"":""stuff"",""type"":{
                        ""name"":""stuffs"",
                        ""type"":""record"",
                        ""fields"":[
                            {""name"":""key"",""type"":""bytes""},
                            {""name"":""keytype"",""type"":{
                                ""name"":""keyenum"",
                                ""type"":""enum"",
                                ""symbols"":[""ANALOG"",""DIGITAL""]
                            }}
                        ]
                    }},
                    {""name"":""NullableThing"",""type"":[""null"",""string""]},
                    {""name"":""Tags"",""type"":{
                        ""type"":""array"",""items"":""string""
                    }}
                ]
            }") as RecordSchema;

            var stuffSchema = schema.First(r => r.Name == "stuff").Type as RecordSchema;
            var enumSchema  = stuffSchema.First(r => r.Name == "keytype").Type as EnumSchema;


            var genericRecord = new GenericRecord(schema);
            var reader        = new DatumReader <GenericRecord>(schema);
            var writer        = new DatumWriter <GenericRecord>(schema);


            var delimiter     = Environment.NewLine;
            var stringBuilder = new StringBuilder();

            using (var stream = new StringWriter(stringBuilder))
                using (var encoder = new JsonEncoder(stream, schema, delimiter))
                {
                    for (int i = 0; i < 10; i++)
                    {
                        var record = new GenericRecord(genericRecord);
                        record[0] = i;
                        record[1] = $"foo{i}";
                        record[2] = null;
                        record[3] = new Dictionary <string, string>()
                        {
                            { "Brother", "John" }
                        };

                        var stuffRecord = new GenericRecord(stuffSchema);
                        var keyEnum     = new GenericEnum(enumSchema, i % enumSchema.Symbols.Count);

                        stuffRecord[0] = Guid.NewGuid().ToByteArray();
                        stuffRecord[1] = keyEnum;

                        record[4] = stuffRecord;

                        record[5] = (i % 2) == 0 ? "ToNullOrNotToNull" : null;
                        record[6] = new List <string> {
                        };
                        writer.Write(encoder, record);

                        var singleJson = stringBuilder.ToString();
                    }
                }
            var allJson = stringBuilder.ToString();


            using (var stream = new StringReader(allJson))
                using (var decoder = new JsonDecoder(stream, schema, delimiter))
                {
                    for (int i = 0; i < 10; i++)
                    {
                        var record = reader.Read(decoder);
                        Debug.WriteLine(record.ToString());
                    }
                }
        }
Ejemplo n.º 30
0
 /// <summary>
 /// Open a new writer instance to append to a file path.
 /// </summary>
 /// <param name="writer">Datum writer to use.</param>
 /// <param name="path">Path to the file.</param>
 /// <returns>
 /// A new file writer.
 /// </returns>
 public static IFileWriter <T> OpenAppendWriter(DatumWriter <T> writer, string path)
 {
     return(new DataFileWriter <T>(writer).AppendTo(path));
 }