public void RecordTest() { var expectedValue = new TestRecord() { FieldA = 123, FieldB = "Test", FieldC = new TestSubRecord() { FieldD = false }, FieldX = TestEnum.B, TestFixed = new TestFixed() }; var writer = new DatumWriter <TestRecord>(expectedValue.Schema); var reader = new DatumReader <TestRecord>(expectedValue.Schema, expectedValue.Schema); using (var stream = new MemoryStream()) using (var encoder = new BinaryEncoder(stream)) using (var decoder = new BinaryDecoder(stream)) { writer.Write(encoder, expectedValue); stream.Seek(0, SeekOrigin.Begin); var actualValue = reader.Read(decoder); for (int i = 0; i < expectedValue.FieldCount; i++) { Assert.AreEqual(expectedValue.Get(i), actualValue.Get(i)); } } }
private async ValueTask <object> DeserializeInternalAsync( Stream stream, Type returnType, bool async, CancellationToken cancellationToken) { Argument.AssertNotNull(stream, nameof(stream)); Argument.AssertNotNull(returnType, nameof(returnType)); SupportedType supportedType = GetSupportedTypeOrThrow(returnType); ReadOnlyMemory <byte> message = CopyToReadOnlyMemory(stream); ValidateRecordFormatIdentifier(message); byte[] schemaIdBytes = message.Slice(RecordFormatIndicatorLength, SchemaIdLength).ToArray(); string schemaId = Utf8Encoding.GetString(schemaIdBytes); Schema schema; if (async) { schema = await GetSchemaByIdAsync(schemaId, true, cancellationToken).ConfigureAwait(false); } else { schema = GetSchemaByIdAsync(schemaId, false, cancellationToken).EnsureCompleted(); } using var valueStream = new MemoryStream(message.Slice(PayloadStartPosition, message.Length - PayloadStartPosition).ToArray()); var binaryDecoder = new BinaryDecoder(valueStream); DatumReader <object> reader = GetReader(schema, supportedType); return(reader.Read(reuse: null, binaryDecoder)); }
public void RecordTestWithOverlap() { var expectedValue = new TestRecordWithDefault() { Name = "Test" }; var writeValue = new TestRecordWithExtraField() { Name = "Test", Desc = "Description" }; var writer = new DatumWriter <TestRecordWithExtraField>(writeValue.Schema); var reader = new DatumReader <TestRecordWithDefault>(expectedValue.Schema, writeValue.Schema); using (var stream = new MemoryStream()) using (var encoder = new BinaryEncoder(stream)) using (var decoder = new BinaryDecoder(stream)) { writer.Write(encoder, writeValue); stream.Seek(0, SeekOrigin.Begin); var actualValue = reader.Read(decoder); for (int i = 0; i < expectedValue.FieldCount; i++) { Assert.AreEqual(expectedValue.Get(i), actualValue.Get(i)); } } }
public void TestReadWrite() { using (var stream = new MemoryStream()) using (var encoder = new BinaryEncoder(stream)) using (var decoder = new BinaryDecoder(stream)) { _specificWriter.Write(encoder, _writeValue); var expectedPosition = stream.Position; stream.Seek(0, SeekOrigin.Begin); var actualValue = _specificReader.Read(decoder); Assert.AreEqual(_readValue, actualValue); stream.Seek(0, SeekOrigin.Begin); actualValue = _specificReader.Read(decoder, ref actualValue); Assert.AreEqual(_readValue, actualValue); stream.Seek(0, SeekOrigin.Begin); _specificReader.Skip(decoder); Assert.AreEqual(stream.Position, expectedPosition); } }
public void NullableArbitraryUnionTest() { var writerSchema = new UnionSchema(new StringSchema(), new DoubleSchema(), new UuidSchema(), new NullSchema()); var readerSchema = new UnionSchema(new StringSchema(), new NullSchema()); var writer = new DatumWriter <object>(writerSchema); var reader = new DatumReader <string>(readerSchema, writerSchema); var expectedValueNotNull = "Some String"; var expectedValueNull = null as string; using (var stream = new MemoryStream()) using (var encoder = new BinaryEncoder(stream)) using (var decoder = new BinaryDecoder(stream)) { writer.Write(encoder, expectedValueNotNull); stream.Seek(0, SeekOrigin.Begin); Assert.AreEqual(expectedValueNotNull, reader.Read(decoder)); stream.Seek(0, SeekOrigin.Begin); writer.Write(encoder, expectedValueNull); stream.Seek(0, SeekOrigin.Begin); Assert.AreEqual(expectedValueNull, reader.Read(decoder)); } }
public void NullableValueTest() { var writerSchema = new UnionSchema(new FloatSchema(), new NullSchema()); var readerSchema = new UnionSchema(new NullSchema(), new FloatSchema()); var writer = new DatumWriter <float?>(writerSchema); var reader = new DatumReader <float?>(readerSchema, writerSchema); var expectedValueNotNull = new float?(56.45F); var expectedValueNull = new float?(); using (var stream = new MemoryStream()) using (var encoder = new BinaryEncoder(stream)) using (var decoder = new BinaryDecoder(stream)) { writer.Write(encoder, expectedValueNotNull); stream.Seek(0, SeekOrigin.Begin); Assert.AreEqual(expectedValueNotNull, reader.Read(decoder)); stream.Seek(0, SeekOrigin.Begin); writer.Write(encoder, expectedValueNull); stream.Seek(0, SeekOrigin.Begin); Assert.AreEqual(expectedValueNull, reader.Read(decoder)); } }
public void EnumTest() { var enumSchema = new EnumSchema(nameof(TestEnum), typeof(TestEnum).Namespace, Enum.GetNames(typeof(TestEnum))); var writer = new DatumWriter <TestEnum>(enumSchema); var reader = new DatumReader <TestEnum>(enumSchema, enumSchema); var expectedValue = TestEnum.A; using (var stream = new MemoryStream()) using (var encoder = new BinaryEncoder(stream)) using (var decoder = new BinaryDecoder(stream)) { writer.Write(encoder, expectedValue); stream.Seek(0, SeekOrigin.Begin); var actualValue = reader.Read(decoder); Assert.AreEqual(expectedValue, actualValue); } }
public void UnionToUnionMixedTest() { var writerSchema = new UnionSchema(new BooleanSchema(), new LongSchema(), new DoubleSchema(), new StringSchema()); var readerSchema = new UnionSchema(new NullSchema(), new IntSchema(), new FloatSchema(), new BytesSchema()); var writer = new DatumWriter <object>(writerSchema); var reader = new DatumReader <object>(readerSchema, writerSchema); var expectedValue = 56723234L; using (var stream = new MemoryStream()) using (var encoder = new BinaryEncoder(stream)) using (var decoder = new BinaryDecoder(stream)) { writer.Write(encoder, expectedValue); stream.Seek(0, SeekOrigin.Begin); Assert.AreEqual(expectedValue, reader.Read(decoder)); } }
public void NonUnionValueToUnionTest() { var writerSchema = new FloatSchema(); var readerSchema = new UnionSchema(new FloatSchema(), new StringSchema(), new BytesSchema()); var writer = new DatumWriter <float>(writerSchema); var reader = new DatumReader <object>(readerSchema, writerSchema); var expectedValue = 123.456F; using (var stream = new MemoryStream()) using (var encoder = new BinaryEncoder(stream)) using (var decoder = new BinaryDecoder(stream)) { writer.Write(encoder, expectedValue); stream.Seek(0, SeekOrigin.Begin); Assert.AreEqual(expectedValue, reader.Read(decoder)); } }
public void MapTest() { var writerSchema = new MapSchema(new FloatSchema()); var readerSchema = new MapSchema(new DoubleSchema()); var writer = new DatumWriter <IDictionary <string, float> >(writerSchema); var reader = new DatumReader <IDictionary <string, double> >(readerSchema, writerSchema); var expectedMap = new Dictionary <string, float> { { "Key1", 1.1F }, { "Key2", 2.2F }, { "Key3", 3.3F }, { "Key4", 4.4F }, { "Key5", 5.5F } }; using (var stream = new MemoryStream()) using (var encoder = new BinaryEncoder(stream)) using (var decoder = new BinaryDecoder(stream)) { stream.Seek(0, SeekOrigin.Begin); writer.Write(encoder, expectedMap); stream.Seek(0, SeekOrigin.Begin); Assert.AreEqual(expectedMap, reader.Read(decoder)); } }
private T Next(T reuse) { try { if (!HasNext()) { throw new AvroRuntimeException("No more datum objects remaining in block!"); } T result = _reader.Read(reuse, _datumDecoder); if (--_blockRemaining == 0) { BlockFinished(); } return(result); } catch (Exception e) { throw new AvroRuntimeException(string.Format("Error fetching next object from block: {0}", e)); } }
public void ArrayTest() { var writerSchema = new ArraySchema(new IntSchema()); var readerSchema = new ArraySchema(new LongSchema()); var writer = new DatumWriter <IList <int> >(writerSchema); var reader = new DatumReader <IList <long> >(readerSchema, writerSchema); var expectedArray = new List <int> { 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0 }; using (var stream = new MemoryStream()) using (var encoder = new BinaryEncoder(stream)) using (var decoder = new BinaryDecoder(stream)) { stream.Seek(0, SeekOrigin.Begin); writer.Write(encoder, expectedArray); stream.Seek(0, SeekOrigin.Begin); Assert.AreEqual(expectedArray, reader.Read(decoder)); } }
public void RecordGenericTest() { var enumSchema = AvroParser.ReadSchema <EnumSchema>(@"{""name"":""Avro.Test.Generic.TestEnum"",""type"":""enum"",""symbols"":[""A"",""B"",""C""]}"); var fixedSchema = AvroParser.ReadSchema <FixedSchema>(@"{""name"":""Avro.Test.Generic.TestFixed"",""type"":""fixed"",""size"":40}"); var subRecordSchema = AvroParser.ReadSchema <RecordSchema>(@"{""name"":""Avro.Test.Generic.TestSubRecord"",""type"":""record"",""fields"":[{""name"":""FieldD"",""type"":""boolean""}]}"); var recordSchema = AvroParser.ReadSchema <RecordSchema>(@"{""name"":""Avro.Test.Generic.TestRecord"",""type"":""record"",""fields"":[{""name"":""FieldA"",""type"":""int""},{""name"":""FieldB"",""type"":""string""},{""name"":""FieldC"",""type"":{""name"":""Avro.Test.Generic.TestSubRecord"",""type"":""record"",""fields"":[{""name"":""FieldD"",""type"":""boolean""}]}},{""name"":""FieldX"",""type"":{""name"":""Avro.Test.Generic.TestEnum"",""type"":""enum"",""symbols"":[""A"",""B"",""C""]}},{""name"":""TestFixed"",""type"":{""name"":""Avro.Test.Generic.TestFixed"",""type"":""fixed"",""size"":40}}]}"); var subRecordInstance = new GenericRecord(subRecordSchema); var fixedInstance = new GenericFixed(fixedSchema); var enumInstance = new GenericEnum(enumSchema, "B"); subRecordInstance["FieldD"] = false; var expectedValue = new GenericRecord(recordSchema); expectedValue["FieldA"] = 123; expectedValue["FieldB"] = "Test"; expectedValue["FieldC"] = subRecordInstance; expectedValue["FieldX"] = enumInstance; expectedValue["TestFixed"] = fixedInstance; var writer = new DatumWriter <GenericRecord>(recordSchema); var reader = new DatumReader <GenericRecord>(recordSchema, recordSchema); using (var stream = new MemoryStream()) using (var encoder = new BinaryEncoder(stream)) using (var decoder = new BinaryDecoder(stream)) { writer.Write(encoder, expectedValue); stream.Seek(0, SeekOrigin.Begin); var actualValue = reader.Read(decoder); for (int i = 0; i < expectedValue.FieldCount; i++) { Assert.AreEqual(expectedValue[i], actualValue[i]); } } }
public void FixedTest() { var expectedValue = new TestFixed(); expectedValue[1] = 1; for (int i = 2; i < expectedValue.Size; i++) { expectedValue[i] = (byte)((expectedValue[i - 2] + expectedValue[i - 1]) % byte.MaxValue); } var writer = new DatumWriter <TestFixed>(expectedValue.Schema); var reader = new DatumReader <TestFixed>(expectedValue.Schema, expectedValue.Schema); using (var stream = new MemoryStream()) using (var encoder = new BinaryEncoder(stream)) using (var decoder = new BinaryDecoder(stream)) { writer.Write(encoder, expectedValue); stream.Seek(0, SeekOrigin.Begin); var actualValue = reader.Read(decoder); Assert.AreEqual(expectedValue, actualValue); } }
public void EnumGenericTest() { var enumSchema = AvroParser.ReadSchema <EnumSchema>(@"{""name"":""Avro.Test.Generic.TestEnum"",""type"":""enum"",""symbols"":[""A"",""B"",""C""]}"); var writer = new DatumWriter <GenericEnum>(enumSchema); var reader = new DatumReader <GenericEnum>(enumSchema, enumSchema); var expectedValue = new GenericEnum(enumSchema, "B"); using (var stream = new MemoryStream()) using (var encoder = new BinaryEncoder(stream)) using (var decoder = new BinaryDecoder(stream)) { writer.Write(encoder, expectedValue); stream.Seek(0, SeekOrigin.Begin); var actualValue = reader.Read(decoder); Assert.AreEqual(expectedValue, actualValue); var expectedPosition = stream.Position; stream.Seek(0, SeekOrigin.Begin); reader.Skip(decoder); Assert.AreEqual(expectedPosition, stream.Position); } }
private static S Read <S>(DatumReader <S> reader, Decoder d) { S reuse = default(S); return(reader.Read(reuse, d)); }
/// <summary>Get the next event from the stream</summary> /// <returns>the next event</returns> /// <exception cref="System.IO.IOException"/> public virtual HistoryEvent GetNextEvent() { Event wrapper; try { wrapper = (Event)reader.Read(null, decoder); } catch (EOFException) { // at EOF return(null); } HistoryEvent result; switch (wrapper.type) { case EventType.JobSubmitted: { result = new JobSubmittedEvent(); break; } case EventType.JobInited: { result = new JobInitedEvent(); break; } case EventType.JobFinished: { result = new JobFinishedEvent(); break; } case EventType.JobPriorityChanged: { result = new JobPriorityChangeEvent(); break; } case EventType.JobQueueChanged: { result = new JobQueueChangeEvent(); break; } case EventType.JobStatusChanged: { result = new JobStatusChangedEvent(); break; } case EventType.JobFailed: { result = new JobUnsuccessfulCompletionEvent(); break; } case EventType.JobKilled: { result = new JobUnsuccessfulCompletionEvent(); break; } case EventType.JobError: { result = new JobUnsuccessfulCompletionEvent(); break; } case EventType.JobInfoChanged: { result = new JobInfoChangeEvent(); break; } case EventType.TaskStarted: { result = new TaskStartedEvent(); break; } case EventType.TaskFinished: { result = new TaskFinishedEvent(); break; } case EventType.TaskFailed: { result = new TaskFailedEvent(); break; } case EventType.TaskUpdated: { result = new TaskUpdatedEvent(); break; } case EventType.MapAttemptStarted: { result = new TaskAttemptStartedEvent(); break; } case EventType.MapAttemptFinished: { result = new MapAttemptFinishedEvent(); break; } case EventType.MapAttemptFailed: { result = new TaskAttemptUnsuccessfulCompletionEvent(); break; } case EventType.MapAttemptKilled: { result = new TaskAttemptUnsuccessfulCompletionEvent(); break; } case EventType.ReduceAttemptStarted: { result = new TaskAttemptStartedEvent(); break; } case EventType.ReduceAttemptFinished: { result = new ReduceAttemptFinishedEvent(); break; } case EventType.ReduceAttemptFailed: { result = new TaskAttemptUnsuccessfulCompletionEvent(); break; } case EventType.ReduceAttemptKilled: { result = new TaskAttemptUnsuccessfulCompletionEvent(); break; } case EventType.SetupAttemptStarted: { result = new TaskAttemptStartedEvent(); break; } case EventType.SetupAttemptFinished: { result = new TaskAttemptFinishedEvent(); break; } case EventType.SetupAttemptFailed: { result = new TaskAttemptUnsuccessfulCompletionEvent(); break; } case EventType.SetupAttemptKilled: { result = new TaskAttemptUnsuccessfulCompletionEvent(); break; } case EventType.CleanupAttemptStarted: { result = new TaskAttemptStartedEvent(); break; } case EventType.CleanupAttemptFinished: { result = new TaskAttemptFinishedEvent(); break; } case EventType.CleanupAttemptFailed: { result = new TaskAttemptUnsuccessfulCompletionEvent(); break; } case EventType.CleanupAttemptKilled: { result = new TaskAttemptUnsuccessfulCompletionEvent(); break; } case EventType.AmStarted: { result = new AMStartedEvent(); break; } default: { throw new RuntimeException("unexpected event type: " + wrapper.type); } } result.SetDatum(wrapper.@event); return(result); }
public void TestSimple() { var schema = AvroParser.ReadSchema(@"{ ""name"":""Test.Foobar.Record.Thing"", ""type"":""record"", ""fields"":[ {""name"":""ID"",""type"":""int""}, {""name"":""Name"",""type"":""string""}, {""name"":""Nuller"",""type"":""null""}, {""name"":""family"",""type"":{ ""type"":""map"",""values"":""string"" }}, {""name"":""stuff"",""type"":{ ""name"":""stuffs"", ""type"":""record"", ""fields"":[ {""name"":""key"",""type"":""bytes""}, {""name"":""keytype"",""type"":{ ""name"":""keyenum"", ""type"":""enum"", ""symbols"":[""ANALOG"",""DIGITAL""] }} ] }}, {""name"":""NullableThing"",""type"":[""null"",""string""]}, {""name"":""Tags"",""type"":{ ""type"":""array"",""items"":""string"" }} ] }") as RecordSchema; var stuffSchema = schema.First(r => r.Name == "stuff").Type as RecordSchema; var enumSchema = stuffSchema.First(r => r.Name == "keytype").Type as EnumSchema; var genericRecord = new GenericRecord(schema); var reader = new DatumReader <GenericRecord>(schema); var writer = new DatumWriter <GenericRecord>(schema); var delimiter = Environment.NewLine; var stringBuilder = new StringBuilder(); using (var stream = new StringWriter(stringBuilder)) using (var encoder = new JsonEncoder(stream, schema, delimiter)) { for (int i = 0; i < 10; i++) { var record = new GenericRecord(genericRecord); record[0] = i; record[1] = $"foo{i}"; record[2] = null; record[3] = new Dictionary <string, string>() { { "Brother", "John" } }; var stuffRecord = new GenericRecord(stuffSchema); var keyEnum = new GenericEnum(enumSchema, i % enumSchema.Symbols.Count); stuffRecord[0] = Guid.NewGuid().ToByteArray(); stuffRecord[1] = keyEnum; record[4] = stuffRecord; record[5] = (i % 2) == 0 ? "ToNullOrNotToNull" : null; record[6] = new List <string> { }; writer.Write(encoder, record); var singleJson = stringBuilder.ToString(); } } var allJson = stringBuilder.ToString(); using (var stream = new StringReader(allJson)) using (var decoder = new JsonDecoder(stream, schema, delimiter)) { for (int i = 0; i < 10; i++) { var record = reader.Read(decoder); Debug.WriteLine(record.ToString()); } } }