public void AvroRecord_CreateWithNonRecordTypeSchema() { var schema = TypeSchema.Create(@"""float"""); Utilities.ShouldThrow<ArgumentException>(() => { var _ = new AvroRecord(schema); Assert.IsNotNull(_); }); }
private void WriteEntryImplementation(ODataResource entry) { if (this.schema == null) { this.schema = this.outputContext.AvroWriter.UpdateSchema(entry, null, this.writingFeed); } if (this.scopes.Count > 0) { var parent = this.scopes.Pop() as ODataNestedResourceInfo; var obj = this.currentEntityObject; ODataAvroConvert.UpdateNestedInfoFromODataObject(obj, entry, parent, schema); } else { var obj = (AvroRecord)ODataAvroConvert.FromODataObject(entry, this.schema); if (this.writingFeed) { this.entityObjectList.Add(obj); this.currentEntityObject = obj; } else { this.currentEntityObject = obj; } } }
/// <summary> /// Converts an Avro generic record to a managed object based on the record's schema or supplied schema /// </summary> /// <param name="avroRecord"></param> /// <param name="recordSchema"></param> /// <param name="typeCacheFilter"></param> /// <param name="customValueSetter"></param> /// <returns></returns> public static T FromAvroRecord <T>( this AvroRecord avroRecord, string recordSchema = null, Func <Type, bool> typeCacheFilter = null, ICustomValueSetter customValueSetter = null) where T : class { TypeSchema typeSchema = avroRecord.Schema; if (recordSchema != null) { typeSchema = new JsonSchemaBuilder().BuildSchema(recordSchema); if (!(typeSchema is RecordSchema)) { throw new ApplicationException("Invalid record schema"); } } _typeCacheFilter = typeCacheFilter; _customValueSetter = customValueSetter; RefreshTypeCache(typeof(T)); return((T)FromAvroRecord(avroRecord , typeof(T) , typeSchema)); }
public void SchemaDefaultValues_RecordWithIntFieldHavingDefaultValue() { const string WriterSchema = @"{ ""type"":""record"", ""name"":""Microsoft.Hadoop.Avro.Tests.SimpleIntClass"", ""fields"": [ { ""name"":""IntField"", ""type"":""int"", ""default"": 0 } ] }"; var serializer = AvroSerializer.CreateGeneric(WriterSchema); var deserializer = AvroSerializer.CreateGenericDeserializerOnly(WriterSchema, WriterSchema); using (var stream = new MemoryStream()) { dynamic expected = new AvroRecord(serializer.ReaderSchema); expected.IntField = 1; serializer.Serialize(stream, expected); stream.Seek(0, SeekOrigin.Begin); dynamic actual = deserializer.Deserialize(stream); Assert.Equal(expected.IntField, actual.IntField); } }
public AvroRecord ToAvroRecord(Schema schema) { var rs = schema as RecordSchema; var avroEnum = new AvroEnum(rs.GetField("EnumMember").TypeSchema) { Value = this.EnumMember.ToString() }; var result = new AvroRecord(schema); result["EnumMember"] = avroEnum; result["StringMember"] = this.StringMember; result["FloatMember"] = this.FloatMember; result["IntMember"] = this.IntMember; result["LongMember"] = this.LongMember; result["DoubleMember"] = this.DoubleMember; result["BoolMember"] = this.BoolMember; result["DecimalMember"] = this.DecimalMember.ToString(); result["GuidMember"] = this.GuidMember.ToByteArray(); result["DateTimeMember"] = DateTimeSerializer.ConvertDateTimeToPosixTime(this.DateTimeMember); result["IntArrayMember"] = this.IntArrayMember; result["SByteArrayMember"] = this.SByteArrayMember; result["IntListMember"] = this.IntListMember.ToArray(); result["IntMapMember"] = this.IntMapMember; return(result); }
public override bool Read() { switch (this.State) { case ODataParameterReaderState.Start: if (!this.inputContext.AvroReader.MoveNext()) { this.state = ODataParameterReaderState.Completed; return false; } this.avroRecord = this.inputContext.AvroReader.Current as AvroRecord; if (this.avroRecord == null) { this.state = ODataParameterReaderState.Exception; return false; } this.enumerator = this.avroRecord.Schema.Fields.GetEnumerator(); return this.UpdateState(); case ODataParameterReaderState.Value: case ODataParameterReaderState.Entry: case ODataParameterReaderState.Feed: return this.UpdateState(); default: throw new ApplicationException("Invalid state " + this.State); } }
private IUpdatableRow TransformEvent(AvroRecord aEventData, IUpdatableRow output) { var body = aEventData[BODY]; output.Set(BODY, body); return(output); }
public void GenericSerializer_SerializeRecursiveRecord() { const string StringSchema = @"{ ""type"":""record"", ""name"":""Microsoft.Hadoop.Avro.Tests.Recursive"", ""fields"":[ {""name"":""IntField"",""type"":""int""}, {""name"":""RecursiveField"",""type"":[ ""null"", ""Microsoft.Hadoop.Avro.Tests.Recursive"" ] } ]}"; var serializer = AvroSerializer.CreateGeneric(StringSchema); using (var stream = new MemoryStream()) { dynamic expected = new AvroRecord(serializer.WriterSchema); expected.IntField = 5; expected.RecursiveField = new AvroRecord(((serializer.ReaderSchema as RecordSchema).GetField("RecursiveField").TypeSchema as UnionSchema).Schemas[1]); expected.RecursiveField.IntField = 3; expected.RecursiveField.RecursiveField = null; serializer.Serialize(stream, expected); stream.Position = 0; var actual = serializer.Deserialize(stream) as AvroRecord; Assert.IsTrue(ShallowlyEqual(expected, actual)); } }
public void GenericSerializer_SerializeRecordWithUnion() { const string StringSchema = @"{ ""name"":""Category"", ""namespace"":""ApacheAvro.Types"", ""type"":""record"", ""fields"": [ {""name"":""CategoryName"", ""type"":""string""}, {""name"":""Description"", ""type"":[""string"",""null""]}, {""name"":""Picture"", ""type"":[""bytes"", ""null""]}, {""name"":""Id"", ""type"":[""int"", ""null""]} ] }"; var serializer = AvroSerializer.CreateGeneric(StringSchema); using (var stream = new MemoryStream()) { dynamic expected = new AvroRecord(serializer.WriterSchema); expected.CategoryName = "Test"; expected.Description = "Test"; expected.Picture = null; expected.Id = 1; serializer.Serialize(stream, expected); stream.Seek(0, SeekOrigin.Begin); var actual = serializer.Deserialize(stream) as AvroRecord; Assert.IsTrue(ShallowlyEqual(expected, actual)); } }
public void GenericSerializer_SerializeEvolvedRecordWithPromotedField() { const string WriterSchema = @"{ ""type"":""record"", ""name"":""Microsoft.Hadoop.Avro.Tests.SimpleIntClass"", ""fields"": [ {""name"":""IntField"", ""type"":""int""} ] }"; const string ReaderSchema = @"{ ""type"":""record"", ""name"":""Microsoft.Hadoop.Avro.Tests.SimpleIntClass"", ""fields"": [ {""name"":""IntField"",""type"":""long""} ] }"; var serializer = AvroSerializer.CreateGeneric(WriterSchema); var deserializer = AvroSerializer.CreateGenericDeserializerOnly(WriterSchema, ReaderSchema); using (var stream = new MemoryStream()) { dynamic expected = new AvroRecord(serializer.ReaderSchema); expected.IntField = 1; serializer.Serialize(stream, expected); stream.Seek(0, SeekOrigin.Begin); dynamic actual = deserializer.Deserialize(stream); Assert.AreEqual(expected.IntField, actual.IntField); } }
public override bool Read() { switch (this.State) { case ODataParameterReaderState.Start: if (!this.inputContext.AvroReader.MoveNext()) { this.state = ODataParameterReaderState.Completed; return(false); } this.avroRecord = this.inputContext.AvroReader.Current as AvroRecord; if (this.avroRecord == null) { this.state = ODataParameterReaderState.Exception; return(false); } this.enumerator = this.avroRecord.Schema.Fields.GetEnumerator(); return(this.UpdateState()); case ODataParameterReaderState.Value: case ODataParameterReaderState.Resource: case ODataParameterReaderState.ResourceSet: return(this.UpdateState()); default: throw new ApplicationException("Invalid state " + this.State); } }
public void AvroRecord_CreateUnionOfAvroRecordAndNull() { const string Schema = @"[ ""null"", { ""type"":""record"", ""name"":""Microsoft.Hadoop.Avro.Tests.SimpleIntClass"", ""fields"": [ { ""name"":""IntField"", ""type"":""int"" }, ] } ]"; var serializer = AvroSerializer.CreateGeneric(Schema); var unionSchema = (UnionSchema)serializer.WriterSchema; var expected = new AvroRecord(unionSchema.Schemas[1]); expected["IntField"] = Utilities.GetRandom <int>(false); serializer.Serialize(this.stream, null); serializer.Serialize(this.stream, expected); this.stream.Seek(0, SeekOrigin.Begin); var actual1 = serializer.Deserialize(this.stream); Assert.Null(actual1); var actual2 = (AvroRecord)serializer.Deserialize(this.stream); Assert.NotNull(actual2); Assert.Equal(expected["IntField"], actual2["IntField"]); }
public void ReadAvroAsError() { const string Schema = @"{ ""type"":""record"", ""name"":""OData.Error"", ""fields"": [ { ""name"":""ErrorCode"", ""type"":""string"" }, { ""name"":""Message"" , ""type"":""string"" }, ] }"; var stream = new MemoryStream(); using (var writer = AvroContainer.CreateGenericWriter(Schema, stream, /*leave open*/ true, Codec.Null)) using (var seqWriter = new SequentialWriter <object>(writer, 24)) { RecordSchema parameterSchema = (RecordSchema)AvroSerializer.CreateGeneric(Schema).WriterSchema; AvroRecord ar = new AvroRecord(parameterSchema); ar["ErrorCode"] = "e1"; ar["Message"] = "m1"; seqWriter.Write(ar); seqWriter.Flush(); } stream.Flush(); stream.Seek(0, SeekOrigin.Begin); var error = this.CreateODataInputContext(stream).ReadError(); Assert.AreEqual("e1", error.ErrorCode); Assert.AreEqual("m1", error.Message); }
public void AvroRecord_CreateUnionOfAvroRecordAndNull() { const string Schema = @"[ ""null"", { ""type"":""record"", ""name"":""Microsoft.Hadoop.Avro.Tests.SimpleIntClass"", ""fields"": [ { ""name"":""IntField"", ""type"":""int"" }, ] } ]"; var serializer = AvroSerializer.CreateGeneric(Schema); var unionSchema = (UnionSchema)serializer.WriterSchema; var expected = new AvroRecord(unionSchema.Schemas[1]); expected["IntField"] = Utilities.GetRandom<int>(false); serializer.Serialize(this.stream, null); serializer.Serialize(this.stream, expected); this.stream.Seek(0, SeekOrigin.Begin); var actual1 = serializer.Deserialize(this.stream); Assert.IsNull(actual1); var actual2 = (AvroRecord)serializer.Deserialize(this.stream); Assert.IsNotNull(actual2); Assert.AreEqual(expected["IntField"], actual2["IntField"]); }
public void GenericSerializer_SerializeUnionWithMapAndNull() { const string StringSchema = @"{ ""type"":""record"", ""name"":""Microsoft.Hadoop.Avro.Tests.SimpleIntClass"", ""fields"": [ {""name"":""LongMap"", ""type"": [""null"", {""type"":""map"", ""values"":""long""}]}, ] }"; var serializer = AvroSerializer.CreateGeneric(StringSchema); using (var stream = new MemoryStream()) { dynamic expected = new AvroRecord(serializer.WriterSchema); expected.LongMap = new Dictionary <string, long> { { "test", 1 } }; serializer.Serialize(stream, expected); stream.Position = 0; dynamic actual = serializer.Deserialize(stream); Utilities.DictionaryEquals <string, long>(expected.LongMap as IDictionary <string, long>, actual.LongMap as IDictionary <string, long>); } }
public void SchemaDefaultValues_RecordWithInvalidFields() { const string DefaultSchema = @"{""type"":""record"",""name"":""SomeRecord"",""fields"":[{""name"":""Field1"",""type"":""int""}, {""name"":""Field2"", ""type"":""string""}]}"; var tempSerializer = AvroSerializer.CreateGeneric(DefaultSchema); dynamic defaultValue = new AvroRecord(tempSerializer.WriterSchema); defaultValue.Field1 = Utilities.GetRandom <int>(false); defaultValue.Field2 = "some string " + Utilities.GetRandom <int>(false); var defaultValueJson = @"{""Field1"":" + defaultValue.Field1 + @", ""Field2"":""" + defaultValue.Field2 + @""", ""WRONGFIELD"":""WRONGVALUE""}"; Assert.ThrowsException <SerializationException>(() => { RoundTripSerializationWithDefaultsAndCheck <AvroRecord>( DefaultSchema, defaultValue, defaultValueJson, (Action <dynamic, dynamic>)((expected, actual) => { Assert.AreEqual(expected.Field1, actual.Field1); Assert.AreEqual(expected.Field2, actual.Field2); })); }); }
static void Main(string[] args) { var schemaString = "{\"type\": \"record\", " + "\"name\": \"kevent\"," + "\"fields\": [" + "{\"name\": \"timestamp\", \"type\": \"long\"}," + "{\"name\": \"payload\", \"type\": \"bytes\"}" + "]}"; var avroRecord = new AvroRecord(TypeSchema.Create(schemaString)); avroRecord["timestamp"] = DateTime.UtcNow.Ticks; avroRecord["payload"] = Enumerable.Range(0, 10).Select(x => (byte)x).ToArray(); var totalMilliseconds = DateTime.Now.Subtract(new DateTime(1970, 01, 01)).TotalMilliseconds; var dateTime = new DateTime(1970, 01, 01).Add(TimeSpan.FromMilliseconds(1504007964797L)); var avroSerializer = AvroSerializer.CreateGeneric(schemaString); //byte[] bytes; //using (var memoryStream = new MemoryStream()) //{ // avroSerializer.Serialize(memoryStream, avroRecord); // bytes = memoryStream.ToArray(); // var base64String = Convert.ToBase64String(bytes); //} try { var bytes = Convert.FromBase64String("AAAAACn6oeLdxVcUUy+TAh26uvgGwg=="); using (var memoryStream = new MemoryStream(bytes)) { var deserialize = avroSerializer.Deserialize(memoryStream); } } catch (Exception e) { } }
public void GenericSerializer_SerializeUsingDifferentReaderWriterType() { const string StringSchema = @"{ ""type"":""record"", ""name"":""Microsoft.Hadoop.Avro.Tests.SimpleIntClass"", ""fields"": [ { ""name"":""IntField"", ""type"":""int"" } ] }"; var serializer = AvroSerializer.CreateGeneric(StringSchema); var deserializer = AvroSerializer.CreateGenericDeserializerOnly(StringSchema, StringSchema); using (var stream = new MemoryStream()) { dynamic expected = new AvroRecord(serializer.WriterSchema); expected.IntField = 5; serializer.Serialize(stream, expected); stream.Position = 0; var actual = deserializer.Deserialize(stream) as AvroRecord; Assert.IsTrue(expected["IntField"].Equals(actual["IntField"])); } }
public static ODataResource ToODataEntry(AvroRecord record) { return(new ODataResource { TypeName = record.Schema.FullName, Properties = GetProperties(record) }); }
private static CoffeeMachineData CreateCoffeeMachineData(AvroRecord avroRecord) { var body = avroRecord.GetField <byte[]>("Body"); var dataAsJson = Encoding.UTF8.GetString(body); var coffeeMachineData = JsonConvert.DeserializeObject <CoffeeMachineData>(dataAsJson); return(coffeeMachineData); }
public static ODataComplexValue ToODataComplexValue(AvroRecord record) { return(new ODataComplexValue { TypeName = record.Schema.FullName, Properties = GetProperties(record) }); }
public ODataAvroParameterWriter(ODataAvroOutputContext outputContext, IEdmOperation operation) { Debug.Assert(outputContext != null, "outputContext != null"); Debug.Assert(operation != null, "operation != null"); this.outputContext = outputContext; schema = (RecordSchema)this.outputContext.AvroWriter.UpdateSchema(null, this.GetTmpType(operation)); this.record = new AvroRecord(schema); }
public ODataAvroParameterWriter(ODataAvroOutputContext outputContext, IEdmOperation operation) { Debug.Assert(outputContext != null, "outputContext != null"); Debug.Assert(operation != null, "operation != null"); this.outputContext = outputContext; schema = (RecordSchema)this.outputContext.AvroWriter.UpdateSchema(null, this.GetTmpType(operation)); this.record = new AvroRecord(schema); }
public static object FromODataObject(object value, Schema schema) { var entry = value as ODataResource; if (entry != null) { RecordSchema recordSchema = schema as RecordSchema; if (recordSchema == null) { var unionSchema = schema as UnionSchema; if (unionSchema != null) { recordSchema = unionSchema.Schemas.OfType <RecordSchema>().Single(); } else { throw new ApplicationException("not supported schema found."); } } var record = new AvroRecord(recordSchema); foreach (var property in entry.Properties) { RecordField field; recordSchema.TryGetField(property.Name, out field); record[property.Name] = FromODataObject(property.Value, field != null ? field.TypeSchema : null); } return(record); } var error = value as ODataError; if (error != null) { var record = new AvroRecord(schema); record["ErrorCode"] = error.ErrorCode; record["Message"] = error.Message; return(record); } var collection = value as ODataCollectionValue; if (collection != null) { if (collection.Items is IEnumerable <object> ) { return(((IEnumerable <object>)collection.Items).ToArray()); } return(collection.Items); } return(value); }
public void GenericSerializer_SerializeNestedRecordWithReferences() { const string StringSchema = @"{ ""name"":""MultiOrderProperties"", ""namespace"":""ApacheAvro.Types"", ""type"":""record"", ""fields"": [ { ""name"":""Orders1"", ""type"": { ""name"":""Order"", ""namespace"":""ApacheAvro.Types"", ""type"":""record"", ""fields"": [ {""name"":""CustomerId"", ""type"":""string""}, {""name"":""EmployeeId"", ""type"":""int""}, ] } }, {""name"":""Orders2"", ""type"":""Order""}, {""name"":""Orders3"", ""type"":""Order""}, {""name"":""Orders4"", ""type"":""Order""}, ] }"; var serializer = AvroSerializer.CreateGeneric(StringSchema); var recordSchema = serializer.ReaderSchema as RecordSchema; Assert.IsNotNull(recordSchema); using (var stream = new MemoryStream()) { dynamic expected = new AvroRecord(serializer.ReaderSchema); expected.Orders1 = new AvroRecord(recordSchema.GetField("Orders1").TypeSchema); expected.Orders1.CustomerId = "1"; expected.Orders1.EmployeeId = 1; expected.Orders2 = new AvroRecord(recordSchema.GetField("Orders2").TypeSchema); expected.Orders2.CustomerId = "1"; expected.Orders2.EmployeeId = 2; expected.Orders3 = new AvroRecord(recordSchema.GetField("Orders3").TypeSchema); expected.Orders3.CustomerId = "1"; expected.Orders3.EmployeeId = 3; expected.Orders4 = new AvroRecord(recordSchema.GetField("Orders4").TypeSchema); expected.Orders4.CustomerId = "1"; expected.Orders4.EmployeeId = 4; serializer.Serialize(stream, expected); stream.Seek(0, SeekOrigin.Begin); var actual = serializer.Deserialize(stream) as AvroRecord; Assert.IsTrue(ShallowlyEqual(expected, actual)); } }
public void SequentialGenericWritingReading_RecursiveRecord() { const string StringSchema = @"{ ""type"":""record"", ""name"":""Microsoft.Hadoop.Avro.Tests.Recursive"", ""fields"":[ {""name"":""IntField"",""type"":""int""}, {""name"":""RecursiveField"",""type"":[ ""null"", ""Microsoft.Hadoop.Avro.Tests.Recursive"" ] } ]}"; using (var stream = new MemoryStream()) { var serializer = AvroSerializer.CreateGeneric(StringSchema); using (var streamWriter = AvroContainer.CreateGenericWriter(StringSchema, stream, Codec.Null)) { using (var writer = new SequentialWriter <object>(streamWriter, 24)) { var expected = new List <AvroRecord>(); var random = new Random(93); for (int i = 0; i < 10; i++) { dynamic record = new AvroRecord(serializer.WriterSchema); record.IntField = random.Next(); record.RecursiveField = new AvroRecord( ((serializer.ReaderSchema as RecordSchema).GetField("RecursiveField").TypeSchema as UnionSchema).Schemas[1]); record.RecursiveField.IntField = random.Next(); record.RecursiveField.RecursiveField = null; expected.Add(record); } expected.ForEach(writer.Write); writer.Flush(); stream.Seek(0, SeekOrigin.Begin); var streamReader = AvroContainer.CreateReader <Recursive>(stream, true, this.dataContractSettings, new CodecFactory()); using (var reader = new SequentialReader <Recursive>(streamReader)) { var j = 0; foreach (var avroRecord in reader.Objects) { Assert.Equal(expected[j]["IntField"], avroRecord.IntField); Assert.Equal(((dynamic)expected[j]["RecursiveField"])["IntField"], avroRecord.RecursiveField.IntField); Assert.Equal( ((dynamic)expected[j++]["RecursiveField"])["RecursiveField"], avroRecord.RecursiveField.RecursiveField); } } } } } }
private static IEnumerable <ODataProperty> GetProperties(AvroRecord record) { return(record.Schema.Fields .Where(field => !(record[field.Name] is AvroRecord)) .Select(field => new ODataProperty { Name = field.Name, Value = ToODataValue(record[field.Name]), })); }
public static ClassWithSByteFields Create(AvroRecord record) { return(new ClassWithSByteFields { SByteField = (sbyte)((dynamic)record).SByteField, MaxSByteField = (sbyte)((dynamic)record).MaxSByteField, MinSByteField = (sbyte)((dynamic)record).MinSByteField, SByteArrayField = ((Array)((dynamic)record).SByteArrayField).OfType <int>().Select(i => (sbyte)i).ToArray() }); }
public void AvroRecord_CreateWithNonRecordTypeSchema() { var schema = TypeSchema.Create(@"""float"""); Utilities.ShouldThrow <ArgumentException>(() => { var _ = new AvroRecord(schema); Assert.NotNull(_); }); }
private static IEnumerable <ODataProperty> GetProperties(AvroRecord record) { return(record.Schema.Fields .Select(field => new ODataProperty { Name = field.Name, Value = record[field.Name] is AvroRecord ? ToODataComplexValue((AvroRecord)record[field.Name]) : ToODataValue(record[field.Name]), })); }
public AvroRecord ToAvroRecord(Schema schema) { var result = new AvroRecord(schema); result["SByteField"] = this.SByteField; result["MaxSByteField"] = this.MaxSByteField; result["MinSByteField"] = this.MinSByteField; result["SByteArrayField"] = this.SByteArrayField; return(result); }
public static ODataNestedResourceInfo GetNestedResourceInfo(AvroRecord record) { return(record.Schema.Fields .Where(field => (record[field.Name] is AvroRecord)) .Select(field => new ODataNestedResourceInfo { Name = field.Name, IsCollection = false }) .SingleOrDefault()); }
public void AvroRecord_CreateMapOfArrayOfAvroRecords() { const string Schema = @" { ""type"" :""map"", ""values"": { ""type"" :""array"", ""items"": { ""type"":""record"", ""name"":""Microsoft.Hadoop.Avro.Tests.SimpleIntClass"", ""fields"": [ { ""name"":""IntField"", ""type"":""int"" }, ] } } }"; var serializer = AvroSerializer.CreateGeneric(Schema); var mapSchema = (MapSchema)serializer.WriterSchema; var arraySchema = (ArraySchema)mapSchema.ValueSchema; var expected = new Dictionary <string, AvroRecord[]>(); for (int i = 0; i < 5; i++) { var key = Utilities.GetRandom <string>(false) + i; var value = new AvroRecord[10]; for (int j = 0; j < 10; j++) { var record = new AvroRecord(arraySchema.ItemSchema); record["IntField"] = j; value[j] = record; } expected.Add(key, value); } serializer.Serialize(this.stream, expected); this.stream.Seek(0, SeekOrigin.Begin); dynamic actual = serializer.Deserialize(this.stream); foreach (var expectedKeyValuePair in expected) { Assert.True(actual.ContainsKey(expectedKeyValuePair.Key)); dynamic actualValue = actual[expectedKeyValuePair.Key]; for (int i = 0; i < 10; i++) { Assert.Equal(expectedKeyValuePair.Value[i]["IntField"], actualValue[i].IntField); } } }
public void GenericSerializer_SerializeSimpleRecord() { const string StringSchema = @"{ ""type"":""record"", ""name"":""Microsoft.Hadoop.Avro.Tests.SimpleIntClass"", ""fields"": [ { ""name"":""IntField"", ""type"":""int"" }, { ""name"":""MyGuid"", ""type"": {""type"":""fixed"", ""size"":4, ""name"": ""q"" } }, { ""name"":""Arr"", ""type"": {""type"":""array"", ""items"":""int""}}, { ""name"":""LongField"", ""type"": ""long""}, {""name"":""LongMap"", ""type"": {""type"":""map"", ""values"":""long""}}, { ""name"":""DoubleField"", ""type"": ""double""}, { ""name"":""FloatField"", ""type"": ""float""}, { ""name"":""BooleanField"", ""type"": ""boolean""}, { ""name"":""BytesField"", ""type"": ""bytes""}, ] }"; var serializer = AvroSerializer.CreateGeneric(StringSchema); using (var stream = new MemoryStream()) { dynamic expected = new AvroRecord(serializer.WriterSchema); expected.IntField = 5; expected.MyGuid = new byte[] { 1, 2, 3, 4 }; expected.Arr = new int[] { 1, 2, 5 }; expected.LongField = (long)14; expected.LongMap = new Dictionary <string, long> { { "test", 1 } }; expected.DoubleField = (double)3; expected.FloatField = (float)4; expected.BooleanField = true; expected.BytesField = new byte[3] { 4, 5, 6 }; serializer.Serialize(stream, expected); stream.Position = 0; dynamic actual = serializer.Deserialize(stream); Assert.AreEqual(expected.IntField, actual.IntField); CollectionAssert.AreEqual(expected.MyGuid, actual.MyGuid); CollectionAssert.AreEqual(expected.Arr, actual.Arr); Assert.AreEqual(expected.LongField, actual.LongField); Utilities.DictionaryEquals(expected.LongMap as IDictionary <string, long>, actual.LongMap as IDictionary <string, long>); Assert.AreEqual(expected.DoubleField, actual.DoubleField); Assert.AreEqual(expected.FloatField, actual.FloatField); Assert.AreEqual(expected.BooleanField, actual.BooleanField); CollectionAssert.AreEqual(expected.BytesField, actual.BytesField); } }
public void GenericSerializer_SerializeSimpleRecord() { const string StringSchema = @"{ ""type"":""record"", ""name"":""Microsoft.Hadoop.Avro.Tests.SimpleIntClass"", ""fields"": [ { ""name"":""IntField"", ""type"":""int"" }, { ""name"":""MyGuid"", ""type"": {""type"":""fixed"", ""size"":4, ""name"": ""q"" } }, { ""name"":""Arr"", ""type"": {""type"":""array"", ""items"":""int""}}, { ""name"":""LongField"", ""type"": ""long""}, {""name"":""LongMap"", ""type"": {""type"":""map"", ""values"":""long""}}, { ""name"":""DoubleField"", ""type"": ""double""}, { ""name"":""FloatField"", ""type"": ""float""}, { ""name"":""BooleanField"", ""type"": ""boolean""}, { ""name"":""BytesField"", ""type"": ""bytes""}, ] }"; var serializer = AvroSerializer.CreateGeneric(StringSchema); using (var stream = new MemoryStream()) { dynamic expected = new AvroRecord(serializer.WriterSchema); expected.IntField = 5; expected.MyGuid = new byte[] { 1, 2, 3, 4 }; expected.Arr = new int[] { 1, 2, 5 }; expected.LongField = (long)14; expected.LongMap = new Dictionary<string, long> { { "test", 1 } }; expected.DoubleField = (double)3; expected.FloatField = (float)4; expected.BooleanField = true; expected.BytesField = new byte[3] { 4, 5, 6 }; serializer.Serialize(stream, expected); stream.Position = 0; dynamic actual = serializer.Deserialize(stream); Assert.AreEqual(expected.IntField, actual.IntField); CollectionAssert.AreEqual(expected.MyGuid, actual.MyGuid); CollectionAssert.AreEqual(expected.Arr, actual.Arr); Assert.AreEqual(expected.LongField, actual.LongField); Utilities.DictionaryEquals(expected.LongMap as IDictionary<string, long>, actual.LongMap as IDictionary<string, long>); Assert.AreEqual(expected.DoubleField, actual.DoubleField); Assert.AreEqual(expected.FloatField, actual.FloatField); Assert.AreEqual(expected.BooleanField, actual.BooleanField); CollectionAssert.AreEqual(expected.BytesField, actual.BytesField); } }
public void SequentialGenericWritingReading_SimpleRecord() { const string StringSchema = @"{ ""type"":""record"", ""name"":""Microsoft.Hadoop.Avro.Tests.ClassOfInt"", ""fields"": [ { ""name"":""PrimitiveInt"", ""type"":""int"" } ] }"; using (var stream = new MemoryStream()) { var serializer = AvroSerializer.CreateGeneric(StringSchema); using (var streamWriter = AvroContainer.CreateGenericWriter(StringSchema, stream, Codec.Null)) { using (var writer = new SequentialWriter<object>(streamWriter, 24)) { var expected = new List<AvroRecord>(); var random = new Random(113); for (int i = 0; i < 10; i++) { dynamic record = new AvroRecord(serializer.WriterSchema); record.PrimitiveInt = random.Next(); expected.Add(record); } expected.ForEach(writer.Write); writer.Flush(); stream.Seek(0, SeekOrigin.Begin); var streamReader = AvroContainer.CreateReader<ClassOfInt>(stream, true, this.dataContractSettings, new CodecFactory()); using (var reader = new SequentialReader<ClassOfInt>(streamReader)) { var j = 0; foreach (var avroRecord in reader.Objects) { Assert.AreEqual(expected[j++]["PrimitiveInt"], avroRecord.PrimitiveInt); } } } } } }
public void GenericSerializer_SerializeRecordWithUknownField() { const string StringSchema = @"{ ""name"":""Category"", ""namespace"":""ApacheAvro.Types"", ""type"":""record"", ""fields"": [ {""name"":""CategoryName"", ""type"":""string""}, {""name"":""Description"", ""type"":[""string"",""null""]}, {""name"":""Picture"", ""type"":[""bytes"", ""null""]}, {""name"":""Id"", ""type"":[""int"", ""null""]} ] }"; var serializer = AvroSerializer.CreateGeneric(StringSchema); dynamic expected = new AvroRecord(serializer.WriterSchema); expected.UknownField = 5; }
public void GenericSerializer_SerializeUnionWithMapAndNull() { const string StringSchema = @"{ ""type"":""record"", ""name"":""Microsoft.Hadoop.Avro.Tests.SimpleIntClass"", ""fields"": [ {""name"":""LongMap"", ""type"": [""null"", {""type"":""map"", ""values"":""long""}]}, ] }"; var serializer = AvroSerializer.CreateGeneric(StringSchema); using (var stream = new MemoryStream()) { dynamic expected = new AvroRecord(serializer.WriterSchema); expected.LongMap = new Dictionary<string, long> { { "test", 1 } }; serializer.Serialize(stream, expected); stream.Position = 0; dynamic actual = serializer.Deserialize(stream); Utilities.DictionaryEquals<string, long>(expected.LongMap as IDictionary<string, long>, actual.LongMap as IDictionary<string, long>); } }
public void GenericSerializer_SerializeUnionWithArrayAndNull() { const string StringSchema = @"{ ""type"":""record"", ""name"":""Microsoft.Hadoop.Avro.Tests.SimpleIntClass"", ""fields"": [ { ""name"":""Arr"", ""type"": [""null"", {""type"":""array"", ""items"":""int""}]} ] }"; var serializer = AvroSerializer.CreateGeneric(StringSchema); using (var stream = new MemoryStream()) { dynamic expected = new AvroRecord(serializer.WriterSchema); expected.Arr = new int[] { 1, 2, 5 }; serializer.Serialize(stream, expected); stream.Position = 0; dynamic actual = serializer.Deserialize(stream); CollectionAssert.AreEqual(expected.Arr, actual.Arr); } }
public void Container_SchemaEvolution_RecordWithPermutatedAndMissingWriterFields() { const string WriterSchema = @"{ ""name"":""WriterClass"", ""namespace"":""Microsoft.Hadoop.Avro.Tests"", ""type"":""record"", ""fields"": [ {""name"":""BoolA"", ""type"":""boolean""}, {""name"":""BoolB"", ""type"":""boolean""}, {""name"":""FloatA"", ""type"":""float""}, {""name"":""FloatB"", ""type"":""float""}, {""name"":""DoubleA"", ""type"":""double""}, {""name"":""DoubleB"", ""type"":""double""}, {""name"":""IntA"", ""type"":""int""}, {""name"":""IntB"", ""type"":""int""}, {""name"":""MyGuid"", ""type"": {""type"":""fixed"", ""size"":16, ""name"": ""q"" }}, {""name"": ""classField"", ""type"" : [ ""null"", ""Microsoft.Hadoop.Avro.Tests.WriterClass""] }, {""name"":""Arr"", ""type"": {""type"":""array"", ""items"":""int""}}, {""name"":""LongField"", ""type"":""long""}, {""name"":""LongMap"", ""type"": {""type"":""map"", ""values"":""long""}}, {""name"":""BytesField"", ""type"":""bytes""}, ] }"; const string ReaderSchema = @"{ ""name"":""WriterClass"", ""namespace"":""Microsoft.Hadoop.Avro.Tests"", ""type"":""record"", ""fields"": [ {""name"":""DoubleB"", ""type"":""double""}, {""name"":""FloatB"", ""type"":""float""}, {""name"":""FloatA"", ""type"":""float""}, {""name"":""BoolB"", ""type"":""boolean""}, {""name"":""BoolA"", ""type"":""boolean""}, {""name"":""DoubleA"", ""type"":""double""}, ] }"; const int RecordsCount = 100; var serializer = AvroSerializer.CreateGeneric(WriterSchema); var schema = serializer.WriterSchema; var expected = new List<AvroRecord>(); for (int counter = 0; counter < RecordsCount; counter++) { dynamic avroRecord = new AvroRecord(schema); avroRecord.BoolA = Utilities.GetRandom<bool>(false); avroRecord.BoolB = Utilities.GetRandom<bool>(false); avroRecord.FloatA = Utilities.GetRandom<float>(false); avroRecord.FloatB = Utilities.GetRandom<float>(false); avroRecord.DoubleA = Utilities.GetRandom<double>(false); avroRecord.DoubleB = Utilities.GetRandom<double>(false); avroRecord.IntA = Utilities.GetRandom<int>(false); avroRecord.IntB = Utilities.GetRandom<int>(false); avroRecord.MyGuid = Utilities.GetRandom<Guid>(false).ToByteArray(); avroRecord.Arr = Utilities.GetRandom<int[]>(false); avroRecord.LongField = Utilities.GetRandom<long>(false); avroRecord.LongMap = Utilities.GetRandom<Dictionary<string, long>>(false); avroRecord.BytesField = Utilities.GetRandom<byte[]>(false); avroRecord.classField = new AvroRecord(serializer.WriterSchema); avroRecord.classField.BoolA = Utilities.GetRandom<bool>(false); avroRecord.classField.BoolB = Utilities.GetRandom<bool>(false); avroRecord.classField.FloatA = Utilities.GetRandom<float>(false); avroRecord.classField.FloatB = Utilities.GetRandom<float>(false); avroRecord.classField.DoubleA = Utilities.GetRandom<double>(false); avroRecord.classField.DoubleB = Utilities.GetRandom<double>(false); avroRecord.classField.IntA = Utilities.GetRandom<int>(false); avroRecord.classField.IntB = Utilities.GetRandom<int>(false); avroRecord.classField.MyGuid = Utilities.GetRandom<Guid>(false).ToByteArray(); avroRecord.classField.Arr = Utilities.GetRandom<int[]>(false); avroRecord.classField.classField = null; avroRecord.classField.LongField = Utilities.GetRandom<long>(false); avroRecord.classField.LongMap = Utilities.GetRandom<Dictionary<string, long>>(false); avroRecord.classField.BytesField = Utilities.GetRandom<byte[]>(false); expected.Add(avroRecord); } using (var memoryStream = new MemoryStream()) { var writer = AvroContainer.CreateGenericWriter(WriterSchema, memoryStream, Codec.Null); var i = 0; while (i < expected.Count) { var block = writer.CreateBlockAsync().Result; for (var j = 0; j < 2; j++) { if (i >= expected.Count) { break; } block.Write(expected[i]); i++; } writer.WriteBlockAsync(block).Wait(); } writer.Dispose(); memoryStream.Seek(0, SeekOrigin.Begin); var reader = AvroContainer.CreateGenericReader(ReaderSchema, memoryStream, true, new CodecFactory()); var actual = new List<AvroRecord>(); while (reader.MoveNext()) { actual.AddRange(reader.Current.Objects.Cast<AvroRecord>()); } for (var k = 0; k < expected.Count; ++k) { Assert.AreEqual(((dynamic)expected[k]).DoubleB, ((dynamic)actual[k]).DoubleB); Assert.AreEqual(((dynamic)expected[k]).FloatB, ((dynamic)actual[k]).FloatB); Assert.AreEqual(((dynamic)expected[k]).FloatA, ((dynamic)actual[k]).FloatA); Assert.AreEqual(((dynamic)expected[k]).BoolB, ((dynamic)actual[k]).BoolB); Assert.AreEqual(((dynamic)expected[k]).BoolA, ((dynamic)actual[k]).BoolA); Assert.AreEqual(((dynamic)expected[k]).DoubleA, ((dynamic)actual[k]).DoubleA); } } }
internal static bool ShallowlyEqual(AvroRecord r1, AvroRecord r2) { if (r1 == r2) { return true; } if (r1 == null || r2 == null) { return false; } var keys1 = r1.Schema.Fields.Select(f => f.Name).ToList(); var keys2 = r2.Schema.Fields.Select(f => f.Name).ToList(); if (!keys1.SequenceEqual(keys2)) { return false; } foreach (var key in keys1) { if (r1[key] == r2[key]) { continue; } if (r1[key] is AvroRecord) { if (!ShallowlyEqual(r1[key] as AvroRecord, r2[key] as AvroRecord)) { return false; } } else if (!r1[key].Equals(r2[key])) { return false; } } return true; }
public void GenericSerializer_SerializeNestedRecordWithReferences() { const string StringSchema = @"{ ""name"":""MultiOrderProperties"", ""namespace"":""ApacheAvro.Types"", ""type"":""record"", ""fields"": [ { ""name"":""Orders1"", ""type"": { ""name"":""Order"", ""namespace"":""ApacheAvro.Types"", ""type"":""record"", ""fields"": [ {""name"":""CustomerId"", ""type"":""string""}, {""name"":""EmployeeId"", ""type"":""int""}, ] } }, {""name"":""Orders2"", ""type"":""Order""}, {""name"":""Orders3"", ""type"":""Order""}, {""name"":""Orders4"", ""type"":""Order""}, ] }"; var serializer = AvroSerializer.CreateGeneric(StringSchema); var recordSchema = serializer.ReaderSchema as RecordSchema; Assert.IsNotNull(recordSchema); using (var stream = new MemoryStream()) { dynamic expected = new AvroRecord(serializer.ReaderSchema); expected.Orders1 = new AvroRecord(recordSchema.GetField("Orders1").TypeSchema); expected.Orders1.CustomerId = "1"; expected.Orders1.EmployeeId = 1; expected.Orders2 = new AvroRecord(recordSchema.GetField("Orders2").TypeSchema); expected.Orders2.CustomerId = "1"; expected.Orders2.EmployeeId = 2; expected.Orders3 = new AvroRecord(recordSchema.GetField("Orders3").TypeSchema); expected.Orders3.CustomerId = "1"; expected.Orders3.EmployeeId = 3; expected.Orders4 = new AvroRecord(recordSchema.GetField("Orders4").TypeSchema); expected.Orders4.CustomerId = "1"; expected.Orders4.EmployeeId = 4; serializer.Serialize(stream, expected); stream.Seek(0, SeekOrigin.Begin); var actual = serializer.Deserialize(stream) as AvroRecord; Assert.IsTrue(ShallowlyEqual(expected, actual)); } }
public void SchemaEvolution_CompatibleConversionOfDictionaryRecordToDictionaryClass() { const string WriterSchema = @"{ ""type"":""record"", ""name"":""SimpleMap"", ""namespace"":""Microsoft.Hadoop.Avro.Tests"", ""fields"":[ {""name"":""Values"",""type"":{""type"": ""map"",""values"": ""string""}} ]}"; var serializer = AvroSerializer.CreateGeneric(WriterSchema); var deserializer = AvroSerializer.CreateDeserializerOnly<SimpleMap>(WriterSchema, this.dataContractSettings); using (var stream = new MemoryStream()) { dynamic expected = new AvroRecord(serializer.WriterSchema); expected.Values = new Dictionary<string, object>(); expected.Values.Add("key1", "value1"); expected.Values.Add("key2", "value2"); serializer.Serialize(stream, expected); stream.Seek(0, SeekOrigin.Begin); var actual = deserializer.Deserialize(stream); Assert.AreEqual(expected.Values["key1"], actual.Values["key1"]); Assert.AreEqual(expected.Values["key2"], actual.Values["key2"]); Assert.AreEqual(expected.Values.Count, actual.Values.Count); Assert.AreEqual(2, actual.Values.Count); } }
public void TestRecordEvolutionWithExtraReaderFieldsAndDefaultValues() { const string WriterSchema = @"{ ""type"":""record"", ""name"":""Microsoft.Hadoop.Avro.Tests.ClassOfInt"", ""fields"": [ { ""name"":""PrimitiveInt"", ""type"":""int"" } ] }"; var serializer = AvroSerializer.CreateGeneric(WriterSchema); var deserializer = AvroSerializer.CreateDeserializerOnly<SimpleIntClassWithExtraFields>(WriterSchema, this.dataContractSettings); using (var stream = new MemoryStream()) { dynamic expected = new AvroRecord(serializer.WriterSchema); expected.IntField = 4; serializer.Serialize(stream, expected); stream.Seek(0, SeekOrigin.Begin); dynamic actual = deserializer.Deserialize(stream); Assert.AreEqual(actual.ExtraFieldInt, 0); Assert.AreEqual(actual.ExtraFieldLong, 0); Assert.AreEqual(actual.ExtraFieldFloat, 0.0f); Assert.AreEqual(actual.ExtraFieldDouble, 0.0d); Assert.AreEqual(actual.ExtraFieldString, null); Assert.AreEqual(actual.ExtraFieldBool, false); Assert.AreEqual(expected.IntField, actual.IntField); } }
public void GenericSerializer_SerializeRecursiveRecord() { const string StringSchema = @"{ ""type"":""record"", ""name"":""Microsoft.Hadoop.Avro.Tests.Recursive"", ""fields"":[ {""name"":""IntField"",""type"":""int""}, {""name"":""RecursiveField"",""type"":[ ""null"", ""Microsoft.Hadoop.Avro.Tests.Recursive"" ] } ]}"; var serializer = AvroSerializer.CreateGeneric(StringSchema); using (var stream = new MemoryStream()) { dynamic expected = new AvroRecord(serializer.WriterSchema); expected.IntField = 5; expected.RecursiveField = new AvroRecord(((serializer.ReaderSchema as RecordSchema).GetField("RecursiveField").TypeSchema as UnionSchema).Schemas[1]); expected.RecursiveField.IntField = 3; expected.RecursiveField.RecursiveField = null; serializer.Serialize(stream, expected); stream.Position = 0; var actual = serializer.Deserialize(stream) as AvroRecord; Assert.IsTrue(ShallowlyEqual(expected, actual)); } }
//Serialize and deserialize sample data set using Generic Record. //Generic Record is a special class with the schema explicitly defined in JSON. //All serialized data should be mapped to the fields of Generic Record, //which in turn will be then serialized. // //This approach is generally slower than using Reflection, //but comes very handy in cases, where for example the data is represented //as Comma Separated Values (CSV) files rather than objects. // //In SerializeDeserializeObjectUsingGenericRecords() we will emulate this case //though explicit filling up the fields of the Generic Record public void SerializeDeserializeObjectUsingGenericRecords() { Console.WriteLine("SERIALIZATION USING GENERIC RECORD\n"); Console.WriteLine("Defining the Schema and creating Sample Data Set..."); //Define the schema in JSON const string Schema = @"{ ""type"":""record"", ""name"":""Microsoft.Hadoop.Avro.Specifications.SensorData"", ""fields"": [ { ""name"":""Location"", ""type"": { ""type"":""record"", ""name"":""Microsoft.Hadoop.Avro.Specifications.Location"", ""fields"": [ { ""name"":""Floor"", ""type"":""int"" }, { ""name"":""Room"", ""type"":""int"" } ] } }, { ""name"":""Value"", ""type"":""bytes"" } ] }"; //Create a generic serializer based on the schema var serializer = AvroSerializer.CreateGeneric(Schema); var rootSchema = serializer.WriterSchema as RecordSchema; //Create a Memory Stream buffer using (var stream = new MemoryStream()) { //Create a generic record to represent the data dynamic location = new AvroRecord(rootSchema.GetField("Location").TypeSchema); location.Floor = 1; location.Room = 243; dynamic expected = new AvroRecord(serializer.WriterSchema); expected.Location = location; expected.Value = new byte[] { 1, 2, 3, 4, 5 }; Console.WriteLine("Serializing Sample Data Set..."); //Serialize the data serializer.Serialize(stream, expected); stream.Seek(0, SeekOrigin.Begin); Console.WriteLine("Deserializing Sample Data Set..."); //Deserialize the data into a generic record dynamic actual = serializer.Deserialize(stream); Console.WriteLine("Comparing Initial and Deserialized Data Sets..."); //Finally, verify the results bool isEqual = expected.Location.Floor.Equals(actual.Location.Floor); isEqual = isEqual && expected.Location.Room.Equals(actual.Location.Room); isEqual = isEqual && ((byte[])expected.Value).SequenceEqual((byte[])actual.Value); Console.WriteLine("Result of Data Set Identity Comparison is {0}", isEqual); } }
private void WriteEntryImplementation(ODataEntry entry) { if (this.schema == null) { this.schema = this.outputContext.AvroWriter.UpdateSchema(entry, null, this.writingFeed); } var obj = (AvroRecord)ODataAvroConvert.FromODataObject(entry, this.schema); if (this.writingFeed) { this.entityObjectList.Add(obj); } else { this.currentEntityObject = obj; } }
public void SchemaEvolution_RecordWithFullyQualifiedAlias() { const string WriterSchema = @"{ ""type"":""record"", ""name"":""Microsoft.Hadoop.Avro.Tests.SimpleIntClass"", ""fields"": [ {""name"":""IntField"", ""type"":""int""} ] }"; const string ReaderSchema = @"{ ""type"":""record"", ""name"":""Microsoft.Hadoop.Avro.Tests.IntClassNewName"", ""aliases"":[""Microsoft.Hadoop.Avro.Tests.SimpleIntClass""], ""fields"": [ {""name"":""IntField"",""type"":""int""}, ] }"; var serializer = AvroSerializer.CreateGeneric(WriterSchema); var deserializer = AvroSerializer.CreateGenericDeserializerOnly(WriterSchema, ReaderSchema); using (var stream = new MemoryStream()) { dynamic expected = new AvroRecord(serializer.WriterSchema); expected.IntField = 1; serializer.Serialize(stream, expected); stream.Seek(0, SeekOrigin.Begin); dynamic actual = deserializer.Deserialize(stream); Assert.AreEqual(expected.IntField, actual.IntField); } }
private dynamic CreateDynamicData(RecordSchema recordSchema) { dynamic expected1 = new AvroRecord(recordSchema); dynamic location1 = new AvroRecord(recordSchema.GetField("Location").TypeSchema); location1.Floor = 1; location1.Room = 243; expected1.Location = location1; expected1.Value = new byte[] {1, 2, 3, 4, 5}; return expected1; }
public void SchemaEvolution_RecordWithPromotionalNumericFields() { const string WriterSchema = @"{ ""name"":""ClassWithPromotionalFields"", ""namespace"":""Microsoft.Hadoop.Avro.Tests"", ""type"":""record"", ""fields"": [ {""name"":""IntToLongField"", ""type"":""int""}, {""name"":""IntToFloatField"", ""type"":""int""}, {""name"":""IntToDoubleField"", ""type"":""int""}, {""name"":""LongToFloatField"", ""type"":""long""}, {""name"":""LongToDoubleField"", ""type"":""long""}, {""name"":""FloatToDoubleField"", ""type"":""float""} ] }"; var serializer = AvroSerializer.CreateGeneric(WriterSchema); var deserializer = AvroSerializer.CreateDeserializerOnly<ReaderClassWithPromotionalFields>(WriterSchema, this.dataContractSettings); using (var stream = new MemoryStream()) { var random = new Random(13); dynamic expected = new AvroRecord(serializer.WriterSchema); expected.IntToLongField = random.Next(); expected.IntToFloatField = random.Next(); expected.IntToDoubleField = random.Next(); expected.LongToFloatField = (long)random.Next(); expected.LongToDoubleField = (long)random.Next(); expected.FloatToDoubleField = (float)random.NextDouble(); serializer.Serialize(stream, expected); stream.Position = 0; var actual = deserializer.Deserialize(stream); Assert.AreEqual(expected.IntToLongField, actual.IntToLongField); Assert.AreEqual(expected.IntToFloatField, actual.IntToFloatField); Assert.AreEqual(expected.IntToDoubleField, actual.IntToDoubleField); Assert.AreEqual(expected.LongToFloatField, actual.LongToFloatField); Assert.AreEqual(expected.LongToDoubleField, actual.LongToDoubleField); Assert.AreEqual(expected.FloatToDoubleField, actual.FloatToDoubleField); } }
public void GenericSerializer_SerializeEvolvedRecordWithPromotedField() { const string WriterSchema = @"{ ""type"":""record"", ""name"":""Microsoft.Hadoop.Avro.Tests.SimpleIntClass"", ""fields"": [ {""name"":""IntField"", ""type"":""int""} ] }"; const string ReaderSchema = @"{ ""type"":""record"", ""name"":""Microsoft.Hadoop.Avro.Tests.SimpleIntClass"", ""fields"": [ {""name"":""IntField"",""type"":""long""} ] }"; var serializer = AvroSerializer.CreateGeneric(WriterSchema); var deserializer = AvroSerializer.CreateGenericDeserializerOnly(WriterSchema, ReaderSchema); using (var stream = new MemoryStream()) { dynamic expected = new AvroRecord(serializer.ReaderSchema); expected.IntField = 1; serializer.Serialize(stream, expected); stream.Seek(0, SeekOrigin.Begin); dynamic actual = deserializer.Deserialize(stream); Assert.AreEqual(expected.IntField, actual.IntField); } }
public void Container_SchemaEvolution_RecordContainingArrayWithWriterIntPromotedToReaderLong() { const string WriterSchema = @"{ ""name"":""RecordContainingArray"", ""type"":""record"", ""fields"": [ {""name"":""ArrayField"", ""type"":{""type"":""array"", ""items"":""int""}}, ] }"; const string ReaderSchema = @"{ ""name"":""RecordContainingArray"", ""type"":""record"", ""fields"": [ {""name"":""ArrayField"", ""type"":{""type"":""array"", ""items"":""long""}}, ] }"; var serializer = AvroSerializer.CreateGeneric(WriterSchema); var schema = serializer.WriterSchema; var randomArrays = Utilities.GetRandom<List<int[]>>(false); var expected = new List<AvroRecord>(); foreach (var array in randomArrays) { dynamic avroRecord = new AvroRecord(schema); avroRecord.ArrayField = array; expected.Add(avroRecord); } using (var memoryStream = new MemoryStream()) { var writer = AvroContainer.CreateGenericWriter(WriterSchema, memoryStream, Codec.Null); var i = 0; while (i < expected.Count) { var block = writer.CreateBlockAsync().Result; for (var j = 0; j < 2; j++) { if (i >= expected.Count) { break; } block.Write(expected[i]); i++; } writer.WriteBlockAsync(block).Wait(); } writer.Dispose(); memoryStream.Seek(0, SeekOrigin.Begin); var reader = AvroContainer.CreateGenericReader(ReaderSchema, memoryStream, true, new CodecFactory()); var actual = new List<AvroRecord>(); while (reader.MoveNext()) { actual.AddRange(reader.Current.Objects.Cast<AvroRecord>()); } for (var k = 0; k < expected.Count; ++k) { var randomArray = randomArrays[k]; Assert.AreEqual(randomArray.Length, ((dynamic)actual[k]).ArrayField.Length); for (int t = 0; t < randomArray.Length; t++) { Assert.AreEqual(randomArray[t], ((dynamic)actual[k]).ArrayField.GetValue(t)); } } } }
public void GenericSerializer_SerializeEvolvedRecordWithExtraFieldHavingDefaultValue() { const string WriterSchema = @"{ ""type"":""record"", ""name"":""Microsoft.Hadoop.Avro.Tests.SimpleIntClass"", ""fields"": [ {""name"":""IntField"", ""type"":""int""} ] }"; const string ReaderSchema = @"{ ""type"":""record"", ""name"":""Microsoft.Hadoop.Avro.Tests.SimpleIntClass"", ""fields"": [ {""name"":""IntField"",""type"":""int""}, {""name"":""ExtraField"",""type"":""string"",""default"":""Default""} ] }"; var serializer = AvroSerializer.CreateGeneric(WriterSchema); var deserializer = AvroSerializer.CreateGenericDeserializerOnly(WriterSchema, ReaderSchema); using (var stream = new MemoryStream()) { dynamic expected = new AvroRecord(serializer.ReaderSchema); expected.IntField = Utilities.GetRandom<int>(false); serializer.Serialize(stream, expected); stream.Seek(0, SeekOrigin.Begin); dynamic actual = deserializer.Deserialize(stream); Assert.AreEqual(expected.IntField, actual.IntField); Assert.AreEqual("Default", actual.ExtraField); } }
public void Container_SchemaEvolution_RecordWithPromotionalLongAndFloatFields() { const string WriterSchema = @"{ ""name"":""ClassWithPromotionalFields"", ""namespace"":""Microsoft.Hadoop.Avro.Tests"", ""type"":""record"", ""fields"": [ {""name"":""LongToFloatField"", ""type"":""long""}, {""name"":""LongToDoubleField"", ""type"":""long""}, {""name"":""FloatToDoubleField"", ""type"":""float""} ] }"; const string ReaderSchema = @"{ ""name"":""ClassWithPromotionalFields"", ""namespace"":""Microsoft.Hadoop.Avro.Tests"", ""type"":""record"", ""fields"": [ {""name"":""LongToFloatField"", ""type"":""float""}, {""name"":""LongToDoubleField"", ""type"":""double""}, {""name"":""FloatToDoubleField"", ""type"":""double""} ] }"; const int RecordsCount = 100; var serializer = AvroSerializer.CreateGeneric(WriterSchema); var schema = serializer.WriterSchema; var expected = new List<AvroRecord>(); for (int counter = 0; counter < RecordsCount; counter++) { dynamic avroRecord = new AvroRecord(schema); avroRecord.LongToFloatField = Utilities.GetRandom<long>(false); avroRecord.LongToDoubleField = Utilities.GetRandom<long>(false); avroRecord.FloatToDoubleField = Utilities.GetRandom<float>(false); expected.Add(avroRecord); } using (var memoryStream = new MemoryStream()) { var writer = AvroContainer.CreateGenericWriter(WriterSchema, memoryStream, Codec.Null); var i = 0; while (i < expected.Count) { var block = writer.CreateBlockAsync().Result; for (var j = 0; j < 2; j++) { if (i >= expected.Count) { break; } block.Write(expected[i]); i++; } writer.WriteBlockAsync(block).Wait(); } writer.Dispose(); memoryStream.Seek(0, SeekOrigin.Begin); var reader = AvroContainer.CreateGenericReader(ReaderSchema, memoryStream, true, new CodecFactory()); var actual = new List<AvroRecord>(); while (reader.MoveNext()) { actual.AddRange(reader.Current.Objects.Cast<AvroRecord>()); } for (var k = 0; k < expected.Count; ++k) { Assert.AreEqual(((dynamic)expected[k]).LongToFloatField, ((dynamic)actual[k]).LongToFloatField); Assert.AreEqual(((dynamic)expected[k]).LongToDoubleField, ((dynamic)actual[k]).LongToDoubleField); Assert.AreEqual(((dynamic)expected[k]).FloatToDoubleField, ((dynamic)actual[k]).FloatToDoubleField); } } }
public void SchemaEvolution_RecordWithIdenticalFields() { const string WriterSchema = @"{ ""type"":""record"", ""name"":""Microsoft.Hadoop.Avro.Tests.ClassOfInt"", ""fields"": [ {""name"":""PrimitiveInt"", ""type"":""int""} ] }"; var serializer = AvroSerializer.CreateGeneric(WriterSchema); var deserializer = AvroSerializer.CreateDeserializerOnly<ClassOfInt>(WriterSchema, this.dataContractSettings); using (var stream = new MemoryStream()) { dynamic expected = new AvroRecord(serializer.WriterSchema); expected.PrimitiveInt = 5; serializer.Serialize(stream, expected); stream.Position = 0; var actual = deserializer.Deserialize(stream); Assert.AreEqual(expected.PrimitiveInt, actual.PrimitiveInt); } }
public void SchemaEvolution_RecordWithWrongUnionReaderAndNonUnionWriter() { const string WriterSchema = @"{ ""name"":""Category"", ""namespace"":""ApacheAvro.Types"", ""type"":""record"", ""fields"": [ {""name"":""CategoryName"", ""type"":""string""}, {""name"":""Description"", ""type"":""string""}, {""name"":""Picture"", ""type"":""bytes""}, {""name"":""Id"", ""type"":""int""} ] }"; const string ReaderSchema = @"{ ""name"":""Category"", ""namespace"":""ApacheAvro.Types"", ""type"":""record"", ""fields"": [ {""name"":""CategoryName"", ""type"":""string""}, {""name"":""Id"", ""type"":[""string"", ""null""]} ] }"; var serializer = AvroSerializer.CreateGeneric(WriterSchema); var deserializer = AvroSerializer.CreateGenericDeserializerOnly(WriterSchema, ReaderSchema); using (var stream = new MemoryStream()) { dynamic expected = new AvroRecord(serializer.WriterSchema); expected.CategoryName = "Test"; expected.Description = "Test"; expected.Picture = new byte[] { 0x10, 0x20, 0x30, 0x40 }; expected.Id = 1; serializer.Serialize(stream, expected); stream.Seek(0, SeekOrigin.Begin); dynamic actual = deserializer.Deserialize(stream); Assert.AreEqual(expected.CategoryName, actual.CategoryName); Assert.AreEqual(expected.Description, actual.Description); CollectionAssert.AreEqual(expected.Picture, actual.Picture); } }
public void SchemaEvolution_MapEvolutionWithRecordContainingPromotionalTypes() { const string WriterSchema = @"{ ""type"":""record"", ""name"":""SimpleMap"", ""namespace"":""Microsoft.Hadoop.Avro.Tests"", ""fields"": [ { ""name"":""Values"", ""type"": { ""type"": ""map"", ""values"": { ""type"":""record"", ""name"":""ClassWithPromotionalFields"", ""namespace"":""Microsoft.Hadoop.Avro.Tests"", ""fields"": [ {""name"":""IntToLongField"",""type"":""int""}, {""name"":""IntToFloatField"",""type"":""int""}, {""name"":""IntToDoubleField"",""type"":""int""}, {""name"":""LongToFloatField"",""type"":""long""}, {""name"":""LongToDoubleField"",""type"":""long""}, {""name"":""FloatToDoubleField"",""type"":""float""} ] } } } ] }"; var serializer = AvroSerializer.CreateGeneric(WriterSchema); var deserializer = AvroSerializer.CreateDeserializerOnly<SimpleMapWithReaderClassWithPromotionalFields>(WriterSchema, this.dataContractSettings); using (var stream = new MemoryStream()) { var valueSchema = ((serializer.WriterSchema as RecordSchema).GetField("Values").TypeSchema as MapSchema).ValueSchema; dynamic expected = new AvroRecord(serializer.WriterSchema); dynamic valueRecord = new AvroRecord(valueSchema); valueRecord.IntToLongField = 1; valueRecord.IntToFloatField = 2; valueRecord.IntToDoubleField = 3; valueRecord.LongToFloatField = 4L; valueRecord.LongToDoubleField = 5L; valueRecord.FloatToDoubleField = 6.0f; expected.Values = new Dictionary<string, AvroRecord>(); expected.Values.Add("key1", valueRecord); serializer.Serialize(stream, expected); stream.Seek(0, SeekOrigin.Begin); var actual = deserializer.Deserialize(stream); Assert.AreEqual(expected.Values["key1"].IntToLongField, actual.Values["key1"].IntToLongField); Assert.AreEqual(expected.Values["key1"].IntToFloatField, actual.Values["key1"].IntToFloatField); Assert.AreEqual(expected.Values["key1"].IntToDoubleField, actual.Values["key1"].IntToDoubleField); Assert.AreEqual(expected.Values["key1"].LongToFloatField, actual.Values["key1"].LongToFloatField); Assert.AreEqual(expected.Values["key1"].LongToDoubleField, actual.Values["key1"].LongToDoubleField); Assert.AreEqual(expected.Values["key1"].FloatToDoubleField, actual.Values["key1"].FloatToDoubleField); } }
public void SchemaEvolution_RecordWithPermutatedAndMissingWriterFields() { const string WriterSchema = @"{ ""name"":""WriterClass"", ""namespace"":""Microsoft.Hadoop.Avro.Tests"", ""type"":""record"", ""fields"": [ {""name"":""BoolA"", ""type"":""boolean""}, {""name"":""BoolB"", ""type"":""boolean""}, {""name"":""FloatA"", ""type"":""float""}, {""name"":""FloatB"", ""type"":""float""}, {""name"":""DoubleA"", ""type"":""double""}, {""name"":""DoubleB"", ""type"":""double""}, {""name"":""IntA"", ""type"":""int""}, {""name"":""IntB"", ""type"":""int""}, {""name"":""MyGuid"", ""type"": {""type"":""fixed"", ""size"":16, ""name"": ""q"" }}, {""name"": ""classField"", ""type"" : [ ""null"", ""Microsoft.Hadoop.Avro.Tests.WriterClass""] }, {""name"":""Arr"", ""type"": {""type"":""array"", ""items"":""int""}}, {""name"":""LongField"", ""type"":""long""}, {""name"":""LongMap"", ""type"": {""type"":""map"", ""values"":""long""}}, {""name"":""BytesField"", ""type"":""bytes""}, ] }"; var serializer = AvroSerializer.CreateGeneric(WriterSchema); var deserializer = AvroSerializer.CreateDeserializerOnly<WriterClassWithPermutatedFields>(WriterSchema, this.dataContractSettings); using (var stream = new MemoryStream()) { dynamic expected = new AvroRecord(serializer.WriterSchema); expected.BoolA = Utilities.GetRandom<bool>(false); expected.BoolB = Utilities.GetRandom<bool>(false); expected.FloatA = Utilities.GetRandom<float>(false); expected.FloatB = Utilities.GetRandom<float>(false); expected.DoubleA = Utilities.GetRandom<double>(false); expected.DoubleB = Utilities.GetRandom<double>(false); expected.IntA = Utilities.GetRandom<int>(false); expected.IntB = Utilities.GetRandom<int>(false); expected.MyGuid = Utilities.GetRandom<Guid>(false).ToByteArray(); expected.Arr = Utilities.GetRandom<int[]>(false); expected.LongField = Utilities.GetRandom<long>(false); expected.LongMap = Utilities.GetRandom<Dictionary<string, long>>(false); expected.BytesField = Utilities.GetRandom<byte[]>(false); expected.classField = new AvroRecord(serializer.WriterSchema); expected.classField.BoolA = Utilities.GetRandom<bool>(false); expected.classField.BoolB = Utilities.GetRandom<bool>(false); expected.classField.FloatA = Utilities.GetRandom<float>(false); expected.classField.FloatB = Utilities.GetRandom<float>(false); expected.classField.DoubleA = Utilities.GetRandom<double>(false); expected.classField.DoubleB = Utilities.GetRandom<double>(false); expected.classField.IntA = Utilities.GetRandom<int>(false); expected.classField.IntB = Utilities.GetRandom<int>(false); expected.classField.MyGuid = Utilities.GetRandom<Guid>(false).ToByteArray(); expected.classField.Arr = Utilities.GetRandom<int[]>(false); expected.classField.classField = null; expected.classField.LongField = Utilities.GetRandom<long>(false); expected.classField.LongMap = Utilities.GetRandom<Dictionary<string, long>>(false); expected.classField.BytesField = Utilities.GetRandom<byte[]>(false); serializer.Serialize(stream, expected); stream.Position = 0; var actual = deserializer.Deserialize(stream); Assert.AreEqual(expected.DoubleB, actual.DoubleB); Assert.AreEqual(expected.FloatB, actual.FloatB); Assert.AreEqual(expected.FloatA, actual.FloatA); Assert.AreEqual(expected.BoolB, actual.BoolB); Assert.AreEqual(expected.BoolA, actual.BoolA); Assert.AreEqual(expected.DoubleA, actual.DoubleA); } }
//Serializes and deserializes sample data set using Generic Record and Avro Object Container Files //Serialized data is not compressed // //This sample uses Memory Stream for all operations related to serialization, deserialization and //Object Container manipulation, though File Stream could be easily used. public void SerializeDeserializeUsingObjectContainersGenericRecord() { Console.WriteLine("SERIALIZATION USING GENERIC RECORD AND AVRO OBJECT CONTAINER FILES\n"); //Path for Avro Object Container File string path = "AvroSampleGenericRecordNullCodec.avro"; Console.WriteLine("Defining the Schema and creating Sample Data Set..."); //Define the schema in JSON const string Schema = @"{ ""type"":""record"", ""name"":""Microsoft.Hadoop.Avro.Specifications.SensorData"", ""fields"": [ { ""name"":""Location"", ""type"": { ""type"":""record"", ""name"":""Microsoft.Hadoop.Avro.Specifications.Location"", ""fields"": [ { ""name"":""Floor"", ""type"":""int"" }, { ""name"":""Room"", ""type"":""int"" } ] } }, { ""name"":""Value"", ""type"":""bytes"" } ] }"; //Create a generic serializer based on the schema var serializer = AvroSerializer.CreateGeneric(Schema); var rootSchema = serializer.WriterSchema as RecordSchema; //Create a generic record to represent the data var testData = new List<AvroRecord>(); dynamic expected1 = new AvroRecord(rootSchema); dynamic location1 = new AvroRecord(rootSchema.GetField("Location").TypeSchema); location1.Floor = 1; location1.Room = 243; expected1.Location = location1; expected1.Value = new byte[] { 1, 2, 3, 4, 5 }; testData.Add(expected1); dynamic expected2 = new AvroRecord(rootSchema); dynamic location2 = new AvroRecord(rootSchema.GetField("Location").TypeSchema); location2.Floor = 1; location2.Room = 244; expected2.Location = location2; expected2.Value = new byte[] { 6, 7, 8, 9 }; testData.Add(expected2); //Serializing and saving data to file //Create a MemoryStream buffer using (var buffer = new MemoryStream()) { Console.WriteLine("Serializing Sample Data Set..."); //Create a SequentialWriter instance for type SensorData which can serialize a sequence of SensorData objects to stream //Data will not be compressed (Null compression codec) using (var writer = AvroContainer.CreateGenericWriter(Schema, buffer, Codec.Null)) { using (var streamWriter = new SequentialWriter<object>(writer, 24)) { // Serialize the data to stream using the sequential writer testData.ForEach(streamWriter.Write); } } Console.WriteLine("Saving serialized data to file..."); //Save stream to file if (!WriteFile(buffer, path)) { Console.WriteLine("Error during file operation. Quitting method"); return; } } //Reading and deserializng the data //Create a Memory Stream buffer using (var buffer = new MemoryStream()) { Console.WriteLine("Reading data from file..."); //Reading data from Object Container File if (!ReadFile(buffer, path)) { Console.WriteLine("Error during file operation. Quitting method"); return; } Console.WriteLine("Deserializing Sample Data Set..."); //Prepare the stream for deserializing the data buffer.Seek(0, SeekOrigin.Begin); //Create a SequentialReader for type SensorData which will derserialize all serialized objects from the given stream //It allows iterating over the deserialized objects because it implements IEnumerable<T> interface using (var reader = AvroContainer.CreateGenericReader(buffer)) { using (var streamReader = new SequentialReader<object>(reader)) { var results = streamReader.Objects; Console.WriteLine("Comparing Initial and Deserialized Data Sets..."); //Finally, verify the results var pairs = testData.Zip(results, (serialized, deserialized) => new { expected = (dynamic)serialized, actual = (dynamic)deserialized }); int count = 1; foreach (var pair in pairs) { bool isEqual = pair.expected.Location.Floor.Equals(pair.actual.Location.Floor); isEqual = isEqual && pair.expected.Location.Room.Equals(pair.actual.Location.Room); isEqual = isEqual && ((byte[])pair.expected.Value).SequenceEqual((byte[])pair.actual.Value); Console.WriteLine("For Pair {0} result of Data Set Identity Comparison is {1}", count, isEqual.ToString()); count++; } } } } //Delete the file RemoveFile(path); }
public void SchemaEvolution_RecordSkippers() { const string WriterSchema = @"{ ""name"":""WriterClass"", ""namespace"":""Microsoft.Hadoop.Avro.Tests"", ""type"":""record"", ""fields"": [ {""name"":""Existing"", ""type"":""boolean""}, {""name"":""BoolField"", ""type"":""boolean""}, {""name"":""FloatField"", ""type"":""float""}, {""name"":""DoubleField"", ""type"":""double""}, {""name"":""IntField"", ""type"":""int""}, {""name"":""GuidField"", ""type"": {""type"":""fixed"", ""size"":16, ""name"": ""q"" }}, {""name"": ""classField"", ""type"" : [ ""null"", ""Microsoft.Hadoop.Avro.Tests.WriterClass""] }, {""name"":""ArrayField"", ""type"": {""type"":""array"", ""items"":""int""}}, {""name"":""LongField"", ""type"":""long""}, {""name"":""LongMap"", ""type"": {""type"":""map"", ""values"":""long""}}, {""name"":""BytesField"", ""type"":""bytes""}, {""name"":""AnotherExisting"", ""type"":""int""}, ] }"; const string ReaderSchema = @"{ ""name"":""WriterClass"", ""namespace"":""Microsoft.Hadoop.Avro.Tests"", ""type"":""record"", ""fields"": [ {""name"":""Existing"", ""type"":""boolean""}, {""name"":""AnotherExisting"", ""type"":""int""}, ] }"; var serializer = AvroSerializer.CreateGeneric(WriterSchema); var deserializer = AvroSerializer.CreateGenericDeserializerOnly(WriterSchema, ReaderSchema); using (var stream = new MemoryStream()) { dynamic expected = new AvroRecord(serializer.WriterSchema); expected.Existing = true; expected.BoolField = Utilities.GetRandom<bool>(false); expected.FloatField = Utilities.GetRandom<float>(false); expected.DoubleField = Utilities.GetRandom<double>(false); expected.IntField = Utilities.GetRandom<int>(false); expected.GuidField = new byte[16]; expected.ArrayField = new int[] { 1, 2, 3 }; expected.LongField = (long)13; expected.LongMap = new Dictionary<string, long> { { "test", 1 } }; expected.BytesField = new byte[3] { 4, 5, 6 }; expected.AnotherExisting = Utilities.GetRandom<int>(false); expected.classField = new AvroRecord(serializer.WriterSchema); expected.classField.Existing = Utilities.GetRandom<bool>(false); expected.classField.BoolField = Utilities.GetRandom<bool>(false); expected.classField.FloatField = Utilities.GetRandom<float>(false); expected.classField.DoubleField = Utilities.GetRandom<double>(false); expected.classField.IntField = Utilities.GetRandom<int>(false); expected.classField.GuidField = new byte[16]; expected.classField.ArrayField = new int[] { 1, 2, 3 }; expected.classField.LongField = (long)13; expected.classField.LongMap = new Dictionary<string, long> { { "test", 1 } }; expected.classField.BytesField = new byte[3] { 4, 5, 6 }; expected.classField.AnotherExisting = 13; expected.classField.classField = null; serializer.Serialize(stream, expected); stream.Position = 0; dynamic actual = deserializer.Deserialize(stream); Assert.AreEqual(expected.Existing, actual.Existing); Assert.AreEqual(expected.AnotherExisting, actual.AnotherExisting); } }