public InheritedEntity DeserializeAvroMsft(byte[] bytes) { var serializer = new AvroSerializer(typeof(InheritedEntity)); using (var ms = new MemoryStream(bytes)) { return serializer.Deserialize<InheritedEntity>(ms); } }
public byte[] SerializeAvroMsft(InheritedEntity thisObj) { var serializer = new AvroSerializer(thisObj.GetType()); using (var ms = new MemoryStream()) { serializer.Serialize(thisObj, ms); return ms.ToArray(); } }
public void TestReadAvroAsODataFeed() { const string Schema = @" {""type"":""array"", ""items"": { ""type"":""record"", ""name"":""TestNS.Person"", ""fields"": [ { ""name"":""Id"", ""type"":""int"" }, { ""name"":""Title"", ""type"":""string"" }, ] } }"; var serializer = AvroSerializer.CreateGeneric(Schema); using (var stream = new MemoryStream()) { var arraySchema = (ArraySchema)serializer.WriterSchema; var recordSchema = arraySchema.ItemSchema; var rec1 = new AvroRecord(recordSchema); rec1["Id"] = 1; rec1["Title"] = "s1"; var rec2 = new AvroRecord(recordSchema); rec2["Id"] = 2; rec2["Title"] = "s2"; var array = new[] { rec1, rec2 }; using (var writer = AvroContainer.CreateGenericWriter(Schema, stream, /*leave open*/ true, Codec.Null)) using (var seqWriter = new SequentialWriter <object>(writer, 24)) { seqWriter.Write(array); } stream.Seek(0, SeekOrigin.Begin); var avroReader = new ODataAvroReader(this.CreateODataInputContext(stream), true); Assert.AreEqual(ODataReaderState.Start, avroReader.State); Assert.IsTrue(avroReader.Read()); Assert.AreEqual(ODataReaderState.ResourceSetStart, avroReader.State); Assert.IsTrue(avroReader.Read()); // Entry 1 Assert.AreEqual(ODataReaderState.ResourceStart, avroReader.State); Assert.IsTrue(avroReader.Read()); Assert.AreEqual(ODataReaderState.ResourceEnd, avroReader.State); var entry = avroReader.Item as ODataResource; Assert.IsNotNull(entry); var properties = entry.Properties.ToList(); Assert.AreEqual(2, properties.Count); Assert.AreEqual("Id", properties[0].Name); Assert.AreEqual(1, properties[0].Value); Assert.AreEqual("Title", properties[1].Name); Assert.AreEqual("s1", properties[1].Value); // Entry 2 Assert.IsTrue(avroReader.Read()); Assert.AreEqual(ODataReaderState.ResourceStart, avroReader.State); Assert.IsTrue(avroReader.Read()); Assert.AreEqual(ODataReaderState.ResourceEnd, avroReader.State); entry = avroReader.Item as ODataResource; Assert.IsNotNull(entry); properties = entry.Properties.ToList(); Assert.AreEqual(2, properties.Count); Assert.AreEqual("Id", properties[0].Name); Assert.AreEqual(2, properties[0].Value); Assert.AreEqual("Title", properties[1].Name); Assert.AreEqual("s2", properties[1].Value); Assert.IsTrue(avroReader.Read()); Assert.AreEqual(ODataReaderState.ResourceSetEnd, avroReader.State); Assert.IsFalse(avroReader.Read()); Assert.AreEqual(ODataReaderState.Completed, avroReader.State); } }
public AvroDeserializer() { avroSerializer = AvroSerializer.Create <T>(); }
public static void SerializeDynamicSampleFile(string path) { Console.WriteLine("SERIALIZATION USING GENERIC RECORD AND AVRO OBJECT CONTAINER FILES\n"); Console.WriteLine("Defining the Schema and creating Sample Data Set..."); //Define the schema in JSON const string Schema = @"{ ""type"":""record"", ""name"":""Microsoft.Hadoop.Avro.Specifications.SensorData"", ""fields"": [ { ""name"":""Location"", ""type"": { ""type"":""record"", ""name"":""Microsoft.Hadoop.Avro.Specifications.Location"", ""fields"": [ { ""name"":""Floor"", ""type"":""int"" }, { ""name"":""Room"", ""type"":""int"" } ] } }, { ""name"":""Value"", ""type"":""bytes"" } ] }"; //Create a generic serializer based on the schema var serializer = AvroSerializer.CreateGeneric(Schema); var rootSchema = serializer.WriterSchema as RecordSchema; //Create a generic record to represent the data var testData = new List <AvroRecord>(); dynamic expected1 = new AvroRecord(rootSchema); dynamic location1 = new AvroRecord(rootSchema.GetField("Location").TypeSchema); location1.Floor = 1; location1.Room = 243; expected1.Location = location1; expected1.Value = new byte[] { 1, 2, 3, 4, 5 }; testData.Add(expected1); dynamic expected2 = new AvroRecord(rootSchema); dynamic location2 = new AvroRecord(rootSchema.GetField("Location").TypeSchema); location2.Floor = 1; location2.Room = 244; expected2.Location = location2; expected2.Value = new byte[] { 6, 7, 8, 9 }; testData.Add(expected2); //Serializing and saving data to file //Create a MemoryStream buffer using (var buffer = new MemoryStream()) { Console.WriteLine("Serializing Sample Data Set..."); //Create a SequentialWriter instance for type SensorData which can serialize a sequence of SensorData objects to stream //Data will not be compressed (Null compression codec) using (var writer = AvroContainer.CreateGenericWriter(Schema, buffer, Codec.Null)) { using (var streamWriter = new SequentialWriter <object>(writer, 24)) { // Serialize the data to stream using the sequential writer testData.ForEach(streamWriter.Write); } } Console.WriteLine("Saving serialized data to file..."); //Save stream to file if (!WriteFile(buffer, path)) { Console.WriteLine("Error during file operation. Quitting method"); return; } } }
public void Container_SchemaEvolution_RecordWithPermutatedAndMissingWriterFields() { const string WriterSchema = @"{ ""name"":""WriterClass"", ""namespace"":""Microsoft.Hadoop.Avro.Tests"", ""type"":""record"", ""fields"": [ {""name"":""BoolA"", ""type"":""boolean""}, {""name"":""BoolB"", ""type"":""boolean""}, {""name"":""FloatA"", ""type"":""float""}, {""name"":""FloatB"", ""type"":""float""}, {""name"":""DoubleA"", ""type"":""double""}, {""name"":""DoubleB"", ""type"":""double""}, {""name"":""IntA"", ""type"":""int""}, {""name"":""IntB"", ""type"":""int""}, {""name"":""MyGuid"", ""type"": {""type"":""fixed"", ""size"":16, ""name"": ""q"" }}, {""name"": ""classField"", ""type"" : [ ""null"", ""Microsoft.Hadoop.Avro.Tests.WriterClass""] }, {""name"":""Arr"", ""type"": {""type"":""array"", ""items"":""int""}}, {""name"":""LongField"", ""type"":""long""}, {""name"":""LongMap"", ""type"": {""type"":""map"", ""values"":""long""}}, {""name"":""BytesField"", ""type"":""bytes""}, ] }"; const string ReaderSchema = @"{ ""name"":""WriterClass"", ""namespace"":""Microsoft.Hadoop.Avro.Tests"", ""type"":""record"", ""fields"": [ {""name"":""DoubleB"", ""type"":""double""}, {""name"":""FloatB"", ""type"":""float""}, {""name"":""FloatA"", ""type"":""float""}, {""name"":""BoolB"", ""type"":""boolean""}, {""name"":""BoolA"", ""type"":""boolean""}, {""name"":""DoubleA"", ""type"":""double""}, ] }"; const int RecordsCount = 100; var serializer = AvroSerializer.CreateGeneric(WriterSchema); var schema = serializer.WriterSchema; var expected = new List <AvroRecord>(); for (int counter = 0; counter < RecordsCount; counter++) { dynamic avroRecord = new AvroRecord(schema); avroRecord.BoolA = Utilities.GetRandom <bool>(false); avroRecord.BoolB = Utilities.GetRandom <bool>(false); avroRecord.FloatA = Utilities.GetRandom <float>(false); avroRecord.FloatB = Utilities.GetRandom <float>(false); avroRecord.DoubleA = Utilities.GetRandom <double>(false); avroRecord.DoubleB = Utilities.GetRandom <double>(false); avroRecord.IntA = Utilities.GetRandom <int>(false); avroRecord.IntB = Utilities.GetRandom <int>(false); avroRecord.MyGuid = Utilities.GetRandom <Guid>(false).ToByteArray(); avroRecord.Arr = Utilities.GetRandom <int[]>(false); avroRecord.LongField = Utilities.GetRandom <long>(false); avroRecord.LongMap = Utilities.GetRandom <Dictionary <string, long> >(false); avroRecord.BytesField = Utilities.GetRandom <byte[]>(false); avroRecord.classField = new AvroRecord(serializer.WriterSchema); avroRecord.classField.BoolA = Utilities.GetRandom <bool>(false); avroRecord.classField.BoolB = Utilities.GetRandom <bool>(false); avroRecord.classField.FloatA = Utilities.GetRandom <float>(false); avroRecord.classField.FloatB = Utilities.GetRandom <float>(false); avroRecord.classField.DoubleA = Utilities.GetRandom <double>(false); avroRecord.classField.DoubleB = Utilities.GetRandom <double>(false); avroRecord.classField.IntA = Utilities.GetRandom <int>(false); avroRecord.classField.IntB = Utilities.GetRandom <int>(false); avroRecord.classField.MyGuid = Utilities.GetRandom <Guid>(false).ToByteArray(); avroRecord.classField.Arr = Utilities.GetRandom <int[]>(false); avroRecord.classField.classField = null; avroRecord.classField.LongField = Utilities.GetRandom <long>(false); avroRecord.classField.LongMap = Utilities.GetRandom <Dictionary <string, long> >(false); avroRecord.classField.BytesField = Utilities.GetRandom <byte[]>(false); expected.Add(avroRecord); } using (var memoryStream = new MemoryStream()) { var writer = AvroContainer.CreateGenericWriter(WriterSchema, memoryStream, Codec.Null); var i = 0; while (i < expected.Count) { var block = writer.CreateBlockAsync().Result; for (var j = 0; j < 2; j++) { if (i >= expected.Count) { break; } block.Write(expected[i]); i++; } writer.WriteBlockAsync(block).Wait(); } writer.Dispose(); memoryStream.Seek(0, SeekOrigin.Begin); var reader = AvroContainer.CreateGenericReader(ReaderSchema, memoryStream, true, new CodecFactory()); var actual = new List <AvroRecord>(); while (reader.MoveNext()) { actual.AddRange(reader.Current.Objects.Cast <AvroRecord>()); } for (var k = 0; k < expected.Count; ++k) { Assert.AreEqual(((dynamic)expected[k]).DoubleB, ((dynamic)actual[k]).DoubleB); Assert.AreEqual(((dynamic)expected[k]).FloatB, ((dynamic)actual[k]).FloatB); Assert.AreEqual(((dynamic)expected[k]).FloatA, ((dynamic)actual[k]).FloatA); Assert.AreEqual(((dynamic)expected[k]).BoolB, ((dynamic)actual[k]).BoolB); Assert.AreEqual(((dynamic)expected[k]).BoolA, ((dynamic)actual[k]).BoolA); Assert.AreEqual(((dynamic)expected[k]).DoubleA, ((dynamic)actual[k]).DoubleA); } } }
public void Container_SchemaEvolution_RecordContainingArrayWithWriterIntPromotedToReaderLong() { const string WriterSchema = @"{ ""name"":""RecordContainingArray"", ""type"":""record"", ""fields"": [ {""name"":""ArrayField"", ""type"":{""type"":""array"", ""items"":""int""}}, ] }"; const string ReaderSchema = @"{ ""name"":""RecordContainingArray"", ""type"":""record"", ""fields"": [ {""name"":""ArrayField"", ""type"":{""type"":""array"", ""items"":""long""}}, ] }"; var serializer = AvroSerializer.CreateGeneric(WriterSchema); var schema = serializer.WriterSchema; var randomArrays = Utilities.GetRandom <List <int[]> >(false); var expected = new List <AvroRecord>(); foreach (var array in randomArrays) { dynamic avroRecord = new AvroRecord(schema); avroRecord.ArrayField = array; expected.Add(avroRecord); } using (var memoryStream = new MemoryStream()) { var writer = AvroContainer.CreateGenericWriter(WriterSchema, memoryStream, Codec.Null); var i = 0; while (i < expected.Count) { var block = writer.CreateBlockAsync().Result; for (var j = 0; j < 2; j++) { if (i >= expected.Count) { break; } block.Write(expected[i]); i++; } writer.WriteBlockAsync(block).Wait(); } writer.Dispose(); memoryStream.Seek(0, SeekOrigin.Begin); var reader = AvroContainer.CreateGenericReader(ReaderSchema, memoryStream, true, new CodecFactory()); var actual = new List <AvroRecord>(); while (reader.MoveNext()) { actual.AddRange(reader.Current.Objects.Cast <AvroRecord>()); } for (var k = 0; k < expected.Count; ++k) { var randomArray = randomArrays[k]; Assert.AreEqual(randomArray.Length, ((dynamic)actual[k]).ArrayField.Length); for (int t = 0; t < randomArray.Length; t++) { Assert.AreEqual(randomArray[t], ((dynamic)actual[k]).ArrayField.GetValue(t)); } } } }
public void Container_CreateBufferReaderBlockNullArguments() { Utilities.ShouldThrow <ArgumentNullException>(() => new AvroBufferReaderBlock <ClassOfInt>(null, Codec.Null, new byte[] { }, 1)); Utilities.ShouldThrow <ArgumentNullException>(() => new AvroBufferReaderBlock <ClassOfInt>(AvroSerializer.Create <ClassOfInt>(), null, new byte[] { }, 1)); Utilities.ShouldThrow <ArgumentNullException>(() => new AvroBufferReaderBlock <ClassOfInt>(AvroSerializer.Create <ClassOfInt>(), Codec.Null, null, 1)); Utilities.ShouldThrow <ArgumentOutOfRangeException>(() => new AvroBufferReaderBlock <ClassOfInt>(AvroSerializer.Create <ClassOfInt>(), Codec.Null, new byte[] { }, -1)); }
public string GetSchema() { var serializer = AvroSerializer.Create <AvroConfiguration>(); return(serializer.WriterSchema.ToString()); }
public static void SerializeDeserializeObjectUsingReflectionStream() { string blobName = "aaa/avrotest/test008"; CloudBlobClient client = new Microsoft.WindowsAzure.Storage.Blob.CloudBlobClient( new Uri("http://hanzstorage.blob.core.windows.net"), new Microsoft.WindowsAzure.Storage.Auth.StorageCredentials( "hanzstorage", "w9TEpvGTusvFlGAdCoWdDrwqLzy6er0Zm5YKdDD0YTkQdOj3WufeVrgd2c8q8amLR0o6xD0tBChcIIA+DCgxXA==" )); CloudBlobContainer container = client.GetContainerReference("hanzhdi"); CloudBlockBlob blockBlob = container.GetBlockBlobReference(blobName); foreach (var md in blockBlob.Metadata) { Console.WriteLine("{0} {1}", md.Key, md.Value); } Console.WriteLine("Serializing Sample Data Set USING REFLECTION\n"); AvroBlobAppender.AvroBlobAppenderWriter <MyDataType> writer = new AvroBlobAppender.AvroBlobAppenderWriter <MyDataType>(blockBlob, false, AvroSerializer.Create <MyDataType>(), Codec.Null); Microsoft.Hadoop.Avro.Container.SequentialWriter <MyDataType> sequentialWriter = new SequentialWriter <MyDataType>(writer, 10000); List <MyDataType> myDataList = MyDataType.GenerateData(555, 10); foreach (var myData in myDataList) { sequentialWriter.Write(myData); } sequentialWriter.Flush(); sequentialWriter.Dispose(); #region commented code //blockBlob.DownloadToFile(blobName, FileMode.Create); //using (Stream stream = File.OpenRead(blobName)) //{ // Microsoft.Hadoop.Avro.Container.SequentialReader<MyDataType> reader = // new Microsoft.Hadoop.Avro.Container.SequentialReader<MyDataType>(AvroContainer.CreateReader<MyDataType>(stream)); // List<MyDataType> actuals = reader.Objects.ToList(); // Console.WriteLine("Number of objects: {0}", actuals.Count); // for (int i = 0; i < actuals.Count; i++) // { // var actual = actuals[i]; // MyDataType exp = null; // switch (i) // { // case 0: // exp = expected; // break; // case 1: // exp = expected2; // break; // default: // Console.WriteLine("No expected for object {0}", i); // continue; // } // Console.WriteLine("Result of Data Set Identity Comparison is {0}", Program.Equal(exp, actual)); // } //} #endregion }
public static void Run([BlobTrigger("telemetry-archive/willowtelemetry01/{name}", Connection = "StorageAccount")] Stream myBlob, string name, TraceWriter log) { DataSet ds = new DataSet( new SchemaElement <int>("ObjectID"), new SchemaElement <string>("Value"), new SchemaElement <string>("ClientID"), new SchemaElement <string>("TimeStamp") ); if (myBlob.Length == 508) { return; } // empty dataset //log.Info($"C# Blob trigger function Processed blob\n Name:{name} \n Size: {myBlob.Length} Bytes"); var serializer = AvroSerializer.CreateGeneric(Schema); var jsonSerializer = new JsonSerializer(); using (var reader = AvroContainer.CreateGenericReader(myBlob)) { using (var streamReader = new SequentialReader <object>(reader)) { var results = streamReader.Objects; foreach (var item in results) { var body = ((AvroRecord)item)[5]; // ["Body"] is the 5th element var json = System.Text.Encoding.Default.GetString((byte[])body); var array = (JArray)JsonConvert.DeserializeObject(json); var telemetry = jsonSerializer.Deserialize <Telemetry>(array[0].CreateReader()); ds.Add(telemetry.ObjectID, telemetry.Value, telemetry.ClientID, telemetry.TimeStamp.ToString()); } } } //log.Info($"{ds.RowCount} Records processed"); if (ds.RowCount == 0) { return; } storageAccount = CloudStorageAccount.Parse(storageAcct); CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient(); CloudBlobContainer blobContainer = blobClient.GetContainerReference(parquetContainerName); blobContainer.CreateIfNotExists(); name = name + ".parquet"; CloudBlob blob = blobContainer.GetBlobReference(blobName); CloudAppendBlob appendBlob = blobContainer.GetAppendBlobReference(name); if (appendBlob.Exists()) { appendBlob.Delete(DeleteSnapshotsOption.IncludeSnapshots); } log.Info($"===> {ds.RowCount} Records, {name}"); using (var ms = new MemoryStream()) { using (var writer = new ParquetWriter(ms)) { writer.Write(ds); } ms.Seek(0, SeekOrigin.Begin); appendBlob.UploadFromStream(ms); } }
public SampleAvroSerializer() { avroSerializer = AvroSerializer.Create <T>(); }
private object CreateSerializer <T>() { return(AvroSerializer.Create <List <T> >()); }
public void ReadAvroAsParameter() { EdmEntityType personType = new EdmEntityType("TestNS", "Person"); personType.AddStructuralProperty("Id", EdmPrimitiveTypeKind.Int32); personType.AddStructuralProperty("Title", EdmPrimitiveTypeKind.String); var operation = new EdmAction("NS", "op1", null); operation.AddParameter("p1", EdmCoreModel.Instance.GetString(false)); operation.AddParameter("p2", new EdmEntityTypeReference(personType, false)); const string Schema = @"{ ""type"":""record"", ""name"":""NS.op1Parameter"", ""fields"": [ { ""name"":""p1"", ""type"":""string"" }, { ""name"":""p2"", ""type"": {""type"":""record"", ""name"":""TestNS.Person"", ""fields"": [ { ""name"":""Id"", ""type"":""int"" }, { ""name"":""Title"", ""type"":""string"" }, ] } } ] }"; var stream = new MemoryStream(); using (var writer = AvroContainer.CreateGenericWriter(Schema, stream, /*leave open*/ true, Codec.Null)) using (var seqWriter = new SequentialWriter <object>(writer, 24)) { RecordSchema parameterSchema = (RecordSchema)AvroSerializer.CreateGeneric(Schema).WriterSchema; AvroRecord ar = new AvroRecord(parameterSchema); ar["p1"] = "dat"; var personSchema = parameterSchema.GetField("p2").TypeSchema; AvroRecord person = new AvroRecord(personSchema); person["Id"] = 5; person["Title"] = "per1"; ar["p2"] = person; seqWriter.Write(ar); seqWriter.Flush(); } stream.Flush(); stream.Seek(0, SeekOrigin.Begin); var reader = new ODataAvroParameterReader(this.CreateODataInputContext(stream), operation); Assert.AreEqual(ODataParameterReaderState.Start, reader.State); Assert.IsTrue(reader.Read()); Assert.AreEqual(ODataParameterReaderState.Value, reader.State); Assert.AreEqual("p1", reader.Name); Assert.AreEqual("dat", reader.Value); Assert.IsTrue(reader.Read()); Assert.AreEqual(ODataParameterReaderState.Resource, reader.State); Assert.AreEqual("p2", reader.Name); var ew = reader.CreateResourceReader(); Assert.AreEqual(ODataReaderState.Start, ew.State); Assert.IsTrue(ew.Read()); Assert.AreEqual(ODataReaderState.ResourceStart, ew.State); Assert.IsTrue(ew.Read()); Assert.AreEqual(ODataReaderState.ResourceEnd, ew.State); var entry = ew.Item as ODataResource; Assert.IsFalse(ew.Read()); Assert.AreEqual(ODataReaderState.Completed, ew.State); Assert.IsNotNull(entry); var properties = entry.Properties.ToList(); Assert.AreEqual(2, properties.Count); Assert.AreEqual("Id", properties[0].Name); Assert.AreEqual(5, properties[0].Value); Assert.AreEqual("Title", properties[1].Name); Assert.AreEqual("per1", properties[1].Value); Assert.IsFalse(reader.Read()); Assert.AreEqual(ODataParameterReaderState.Completed, reader.State); }
public void TestReadAvroAsODataEntry() { const string Schema = @" { ""type"":""record"", ""name"":""TestNS.Person"", ""fields"": [ { ""name"":""Id"", ""type"":""int"" }, { ""name"":""Title"", ""type"":""string"" }, { ""name"":""Address"", ""type"":{ ""name"":""TestNS.Address"", ""type"":""record"", ""fields"":[ { ""name"":""ZipCode"", ""type"":""long"" }, ] } }, ] }"; var serializer = AvroSerializer.CreateGeneric(Schema); using (var stream = new MemoryStream()) { var expected = new AvroRecord(serializer.WriterSchema); expected["Id"] = -5; expected["Title"] = "set"; var cpxSchema = ((RecordSchema)serializer.WriterSchema).GetField("Address").TypeSchema; var cpx = new AvroRecord(cpxSchema); cpx["ZipCode"] = 5L; expected["Address"] = cpx; using (var writer = AvroContainer.CreateGenericWriter(Schema, stream, /*leave open*/ true, Codec.Null)) { using (var streamWriter = new SequentialWriter <object>(writer, 24)) { // Serialize the data to stream using the sequential writer streamWriter.Write(expected); } } stream.Seek(0, SeekOrigin.Begin); var avroReader = new ODataAvroReader(this.CreateODataInputContext(stream), false); Assert.AreEqual(ODataReaderState.Start, avroReader.State); Assert.IsTrue(avroReader.Read()); Assert.AreEqual(ODataReaderState.ResourceStart, avroReader.State); Assert.IsTrue(avroReader.Read()); Assert.AreEqual(ODataReaderState.NestedResourceInfoStart, avroReader.State); Assert.IsTrue(avroReader.Read()); Assert.AreEqual(ODataReaderState.ResourceStart, avroReader.State); Assert.IsTrue(avroReader.Read()); Assert.AreEqual(ODataReaderState.ResourceEnd, avroReader.State); var entry = avroReader.Item as ODataResource; Assert.IsNotNull(entry); Assert.AreEqual("TestNS.Address", entry.TypeName); var zip = entry.Properties.Single(); Assert.AreEqual(5L, zip.Value); Assert.IsTrue(avroReader.Read()); Assert.AreEqual(ODataReaderState.NestedResourceInfoEnd, avroReader.State); Assert.IsTrue(avroReader.Read()); Assert.AreEqual(ODataReaderState.ResourceEnd, avroReader.State); entry = avroReader.Item as ODataResource; Assert.IsNotNull(entry); Assert.AreEqual("TestNS.Person", entry.TypeName); var properties = entry.Properties.ToList(); Assert.AreEqual(2, properties.Count); Assert.AreEqual("Id", properties[0].Name); Assert.AreEqual(-5, properties[0].Value); Assert.AreEqual("Title", properties[1].Name); Assert.AreEqual("set", properties[1].Value); Assert.IsFalse(avroReader.Read()); Assert.AreEqual(ODataReaderState.Completed, avroReader.State); } }
public Schema GetSchema() { return(AvroSerializer.Create <LargeClass>(new AvroSerializerSettings { UsePosixTime = true }).WriterSchema); }
//Serialize and deserialize sample data set using Generic Record. //Generic Record is a special class with the schema explicitly defined in JSON. //All serialized data should be mapped to the fields of Generic Record, //which in turn will be then serialized. // //This approach is generally slower than using Reflection, //but comes very handy in cases, where for example the data is represented //as Comma Separated Values (CSV) files rather than objects. // //In SerializeDeserializeObjectUsingGenericRecords() we will emulate this case //though explicit filling up the fields of the Generic Record public void SerializeDeserializeObjectUsingGenericRecords() { Console.WriteLine("SERIALIZATION USING GENERIC RECORD\n"); Console.WriteLine("Defining the Schema and creating Sample Data Set..."); //Define the schema in JSON const string Schema = @"{ ""type"":""record"", ""name"":""Microsoft.Hadoop.Avro.Specifications.SensorData"", ""fields"": [ { ""name"":""Location"", ""type"": { ""type"":""record"", ""name"":""Microsoft.Hadoop.Avro.Specifications.Location"", ""fields"": [ { ""name"":""Floor"", ""type"":""int"" }, { ""name"":""Room"", ""type"":""int"" } ] } }, { ""name"":""Value"", ""type"":""bytes"" } ] }"; //Create a generic serializer based on the schema var serializer = AvroSerializer.CreateGeneric(Schema); var rootSchema = serializer.WriterSchema as RecordSchema; //Create a Memory Stream buffer using (var stream = new MemoryStream()) { //Create a generic record to represent the data dynamic location = new AvroRecord(rootSchema.GetField("Location").TypeSchema); location.Floor = 1; location.Room = 243; dynamic expected = new AvroRecord(serializer.WriterSchema); expected.Location = location; expected.Value = new byte[] { 1, 2, 3, 4, 5 }; Console.WriteLine("Serializing Sample Data Set..."); //Serialize the data serializer.Serialize(stream, expected); stream.Seek(0, SeekOrigin.Begin); Console.WriteLine("Deserializing Sample Data Set..."); //Deserialize the data into a generic record dynamic actual = serializer.Deserialize(stream); Console.WriteLine("Comparing Initial and Deserialized Data Sets..."); //Finally, verify the results bool isEqual = expected.Location.Floor.Equals(actual.Location.Floor); isEqual = isEqual && expected.Location.Room.Equals(actual.Location.Room); isEqual = isEqual && ((byte[])expected.Value).SequenceEqual((byte[])actual.Value); Console.WriteLine("Result of Data Set Identity Comparison is {0}", isEqual); } }
public Serializer() { _serializer = AvroSerializer.Create <T>(); }
//Serializes and deserializes sample data set using Generic Record and Avro Object Container Files //Serialized data is not compressed // //This sample uses Memory Stream for all operations related to serialization, deserialization and //Object Container manipulation, though File Stream could be easily used. public void SerializeDeserializeUsingObjectContainersGenericRecord() { Console.WriteLine("SERIALIZATION USING GENERIC RECORD AND AVRO OBJECT CONTAINER FILES\n"); //Path for Avro Object Container File string path = "AvroSampleGenericRecordNullCodec.avro"; Console.WriteLine("Defining the Schema and creating Sample Data Set..."); //Define the schema in JSON const string Schema = @"{ ""type"":""record"", ""name"":""Microsoft.Hadoop.Avro.Specifications.SensorData"", ""fields"": [ { ""name"":""Location"", ""type"": { ""type"":""record"", ""name"":""Microsoft.Hadoop.Avro.Specifications.Location"", ""fields"": [ { ""name"":""Floor"", ""type"":""int"" }, { ""name"":""Room"", ""type"":""int"" } ] } }, { ""name"":""Value"", ""type"":""bytes"" } ] }"; //Create a generic serializer based on the schema var serializer = AvroSerializer.CreateGeneric(Schema); var rootSchema = serializer.WriterSchema as RecordSchema; //Create a generic record to represent the data var testData = new List <AvroRecord>(); dynamic expected1 = new AvroRecord(rootSchema); dynamic location1 = new AvroRecord(rootSchema.GetField("Location").TypeSchema); location1.Floor = 1; location1.Room = 243; expected1.Location = location1; expected1.Value = new byte[] { 1, 2, 3, 4, 5 }; testData.Add(expected1); dynamic expected2 = new AvroRecord(rootSchema); dynamic location2 = new AvroRecord(rootSchema.GetField("Location").TypeSchema); location2.Floor = 1; location2.Room = 244; expected2.Location = location2; expected2.Value = new byte[] { 6, 7, 8, 9 }; testData.Add(expected2); //Serializing and saving data to file //Create a MemoryStream buffer using (var buffer = new MemoryStream()) { Console.WriteLine("Serializing Sample Data Set..."); //Create a SequentialWriter instance for type SensorData which can serialize a sequence of SensorData objects to stream //Data will not be compressed (Null compression codec) using (var writer = AvroContainer.CreateGenericWriter(Schema, buffer, Codec.Null)) { using (var streamWriter = new SequentialWriter <object>(writer, 24)) { // Serialize the data to stream using the sequential writer testData.ForEach(streamWriter.Write); } } Console.WriteLine("Saving serialized data to file..."); //Save stream to file if (!WriteFile(buffer, path)) { Console.WriteLine("Error during file operation. Quitting method"); return; } } //Reading and deserializng the data //Create a Memory Stream buffer using (var buffer = new MemoryStream()) { Console.WriteLine("Reading data from file..."); //Reading data from Object Container File if (!ReadFile(buffer, path)) { Console.WriteLine("Error during file operation. Quitting method"); return; } Console.WriteLine("Deserializing Sample Data Set..."); //Prepare the stream for deserializing the data buffer.Seek(0, SeekOrigin.Begin); //Create a SequentialReader for type SensorData which will derserialize all serialized objects from the given stream //It allows iterating over the deserialized objects because it implements IEnumerable<T> interface using (var reader = AvroContainer.CreateGenericReader(buffer)) { using (var streamReader = new SequentialReader <object>(reader)) { var results = streamReader.Objects; Console.WriteLine("Comparing Initial and Deserialized Data Sets..."); //Finally, verify the results var pairs = testData.Zip(results, (serialized, deserialized) => new { expected = (dynamic)serialized, actual = (dynamic)deserialized }); int count = 1; foreach (var pair in pairs) { bool isEqual = pair.expected.Location.Floor.Equals(pair.actual.Location.Floor); isEqual = isEqual && pair.expected.Location.Room.Equals(pair.actual.Location.Room); isEqual = isEqual && ((byte[])pair.expected.Value).SequenceEqual((byte[])pair.actual.Value); Console.WriteLine("For Pair {0} result of Data Set Identity Comparison is {1}", count, isEqual.ToString()); count++; } } } } //Delete the file RemoveFile(path); }
public void Container_CreateBufferWriterBlockNullArguments() { Utilities.ShouldThrow <ArgumentNullException>(() => new AvroBufferWriterBlock <ClassOfInt>(null, Codec.Null)); Utilities.ShouldThrow <ArgumentNullException>(() => new AvroBufferWriterBlock <ClassOfInt>(AvroSerializer.Create <ClassOfInt>(), null)); }
public async Task process(IConfiguration config) { // creando AvroSchemas from clases AvroSerializerSettings settings = new AvroSerializerSettings(); settings.Resolver = new AvroPublicMemberContractResolver(); var endpontSchema = AvroSerializer.Create <Endpoint> (settings).WriterSchema.ToString(); var messageDSchema = AvroSerializer.Create <MessageDestination> (settings).WriterSchema.ToString(); Console.WriteLine("Endpoint Schema: " + endpontSchema); Console.WriteLine("Message Destination Schema: " + messageDSchema); Console.WriteLine("RouterProcess"); var sConfig = new StreamConfig <StringSerDes, StringSerDes> (); sConfig.ApplicationId = config["SPRING_CLOUD_APPLICATION_GROUP"]; sConfig.BootstrapServers = config["SPRING_CLOUD_STREAM_KAFKA_BINDER_BROKERS"]; sConfig.AutoOffsetReset = AutoOffsetReset.Earliest; sConfig.SchemaRegistryUrl = config["SchemaRegistryUrl"]; sConfig.AutoRegisterSchemas = true; sConfig.NumStreamThreads = 1; sConfig.Acks = Acks.All; //sConfig.Debug = "consumer,cgrp,topic,fetch"; sConfig.AddConsumerConfig("allow.auto.create.topics", "true"); sConfig.MaxTaskIdleMs = 50; sConfig.InnerExceptionHandler = (e) => ExceptionHandlerResponse.CONTINUE; var timeout = TimeSpan.FromSeconds(10); DateTime dt = DateTime.Now; MessageDestination op = new MessageDestination(); var serializer = new SchemaAvroSerDes <OrderProduct>(); StreamBuilder builder = new StreamBuilder(); var table = builder.Table(config["endpoints"], new Int32SerDes(), new SchemaAvroSerDes <Endpoint> (), InMemory <int, Endpoint> .As(config["endpoints-table"])); builder.Stream <int, OrderProduct, Int32SerDes, SchemaAvroSerDes <OrderProduct> >(config["spring.cloud.stream.bindings.input.destination"]) .Map <int, OrderProduct>((k, v) => { return(KeyValuePair.Create(v.product_id, v)); }) .Peek((k, v) => { Console.WriteLine($"Sending message {k} to endpoint {v.product_id}"); //calcular metrica }) .Join(table, (orderProduct, endpoint) => { Console.WriteLine("OrderProduct: " + orderProduct?.order_id); Console.WriteLine("Endpoint: " + endpoint?.endpoint_id); op = new MessageDestination { messageId = orderProduct.order_id, endpoint = endpoint, payload = orderProduct }; return(op); }) .Peek((k, v) => { Console.WriteLine($"Sending message {k} to endpoint {v.endpoint.endpoint_url}"); // crear metricas if (_messageCounterList != null) { var counterMessage = Metrics .CreateCounter($"router_{v.endpoint.endpoint_id}_processed_total", $"Number of messages sent to {v.endpoint.endpoint_url}"); counterMessage.Inc(); _messageCounterList.Add(counterMessage); } }) .Print(Printed <int, MessageDestination> .ToOut()); Topology t = builder.Build(); Console.WriteLine(t.Describe()); KafkaStream stream = new KafkaStream(t, sConfig); bool isRunningState = false; stream.StateChanged += (old, @new) => { if (@new.Equals(KafkaStream.State.RUNNING)) { isRunningState = true; } }; await stream.StartAsync(); while (!isRunningState) { Thread.Sleep(250); if (DateTime.Now > dt + timeout) { break; } } if (isRunningState) { Console.WriteLine("Stream running state is " + isRunningState.ToString()); } }
public void Container_SchemaEvolution_RecordWithPromotionalLongAndFloatFields() { const string WriterSchema = @"{ ""name"":""ClassWithPromotionalFields"", ""namespace"":""Microsoft.Hadoop.Avro.Tests"", ""type"":""record"", ""fields"": [ {""name"":""LongToFloatField"", ""type"":""long""}, {""name"":""LongToDoubleField"", ""type"":""long""}, {""name"":""FloatToDoubleField"", ""type"":""float""} ] }"; const string ReaderSchema = @"{ ""name"":""ClassWithPromotionalFields"", ""namespace"":""Microsoft.Hadoop.Avro.Tests"", ""type"":""record"", ""fields"": [ {""name"":""LongToFloatField"", ""type"":""float""}, {""name"":""LongToDoubleField"", ""type"":""double""}, {""name"":""FloatToDoubleField"", ""type"":""double""} ] }"; const int RecordsCount = 100; var serializer = AvroSerializer.CreateGeneric(WriterSchema); var schema = serializer.WriterSchema; var expected = new List <AvroRecord>(); for (int counter = 0; counter < RecordsCount; counter++) { dynamic avroRecord = new AvroRecord(schema); avroRecord.LongToFloatField = Utilities.GetRandom <long>(false); avroRecord.LongToDoubleField = Utilities.GetRandom <long>(false); avroRecord.FloatToDoubleField = Utilities.GetRandom <float>(false); expected.Add(avroRecord); } using (var memoryStream = new MemoryStream()) { var writer = AvroContainer.CreateGenericWriter(WriterSchema, memoryStream, Codec.Null); var i = 0; while (i < expected.Count) { var block = writer.CreateBlockAsync().Result; for (var j = 0; j < 2; j++) { if (i >= expected.Count) { break; } block.Write(expected[i]); i++; } writer.WriteBlockAsync(block).Wait(); } writer.Dispose(); memoryStream.Seek(0, SeekOrigin.Begin); var reader = AvroContainer.CreateGenericReader(ReaderSchema, memoryStream, true, new CodecFactory()); var actual = new List <AvroRecord>(); while (reader.MoveNext()) { actual.AddRange(reader.Current.Objects.Cast <AvroRecord>()); } for (var k = 0; k < expected.Count; ++k) { Assert.AreEqual(((dynamic)expected[k]).LongToFloatField, ((dynamic)actual[k]).LongToFloatField); Assert.AreEqual(((dynamic)expected[k]).LongToDoubleField, ((dynamic)actual[k]).LongToDoubleField); Assert.AreEqual(((dynamic)expected[k]).FloatToDoubleField, ((dynamic)actual[k]).FloatToDoubleField); } } }
public void GenericSerializer_CreateWithNullSchema() { AvroSerializer.CreateGeneric(null); }
/// <summary> /// Avro schema /// </summary> /// <returns></returns> private string GetSchema() { var serializer = AvroSerializer.Create <AvroNode>(); return(serializer.WriterSchema.ToString()); }
internal MSHadoopAvroSerializer() : base(new Catalog()) { serializer = AvroSerializer.Create <Catalog>(); }
static void SerializeAndDeserializeDynamicTest() { string path = "AvroSampleReflection.avro"; //SerializeDynamicSampleFile(path); var dict = new Dictionary <string, object>(); dict.Add("1", 3); dict.Add("2", new Location { Room = 243, Floor = 1 }); ChoAvroRecordConfiguration config = null; AvroSerializerSettings sett1 = null; using (var w = new ChoAvroWriter(path) .WithAvroSerializer(AvroSerializer.Create <Dictionary <string, object> >(new AvroSerializerSettings() { Resolver = new ChoAvroPublicMemberContractResolver() })) .Configure(c => c.KnownTypes = new List <Type> { typeof(Location), typeof(string), typeof(int) }) //.Configure(c => c.UseAvroSerializer = true) //.Configure(c => c.AvroSerializerSettings.Resolver = new AvroDataContractResolverEx()) ) { sett1 = w.Configuration.AvroSerializerSettings; config = w.Configuration; w.Write(dict); w.Write(dict); w.Write(dict); } //var sett = new AvroSerializerSettings(); //sett.Resolver = new ChoAvroPublicMemberContractResolver(); // false) { Configuration = config }; //sett.KnownTypes = new List<Type> { typeof(Location), typeof(string), typeof(int) }; //var avroSerializer = AvroSerializer.Create<Dictionary<string, object>>(sett1); //using (var r = new StreamReader(path)) //{ // var rec = avroSerializer.Deserialize(r.BaseStream); // var rec2 = avroSerializer.Deserialize(r.BaseStream); // var rec3 = avroSerializer.Deserialize(r.BaseStream); // Console.WriteLine(rec.Dump()); // Console.WriteLine(rec2.Dump()); // Console.WriteLine(rec3.Dump()); // //var rec4 = avroSerializer.Deserialize(r); //} StringBuilder json = new StringBuilder(); using (var r = new ChoAvroReader(path) .Configure(c => c.KnownTypes = new List <Type> { typeof(Location), typeof(string), typeof(int) }) .Configure(c => c.UseAvroSerializer = true) //.Configure(c => c.AvroSerializerSettings = sett1) .Configure(c => c.NestedColumnSeparator = '_') ) { //var dt = r.AsDataTable(); //Console.WriteLine(dt.Dump()); //return; //foreach (var rec in r) //{ // Console.WriteLine(rec.Dump()); //} //return; using (var w = new ChoJSONWriter(json) .Configure(c => c.TurnOnAutoDiscoverJsonConverters = true) ) { w.Write(r); } } Console.WriteLine(json.ToString()); }
private void CompareSchemas() { var eCodeGenEntity = new InheritedEntityAvro(); // Extra test for schema compare, save schema to compare later string avroSchema = eCodeGenEntity.Schema.ToString(); // Extra test for schema compare, save schema to compare later var serializer = new AvroSerializer(typeof(InheritedEntity)); string avroMsftSchema = serializer.Schema; Console.Write("\nThe Avro MSFT schema and the Avro Apache schema "); if (String.IsNullOrEmpty(avroSchema) || (avroSchema != avroMsftSchema)) { Console.WriteLine("do NOT MATCH !!!\n"); Console.WriteLine("Avro Apache schema: {0}", avroSchema); Console.WriteLine("Avro MSFT schema : {0}", avroMsftSchema); } else Console.WriteLine("match.\n"); }
public void SequentialGenericWritingReading_NestedRecord() { const string StringSchema = @"{ ""type"":""record"", ""name"":""Microsoft.Hadoop.Avro.Tests.NestedClass"", ""fields"":[ { ""name"":""ClassOfIntReference"", ""type"":{ ""type"":""record"", ""name"":""Microsoft.Hadoop.Avro.Tests.ClassOfInt"", ""fields"":[ { ""name"":""PrimitiveInt"", ""type"":""int"" } ] } }, { ""name"":""PrimitiveInt"",""type"":""int"" } ] }"; using (var stream = new MemoryStream()) { var serializer = AvroSerializer.CreateGeneric(StringSchema); using (var streamWriter = AvroContainer.CreateGenericWriter(StringSchema, stream, Codec.Null)) { using (var writer = new SequentialWriter <object>(streamWriter, 24)) { var expected = new List <AvroRecord>(); var random = new Random(83); for (int i = 0; i < 10; i++) { dynamic record = new AvroRecord(serializer.WriterSchema); record.PrimitiveInt = random.Next(); record.ClassOfIntReference = new AvroRecord((serializer.WriterSchema as RecordSchema).GetField("ClassOfIntReference").TypeSchema); record.ClassOfIntReference.PrimitiveInt = random.Next(); expected.Add(record); } expected.ForEach(writer.Write); writer.Flush(); stream.Seek(0, SeekOrigin.Begin); var streamReader = AvroContainer.CreateReader <NestedClass>(stream, true, this.dataContractSettings, new CodecFactory()); using (var reader = new SequentialReader <NestedClass>(streamReader)) { var j = 0; foreach (var avroRecord in reader.Objects) { Assert.Equal(expected[j]["PrimitiveInt"], avroRecord.PrimitiveInt); Assert.Equal(((dynamic)expected[j++]["ClassOfIntReference"])["PrimitiveInt"], avroRecord.ClassOfIntReference.PrimitiveInt); } } } } } }
public void GenericSerializer_CreateWithNullSchema() { Assert.ThrowsException <ArgumentNullException>(() => AvroSerializer.CreateGeneric(null)); }