public bool Export() { using (var buffer = new MemoryStream()) { //Data is compressed using the Deflate codec. using (var w = AvroContainer.CreateWriter <Models.Claim>(buffer, Codec.Deflate)) { using (var writer = new SequentialWriter <Models.Claim>(w, 24)) { // Serialize the data to stream by using the sequential writer values.ToList().ForEach(writer.Write); } } //Save stream to file Console.WriteLine("Saving serialized data to file..."); if (!WriteFile(buffer)) { Console.WriteLine("Error during file operation. Quitting method"); } else { IsImported = true; } } return(IsImported); }
public void ReadAvroAsError() { const string Schema = @"{ ""type"":""record"", ""name"":""OData.Error"", ""fields"": [ { ""name"":""ErrorCode"", ""type"":""string"" }, { ""name"":""Message"" , ""type"":""string"" }, ] }"; var stream = new MemoryStream(); using (var writer = AvroContainer.CreateGenericWriter(Schema, stream, /*leave open*/ true, Codec.Null)) using (var seqWriter = new SequentialWriter <object>(writer, 24)) { RecordSchema parameterSchema = (RecordSchema)AvroSerializer.CreateGeneric(Schema).WriterSchema; AvroRecord ar = new AvroRecord(parameterSchema); ar["ErrorCode"] = "e1"; ar["Message"] = "m1"; seqWriter.Write(ar); seqWriter.Flush(); } stream.Flush(); stream.Seek(0, SeekOrigin.Begin); var error = this.CreateODataInputContext(stream).ReadError(); Assert.AreEqual("e1", error.ErrorCode); Assert.AreEqual("m1", error.Message); }
public TypeSchema UpdateSchema(object value, IEdmType edmType, bool collection = false) { TypeSchema schema = null; if (edmType != null) { try { schema = ODataAvroSchemaGen.GetSchemaFromModel(edmType); } catch (ApplicationException) { } } if (schema == null) { if (value == null) { return null; } schema = ODataAvroSchemaGen.GetSchema(value); } TypeSchema single = AvroSerializer.CreateGeneric(schema.ToString()).WriterSchema; if (collection) { schema = ODataAvroSchemaGen.GetArraySchema(schema); single = ((ArraySchema)AvroSerializer.CreateGeneric(schema.ToString()).WriterSchema).ItemSchema; } var writer = AvroContainer.CreateGenericWriter(schema.ToString(), stream, true, Codec.Null); this.seqWriter = new SequentialWriter<object>(writer, 16); return single; }
private static async Task PutBlockAsync(String blockId, CloudBlockBlob blob, int start, int end, CancellationToken token) { var initialMessages = Enumerable .Range(start, end) .Select(x => new MyDataType(x)); using (var buffer = new MemoryStream()) { using (var w = AvroContainer.CreateWriter <MyDataType>(buffer, Codec.Null)) { using (var writer = new SequentialWriter <MyDataType>(w, 24)) { initialMessages .ToList() .ForEach(writer.Write); } } await buffer.FlushAsync(token); //var blockId = "AAAAA"; buffer.Seek(0, SeekOrigin.Begin); await blob.PutBlockAsync(blockId, buffer, string.Empty, token); //await blob.PutBlockAsync(blockId, new MemoryStream(Encoding.Default.GetBytes("Hello! " + start.ToString() + " - " + end.ToString() + "\r\n")), string.Empty, token); //await blob.PutBlockAsync(blockId, buffer, null); //await blob.PutBlockAsync(blockId, new MemoryStream(Encoding.Default.GetBytes("Hello! " + start.ToString() + " - " + end.ToString() + "\r\n")), null); } }
/// <summary> /// Reads CSV file data and converts records to instances of Stock class then serializes them using Reflection. /// Stock type definition is created from JSON schema using Microsoft Avro Library Code Generation utility /// </summary> /// <param name="inputFilePath">The input file path.</param> /// <param name="outputStream">The output stream.</param> private void SerializeCsv(string inputFilePath, Stream outputStream) { try { // Create an Avro writer using Null Codec var writer = AvroContainer.CreateWriter <Stock>( outputStream, new AvroSerializerSettings { Resolver = new AvroDataContractResolver(true) }, Codec.Null); // Create a new sequential writer and use it to write Avro file using a block length of 100 using (var seq = new SequentialWriter <Stock>(writer, 100)) { foreach (var stock in this.ReadCsv(inputFilePath)) { seq.Write(stock); } } } catch (Exception e) { ReportError("Error while creating Avro file(s) from CSV file(s)" + e); } }
static void Main(string[] args) { SampleData obj = new SampleData(); using (FileStream xmlStream = new FileStream(@"SampleData.xml", FileMode.Open)) { using (XmlReader xmlReader = XmlReader.Create(xmlStream)) { obj.ReadXml(xmlReader); } } // hi ahmed var serializer = new AvroSerializer(typeof(Person)); StreamWriter schemafs = new StreamWriter(File.Create(@"SampleData.avsc")); schemafs.Write(serializer.Schema.ToString()); schemafs.Close(); var resultStream = new FileStream(@"SampleData.avro", FileMode.OpenOrCreate); var writer = new SequentialWriter <Person>(resultStream, serializer, Codec.Create("null"), 24); int i; foreach (DataRow item in obj.Tables["Person"].Rows) { var p = new Person { First_Name = item["Name"].ToString(), Street = item["Street"].ToString(), City = item["City"].ToString(), State = item["State"].ToString() }; writer.Write(p); } writer.Flush(); }
public static Tuple <byte[], byte[]> CreateAvroHeaderAndBlocks <TSchema>(List <TSchema> dataList, string schema, string codec) { byte[] avroHeaderAndBlocksByteArray = null; byte[] syncMarker = new byte[16]; new Random().NextBytes(syncMarker); var objContainerHeader = new ObjectContainerHeader(syncMarker) { CodecName = codec, Schema = schema }; using (var buffer = new MemoryStream()) { // TODO refactor the above AvroBlobAppenderWriter class to expose the syncMarker AvroBlobAppender.AvroBlobAppenderWriter <TSchema> avroWriter = new AvroBlobAppender.AvroBlobAppenderWriter <TSchema>(null, false, AvroSerializer.Create <TSchema>(), Codec.Null); syncMarker = avroWriter.syncMarker; //using (var avroWriter = AvroContainer.CreateWriter<TSchema>(buffer, Codec.Null)) { using (var sequentialWriter = new SequentialWriter <TSchema>(avroWriter, 24)) { dataList.ToList().ForEach(sequentialWriter.Write); } } buffer.Seek(0, SeekOrigin.Begin); avroHeaderAndBlocksByteArray = buffer.ToArray(); } return(new Tuple <byte[], byte[]>(avroHeaderAndBlocksByteArray, syncMarker)); }
public void SequentialWriter_MicrosoftWriterApacherReaderOfDictionary() { var expected = new List <ContainingDictionaryClass <string, string> >(); for (var i = 0; i < 7; i++) { expected.Add(ContainingDictionaryClass <string, string> .Create( new Dictionary <string, string> { { "testkey" + i, "testvalue" + i } })); } var w = AvroContainer.CreateWriter <ContainingDictionaryClass <string, string> >(this.resultStream, Codec.Deflate); using (var writer = new SequentialWriter <ContainingDictionaryClass <string, string> >(w, 2)) { expected.ForEach(writer.Write); } this.resultStream.Seek(0, SeekOrigin.Begin); var reader = DataFileReader <GenericRecord> .OpenReader(this.resultStream); var actual = new List <GenericRecord>(reader); Assert.AreEqual(expected.Count, actual.Count); for (var i = 0; i < expected.Count; ++i) { var actualValue = actual[i]["Property"] as Dictionary <string, object>; Assert.IsNotNull(actualValue); Assert.AreEqual(actualValue["testkey" + i] as string, expected[i].Property["testkey" + i]); } }
public void SequentialReaderWriter_Reset() { var expected = new List <ClassOfListOfGuid>(); for (var i = 0; i < 7; i++) { expected.Add(ClassOfListOfGuid.Create(true)); } var w = AvroContainer.CreateWriter <ClassOfListOfGuid>(this.resultStream, Codec.Deflate); using (var writer = new SequentialWriter <ClassOfListOfGuid>(w, 3)) { expected.ForEach(writer.Write); } this.resultStream.Seek(0, SeekOrigin.Begin); var r = AvroContainer.CreateReader <ClassOfListOfGuid>(this.resultStream); using (var reader = new SequentialReader <ClassOfListOfGuid>(r)) { Assert.IsTrue(expected.SequenceEqual(reader.Objects)); var enumerator = (IEnumerator)reader.Objects.GetEnumerator(); Assert.IsFalse(enumerator.MoveNext()); } }
public void SequentialWriter_MicrosoftWriterApacheReader() { var expected = new List <ClassOfInt>(); for (var i = 0; i < 7; i++) { expected.Add(ClassOfInt.Create(true)); } var w = AvroContainer.CreateWriter <ClassOfInt>(this.resultStream, Codec.Deflate); using (var writer = new SequentialWriter <ClassOfInt>(w, 2)) { expected.ForEach(writer.Write); } this.resultStream.Seek(0, SeekOrigin.Begin); var reader = DataFileReader <GenericRecord> .OpenReader(this.resultStream); var actual = new List <GenericRecord>(reader); for (var i = 0; i < expected.Count; ++i) { Assert.AreEqual(expected[i].PrimitiveInt, actual[i]["PrimitiveInt"]); } }
public void TestReadCollection() { MemoryStream ms = new MemoryStream(); using (var writer = AvroContainer.CreateWriter <string[]>(ms, true, new AvroSerializerSettings(), Codec.Null)) using (var seqWriter = new SequentialWriter <string[]>(writer, 24)) { seqWriter.Write(new[] { "p9t", "cjk如" }); } ms.Seek(0, SeekOrigin.Begin); var values = new List <string>(); using (var omr = TestHelper.CreateMessageReader(ms, "avro/binary", AvroMediaTypeResolver.Instance)) { var reader = omr.CreateODataCollectionReader(); while (reader.Read()) { if (reader.State == ODataCollectionReaderState.Value) { values.Add((string)reader.Item); } } } Assert.AreEqual(2, values.Count); Assert.AreEqual("p9t", values[0]); Assert.AreEqual("cjk如", values[1]); }
public void SequentialReaderWriter_SerializeHugeObject() { var single = new SimpleFlatClass { StringField = new string('a', 16254), ByteArrayField = Encoding.ASCII.GetBytes(new string('b', 65666)), ZeroByteArrayField = Encoding.ASCII.GetBytes(new string('c', 128344)) }; var expected = new List <SimpleFlatClass> { single, single, single, }; var w = AvroContainer.CreateWriter <SimpleFlatClass>(this.resultStream, new AvroSerializerSettings { Resolver = new AvroDataContractResolver(true) }, Codec.Null); using (var writer = new SequentialWriter <SimpleFlatClass>(w, 24)) { expected.ForEach(writer.Write); } this.resultStream.Seek(0, SeekOrigin.Begin); var r = AvroContainer.CreateReader <SimpleFlatClass>(this.resultStream, true, new AvroSerializerSettings { Resolver = new AvroDataContractResolver(true) }, new CodecFactory()); using (var reader = new SequentialReader <SimpleFlatClass>(r)) { Assert.IsTrue(expected.SequenceEqual(reader.Objects)); } }
public void SequentialReaderWriter_CreateReaderForSchemaWithNullableField() { var expected = new ClassWithNullableIntField { NullableIntField = 10 }; using (var writer = AvroContainer.CreateWriter <ClassWithNullableIntField>(this.resultStream, Codec.Deflate)) { using (var swriter = new SequentialWriter <ClassWithNullableIntField>(writer, 2)) { swriter.Write(expected); swriter.Write(expected); swriter.Write(expected); swriter.Write(expected); } } this.resultStream.Seek(0, SeekOrigin.Begin); using (var reader = AvroContainer.CreateReader <ClassWithNullableIntField>(this.resultStream)) { using (var sreader = new SequentialReader <ClassWithNullableIntField>(reader)) { foreach (var actual in sreader.Objects) { Assert.AreEqual(expected.NullableIntField, actual.NullableIntField); } } } }
public void TestReadInvalidAvroInput() { string Schema = @" {""type"":""array"", ""items"": { ""type"":""record"", ""name"":""TestNS.Person"", ""fields"": [ { ""name"":""Id"", ""type"":""int"" }, { ""name"":""Title"", ""type"":""string"" }, ] } }"; using (var stream = new MemoryStream()) { using (var writer = AvroContainer.CreateGenericWriter(Schema, stream, /*leave open*/ true, Codec.Null)) using (var seqWriter = new SequentialWriter <object>(writer, 24)) { } stream.Seek(0, SeekOrigin.Begin); var avroReader = new ODataAvroReader(this.CreateODataInputContext(stream), true); Assert.AreEqual(ODataReaderState.Start, avroReader.State); Assert.IsFalse(avroReader.Read()); Assert.AreEqual(ODataReaderState.Exception, avroReader.State); } }
public void TestReadAvroAsCollection() { const string Schema = @"{""type"":""array"", ""items"":""long""}"; var stream = new MemoryStream(); using (var writer = AvroContainer.CreateGenericWriter(Schema, stream, /*leave open*/ true, Codec.Null)) using (var seqWriter = new SequentialWriter <object>(writer, 24)) { // Serialize the data to stream using the sequential writer seqWriter.Write(new[] { 6L, 8 }); seqWriter.Flush(); } stream.Flush(); stream.Seek(0, SeekOrigin.Begin); var reader = new ODataAvroCollectionReader(this.CreateODataInputContext(stream)); Assert.AreEqual(ODataCollectionReaderState.Start, reader.State); Assert.IsTrue(reader.Read()); Assert.AreEqual(ODataCollectionReaderState.CollectionStart, reader.State); Assert.IsTrue(reader.Read()); Assert.AreEqual(ODataCollectionReaderState.Value, reader.State); Assert.AreEqual(6L, (long)reader.Item); Assert.IsTrue(reader.Read()); Assert.AreEqual(ODataCollectionReaderState.Value, reader.State); Assert.IsTrue(reader.Read()); Assert.AreEqual(ODataCollectionReaderState.CollectionEnd, reader.State); Assert.IsFalse(reader.Read()); Assert.AreEqual(ODataCollectionReaderState.Completed, reader.State); }
public static void Main(string[] args) { string line = Environment.NewLine; string fileName = "Messages.avro"; string filePath = null; if (Environment.NewLine.Contains("\r")) { filePath = new DirectoryInfo(".") + @"\" + fileName; } else { filePath = new DirectoryInfo(".") + @"/" + fileName; } List <TestMsg> msgs = new List <TestMsg>(); msgs.Add(new AvroTestApp.TestMsg(1, 189.12)); msgs.Add(new AvroTestApp.TestMsg(2, 345.94)); using (var dataStream = new FileStream(filePath, FileMode.Create)) { using (var avroWriter = AvroContainer.CreateWriter <TestMsg>(dataStream, Codec.Deflate)) { using (var seqWriter = new SequentialWriter <TestMsg>(avroWriter, msgs.Count)) { msgs.ForEach(seqWriter.Write); } } dataStream.Dispose(); } }
public void SequentialWriter_AddMetadataWithNullValue() { var writer = AvroContainer.CreateWriter <ClassOfInt>(this.resultStream, Codec.Null); using (var sequentialWriter = new SequentialWriter <ClassOfInt>(writer, 2)) { sequentialWriter.AddMetadata("Key", null); } }
public void SequentialWriter_AddMetadataWithNullKey() { var writer = AvroContainer.CreateWriter <ClassOfInt>(this.resultStream, Codec.Null); using (var sequentialWriter = new SequentialWriter <ClassOfInt>(writer, 2)) { sequentialWriter.AddMetadata(null, Utilities.GetRandom <byte[]>(false)); } }
public void SequentialGenericWritingReading_RecursiveRecord() { const string StringSchema = @"{ ""type"":""record"", ""name"":""Microsoft.Hadoop.Avro.Tests.Recursive"", ""fields"":[ {""name"":""IntField"",""type"":""int""}, {""name"":""RecursiveField"",""type"":[ ""null"", ""Microsoft.Hadoop.Avro.Tests.Recursive"" ] } ]}"; using (var stream = new MemoryStream()) { var serializer = AvroSerializer.CreateGeneric(StringSchema); using (var streamWriter = AvroContainer.CreateGenericWriter(StringSchema, stream, Codec.Null)) { using (var writer = new SequentialWriter <object>(streamWriter, 24)) { var expected = new List <AvroRecord>(); var random = new Random(93); for (int i = 0; i < 10; i++) { dynamic record = new AvroRecord(serializer.WriterSchema); record.IntField = random.Next(); record.RecursiveField = new AvroRecord( ((serializer.ReaderSchema as RecordSchema).GetField("RecursiveField").TypeSchema as UnionSchema).Schemas[1]); record.RecursiveField.IntField = random.Next(); record.RecursiveField.RecursiveField = null; expected.Add(record); } expected.ForEach(writer.Write); writer.Flush(); stream.Seek(0, SeekOrigin.Begin); var streamReader = AvroContainer.CreateReader <Recursive>(stream, true, this.dataContractSettings, new CodecFactory()); using (var reader = new SequentialReader <Recursive>(streamReader)) { var j = 0; foreach (var avroRecord in reader.Objects) { Assert.Equal(expected[j]["IntField"], avroRecord.IntField); Assert.Equal(((dynamic)expected[j]["RecursiveField"])["IntField"], avroRecord.RecursiveField.IntField); Assert.Equal( ((dynamic)expected[j++]["RecursiveField"])["RecursiveField"], avroRecord.RecursiveField.RecursiveField); } } } } } }
public void SequentialWriter_AddMetadataWithNullValue() { Assert.Throws <ArgumentNullException>(() => { var writer = AvroContainer.CreateWriter <ClassOfInt>(this.resultStream, Codec.Null); using (var sequentialWriter = new SequentialWriter <ClassOfInt>(writer, 2)) { sequentialWriter.AddMetadata("Key", null); } } ); }
public void TestSerializeDeserializeUsingSequentialContainers() { //CreateDeserializerOnly a new AvroSerializer instance and specify a custom serialization strategy AvroDataContractResolver //for handling only properties attributed with DataContract/DateMember. var settings = new AvroSerializerSettings(); //CreateDeserializerOnly a new buffer using (var buffer = new MemoryStream()) { //CreateDeserializerOnly sample data var testData = new List <SensorData> { new SensorData { Value = new byte[] { 1, 2, 3, 4, 5 }, Position = new Location { Room = 243, Floor = 1 } }, new SensorData { Value = new byte[] { 6, 7, 8, 9 }, Position = new Location { Room = 244, Floor = 1 } } }; //CreateDeserializerOnly a SequentialWriter instance for type SensorData which can serialize a sequence of SensorData objects to stream using (var w = AvroContainer.CreateWriter <SensorData>(buffer, settings, Codec.Null)) { using (var writer = new SequentialWriter <SensorData>(w, 24)) { // Serialize the data to stream using the sequential writer testData.ForEach(writer.Write); } } //Prepare the stream for deserializing the data buffer.Seek(0, SeekOrigin.Begin); //CreateDeserializerOnly a SequentialReader for type SensorData which will derserialize all serialized objects from the given stream //It allows iterating over the deserialized objects because it implements IEnumerable<T> interface using (var reader = new SequentialReader <SensorData>( AvroContainer.CreateReader <SensorData>(buffer, true, settings, new CodecFactory()))) { var results = reader.Objects; //Finally, verify that deserialized data matches the original ones var pairs = testData.Zip(results, (serialized, deserialized) => new { expected = serialized, actual = deserialized }); foreach (var pair in pairs) { Assert.True(this.Equal(pair.expected, pair.actual)); } } } }
public override void WriteAvroFile(Stream stream) { using (IAvroWriter <object> streamWriter = AvroContainer.CreateGenericWriter(this.GetSchema().ToString(), stream, this.Codec)) { using (var sequentialWriter = new SequentialWriter <object>(streamWriter, AvroIntegrationWithHiveConfigurations.AvroBlockSize)) { foreach (LargeClass record in this.Expected) { sequentialWriter.Write(record.ToAvroRecord(this.GetSchema())); } } } }
public void SequentialGenericWritingReading_SimpleRecord() { const string StringSchema = @"{ ""type"":""record"", ""name"":""Microsoft.Hadoop.Avro.Tests.ClassOfInt"", ""fields"": [ { ""name"":""PrimitiveInt"", ""type"":""int"" } ] }"; using (var stream = new MemoryStream()) { var serializer = AvroSerializer.CreateGeneric(StringSchema); using (var streamWriter = AvroContainer.CreateGenericWriter(StringSchema, stream, Codec.Null)) { using (var writer = new SequentialWriter<object>(streamWriter, 24)) { var expected = new List<AvroRecord>(); var random = new Random(113); for (int i = 0; i < 10; i++) { dynamic record = new AvroRecord(serializer.WriterSchema); record.PrimitiveInt = random.Next(); expected.Add(record); } expected.ForEach(writer.Write); writer.Flush(); stream.Seek(0, SeekOrigin.Begin); var streamReader = AvroContainer.CreateReader<ClassOfInt>(stream, true, this.dataContractSettings, new CodecFactory()); using (var reader = new SequentialReader<ClassOfInt>(streamReader)) { var j = 0; foreach (var avroRecord in reader.Objects) { Assert.AreEqual(expected[j++]["PrimitiveInt"], avroRecord.PrimitiveInt); } } } } } }
public void SequentialGenericWritingReading_SimpleRecord() { const string StringSchema = @"{ ""type"":""record"", ""name"":""Microsoft.Hadoop.Avro.Tests.ClassOfInt"", ""fields"": [ { ""name"":""PrimitiveInt"", ""type"":""int"" } ] }"; using (var stream = new MemoryStream()) { var serializer = AvroSerializer.CreateGeneric(StringSchema); using (var streamWriter = AvroContainer.CreateGenericWriter(StringSchema, stream, Codec.Null)) { using (var writer = new SequentialWriter <object>(streamWriter, 24)) { var expected = new List <AvroRecord>(); var random = new Random(113); for (int i = 0; i < 10; i++) { dynamic record = new AvroRecord(serializer.WriterSchema); record.PrimitiveInt = random.Next(); expected.Add(record); } expected.ForEach(writer.Write); writer.Flush(); stream.Seek(0, SeekOrigin.Begin); var streamReader = AvroContainer.CreateReader <ClassOfInt>(stream, true, this.dataContractSettings, new CodecFactory()); using (var reader = new SequentialReader <ClassOfInt>(streamReader)) { var j = 0; foreach (var avroRecord in reader.Objects) { Assert.Equal(expected[j++]["PrimitiveInt"], avroRecord.PrimitiveInt); } } } } } }
public void Test_Serialization(bool withSchema) { var schema = Resources.SalesRecord; var avroFile = Path.ChangeExtension(Path.GetTempFileName(), "avro"); using (var fs = new FileStream(avroFile, FileMode.Create)) using (var writer = AvroContainer.CreateGenericWriter(schema, fs, Codec.Deflate)) using (var sequentialWriter = new SequentialWriter <object>(writer, 1000)) using (var sr = new StreamReader("1000 Sales Records.csv")) { sr.ReadLine(); while (!sr.EndOfStream) { var fields = sr.ReadLine()?.Split(','); var salesRecord = new SalesRecord { Region = fields[0], Country = fields[1], ItemType = fields[2], SalesChannel = Enum.Parse <SalesChannelType>(fields[3]), OrderPriority = Enum.Parse <OrderPriorityType>(fields[4]), OrderDate = DateTime.Parse(fields[5]), OrderId = int.Parse(fields[6]), ShipDate = DateTime.Parse(fields[7]), UnitsSold = int.Parse(fields[8]), UnitPrice = double.Parse(fields[9]), UnitCost = double.Parse(fields[10]), TotalRevenue = double.Parse(fields[11]), TotalCost = double.Parse(fields[12]), TotalProfit = double.Parse(fields[13]) }; sequentialWriter.Write(withSchema ? salesRecord.ToAvroRecord(schema) : salesRecord.ToAvroRecord()); } } var salesRecords = new List <SalesRecord>(); using (var fs = new FileStream(avroFile, FileMode.Open)) using (var reader = AvroContainer.CreateGenericReader(fs)) using (var sequentialReader = new SequentialReader <object>(reader)) { salesRecords.AddRange(sequentialReader.Objects.Cast <AvroRecord>() .Select(o => withSchema ? o.FromAvroRecord <SalesRecord>(schema) : o.FromAvroRecord <SalesRecord>())); } File.Delete(avroFile); }
public override void WriteAvroFile(Stream stream) { IAvroWriter <LargeClass> avroWriter = AvroContainer.CreateWriter <LargeClass>( stream, new AvroSerializerSettings { UsePosixTime = true }, this.Codec); using (var writer = new SequentialWriter <LargeClass>(avroWriter, AvroIntegrationWithHiveConfigurations.AvroBlockSize)) { foreach (LargeClass record in this.Expected) { writer.Write(record); } } }
public void TestAvroSerializeUsingObjectContainersGenericRecord() { const string schema = @"{ ""type"":""record"", ""name"":""Microsoft.Hadoop.Avro.Specifications.SensorData"", ""fields"": [ { ""name"":""Location"", ""type"": { ""type"":""record"", ""name"":""Microsoft.Hadoop.Avro.Specifications.Location"", ""fields"": [ { ""name"":""Floor"", ""type"":""int"" }, { ""name"":""Room"", ""type"":""int"" } ] } }, { ""name"":""Value"", ""type"":""bytes"" } ] }"; //Create a generic serializer based on the schema var serializer = AvroSerializer.CreateGeneric(schema); var rootSchema = serializer.WriterSchema as RecordSchema; dynamic sensorData = CreateDynamicData(rootSchema); PerformanceHarness.Test(() => { using (var buffer = new MemoryStream()) { using (var avroWriter = AvroContainer.CreateGenericWriter(schema, buffer, Codec.Null)) { using (var writer = new SequentialWriter<object>(avroWriter, 24)) { writer.Write(sensorData); } } } }, "Avro serialization through generic records", 10000); }
public static void UseAvroPushToS3Bucket(this SampleContext context) { context.PushToS3Bucket = (int numOfFiles, int numOfRecords) => { //Get compression method var codec = Codec.Null; if (!String.IsNullOrEmpty(context.CompressionMethod)) { if (context.CompressionMethod.ToLower() == "deflate") { codec = Codec.Deflate; } } using (var client = new SampleS3Client(context.Format, context.S3BucketName, context.S3BucketPath, Amazon.RegionEndpoint.USEast1)) { for (int i = 0; i < numOfFiles; i++) { DateTime randDateTime = SampleData.RandDate(); var testData = new List <LogEntry>(); testData.AddRange(SampleData.GetBunchOfData(numOfRecords, randDateTime)); using (MemoryStream buffer = new MemoryStream()) { using (var w = AvroContainer.CreateWriter <LogEntry>(buffer, codec)) { using (var writer = new SequentialWriter <LogEntry>(w, 24)) { testData.ForEach(writer.Write); } } //Objects are push sync. to keep the order. client.PutObject(buffer, randDateTime); } SampleContext.ClearConsoleLine(); Console.Write($"\r{(i + 1f) / (float)numOfFiles,6:P2}"); } } }; }
public void TestReadParameter() { MemoryStream ms = new MemoryStream(); using (var writer = AvroContainer.CreateWriter <AddProductParameter>(ms, true, new AvroSerializerSettings(), Codec.Null)) using (var seqWriter = new SequentialWriter <AddProductParameter>(writer, 24)) { seqWriter.Write(new AddProductParameter { Location = address0, Product = product0 }); } ms.Seek(0, SeekOrigin.Begin); var result = new Dictionary <string, object>(); using (var omr = TestHelper.CreateMessageReader(ms, "avro/binary", AvroMediaTypeResolver.Instance, new EdmModel())) { var reader = omr.CreateODataParameterReader(AddProduct); while (reader.Read()) { switch (reader.State) { case ODataParameterReaderState.Value: result.Add(reader.Name, reader.Value); break; case ODataParameterReaderState.Entry: var entryReader = reader.CreateEntryReader(); while (entryReader.Read()) { if (entryReader.State == ODataReaderState.EntryEnd) { result.Add(reader.Name, entryReader.Item); break; } } break; } } } Assert.AreEqual(2, result.Count); Assert.IsTrue(TestHelper.ComplexValueEqual(complexValue0, (ODataComplexValue)result["Location"])); Assert.IsTrue(TestHelper.EntryEqual(entry0, (ODataEntry)result["Product"])); }
public void TestReadAvroAsPrimitiveCollection() { const string Schema = @"{""type"":""array"", ""items"":""long""}"; var stream = new MemoryStream(); using (var writer = AvroContainer.CreateGenericWriter(Schema, stream, /*leave open*/ true, Codec.Null)) using (var seqWriter = new SequentialWriter <object>(writer, 24)) { // Serialize the data to stream using the sequential writer seqWriter.Write(new[] { 6L, 8 }); seqWriter.Flush(); } stream.Flush(); stream.Seek(0, SeekOrigin.Begin); var property = this.CreateODataInputContext(stream).ReadProperty(null, null); }
public override Stream GetContentStream(IEnumerable <ExpandoObject> records) { var result = new MemoryStream(); // We get a JSON schema for the passed type. Normally, the AvroSerializer expects a type passed that has DataContract/DataMember attributes, but I don't want users of this utility // to have to modify their POCOs in any way, hence building a schema here and proceeding with that. string schema = GetJsonSchema(); var serializer = AvroSerializer.CreateGeneric(schema); var rootSchema = serializer.WriterSchema as RecordSchema; // We'll write the Avro content to a memory stream using (var interim = new MemoryStream()) { // Avro serializer with deflate using (var avroWriter = AvroContainer.CreateGenericWriter(schema, interim, Codec.Deflate)) { using (var seqWriter = new SequentialWriter <object>(avroWriter, SYNCNUM)) { foreach (var record in records) { IDictionary <string, object> recordProperties = record as IDictionary <string, object>; dynamic avroRecord = new AvroRecord(rootSchema); foreach (KeyValuePair <string, object> recordKVP in recordProperties) { avroRecord[recordKVP.Key] = recordKVP.Value; } seqWriter.Write(avroRecord); } seqWriter.Flush(); interim.Seek(0, SeekOrigin.Begin); interim.CopyTo(result); } } } return(result); }
public Stream Compress(IEnumerable <T> documents) { var buffer = new MemoryStream(); var settings = new AvroSerializerSettings { GenerateSerializer = true }; using (var w = AvroContainer.CreateWriter <T>(buffer, settings, Codec.Deflate)) using (var writer = new SequentialWriter <T>(w, 24)) { // Serialize the data to stream by using the sequential writer documents.ToList().ForEach(writer.Write); } buffer.Seek(0, SeekOrigin.Begin); return(buffer); }
public void TestAvroSerializationThroughReflection() { SensorData sensorData = CreateTestData(); PerformanceHarness.Test(() => { using (var buffer = new MemoryStream()) { using (var avroWriter = AvroContainer.CreateWriter<SensorData>(buffer, Codec.Deflate)) { using (var writer = new SequentialWriter<SensorData>(avroWriter, 24)) { writer.Write(sensorData); } } } }, "Avro serialization through reflection", 10000); }
internal void Dispose <T>() { if (_sw != null) { RaiseEndWrite(_sw); } else if (_avroWriter != null) { RaiseEndWrite(_avroWriter); } if (Configuration.UseAvroSerializer) { if (IsDynamicType) { IAvroSerializer <Dictionary <string, object> > avroSerializer = _avroSerializer as IAvroSerializer <Dictionary <string, object> >; } else { IAvroSerializer <T> avroSerializer = _avroSerializer as IAvroSerializer <T>; } } else { if (IsDynamicType) { SequentialWriter <Dictionary <string, object> > avroWriter = _avroWriter as SequentialWriter <Dictionary <string, object> >; if (avroWriter != null) { avroWriter.Dispose(); } } else { SequentialWriter <T> avroWriter = _avroWriter as SequentialWriter <T>; if (avroWriter != null) { avroWriter.Dispose(); } } } }
/// <summary> /// Serialize a ClassHierarchy into a file /// </summary> /// <param name="c"></param> /// <param name="fileName"></param> public void ToFile(IClassHierarchy c, string fileName) { var avronNodeData = ToAvroNode(c); using (var buffer = new MemoryStream()) { using (var w = AvroContainer.CreateWriter<AvroNode>(buffer, Codec.Null)) { using (var writer = new SequentialWriter<AvroNode>(w, 24)) { writer.Write(avronNodeData); } } if (!WriteFile(buffer, fileName)) { var e = new ApplicationException("Error during file operation. Quitting method: " + fileName); Utilities.Diagnostics.Exceptions.Throw(e, LOGGER); } } }
public void SequentialReaderWriter_SerializeThreeObjects() { var expected = new List<ClassOfInt> { ClassOfInt.Create(true), ClassOfInt.Create(true), ClassOfInt.Create(true), }; var w = AvroContainer.CreateWriter<ClassOfInt>(this.resultStream, Codec.Null); using (var writer = new SequentialWriter<ClassOfInt>(w, 24)) { expected.ForEach(writer.Write); } this.resultStream.Seek(0, SeekOrigin.Begin); var r = AvroContainer.CreateReader<ClassOfInt>(this.resultStream); using (var reader = new SequentialReader<ClassOfInt>(r)) { Assert.IsTrue(expected.SequenceEqual(reader.Objects)); } }
public void SequentialWriter_AddMetadataWithNullKey() { var writer = AvroContainer.CreateWriter<ClassOfInt>(this.resultStream, Codec.Null); using (var sequentialWriter = new SequentialWriter<ClassOfInt>(writer, 2)) { sequentialWriter.AddMetadata(null, Utilities.GetRandom<byte[]>(false)); } }
public void SequentialWriter_MicrosoftWriterApacherReaderOfDictionary() { var expected = new List<ContainingDictionaryClass<string, string>>(); for (var i = 0; i < 7; i++) { expected.Add(ContainingDictionaryClass<string, string>.Create( new Dictionary<string, string> { { "testkey" + i, "testvalue" + i } })); } var w = AvroContainer.CreateWriter<ContainingDictionaryClass<string, string>>(this.resultStream, Codec.Deflate); using (var writer = new SequentialWriter<ContainingDictionaryClass<string, string>>(w, 2)) { expected.ForEach(writer.Write); } this.resultStream.Seek(0, SeekOrigin.Begin); var reader = DataFileReader<GenericRecord>.OpenReader(this.resultStream); var actual = new List<GenericRecord>(reader); Assert.AreEqual(expected.Count, actual.Count); for (var i = 0; i < expected.Count; ++i) { var actualValue = actual[i]["Property"] as Dictionary<string, object>; Assert.IsNotNull(actualValue); Assert.AreEqual(actualValue["testkey" + i] as string, expected[i].Property["testkey" + i]); } }
public void SequentialWriter_MicrosoftWriterApacherReaderOfNestedType() { var expected = new List<NestedClass>(); for (var i = 0; i < 7; i++) { expected.Add(NestedClass.Create(true)); } var w = AvroContainer.CreateWriter<NestedClass>(this.resultStream, new AvroSerializerSettings { Resolver = new AvroDataContractResolver(true) }, Codec.Deflate); using (var writer = new SequentialWriter<NestedClass>(w, 2)) { expected.ForEach(writer.Write); } this.resultStream.Seek(0, SeekOrigin.Begin); var reader = DataFileReader<GenericRecord>.OpenReader(this.resultStream); var actual = new List<GenericRecord>(reader); for (var i = 0; i < expected.Count; ++i) { Assert.AreEqual(expected[i].PrimitiveInt, actual[i]["PrimitiveInt"]); if (expected[i].ClassOfIntReference == null) { Assert.IsNull(actual[i]["ClassOfIntReference"]); } else { Assert.IsNotNull(actual[i]["ClassOfIntReference"] as GenericRecord); Assert.AreEqual(expected[i].ClassOfIntReference.PrimitiveInt, (actual[i]["ClassOfIntReference"] as GenericRecord)["PrimitiveInt"]); } } }
public void SequentialWriter_MicrosoftWriterApacheReader() { var expected = new List<ClassOfInt>(); for (var i = 0; i < 7; i++) { expected.Add(ClassOfInt.Create(true)); } var w = AvroContainer.CreateWriter<ClassOfInt>(this.resultStream, Codec.Deflate); using (var writer = new SequentialWriter<ClassOfInt>(w, 2)) { expected.ForEach(writer.Write); } this.resultStream.Seek(0, SeekOrigin.Begin); var reader = DataFileReader<GenericRecord>.OpenReader(this.resultStream); var actual = new List<GenericRecord>(reader); for (var i = 0; i < expected.Count; ++i) { Assert.AreEqual(expected[i].PrimitiveInt, actual[i]["PrimitiveInt"]); } }
public void TestReadParameter() { MemoryStream ms = new MemoryStream(); using (var writer = AvroContainer.CreateWriter<AddProductParameter>(ms, true, new AvroSerializerSettings(), Codec.Null)) using (var seqWriter = new SequentialWriter<AddProductParameter>(writer, 24)) { seqWriter.Write(new AddProductParameter { Location = address0, Product = product0 }); } ms.Seek(0, SeekOrigin.Begin); var result = new Dictionary<string, object>(); using (var omr = TestHelper.CreateMessageReader(ms, "avro/binary", AvroMediaTypeResolver.Instance, new EdmModel())) { var reader = omr.CreateODataParameterReader(AddProduct); while (reader.Read()) { switch (reader.State) { case ODataParameterReaderState.Value: result.Add(reader.Name, reader.Value); break; case ODataParameterReaderState.Entry: var entryReader = reader.CreateEntryReader(); while (entryReader.Read()) { if (entryReader.State == ODataReaderState.EntryEnd) { result.Add(reader.Name, entryReader.Item); break; } } break; } } } Assert.AreEqual(2, result.Count); Assert.IsTrue(TestHelper.ComplexValueEqual(complexValue0, (ODataComplexValue)result["Location"])); Assert.IsTrue(TestHelper.EntryEqual(entry0, (ODataEntry)result["Product"])); }
public void ReadAvroAsParameter() { EdmEntityType personType = new EdmEntityType("TestNS", "Person"); personType.AddStructuralProperty("Id", EdmPrimitiveTypeKind.Int32); personType.AddStructuralProperty("Title", EdmPrimitiveTypeKind.String); var operation = new EdmAction("NS", "op1", null); operation.AddParameter("p1", EdmCoreModel.Instance.GetString(false)); operation.AddParameter("p2", new EdmEntityTypeReference(personType, false)); const string Schema = @"{ ""type"":""record"", ""name"":""NS.op1Parameter"", ""fields"": [ { ""name"":""p1"", ""type"":""string"" }, { ""name"":""p2"", ""type"": {""type"":""record"", ""name"":""TestNS.Person"", ""fields"": [ { ""name"":""Id"", ""type"":""int"" }, { ""name"":""Title"", ""type"":""string"" }, ] } } ] }"; var stream = new MemoryStream(); using (var writer = AvroContainer.CreateGenericWriter(Schema, stream, /*leave open*/true, Codec.Null)) using (var seqWriter = new SequentialWriter<object>(writer, 24)) { RecordSchema parameterSchema = (RecordSchema)AvroSerializer.CreateGeneric(Schema).WriterSchema; AvroRecord ar = new AvroRecord(parameterSchema); ar["p1"] = "dat"; var personSchema = parameterSchema.GetField("p2").TypeSchema; AvroRecord person = new AvroRecord(personSchema); person["Id"] = 5; person["Title"] = "per1"; ar["p2"] = person; seqWriter.Write(ar); seqWriter.Flush(); } stream.Flush(); stream.Seek(0, SeekOrigin.Begin); var reader = new ODataAvroParameterReader(this.CreateODataInputContext(stream), operation); Assert.AreEqual(ODataParameterReaderState.Start, reader.State); Assert.IsTrue(reader.Read()); Assert.AreEqual(ODataParameterReaderState.Value, reader.State); Assert.AreEqual("p1", reader.Name); Assert.AreEqual("dat", reader.Value); Assert.IsTrue(reader.Read()); Assert.AreEqual(ODataParameterReaderState.Entry, reader.State); Assert.AreEqual("p2", reader.Name); var ew = reader.CreateEntryReader(); Assert.AreEqual(ODataReaderState.Start, ew.State); Assert.IsTrue(ew.Read()); Assert.AreEqual(ODataReaderState.EntryStart, ew.State); Assert.IsTrue(ew.Read()); Assert.AreEqual(ODataReaderState.EntryEnd, ew.State); var entry = ew.Item as ODataEntry; Assert.IsFalse(ew.Read()); Assert.AreEqual(ODataReaderState.Completed, ew.State); Assert.IsNotNull(entry); var properties = entry.Properties.ToList(); Assert.AreEqual(2, properties.Count); Assert.AreEqual("Id", properties[0].Name); Assert.AreEqual(5, properties[0].Value); Assert.AreEqual("Title", properties[1].Name); Assert.AreEqual("per1", properties[1].Value); Assert.IsFalse(reader.Read()); Assert.AreEqual(ODataParameterReaderState.Completed, reader.State); }
public void TestReadFeed() { MemoryStream ms = new MemoryStream(); IAvroWriter<Product[]> writer = null; try { writer = AvroContainer.CreateWriter<Product[]>(ms, true, new AvroSerializerSettings(), Codec.Null); using (var seqWriter = new SequentialWriter<Product[]>(writer, 24)) { seqWriter.Write(new[] { product1, product2, product3 }); } } finally { if (writer != null) writer.Dispose(); } ms.Seek(0, SeekOrigin.Begin); var entries = new List<ODataEntry>(); using (var omr = TestHelper.CreateMessageReader(ms, "avro/binary", AvroMediaTypeResolver.Instance)) { var reader = omr.CreateODataFeedReader(); while (reader.Read()) { if (reader.State == ODataReaderState.EntryEnd) { entries.Add((ODataEntry)reader.Item); } } } Assert.AreEqual(3, entries.Count); Assert.IsTrue(TestHelper.EntryEqual(entry1, entries[0])); Assert.IsTrue(TestHelper.EntryEqual(entry2, entries[1])); Assert.IsTrue(TestHelper.EntryEqual(entry3, entries[2])); }
public void ToFile(IConfiguration c, string fileName) { var avronConfigurationData = ToAvroConfiguration(c); using (var buffer = new MemoryStream()) { using (var w = AvroContainer.CreateWriter<AvroConfiguration>(buffer, Codec.Null)) { using (var writer = new SequentialWriter<AvroConfiguration>(w, 24)) { // Serialize the data to stream using the sequential writer writer.Write(avronConfigurationData); } } if (!WriteFile(buffer, fileName)) { var e = new ApplicationException("Error during file operation. Quitting method: " + fileName); Org.Apache.REEF.Utilities.Diagnostics.Exceptions.Throw(e, LOGGER); } } }
static void SerializeReadings() { try { //Serialize GPS data to file Console.WriteLine("Saving GPS data..."); using (var buffer = new FileStream(gpsFile, FileMode.Create)) { //Serialize a sequence of GpsReading objects to stream //Data will be compressed using Deflate codec using (var w = AvroContainer.CreateWriter<GpsReading>(buffer, Codec.Deflate)) { using (var writer = new SequentialWriter<GpsReading>(w, 24)) { // Serialize the data to stream using the sequential writer GpsReadings.ForEach(writer.Write); } } buffer.Close(); } //Serialize Engine data to file Console.WriteLine("Saving engine data..."); using (var buffer = new FileStream(engineFile, FileMode.Create)) { //Serialize a sequence of EngineReading objects to stream //Data will be compressed using Deflate codec using (var w = AvroContainer.CreateWriter<EngineReading>(buffer, Codec.Deflate)) { using (var writer = new SequentialWriter<EngineReading>(w, 24)) { // Serialize the data to stream using the sequential writer EngineReadings.ForEach(writer.Write); } } buffer.Close(); } //Serialize Brake data to file Console.WriteLine("Saving brake data..."); using (var buffer = new FileStream(brakeFile, FileMode.Create)) { //Serialize a sequence of BrakeReading objects to stream //Data will be compressed using Deflate codec using (var w = AvroContainer.CreateWriter<BrakeReading>(buffer, Codec.Deflate)) { using (var writer = new SequentialWriter<BrakeReading>(w, 24)) { // Serialize the data to stream using the sequential writer BrakeReadings.ForEach(writer.Write); } } buffer.Close(); } } catch (Exception ex) { Console.WriteLine("An error occurred while serializing the data"); throw (ex); } }
public void ReadAvroAsError() { const string Schema = @"{ ""type"":""record"", ""name"":""OData.Error"", ""fields"": [ { ""name"":""ErrorCode"", ""type"":""string"" }, { ""name"":""Message"" , ""type"":""string"" }, ] }"; var stream = new MemoryStream(); using (var writer = AvroContainer.CreateGenericWriter(Schema, stream, /*leave open*/true, Codec.Null)) using (var seqWriter = new SequentialWriter<object>(writer, 24)) { RecordSchema parameterSchema = (RecordSchema)AvroSerializer.CreateGeneric(Schema).WriterSchema; AvroRecord ar = new AvroRecord(parameterSchema); ar["ErrorCode"] = "e1"; ar["Message"] = "m1"; seqWriter.Write(ar); seqWriter.Flush(); } stream.Flush(); stream.Seek(0, SeekOrigin.Begin); var error = this.CreateODataInputContext(stream).ReadError(); Assert.AreEqual("e1", error.ErrorCode); Assert.AreEqual("m1", error.Message); }
public void TestReadError() { MemoryStream ms = new MemoryStream(); using (var writer = AvroContainer.CreateWriter<Error>(ms, true, new AvroSerializerSettings(), Codec.Null)) using (var seqWriter = new SequentialWriter<Error>(writer, 24)) { seqWriter.Write(new Error { ErrorCode = "500", Message = "Internal Error" }); } ms.Seek(0, SeekOrigin.Begin); ODataError error = null; using (var omr = TestHelper.CreateMessageReader(ms, "avro/binary", AvroMediaTypeResolver.Instance, new EdmModel(), true)) { error = omr.ReadError(); } Assert.IsNotNull(error); Assert.AreEqual("500", error.ErrorCode); Assert.AreEqual("Internal Error", error.Message); }
public void TestReadParameterWithFeed() { MemoryStream ms = new MemoryStream(); using (var writer = AvroContainer.CreateWriter<GetMaxIdParameter>(ms, true, new AvroSerializerSettings(), Codec.Null)) using (var seqWriter = new SequentialWriter<GetMaxIdParameter>(writer, 24)) { seqWriter.Write(new GetMaxIdParameter { Products = new List<Product>() { product0, product1 } }); } ms.Seek(0, SeekOrigin.Begin); var result = new Dictionary<string, object>(); using (var omr = TestHelper.CreateMessageReader(ms, "avro/binary", AvroMediaTypeResolver.Instance, new EdmModel())) { var reader = omr.CreateODataParameterReader(GetMaxId); while (reader.Read()) { switch (reader.State) { case ODataParameterReaderState.Value: result.Add(reader.Name, reader.Value); break; case ODataParameterReaderState.Entry: var entryReader = reader.CreateEntryReader(); while (entryReader.Read()) { if (entryReader.State == ODataReaderState.EntryEnd) { result.Add(reader.Name, entryReader.Item); break; } } break; case ODataParameterReaderState.Feed: var feedReader = reader.CreateFeedReader(); IList<ODataEntry> entryList = new List<ODataEntry>(); while (feedReader.Read()) { if (feedReader.State == ODataReaderState.EntryEnd) { entryList.Add((ODataEntry)feedReader.Item); } } result.Add(reader.Name, entryList); break; } } } Assert.AreEqual(1, result.Count); var feed = result["Products"] as IList<ODataEntry>; Assert.IsNotNull(feed); Assert.AreEqual(2, feed.Count); Assert.IsTrue(TestHelper.EntryEqual(entry0, feed[0])); Assert.IsTrue(TestHelper.EntryEqual(entry1, feed[1])); }
public void SequentialWriter_AddMetadataWithNullValue() { var writer = AvroContainer.CreateWriter<ClassOfInt>(this.resultStream, Codec.Null); using (var sequentialWriter = new SequentialWriter<ClassOfInt>(writer, 2)) { sequentialWriter.AddMetadata("Key", null); } }
//Serializes and deserializes sample data set using Generic Record and Avro Object Container Files //Serialized data is not compressed // //This sample uses Memory Stream for all operations related to serialization, deserialization and //Object Container manipulation, though File Stream could be easily used. public void SerializeDeserializeUsingObjectContainersGenericRecord() { Console.WriteLine("SERIALIZATION USING GENERIC RECORD AND AVRO OBJECT CONTAINER FILES\n"); //Path for Avro Object Container File string path = "AvroSampleGenericRecordNullCodec.avro"; Console.WriteLine("Defining the Schema and creating Sample Data Set..."); //Define the schema in JSON const string Schema = @"{ ""type"":""record"", ""name"":""Microsoft.Hadoop.Avro.Specifications.SensorData"", ""fields"": [ { ""name"":""Location"", ""type"": { ""type"":""record"", ""name"":""Microsoft.Hadoop.Avro.Specifications.Location"", ""fields"": [ { ""name"":""Floor"", ""type"":""int"" }, { ""name"":""Room"", ""type"":""int"" } ] } }, { ""name"":""Value"", ""type"":""bytes"" } ] }"; //Create a generic serializer based on the schema var serializer = AvroSerializer.CreateGeneric(Schema); var rootSchema = serializer.WriterSchema as RecordSchema; //Create a generic record to represent the data var testData = new List<AvroRecord>(); dynamic expected1 = new AvroRecord(rootSchema); dynamic location1 = new AvroRecord(rootSchema.GetField("Location").TypeSchema); location1.Floor = 1; location1.Room = 243; expected1.Location = location1; expected1.Value = new byte[] { 1, 2, 3, 4, 5 }; testData.Add(expected1); dynamic expected2 = new AvroRecord(rootSchema); dynamic location2 = new AvroRecord(rootSchema.GetField("Location").TypeSchema); location2.Floor = 1; location2.Room = 244; expected2.Location = location2; expected2.Value = new byte[] { 6, 7, 8, 9 }; testData.Add(expected2); //Serializing and saving data to file //Create a MemoryStream buffer using (var buffer = new MemoryStream()) { Console.WriteLine("Serializing Sample Data Set..."); //Create a SequentialWriter instance for type SensorData which can serialize a sequence of SensorData objects to stream //Data will not be compressed (Null compression codec) using (var writer = AvroContainer.CreateGenericWriter(Schema, buffer, Codec.Null)) { using (var streamWriter = new SequentialWriter<object>(writer, 24)) { // Serialize the data to stream using the sequential writer testData.ForEach(streamWriter.Write); } } Console.WriteLine("Saving serialized data to file..."); //Save stream to file if (!WriteFile(buffer, path)) { Console.WriteLine("Error during file operation. Quitting method"); return; } } //Reading and deserializng the data //Create a Memory Stream buffer using (var buffer = new MemoryStream()) { Console.WriteLine("Reading data from file..."); //Reading data from Object Container File if (!ReadFile(buffer, path)) { Console.WriteLine("Error during file operation. Quitting method"); return; } Console.WriteLine("Deserializing Sample Data Set..."); //Prepare the stream for deserializing the data buffer.Seek(0, SeekOrigin.Begin); //Create a SequentialReader for type SensorData which will derserialize all serialized objects from the given stream //It allows iterating over the deserialized objects because it implements IEnumerable<T> interface using (var reader = AvroContainer.CreateGenericReader(buffer)) { using (var streamReader = new SequentialReader<object>(reader)) { var results = streamReader.Objects; Console.WriteLine("Comparing Initial and Deserialized Data Sets..."); //Finally, verify the results var pairs = testData.Zip(results, (serialized, deserialized) => new { expected = (dynamic)serialized, actual = (dynamic)deserialized }); int count = 1; foreach (var pair in pairs) { bool isEqual = pair.expected.Location.Floor.Equals(pair.actual.Location.Floor); isEqual = isEqual && pair.expected.Location.Room.Equals(pair.actual.Location.Room); isEqual = isEqual && ((byte[])pair.expected.Value).SequenceEqual((byte[])pair.actual.Value); Console.WriteLine("For Pair {0} result of Data Set Identity Comparison is {1}", count, isEqual.ToString()); count++; } } } } //Delete the file RemoveFile(path); }
public void SequentialReaderWriter_CreateReaderForSchemaWithNullableField() { var expected = new ClassWithNullableIntField { NullableIntField = 10 }; using (var writer = AvroContainer.CreateWriter<ClassWithNullableIntField>(this.resultStream, Codec.Deflate)) { using (var swriter = new SequentialWriter<ClassWithNullableIntField>(writer, 2)) { swriter.Write(expected); swriter.Write(expected); swriter.Write(expected); swriter.Write(expected); } } this.resultStream.Seek(0, SeekOrigin.Begin); using (var reader = AvroContainer.CreateReader<ClassWithNullableIntField>(this.resultStream)) { using (var sreader = new SequentialReader<ClassWithNullableIntField>(reader)) { foreach (var actual in sreader.Objects) { Assert.AreEqual(expected.NullableIntField, actual.NullableIntField); } } } }
public void TestReadEntry() { MemoryStream ms = new MemoryStream(); IAvroWriter<Product> writer = null; try { writer = AvroContainer.CreateWriter<Product>(ms, true, new AvroSerializerSettings(), Codec.Null); using (var seqWriter = new SequentialWriter<Product>(writer, 24)) { seqWriter.Write(product0); } } finally { if (writer != null) writer.Dispose(); } ms.Seek(0, SeekOrigin.Begin); ODataEntry entry = null; using (var omr = TestHelper.CreateMessageReader(ms, "avro/binary", AvroMediaTypeResolver.Instance)) { var reader = omr.CreateODataEntryReader(); while (reader.Read()) { if (reader.State == ODataReaderState.EntryEnd) { entry = (ODataEntry)reader.Item; } } } Assert.IsTrue(TestHelper.EntryEqual(entry0, entry)); }
public void SequentialReaderWriter_SerializeHugeObject() { var single = new SimpleFlatClass { StringField = new string('a', 16254), ByteArrayField = Encoding.ASCII.GetBytes(new string('b', 65666)), ZeroByteArrayField = Encoding.ASCII.GetBytes(new string('c', 128344)) }; var expected = new List<SimpleFlatClass> { single, single, single, }; var w = AvroContainer.CreateWriter<SimpleFlatClass>(this.resultStream, new AvroSerializerSettings { Resolver = new AvroDataContractResolver(true) }, Codec.Null); using (var writer = new SequentialWriter<SimpleFlatClass>(w, 24)) { expected.ForEach(writer.Write); } this.resultStream.Seek(0, SeekOrigin.Begin); var r = AvroContainer.CreateReader<SimpleFlatClass>(this.resultStream, true, new AvroSerializerSettings { Resolver = new AvroDataContractResolver(true) }, new CodecFactory()); using (var reader = new SequentialReader<SimpleFlatClass>(r)) { Assert.IsTrue(expected.SequenceEqual(reader.Objects)); } }
public void SequentialReaderWriter_Reset() { var expected = new List<ClassOfListOfGuid>(); for (var i = 0; i < 7; i++) { expected.Add(ClassOfListOfGuid.Create(true)); } var w = AvroContainer.CreateWriter<ClassOfListOfGuid>(this.resultStream, Codec.Deflate); using (var writer = new SequentialWriter<ClassOfListOfGuid>(w, 3)) { expected.ForEach(writer.Write); } this.resultStream.Seek(0, SeekOrigin.Begin); var r = AvroContainer.CreateReader<ClassOfListOfGuid>(this.resultStream); using (var reader = new SequentialReader<ClassOfListOfGuid>(r)) { Assert.IsTrue(expected.SequenceEqual(reader.Objects)); var enumerator = (IEnumerator)reader.Objects.GetEnumerator(); Assert.IsFalse(enumerator.MoveNext()); } }
/// <summary> /// Reads CSV file data and converts records to instances of Stock class then serializes them using Reflection. /// Stock type definition is created from JSON schema using Microsoft Avro Library Code Generation utility /// </summary> /// <param name="inputFilePath">The input file path.</param> /// <param name="outputStream">The output stream.</param> private void SerializeCsv(string inputFilePath, Stream outputStream) { try { // Create an Avro writer using Null Codec var writer = AvroContainer.CreateWriter<Stock>( outputStream, new AvroSerializerSettings { Resolver = new AvroDataContractResolver(true) }, Codec.Null); // Create a new sequential writer and use it to write Avro file using a block length of 100 using (var seq = new SequentialWriter<Stock>(writer, 100)) { foreach (var stock in this.ReadCsv(inputFilePath)) { seq.Write(stock); } } } catch (Exception e) { ReportError("Error while creating Avro file(s) from CSV file(s)" + e); } }
public void TestReadCollection() { MemoryStream ms = new MemoryStream(); using (var writer = AvroContainer.CreateWriter<string[]>(ms, true, new AvroSerializerSettings(), Codec.Null)) using (var seqWriter = new SequentialWriter<string[]>(writer, 24)) { seqWriter.Write(new[] { "p9t", "cjk如" }); } ms.Seek(0, SeekOrigin.Begin); var values = new List<string>(); using (var omr = TestHelper.CreateMessageReader(ms, "avro/binary", AvroMediaTypeResolver.Instance)) { var reader = omr.CreateODataCollectionReader(); while (reader.Read()) { if (reader.State == ODataCollectionReaderState.Value) { values.Add((string)reader.Item); } } } Assert.AreEqual(2, values.Count); Assert.AreEqual("p9t", values[0]); Assert.AreEqual("cjk如", values[1]); }
//Serializes and deserializes sample data set using Reflection and Avro Object Container Files //Serialized data is compressed with Deflate codec // //This sample uses Memory Stream for all operations related to serialization, deserialization and //Object Container manipulation, though File Stream could be easily used. public void SerializeDeserializeUsingObjectContainersReflection() { Console.WriteLine("SERIALIZATION USING REFLECTION AND AVRO OBJECT CONTAINER FILES\n"); //Path for Avro Object Container File string path = "AvroSampleReflectionDeflate.avro"; //Create a data set using sample Class and struct var testData = new List<SensorData> { new SensorData { Value = new byte[] { 1, 2, 3, 4, 5 }, Position = new Location { Room = 243, Floor = 1 } }, new SensorData { Value = new byte[] { 6, 7, 8, 9 }, Position = new Location { Room = 244, Floor = 1 } } }; //Serializing and saving data to file //Creating a Memory Stream buffer using (var buffer = new MemoryStream()) { Console.WriteLine("Serializing Sample Data Set..."); //Create a SequentialWriter instance for type SensorData which can serialize a sequence of SensorData objects to stream //Data will be compressed using Deflate codec using (var w = AvroContainer.CreateWriter<SensorData>(buffer, Codec.Deflate)) { using (var writer = new SequentialWriter<SensorData>(w, 24)) { // Serialize the data to stream using the sequential writer testData.ForEach(writer.Write); } } //Save stream to file Console.WriteLine("Saving serialized data to file..."); if (!WriteFile(buffer, path)) { Console.WriteLine("Error during file operation. Quitting method"); return; } } //Reading and deserializing data //Creating a Memory Stream buffer using (var buffer = new MemoryStream()) { Console.WriteLine("Reading data from file..."); //Reading data from Object Container File if (!ReadFile(buffer, path)) { Console.WriteLine("Error during file operation. Quitting method"); return; } Console.WriteLine("Deserializing Sample Data Set..."); //Prepare the stream for deserializing the data buffer.Seek(0, SeekOrigin.Begin); //Create a SequentialReader for type SensorData which will derserialize all serialized objects from the given stream //It allows iterating over the deserialized objects because it implements IEnumerable<T> interface using (var reader = new SequentialReader<SensorData>( AvroContainer.CreateReader<SensorData>(buffer, true))) { var results = reader.Objects; //Finally, verify that deserialized data matches the original one Console.WriteLine("Comparing Initial and Deserialized Data Sets..."); int count = 1; var pairs = testData.Zip(results, (serialized, deserialized) => new { expected = serialized, actual = deserialized }); foreach (var pair in pairs) { bool isEqual = this.Equal(pair.expected, pair.actual); Console.WriteLine("For Pair {0} result of Data Set Identity Comparison is {1}", count, isEqual.ToString()); count++; } } } //Delete the file RemoveFile(path); }
//Serializes and deserializes sample data set using Reflection and Avro Object Container Files //Serialized data is compressed with the Custom compression codec (Deflate of .NET Framework 4.5) // //This sample uses Memory Stream for all operations related to serialization, deserialization and //Object Container manipulation, though File Stream could be easily used. public void SerializeDeserializeUsingObjectContainersReflectionCustomCodec() { Console.WriteLine("SERIALIZATION USING REFLECTION, AVRO OBJECT CONTAINER FILES AND CUSTOM CODEC\n"); //Path for Avro Object Container File string path = "AvroSampleReflectionDeflate45.avro"; //Create a data set using sample Class and struct var testData = new List<SensorData> { new SensorData { Value = new byte[] { 1, 2, 3, 4, 5 }, Position = new Location { Room = 243, Floor = 1 } }, new SensorData { Value = new byte[] { 6, 7, 8, 9 }, Position = new Location { Room = 244, Floor = 1 } } }; //Serializing and saving data to file //Creating a Memory Stream buffer using (var buffer = new MemoryStream()) { Console.WriteLine("Serializing Sample Data Set..."); //Create a SequentialWriter instance for type SensorData which can serialize a sequence of SensorData objects to stream //Here the custom Codec is introduced. For convenience the next commented code line shows how to use built-in Deflate. //Note, that because the sample deals with different IMPLEMENTATIONS of Deflate, built-in and custom codecs are interchangeable //in read-write operations //using (var w = AvroContainer.CreateWriter<SensorData>(buffer, Codec.Deflate)) using (var w = AvroContainer.CreateWriter<SensorData>(buffer, new DeflateCodec45())) { using (var writer = new SequentialWriter<SensorData>(w, 24)) { // Serialize the data to stream using the sequential writer testData.ForEach(writer.Write); } } //Save stream to file Console.WriteLine("Saving serialized data to file..."); if (!WriteFile(buffer, path)) { Console.WriteLine("Error during file operation. Quitting method"); return; } } //Reading and deserializing data //Creating a Memory Stream buffer using (var buffer = new MemoryStream()) { Console.WriteLine("Reading data from file..."); //Reading data from Object Container File if (!ReadFile(buffer, path)) { Console.WriteLine("Error during file operation. Quitting method"); return; } Console.WriteLine("Deserializing Sample Data Set..."); //Prepare the stream for deserializing the data buffer.Seek(0, SeekOrigin.Begin); //Because of SequentialReader<T> constructor signature an AvroSerializerSettings instance is required //when Codec Factory is explicitly specified //You may comment the line below if you want to use built-in Deflate (see next comment) AvroSerializerSettings settings = new AvroSerializerSettings(); //Create a SequentialReader for type SensorData which will derserialize all serialized objects from the given stream //It allows iterating over the deserialized objects because it implements IEnumerable<T> interface //Here the custom Codec Factory is introduced. //For convenience the next commented code line shows how to use built-in Deflate //(no explicit Codec Factory parameter is required in this case). //Note, that because the sample deals with different IMPLEMENTATIONS of Deflate, built-in and custom codecs are interchangeable //in read-write operations //using (var reader = new SequentialReader<SensorData>(AvroContainer.CreateReader<SensorData>(buffer, true))) using (var reader = new SequentialReader<SensorData>( AvroContainer.CreateReader<SensorData>(buffer, true, settings, new CodecFactoryDeflate45()))) { var results = reader.Objects; //Finally, verify that deserialized data matches the original one Console.WriteLine("Comparing Initial and Deserialized Data Sets..."); bool isEqual; int count = 1; var pairs = testData.Zip(results, (serialized, deserialized) => new { expected = serialized, actual = deserialized }); foreach (var pair in pairs) { isEqual = this.Equal(pair.expected, pair.actual); Console.WriteLine("For Pair {0} result of Data Set Identity Comparison is {1}", count, isEqual.ToString()); count++; } } } //Delete the file RemoveFile(path); }
public void SequentialReaderWriter_SyncAfterDeflateCodec() { var expected = new List<ClassOfInt>(); for (var i = 0; i < 7; i++) { expected.Add(ClassOfInt.Create(true)); } var w = AvroContainer.CreateWriter<ClassOfInt>(this.resultStream, Codec.Deflate); using (var writer = new SequentialWriter<ClassOfInt>(w, 2)) { expected.ForEach(writer.Write); } this.resultStream.Seek(0, SeekOrigin.Begin); var r = AvroContainer.CreateReader<ClassOfInt>(this.resultStream); using (var reader = new SequentialReader<ClassOfInt>(r)) { Assert.IsTrue(expected.SequenceEqual(reader.Objects)); } }
public void TestReadAvroAsODataFeed() { const string Schema = @" {""type"":""array"", ""items"": { ""type"":""record"", ""name"":""TestNS.Person"", ""fields"": [ { ""name"":""Id"", ""type"":""int"" }, { ""name"":""Title"", ""type"":""string"" }, ] } }"; var serializer = AvroSerializer.CreateGeneric(Schema); using (var stream = new MemoryStream()) { var arraySchema = (ArraySchema)serializer.WriterSchema; var recordSchema = arraySchema.ItemSchema; var rec1 = new AvroRecord(recordSchema); rec1["Id"] = 1; rec1["Title"] = "s1"; var rec2 = new AvroRecord(recordSchema); rec2["Id"] = 2; rec2["Title"] = "s2"; var array = new[] { rec1, rec2 }; using (var writer = AvroContainer.CreateGenericWriter(Schema, stream, /*leave open*/true, Codec.Null)) using (var seqWriter = new SequentialWriter<object>(writer, 24)) { seqWriter.Write(array); } stream.Seek(0, SeekOrigin.Begin); var avroReader = new ODataAvroReader(this.CreateODataInputContext(stream), true); Assert.AreEqual(ODataReaderState.Start, avroReader.State); Assert.IsTrue(avroReader.Read()); Assert.AreEqual(ODataReaderState.FeedStart, avroReader.State); Assert.IsTrue(avroReader.Read()); // Entry 1 Assert.AreEqual(ODataReaderState.EntryStart, avroReader.State); Assert.IsTrue(avroReader.Read()); Assert.AreEqual(ODataReaderState.EntryEnd, avroReader.State); var entry = avroReader.Item as ODataEntry; Assert.IsNotNull(entry); var properties = entry.Properties.ToList(); Assert.AreEqual(2, properties.Count); Assert.AreEqual("Id", properties[0].Name); Assert.AreEqual(1, properties[0].Value); Assert.AreEqual("Title", properties[1].Name); Assert.AreEqual("s1", properties[1].Value); // Entry 2 Assert.IsTrue(avroReader.Read()); Assert.AreEqual(ODataReaderState.EntryStart, avroReader.State); Assert.IsTrue(avroReader.Read()); Assert.AreEqual(ODataReaderState.EntryEnd, avroReader.State); entry = avroReader.Item as ODataEntry; Assert.IsNotNull(entry); properties = entry.Properties.ToList(); Assert.AreEqual(2, properties.Count); Assert.AreEqual("Id", properties[0].Name); Assert.AreEqual(2, properties[0].Value); Assert.AreEqual("Title", properties[1].Name); Assert.AreEqual("s2", properties[1].Value); Assert.IsTrue(avroReader.Read()); Assert.AreEqual(ODataReaderState.FeedEnd, avroReader.State); Assert.IsFalse(avroReader.Read()); Assert.AreEqual(ODataReaderState.Completed, avroReader.State); } }