public void SequentialReaderWriter_CreateReaderForSchemaWithNullableField() { var expected = new ClassWithNullableIntField { NullableIntField = 10 }; using (var writer = AvroContainer.CreateWriter <ClassWithNullableIntField>(this.resultStream, Codec.Deflate)) { using (var swriter = new SequentialWriter <ClassWithNullableIntField>(writer, 2)) { swriter.Write(expected); swriter.Write(expected); swriter.Write(expected); swriter.Write(expected); } } this.resultStream.Seek(0, SeekOrigin.Begin); using (var reader = AvroContainer.CreateReader <ClassWithNullableIntField>(this.resultStream)) { using (var sreader = new SequentialReader <ClassWithNullableIntField>(reader)) { foreach (var actual in sreader.Objects) { Assert.AreEqual(expected.NullableIntField, actual.NullableIntField); } } } }
static void Main(string[] args) { SampleData obj = new SampleData(); using (FileStream xmlStream = new FileStream(@"SampleData.xml", FileMode.Open)) { using (XmlReader xmlReader = XmlReader.Create(xmlStream)) { obj.ReadXml(xmlReader); } } // hi ahmed var serializer = new AvroSerializer(typeof(Person)); StreamWriter schemafs = new StreamWriter(File.Create(@"SampleData.avsc")); schemafs.Write(serializer.Schema.ToString()); schemafs.Close(); var resultStream = new FileStream(@"SampleData.avro", FileMode.OpenOrCreate); var writer = new SequentialWriter <Person>(resultStream, serializer, Codec.Create("null"), 24); int i; foreach (DataRow item in obj.Tables["Person"].Rows) { var p = new Person { First_Name = item["Name"].ToString(), Street = item["Street"].ToString(), City = item["City"].ToString(), State = item["State"].ToString() }; writer.Write(p); } writer.Flush(); }
public void TestReadCollection() { MemoryStream ms = new MemoryStream(); using (var writer = AvroContainer.CreateWriter <string[]>(ms, true, new AvroSerializerSettings(), Codec.Null)) using (var seqWriter = new SequentialWriter <string[]>(writer, 24)) { seqWriter.Write(new[] { "p9t", "cjk如" }); } ms.Seek(0, SeekOrigin.Begin); var values = new List <string>(); using (var omr = TestHelper.CreateMessageReader(ms, "avro/binary", AvroMediaTypeResolver.Instance)) { var reader = omr.CreateODataCollectionReader(); while (reader.Read()) { if (reader.State == ODataCollectionReaderState.Value) { values.Add((string)reader.Item); } } } Assert.AreEqual(2, values.Count); Assert.AreEqual("p9t", values[0]); Assert.AreEqual("cjk如", values[1]); }
public void ReadAvroAsError() { const string Schema = @"{ ""type"":""record"", ""name"":""OData.Error"", ""fields"": [ { ""name"":""ErrorCode"", ""type"":""string"" }, { ""name"":""Message"" , ""type"":""string"" }, ] }"; var stream = new MemoryStream(); using (var writer = AvroContainer.CreateGenericWriter(Schema, stream, /*leave open*/ true, Codec.Null)) using (var seqWriter = new SequentialWriter <object>(writer, 24)) { RecordSchema parameterSchema = (RecordSchema)AvroSerializer.CreateGeneric(Schema).WriterSchema; AvroRecord ar = new AvroRecord(parameterSchema); ar["ErrorCode"] = "e1"; ar["Message"] = "m1"; seqWriter.Write(ar); seqWriter.Flush(); } stream.Flush(); stream.Seek(0, SeekOrigin.Begin); var error = this.CreateODataInputContext(stream).ReadError(); Assert.AreEqual("e1", error.ErrorCode); Assert.AreEqual("m1", error.Message); }
public void TestReadAvroAsCollection() { const string Schema = @"{""type"":""array"", ""items"":""long""}"; var stream = new MemoryStream(); using (var writer = AvroContainer.CreateGenericWriter(Schema, stream, /*leave open*/ true, Codec.Null)) using (var seqWriter = new SequentialWriter <object>(writer, 24)) { // Serialize the data to stream using the sequential writer seqWriter.Write(new[] { 6L, 8 }); seqWriter.Flush(); } stream.Flush(); stream.Seek(0, SeekOrigin.Begin); var reader = new ODataAvroCollectionReader(this.CreateODataInputContext(stream)); Assert.AreEqual(ODataCollectionReaderState.Start, reader.State); Assert.IsTrue(reader.Read()); Assert.AreEqual(ODataCollectionReaderState.CollectionStart, reader.State); Assert.IsTrue(reader.Read()); Assert.AreEqual(ODataCollectionReaderState.Value, reader.State); Assert.AreEqual(6L, (long)reader.Item); Assert.IsTrue(reader.Read()); Assert.AreEqual(ODataCollectionReaderState.Value, reader.State); Assert.IsTrue(reader.Read()); Assert.AreEqual(ODataCollectionReaderState.CollectionEnd, reader.State); Assert.IsFalse(reader.Read()); Assert.AreEqual(ODataCollectionReaderState.Completed, reader.State); }
/// <summary> /// Reads CSV file data and converts records to instances of Stock class then serializes them using Reflection. /// Stock type definition is created from JSON schema using Microsoft Avro Library Code Generation utility /// </summary> /// <param name="inputFilePath">The input file path.</param> /// <param name="outputStream">The output stream.</param> private void SerializeCsv(string inputFilePath, Stream outputStream) { try { // Create an Avro writer using Null Codec var writer = AvroContainer.CreateWriter <Stock>( outputStream, new AvroSerializerSettings { Resolver = new AvroDataContractResolver(true) }, Codec.Null); // Create a new sequential writer and use it to write Avro file using a block length of 100 using (var seq = new SequentialWriter <Stock>(writer, 100)) { foreach (var stock in this.ReadCsv(inputFilePath)) { seq.Write(stock); } } } catch (Exception e) { ReportError("Error while creating Avro file(s) from CSV file(s)" + e); } }
public override void WriteAvroFile(Stream stream) { using (IAvroWriter <object> streamWriter = AvroContainer.CreateGenericWriter(this.GetSchema().ToString(), stream, this.Codec)) { using (var sequentialWriter = new SequentialWriter <object>(streamWriter, AvroIntegrationWithHiveConfigurations.AvroBlockSize)) { foreach (LargeClass record in this.Expected) { sequentialWriter.Write(record.ToAvroRecord(this.GetSchema())); } } } }
public void Test_Serialization(bool withSchema) { var schema = Resources.SalesRecord; var avroFile = Path.ChangeExtension(Path.GetTempFileName(), "avro"); using (var fs = new FileStream(avroFile, FileMode.Create)) using (var writer = AvroContainer.CreateGenericWriter(schema, fs, Codec.Deflate)) using (var sequentialWriter = new SequentialWriter <object>(writer, 1000)) using (var sr = new StreamReader("1000 Sales Records.csv")) { sr.ReadLine(); while (!sr.EndOfStream) { var fields = sr.ReadLine()?.Split(','); var salesRecord = new SalesRecord { Region = fields[0], Country = fields[1], ItemType = fields[2], SalesChannel = Enum.Parse <SalesChannelType>(fields[3]), OrderPriority = Enum.Parse <OrderPriorityType>(fields[4]), OrderDate = DateTime.Parse(fields[5]), OrderId = int.Parse(fields[6]), ShipDate = DateTime.Parse(fields[7]), UnitsSold = int.Parse(fields[8]), UnitPrice = double.Parse(fields[9]), UnitCost = double.Parse(fields[10]), TotalRevenue = double.Parse(fields[11]), TotalCost = double.Parse(fields[12]), TotalProfit = double.Parse(fields[13]) }; sequentialWriter.Write(withSchema ? salesRecord.ToAvroRecord(schema) : salesRecord.ToAvroRecord()); } } var salesRecords = new List <SalesRecord>(); using (var fs = new FileStream(avroFile, FileMode.Open)) using (var reader = AvroContainer.CreateGenericReader(fs)) using (var sequentialReader = new SequentialReader <object>(reader)) { salesRecords.AddRange(sequentialReader.Objects.Cast <AvroRecord>() .Select(o => withSchema ? o.FromAvroRecord <SalesRecord>(schema) : o.FromAvroRecord <SalesRecord>())); } File.Delete(avroFile); }
public override void WriteAvroFile(Stream stream) { IAvroWriter <LargeClass> avroWriter = AvroContainer.CreateWriter <LargeClass>( stream, new AvroSerializerSettings { UsePosixTime = true }, this.Codec); using (var writer = new SequentialWriter <LargeClass>(avroWriter, AvroIntegrationWithHiveConfigurations.AvroBlockSize)) { foreach (LargeClass record in this.Expected) { writer.Write(record); } } }
public void TestReadParameter() { MemoryStream ms = new MemoryStream(); using (var writer = AvroContainer.CreateWriter <AddProductParameter>(ms, true, new AvroSerializerSettings(), Codec.Null)) using (var seqWriter = new SequentialWriter <AddProductParameter>(writer, 24)) { seqWriter.Write(new AddProductParameter { Location = address0, Product = product0 }); } ms.Seek(0, SeekOrigin.Begin); var result = new Dictionary <string, object>(); using (var omr = TestHelper.CreateMessageReader(ms, "avro/binary", AvroMediaTypeResolver.Instance, new EdmModel())) { var reader = omr.CreateODataParameterReader(AddProduct); while (reader.Read()) { switch (reader.State) { case ODataParameterReaderState.Value: result.Add(reader.Name, reader.Value); break; case ODataParameterReaderState.Entry: var entryReader = reader.CreateEntryReader(); while (entryReader.Read()) { if (entryReader.State == ODataReaderState.EntryEnd) { result.Add(reader.Name, entryReader.Item); break; } } break; } } } Assert.AreEqual(2, result.Count); Assert.IsTrue(TestHelper.ComplexValueEqual(complexValue0, (ODataComplexValue)result["Location"])); Assert.IsTrue(TestHelper.EntryEqual(entry0, (ODataEntry)result["Product"])); }
public void TestAvroSerializeUsingObjectContainersGenericRecord() { const string schema = @"{ ""type"":""record"", ""name"":""Microsoft.Hadoop.Avro.Specifications.SensorData"", ""fields"": [ { ""name"":""Location"", ""type"": { ""type"":""record"", ""name"":""Microsoft.Hadoop.Avro.Specifications.Location"", ""fields"": [ { ""name"":""Floor"", ""type"":""int"" }, { ""name"":""Room"", ""type"":""int"" } ] } }, { ""name"":""Value"", ""type"":""bytes"" } ] }"; //Create a generic serializer based on the schema var serializer = AvroSerializer.CreateGeneric(schema); var rootSchema = serializer.WriterSchema as RecordSchema; dynamic sensorData = CreateDynamicData(rootSchema); PerformanceHarness.Test(() => { using (var buffer = new MemoryStream()) { using (var avroWriter = AvroContainer.CreateGenericWriter(schema, buffer, Codec.Null)) { using (var writer = new SequentialWriter<object>(avroWriter, 24)) { writer.Write(sensorData); } } } }, "Avro serialization through generic records", 10000); }
public void TestReadAvroAsPrimitiveCollection() { const string Schema = @"{""type"":""array"", ""items"":""long""}"; var stream = new MemoryStream(); using (var writer = AvroContainer.CreateGenericWriter(Schema, stream, /*leave open*/ true, Codec.Null)) using (var seqWriter = new SequentialWriter <object>(writer, 24)) { // Serialize the data to stream using the sequential writer seqWriter.Write(new[] { 6L, 8 }); seqWriter.Flush(); } stream.Flush(); stream.Seek(0, SeekOrigin.Begin); var property = this.CreateODataInputContext(stream).ReadProperty(null, null); }
public void TestAvroSerializationThroughReflection() { SensorData sensorData = CreateTestData(); PerformanceHarness.Test(() => { using (var buffer = new MemoryStream()) { using (var avroWriter = AvroContainer.CreateWriter<SensorData>(buffer, Codec.Deflate)) { using (var writer = new SequentialWriter<SensorData>(avroWriter, 24)) { writer.Write(sensorData); } } } }, "Avro serialization through reflection", 10000); }
public override Stream GetContentStream(IEnumerable <ExpandoObject> records) { var result = new MemoryStream(); // We get a JSON schema for the passed type. Normally, the AvroSerializer expects a type passed that has DataContract/DataMember attributes, but I don't want users of this utility // to have to modify their POCOs in any way, hence building a schema here and proceeding with that. string schema = GetJsonSchema(); var serializer = AvroSerializer.CreateGeneric(schema); var rootSchema = serializer.WriterSchema as RecordSchema; // We'll write the Avro content to a memory stream using (var interim = new MemoryStream()) { // Avro serializer with deflate using (var avroWriter = AvroContainer.CreateGenericWriter(schema, interim, Codec.Deflate)) { using (var seqWriter = new SequentialWriter <object>(avroWriter, SYNCNUM)) { foreach (var record in records) { IDictionary <string, object> recordProperties = record as IDictionary <string, object>; dynamic avroRecord = new AvroRecord(rootSchema); foreach (KeyValuePair <string, object> recordKVP in recordProperties) { avroRecord[recordKVP.Key] = recordKVP.Value; } seqWriter.Write(avroRecord); } seqWriter.Flush(); interim.Seek(0, SeekOrigin.Begin); interim.CopyTo(result); } } } return(result); }
public void TestReadFeed() { MemoryStream ms = new MemoryStream(); IAvroWriter <Product[]> writer = null; try { writer = AvroContainer.CreateWriter <Product[]>(ms, true, new AvroSerializerSettings(), Codec.Null); using (var seqWriter = new SequentialWriter <Product[]>(writer, 24)) { seqWriter.Write(new[] { product1, product2, product3 }); } } finally { if (writer != null) { writer.Dispose(); } } ms.Seek(0, SeekOrigin.Begin); var entries = new List <ODataEntry>(); using (var omr = TestHelper.CreateMessageReader(ms, "avro/binary", AvroMediaTypeResolver.Instance)) { var reader = omr.CreateODataFeedReader(); while (reader.Read()) { if (reader.State == ODataReaderState.EntryEnd) { entries.Add((ODataEntry)reader.Item); } } } Assert.AreEqual(3, entries.Count); Assert.IsTrue(TestHelper.EntryEqual(entry1, entries[0])); Assert.IsTrue(TestHelper.EntryEqual(entry2, entries[1])); Assert.IsTrue(TestHelper.EntryEqual(entry3, entries[2])); }
/// <summary> /// Serialize a ClassHierarchy into a file /// </summary> /// <param name="c"></param> /// <param name="fileName"></param> public void ToFile(IClassHierarchy c, string fileName) { var avronNodeData = ToAvroNode(c); using (var buffer = new MemoryStream()) { using (var w = AvroContainer.CreateWriter<AvroNode>(buffer, Codec.Null)) { using (var writer = new SequentialWriter<AvroNode>(w, 24)) { writer.Write(avronNodeData); } } if (!WriteFile(buffer, fileName)) { var e = new ApplicationException("Error during file operation. Quitting method: " + fileName); Utilities.Diagnostics.Exceptions.Throw(e, LOGGER); } } }
/// <summary> /// Serialize a ClassHierarchy into a file /// </summary> /// <param name="c"></param> /// <param name="fileName"></param> public void ToFile(IClassHierarchy c, string fileName) { var avroNodeData = ToAvroNode(c); using (var buffer = new MemoryStream()) { using (var w = AvroContainer.CreateWriter <AvroNode>(buffer, Codec.Null)) { using (var writer = new SequentialWriter <AvroNode>(w, 24)) { writer.Write(avroNodeData); } } if (!WriteFile(buffer, fileName)) { var e = new TangApplicationException("Error during file operation. Quitting method: " + fileName); Utilities.Diagnostics.Exceptions.Throw(e, LOGGER); } } }
public void ToFile(IConfiguration c, string fileName) { var avronConfigurationData = ToAvroConfiguration(c); using (var buffer = new MemoryStream()) { using (var w = AvroContainer.CreateWriter <AvroConfiguration>(buffer, Codec.Null)) { using (var writer = new SequentialWriter <AvroConfiguration>(w, 24)) { // Serialize the data to stream using the sequential writer writer.Write(avronConfigurationData); } } if (!WriteFile(buffer, fileName)) { var e = new ApplicationException("Error during file operation. Quitting method: " + fileName); Org.Apache.REEF.Utilities.Diagnostics.Exceptions.Throw(e, LOGGER); } } }
public override void Output(IRow row, IUnstructuredWriter output) { // First Row if (this.writer == null) { this.writer = new SequentialWriter <object>(AvroContainer.CreateGenericWriter(this.avroSchema, output.BaseStream, Codec.Deflate), 24); this.serializer = AvroSerializer.CreateGeneric(this.avroSchema); } dynamic data = new AvroRecord(this.serializer.WriterSchema); data.station = row.Get <string>(row.Schema[0].Name); data.datekey = row.Get <int?>(row.Schema[1].Name); data.element = row.Get <string>(row.Schema[2].Name); data.value = (double?)row.Get <decimal?>(row.Schema[3].Name); data.mflag = row.Get <string>(row.Schema[4].Name); data.qflag = row.Get <string>(row.Schema[5].Name); data.sflag = row.Get <string>(row.Schema[6].Name); data.timekey = row.Get <int?>(row.Schema[7].Name); writer.Write(data); }
public void TestReadEntry() { MemoryStream ms = new MemoryStream(); IAvroWriter <Product> writer = null; try { writer = AvroContainer.CreateWriter <Product>(ms, true, new AvroSerializerSettings(), Codec.Null); using (var seqWriter = new SequentialWriter <Product>(writer, 24)) { seqWriter.Write(product0); } } finally { if (writer != null) { writer.Dispose(); } } ms.Seek(0, SeekOrigin.Begin); ODataEntry entry = null; using (var omr = TestHelper.CreateMessageReader(ms, "avro/binary", AvroMediaTypeResolver.Instance)) { var reader = omr.CreateODataEntryReader(); while (reader.Read()) { if (reader.State == ODataReaderState.EntryEnd) { entry = (ODataEntry)reader.Item; } } } Assert.IsTrue(TestHelper.EntryEqual(entry0, entry)); }
public void TestReadAvroAsODataEmptyFeed() { string Schema = @" {""type"":""array"", ""items"": { ""type"":""record"", ""name"":""TestNS.Person"", ""fields"": [ { ""name"":""Id"", ""type"":""int"" }, { ""name"":""Title"", ""type"":""string"" }, ] } }"; using (var stream = new MemoryStream()) { using (var writer = AvroContainer.CreateGenericWriter(Schema, stream, /*leave open*/ true, Codec.Null)) using (var seqWriter = new SequentialWriter <object>(writer, 24)) { seqWriter.Write(new AvroRecord[] { }); } stream.Seek(0, SeekOrigin.Begin); var avroReader = new ODataAvroReader(this.CreateODataInputContext(stream), true); Assert.AreEqual(ODataReaderState.Start, avroReader.State); Assert.IsTrue(avroReader.Read()); Assert.AreEqual(ODataReaderState.FeedStart, avroReader.State); Assert.IsTrue(avroReader.Read()); Assert.AreEqual(ODataReaderState.FeedEnd, avroReader.State); Assert.IsFalse(avroReader.Read()); Assert.AreEqual(ODataReaderState.Completed, avroReader.State); } }
public void TestReadError() { MemoryStream ms = new MemoryStream(); using (var writer = AvroContainer.CreateWriter <Error>(ms, true, new AvroSerializerSettings(), Codec.Null)) using (var seqWriter = new SequentialWriter <Error>(writer, 24)) { seqWriter.Write(new Error { ErrorCode = "500", Message = "Internal Error" }); } ms.Seek(0, SeekOrigin.Begin); ODataError error = null; using (var omr = TestHelper.CreateMessageReader(ms, "avro/binary", AvroMediaTypeResolver.Instance, new EdmModel(), true)) { error = omr.ReadError(); } Assert.IsNotNull(error); Assert.AreEqual("500", error.ErrorCode); Assert.AreEqual("Internal Error", error.Message); }
public void TestReadParameter() { MemoryStream ms = new MemoryStream(); using (var writer = AvroContainer.CreateWriter<AddProductParameter>(ms, true, new AvroSerializerSettings(), Codec.Null)) using (var seqWriter = new SequentialWriter<AddProductParameter>(writer, 24)) { seqWriter.Write(new AddProductParameter { Location = address0, Product = product0 }); } ms.Seek(0, SeekOrigin.Begin); var result = new Dictionary<string, object>(); using (var omr = TestHelper.CreateMessageReader(ms, "avro/binary", AvroMediaTypeResolver.Instance, new EdmModel())) { var reader = omr.CreateODataParameterReader(AddProduct); while (reader.Read()) { switch (reader.State) { case ODataParameterReaderState.Value: result.Add(reader.Name, reader.Value); break; case ODataParameterReaderState.Entry: var entryReader = reader.CreateEntryReader(); while (entryReader.Read()) { if (entryReader.State == ODataReaderState.EntryEnd) { result.Add(reader.Name, entryReader.Item); break; } } break; } } } Assert.AreEqual(2, result.Count); Assert.IsTrue(TestHelper.ComplexValueEqual(complexValue0, (ODataComplexValue)result["Location"])); Assert.IsTrue(TestHelper.EntryEqual(entry0, (ODataEntry)result["Product"])); }
public void ReadAvroAsParameter() { EdmEntityType personType = new EdmEntityType("TestNS", "Person"); personType.AddStructuralProperty("Id", EdmPrimitiveTypeKind.Int32); personType.AddStructuralProperty("Title", EdmPrimitiveTypeKind.String); var operation = new EdmAction("NS", "op1", null); operation.AddParameter("p1", EdmCoreModel.Instance.GetString(false)); operation.AddParameter("p2", new EdmEntityTypeReference(personType, false)); const string Schema = @"{ ""type"":""record"", ""name"":""NS.op1Parameter"", ""fields"": [ { ""name"":""p1"", ""type"":""string"" }, { ""name"":""p2"", ""type"": {""type"":""record"", ""name"":""TestNS.Person"", ""fields"": [ { ""name"":""Id"", ""type"":""int"" }, { ""name"":""Title"", ""type"":""string"" }, ] } } ] }"; var stream = new MemoryStream(); using (var writer = AvroContainer.CreateGenericWriter(Schema, stream, /*leave open*/true, Codec.Null)) using (var seqWriter = new SequentialWriter<object>(writer, 24)) { RecordSchema parameterSchema = (RecordSchema)AvroSerializer.CreateGeneric(Schema).WriterSchema; AvroRecord ar = new AvroRecord(parameterSchema); ar["p1"] = "dat"; var personSchema = parameterSchema.GetField("p2").TypeSchema; AvroRecord person = new AvroRecord(personSchema); person["Id"] = 5; person["Title"] = "per1"; ar["p2"] = person; seqWriter.Write(ar); seqWriter.Flush(); } stream.Flush(); stream.Seek(0, SeekOrigin.Begin); var reader = new ODataAvroParameterReader(this.CreateODataInputContext(stream), operation); Assert.AreEqual(ODataParameterReaderState.Start, reader.State); Assert.IsTrue(reader.Read()); Assert.AreEqual(ODataParameterReaderState.Value, reader.State); Assert.AreEqual("p1", reader.Name); Assert.AreEqual("dat", reader.Value); Assert.IsTrue(reader.Read()); Assert.AreEqual(ODataParameterReaderState.Entry, reader.State); Assert.AreEqual("p2", reader.Name); var ew = reader.CreateEntryReader(); Assert.AreEqual(ODataReaderState.Start, ew.State); Assert.IsTrue(ew.Read()); Assert.AreEqual(ODataReaderState.EntryStart, ew.State); Assert.IsTrue(ew.Read()); Assert.AreEqual(ODataReaderState.EntryEnd, ew.State); var entry = ew.Item as ODataEntry; Assert.IsFalse(ew.Read()); Assert.AreEqual(ODataReaderState.Completed, ew.State); Assert.IsNotNull(entry); var properties = entry.Properties.ToList(); Assert.AreEqual(2, properties.Count); Assert.AreEqual("Id", properties[0].Name); Assert.AreEqual(5, properties[0].Value); Assert.AreEqual("Title", properties[1].Name); Assert.AreEqual("per1", properties[1].Value); Assert.IsFalse(reader.Read()); Assert.AreEqual(ODataParameterReaderState.Completed, reader.State); }
public IEnumerable <T> WriteTo <T>(object writer, IEnumerable <object> records, Func <object, bool> predicate = null) { _sw = writer; IAvroWriter <T> sw1 = writer as IAvroWriter <T>; ChoGuard.ArgumentNotNull(sw1, "AvroWriter"); using (SequentialWriter <T> sw = new SequentialWriter <T>(sw1, 24)) { if (records == null) { yield break; } if (!BeginWrite.Value) { yield break; } CultureInfo prevCultureInfo = System.Threading.Thread.CurrentThread.CurrentCulture; System.Threading.Thread.CurrentThread.CurrentCulture = Configuration.Culture; object recOutput = null; var recEnum = records.GetEnumerator(); try { object notNullRecord = GetFirstNotNullRecord(recEnum); if (notNullRecord == null) { yield break; } foreach (object record in GetRecords(recEnum)) { _index++; if (TraceSwitch.TraceVerbose) { if (record is IChoETLNameableObject) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Writing [{0}] object...".FormatString(((IChoETLNameableObject)record).Name)); } else { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Writing [{0}] object...".FormatString(_index)); } } recOutput = record; if (record != null) { if (predicate == null || predicate(record)) { if (!RaiseBeforeRecordWrite(record, _index, ref recOutput)) { yield break; } if (recOutput == null) { continue; } else if (!(recOutput is T)) { continue; } try { if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.ObjectLevel) == ChoObjectValidationMode.ObjectLevel) { record.DoObjectLevelValidation(Configuration, Configuration.AvroRecordFieldConfigurations); } if (recOutput is T) { sw.Write((T)recOutput); if (!RaiseAfterRecordWrite(record, _index, recOutput)) { yield break; } } } catch (Exception ex) { ChoETLFramework.HandleException(ref ex); if (Configuration.ErrorMode == ChoErrorMode.IgnoreAndContinue) { ChoETLFramework.WriteLog(TraceSwitch.TraceError, "Error [{0}] found. Ignoring record...".FormatString(ex.Message)); } else if (Configuration.ErrorMode == ChoErrorMode.ReportAndContinue) { if (!RaiseRecordWriteError(record, _index, recOutput, ex)) { throw; } else { //ChoETLFramework.WriteLog(TraceSwitch.TraceError, "Error [{0}] found. Ignoring record...".FormatString(ex.Message)); } } else { throw; } } } } yield return((T)recOutput); if (Configuration.NotifyAfter > 0 && _index % Configuration.NotifyAfter == 0) { if (RaisedRowsWritten(_index)) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Abort requested."); yield break; } } } } finally { System.Threading.Thread.CurrentThread.CurrentCulture = prevCultureInfo; } } }
public void ReadAvroAsError() { const string Schema = @"{ ""type"":""record"", ""name"":""OData.Error"", ""fields"": [ { ""name"":""ErrorCode"", ""type"":""string"" }, { ""name"":""Message"" , ""type"":""string"" }, ] }"; var stream = new MemoryStream(); using (var writer = AvroContainer.CreateGenericWriter(Schema, stream, /*leave open*/true, Codec.Null)) using (var seqWriter = new SequentialWriter<object>(writer, 24)) { RecordSchema parameterSchema = (RecordSchema)AvroSerializer.CreateGeneric(Schema).WriterSchema; AvroRecord ar = new AvroRecord(parameterSchema); ar["ErrorCode"] = "e1"; ar["Message"] = "m1"; seqWriter.Write(ar); seqWriter.Flush(); } stream.Flush(); stream.Seek(0, SeekOrigin.Begin); var error = this.CreateODataInputContext(stream).ReadError(); Assert.AreEqual("e1", error.ErrorCode); Assert.AreEqual("m1", error.Message); }
public void SequentialReaderWriter_CreateReaderForSchemaWithNullableField() { var expected = new ClassWithNullableIntField { NullableIntField = 10 }; using (var writer = AvroContainer.CreateWriter<ClassWithNullableIntField>(this.resultStream, Codec.Deflate)) { using (var swriter = new SequentialWriter<ClassWithNullableIntField>(writer, 2)) { swriter.Write(expected); swriter.Write(expected); swriter.Write(expected); swriter.Write(expected); } } this.resultStream.Seek(0, SeekOrigin.Begin); using (var reader = AvroContainer.CreateReader<ClassWithNullableIntField>(this.resultStream)) { using (var sreader = new SequentialReader<ClassWithNullableIntField>(reader)) { foreach (var actual in sreader.Objects) { Assert.AreEqual(expected.NullableIntField, actual.NullableIntField); } } } }
public IEnumerable <T> WriteTo <T>(object writer, IEnumerable <object> records, Func <object, bool> predicate = null) { Configuration.Init(); if (records == null) { yield break; } var recEnum = records.GetEnumerator(); object notNullRecord = GetFirstNotNullRecord(recEnum); if (notNullRecord == null) { yield break; } DiscoverKnownTypes(notNullRecord); StreamWriter sw = null; if (writer is Lazy <StreamWriter> ) { var lsw = writer as Lazy <StreamWriter>; ChoGuard.ArgumentNotNull(lsw, "StreamWriter"); _sw = sw = lsw.Value; if (!Configuration.UseAvroSerializer) { if (_avroWriter == null) { _avroWriter = new SequentialWriter <T>(CreateAvroWriter <T>(sw), Configuration.SyncNumberOfObjects); } } else { _avroSerializer = CreateAvroSerializer <T>(); } } else { _avroWriter = writer as IAvroWriter <T>; if (_avroWriter == null) { throw new ChoParserException("Missing valid writer object passed."); } } if (!BeginWrite.Value) { yield break; } CultureInfo prevCultureInfo = System.Threading.Thread.CurrentThread.CurrentCulture; System.Threading.Thread.CurrentThread.CurrentCulture = Configuration.Culture; object recOutput = null; try { foreach (object record in GetRecords(recEnum)) { _index++; if (TraceSwitch.TraceVerbose) { if (record is IChoETLNameableObject) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Writing [{0}] object...".FormatString(((IChoETLNameableObject)record).Name)); } else { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Writing [{0}] object...".FormatString(_index)); } } recOutput = record; if (record != null) { if (predicate == null || predicate(record)) { if (!RaiseBeforeRecordWrite(record, _index, ref recOutput)) { yield break; } if (recOutput == null) { continue; } else if (!(recOutput is T)) { continue; } try { if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.ObjectLevel) == ChoObjectValidationMode.ObjectLevel) { record.DoObjectLevelValidation(Configuration, Configuration.AvroRecordFieldConfigurations); } if (recOutput is T) { if (recOutput is ChoDynamicObject) { recOutput = new Dictionary <string, object>((ChoDynamicObject)recOutput); } if (_sw != null) { if (Configuration.UseAvroSerializer) { IAvroSerializer <T> avroSerializer = _avroSerializer as IAvroSerializer <T>; avroSerializer.Serialize(sw.BaseStream, (T)(recOutput as object)); } else { SequentialWriter <T> avroWriter = _avroWriter as SequentialWriter <T>; avroWriter.Write((T)(recOutput as object)); } } else { SequentialWriter <T> avroWriter = _avroWriter as SequentialWriter <T>; avroWriter.Write((T)(recOutput as object)); } if (!RaiseAfterRecordWrite(record, _index, recOutput)) { yield break; } } } catch (Exception ex) { ChoETLFramework.HandleException(ref ex); if (Configuration.ErrorMode == ChoErrorMode.IgnoreAndContinue) { ChoETLFramework.WriteLog(TraceSwitch.TraceError, "Error [{0}] found. Ignoring record...".FormatString(ex.Message)); } else if (Configuration.ErrorMode == ChoErrorMode.ReportAndContinue) { if (!RaiseRecordWriteError(record, _index, recOutput, ex)) { throw; } else { //ChoETLFramework.WriteLog(TraceSwitch.TraceError, "Error [{0}] found. Ignoring record...".FormatString(ex.Message)); } } else { throw; } } } } yield return((T)recOutput); if (Configuration.NotifyAfter > 0 && _index % Configuration.NotifyAfter == 0) { if (RaisedRowsWritten(_index)) { ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Abort requested."); yield break; } } } } finally { System.Threading.Thread.CurrentThread.CurrentCulture = prevCultureInfo; } }
public void ToFile(IConfiguration c, string fileName) { var avronConfigurationData = ToAvroConfiguration(c); using (var buffer = new MemoryStream()) { using (var w = AvroContainer.CreateWriter<AvroConfiguration>(buffer, Codec.Null)) { using (var writer = new SequentialWriter<AvroConfiguration>(w, 24)) { // Serialize the data to stream using the sequential writer writer.Write(avronConfigurationData); } } if (!WriteFile(buffer, fileName)) { var e = new ApplicationException("Error during file operation. Quitting method: " + fileName); Org.Apache.REEF.Utilities.Diagnostics.Exceptions.Throw(e, LOGGER); } } }
public void TestReadAvroAsODataEntry() { const string Schema = @" { ""type"":""record"", ""name"":""TestNS.Person"", ""fields"": [ { ""name"":""Id"", ""type"":""int"" }, { ""name"":""Title"", ""type"":""string"" }, { ""name"":""Address"", ""type"":{ ""name"":""TestNS.Address"", ""type"":""record"", ""fields"":[ { ""name"":""ZipCode"", ""type"":""long"" }, ] } }, ] }"; var serializer = AvroSerializer.CreateGeneric(Schema); using (var stream = new MemoryStream()) { var expected = new AvroRecord(serializer.WriterSchema); expected["Id"] = -5; expected["Title"] = "set"; var cpxSchema = ((RecordSchema)serializer.WriterSchema).GetField("Address").TypeSchema; var cpx = new AvroRecord(cpxSchema); cpx["ZipCode"] = 5L; expected["Address"] = cpx; using (var writer = AvroContainer.CreateGenericWriter(Schema, stream, /*leave open*/true, Codec.Null)) { using (var streamWriter = new SequentialWriter<object>(writer, 24)) { // Serialize the data to stream using the sequential writer streamWriter.Write(expected); } } stream.Seek(0, SeekOrigin.Begin); var avroReader = new ODataAvroReader(this.CreateODataInputContext(stream), false); Assert.AreEqual(ODataReaderState.Start, avroReader.State); Assert.IsTrue(avroReader.Read()); Assert.AreEqual(ODataReaderState.EntryStart, avroReader.State); Assert.IsTrue(avroReader.Read()); Assert.AreEqual(ODataReaderState.EntryEnd, avroReader.State); var entry = avroReader.Item as ODataEntry; Assert.IsNotNull(entry); Assert.AreEqual("TestNS.Person", entry.TypeName); var properties = entry.Properties.ToList(); Assert.AreEqual(3, properties.Count); Assert.AreEqual("Id", properties[0].Name); Assert.AreEqual(-5, properties[0].Value); Assert.AreEqual("Title", properties[1].Name); Assert.AreEqual("set", properties[1].Value); Assert.AreEqual("Address", properties[2].Name); var address = properties[2].Value as ODataComplexValue; Assert.IsNotNull(address); Assert.AreEqual("TestNS.Address", address.TypeName); var zip = address.Properties.Single(); Assert.AreEqual(5L, zip.Value); Assert.IsFalse(avroReader.Read()); Assert.AreEqual(ODataReaderState.Completed, avroReader.State); } }
public void TestReadAvroAsPrimitiveCollection() { const string Schema = @"{""type"":""array"", ""items"":""long""}"; var stream = new MemoryStream(); using (var writer = AvroContainer.CreateGenericWriter(Schema, stream, /*leave open*/true, Codec.Null)) using (var seqWriter = new SequentialWriter<object>(writer, 24)) { // Serialize the data to stream using the sequential writer seqWriter.Write(new[] { 6L, 8 }); seqWriter.Flush(); } stream.Flush(); stream.Seek(0, SeekOrigin.Begin); var property = this.CreateODataInputContext(stream).ReadProperty(null, null); }
public void TestReadAvroAsODataEntry() { const string Schema = @" { ""type"":""record"", ""name"":""TestNS.Person"", ""fields"": [ { ""name"":""Id"", ""type"":""int"" }, { ""name"":""Title"", ""type"":""string"" }, { ""name"":""Address"", ""type"":{ ""name"":""TestNS.Address"", ""type"":""record"", ""fields"":[ { ""name"":""ZipCode"", ""type"":""long"" }, ] } }, ] }"; var serializer = AvroSerializer.CreateGeneric(Schema); using (var stream = new MemoryStream()) { var expected = new AvroRecord(serializer.WriterSchema); expected["Id"] = -5; expected["Title"] = "set"; var cpxSchema = ((RecordSchema)serializer.WriterSchema).GetField("Address").TypeSchema; var cpx = new AvroRecord(cpxSchema); cpx["ZipCode"] = 5L; expected["Address"] = cpx; using (var writer = AvroContainer.CreateGenericWriter(Schema, stream, /*leave open*/ true, Codec.Null)) { using (var streamWriter = new SequentialWriter <object>(writer, 24)) { // Serialize the data to stream using the sequential writer streamWriter.Write(expected); } } stream.Seek(0, SeekOrigin.Begin); var avroReader = new ODataAvroReader(this.CreateODataInputContext(stream), false); Assert.AreEqual(ODataReaderState.Start, avroReader.State); Assert.IsTrue(avroReader.Read()); Assert.AreEqual(ODataReaderState.EntryStart, avroReader.State); Assert.IsTrue(avroReader.Read()); Assert.AreEqual(ODataReaderState.EntryEnd, avroReader.State); var entry = avroReader.Item as ODataEntry; Assert.IsNotNull(entry); Assert.AreEqual("TestNS.Person", entry.TypeName); var properties = entry.Properties.ToList(); Assert.AreEqual(3, properties.Count); Assert.AreEqual("Id", properties[0].Name); Assert.AreEqual(-5, properties[0].Value); Assert.AreEqual("Title", properties[1].Name); Assert.AreEqual("set", properties[1].Value); Assert.AreEqual("Address", properties[2].Name); var address = properties[2].Value as ODataComplexValue; Assert.IsNotNull(address); Assert.AreEqual("TestNS.Address", address.TypeName); var zip = address.Properties.Single(); Assert.AreEqual(5L, zip.Value); Assert.IsFalse(avroReader.Read()); Assert.AreEqual(ODataReaderState.Completed, avroReader.State); } }
public void Test_ComplexType_Serialization_To_Stream() { dynamic instance = new ShapeBasket { Shapes = new List <IShape> { // the schema for Circle allows for Names and Radius to be null new Circle(), new Circle { Name = "Red Dot", Radius = 15, Color = BasicColor.Red }, new Square { Name = "Blue Square", Width = 20, Color = BasicColor.Blue }, new Triangle { Name = "Bermuda Triangle", SideA = 10, SideB = 20, SideC = 30, Color = BasicColor.Indigo }, new StrangeShape { Name = "Blob", ChildShape = new StrangeShape { Name = "Child Blob", ChildShape = new Square { Name = "Child Square", Width = 50 } } } } }; var schema = Resources.ShapeBasket_v1_0; var avroFile = Path.GetTempFileName(); // serialize using (var fs = new FileStream(avroFile, FileMode.Create)) { using var writer = AvroContainer.CreateGenericWriter(schema, fs, Codec.Deflate); using var sequentialWriter = new SequentialWriter <object>(writer, 1); sequentialWriter.Write(((object)instance).ToAvroRecord(schema)); } dynamic target; // deserialize using (var fs = new FileStream(avroFile, FileMode.Open)) { using var reader = AvroContainer.CreateGenericReader(fs); using var sequentialReader = new SequentialReader <object>(reader); target = sequentialReader.Objects.Cast <AvroRecord>().Select(r => r.FromAvroRecord <ShapeBasket>()) .FirstOrDefault(); } File.Delete(avroFile); Assert.AreEqual(instance.Shapes[0].Name, target.Shapes[0].Name); Assert.AreEqual(instance.Shapes[1].Name, target.Shapes[1].Name); Assert.AreEqual(instance.Shapes[1].Radius, target.Shapes[1].Radius); Assert.AreEqual(instance.Shapes[1].Color, target.Shapes[1].Color); Assert.AreEqual(instance.Shapes[2].Name, target.Shapes[2].Name); Assert.AreEqual(instance.Shapes[2].Width, target.Shapes[2].Width); Assert.AreEqual(instance.Shapes[2].Color, target.Shapes[2].Color); Assert.AreEqual(instance.Shapes[3].Name, target.Shapes[3].Name); Assert.AreEqual(instance.Shapes[3].SideA, target.Shapes[3].SideA); Assert.AreEqual(instance.Shapes[3].SideB, target.Shapes[3].SideB); Assert.AreEqual(instance.Shapes[3].SideC, target.Shapes[3].SideC); Assert.AreEqual(instance.Shapes[3].Color, target.Shapes[3].Color); Assert.AreEqual(instance.Shapes[4].Name, target.Shapes[4].Name); Assert.AreEqual(instance.Shapes[4].ChildShape.Name, target.Shapes[4].ChildShape.Name); Assert.AreEqual(instance.Shapes[4].ChildShape.ChildShape.Name, target.Shapes[4].ChildShape.ChildShape.Name); Assert.AreEqual(instance.Shapes[4].ChildShape.ChildShape.Width, target.Shapes[4].ChildShape.ChildShape.Width); }
public void TestReadAvroAsCollection() { const string Schema = @"{""type"":""array"", ""items"":""long""}"; var stream = new MemoryStream(); using (var writer = AvroContainer.CreateGenericWriter(Schema, stream, /*leave open*/true, Codec.Null)) using (var seqWriter = new SequentialWriter<object>(writer, 24)) { // Serialize the data to stream using the sequential writer seqWriter.Write(new[] { 6L, 8 }); seqWriter.Flush(); } stream.Flush(); stream.Seek(0, SeekOrigin.Begin); var reader = new ODataAvroCollectionReader(this.CreateODataInputContext(stream)); Assert.AreEqual(ODataCollectionReaderState.Start, reader.State); Assert.IsTrue(reader.Read()); Assert.AreEqual(ODataCollectionReaderState.CollectionStart, reader.State); Assert.IsTrue(reader.Read()); Assert.AreEqual(ODataCollectionReaderState.Value, reader.State); Assert.AreEqual(6L, (long)reader.Item); Assert.IsTrue(reader.Read()); Assert.AreEqual(ODataCollectionReaderState.Value, reader.State); Assert.IsTrue(reader.Read()); Assert.AreEqual(ODataCollectionReaderState.CollectionEnd, reader.State); Assert.IsFalse(reader.Read()); Assert.AreEqual(ODataCollectionReaderState.Completed, reader.State); }
static void Main(string[] args) { //Read data from files CloudStorageAccount storageAccount = CloudStorageAccount.Parse(CloudConfigurationManager.GetSetting("storageAccountName")); CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient(); CloudBlobContainer container = blobClient.GetContainerReference(CloudConfigurationManager.GetSetting("eventHubName")); var blobInputInitialPath = CloudConfigurationManager.GetSetting("eventHubName") + "/" + CloudConfigurationManager.GetSetting("eventHubName") + "/"; var blobOutputInitialPath = "outputdata/"; var dt = DateTime.UtcNow.AddHours(-1); var dtStr = dt.ToString("yyyy/MM/dd/HH/"); var ehPartitions = 8; MemoryStream stream = new MemoryStream(); MemoryStream outStream = new MemoryStream(); var outputSchemaJson = "{\"type\":\"record\",\"name\":\"EventDataOut\",\"namespace\":\"Microsoft.ServiceBus.Messaging\",\"fields\":[{\"name\":\"Offset\",\"type\":\"string\"},{\"name\":\"EnqueuedTimeUtc\",\"type\":\"string\"}, {\"name\":\"Body\",\"type\":[\"null\",\"bytes\"]}]}"; var outputSchema = Microsoft.Hadoop.Avro.Schema.TypeSchema.Create(outputSchemaJson); var writer = AvroContainer.CreateGenericWriter(outputSchemaJson, outStream, Codec.Null); for (int i = 0; i < ehPartitions; i++) { var blobPath = blobInputInitialPath + i.ToString() + "/" + dtStr; foreach (IListBlobItem item in container.ListBlobs(blobPath, true)) { CloudBlockBlob cbb = (CloudBlockBlob)item; if (cbb.Properties.Length > 200) { Console.Out.WriteLine("Downloading File: " + cbb.Name); stream = new MemoryStream(); cbb.DownloadToStream(stream); //Setting stream position to zero so we can read from the start stream.Position = 0; var reader = AvroContainer.CreateGenericReader(stream); //Reading the stream AvroRecord avroRecord = new AvroRecord(outputSchema); //Creating a new Avro Record to be written while (reader.MoveNext()) { var streamWriter = new SequentialWriter <object>(writer, reader.Current.Objects.Count()); var testCount = reader.Current.Objects.Count(); Console.Out.WriteLine("Reading downloaded Objects. Count: " + testCount); foreach (dynamic record in reader.Current.Objects) { var eventData = new EventData(record); var eventDataOut = new EventDataOut(eventData); dynamic outRecord = new AvroRecord(outputSchema); outRecord.Offset = eventData.Offset; outRecord.EnqueuedTimeUtc = eventData.EnqueuedTimeUtc.ToString(); outRecord.Body = eventData.Body; streamWriter.Write(outRecord); //Writing the Avro Record } } } } } //Test to Read the Avro File we just wrote outStream.Position = 0; var testreader = AvroContainer.CreateGenericReader(outStream); while (testreader.MoveNext()) { var testCount = testreader.Current.Objects.Count(); Console.Out.WriteLine("Reading Written Objects. Count: " + testCount); foreach (dynamic record in testreader.Current.Objects) { var eventData = new EventDataOut(record); Console.Out.WriteLine("Test - EventData Body: " + Encoding.UTF8.GetString(eventData.Body)); } } outStream.Position = 0; var outputBlob = container.GetBlockBlobReference(blobOutputInitialPath + dtStr + "output.avro"); outputBlob.UploadFromStream(outStream); Console.Out.WriteLine("Test Done"); }
public void TestReadParameterWithFeed() { MemoryStream ms = new MemoryStream(); using (var writer = AvroContainer.CreateWriter<GetMaxIdParameter>(ms, true, new AvroSerializerSettings(), Codec.Null)) using (var seqWriter = new SequentialWriter<GetMaxIdParameter>(writer, 24)) { seqWriter.Write(new GetMaxIdParameter { Products = new List<Product>() { product0, product1 } }); } ms.Seek(0, SeekOrigin.Begin); var result = new Dictionary<string, object>(); using (var omr = TestHelper.CreateMessageReader(ms, "avro/binary", AvroMediaTypeResolver.Instance, new EdmModel())) { var reader = omr.CreateODataParameterReader(GetMaxId); while (reader.Read()) { switch (reader.State) { case ODataParameterReaderState.Value: result.Add(reader.Name, reader.Value); break; case ODataParameterReaderState.Entry: var entryReader = reader.CreateEntryReader(); while (entryReader.Read()) { if (entryReader.State == ODataReaderState.EntryEnd) { result.Add(reader.Name, entryReader.Item); break; } } break; case ODataParameterReaderState.Feed: var feedReader = reader.CreateFeedReader(); IList<ODataEntry> entryList = new List<ODataEntry>(); while (feedReader.Read()) { if (feedReader.State == ODataReaderState.EntryEnd) { entryList.Add((ODataEntry)feedReader.Item); } } result.Add(reader.Name, entryList); break; } } } Assert.AreEqual(1, result.Count); var feed = result["Products"] as IList<ODataEntry>; Assert.IsNotNull(feed); Assert.AreEqual(2, feed.Count); Assert.IsTrue(TestHelper.EntryEqual(entry0, feed[0])); Assert.IsTrue(TestHelper.EntryEqual(entry1, feed[1])); }
public void TestReadError() { MemoryStream ms = new MemoryStream(); using (var writer = AvroContainer.CreateWriter<Error>(ms, true, new AvroSerializerSettings(), Codec.Null)) using (var seqWriter = new SequentialWriter<Error>(writer, 24)) { seqWriter.Write(new Error { ErrorCode = "500", Message = "Internal Error" }); } ms.Seek(0, SeekOrigin.Begin); ODataError error = null; using (var omr = TestHelper.CreateMessageReader(ms, "avro/binary", AvroMediaTypeResolver.Instance, new EdmModel(), true)) { error = omr.ReadError(); } Assert.IsNotNull(error); Assert.AreEqual("500", error.ErrorCode); Assert.AreEqual("Internal Error", error.Message); }
public void ReadAvroAsParameter() { EdmEntityType personType = new EdmEntityType("TestNS", "Person"); personType.AddStructuralProperty("Id", EdmPrimitiveTypeKind.Int32); personType.AddStructuralProperty("Title", EdmPrimitiveTypeKind.String); var operation = new EdmAction("NS", "op1", null); operation.AddParameter("p1", EdmCoreModel.Instance.GetString(false)); operation.AddParameter("p2", new EdmEntityTypeReference(personType, false)); const string Schema = @"{ ""type"":""record"", ""name"":""NS.op1Parameter"", ""fields"": [ { ""name"":""p1"", ""type"":""string"" }, { ""name"":""p2"", ""type"": {""type"":""record"", ""name"":""TestNS.Person"", ""fields"": [ { ""name"":""Id"", ""type"":""int"" }, { ""name"":""Title"", ""type"":""string"" }, ] } } ] }"; var stream = new MemoryStream(); using (var writer = AvroContainer.CreateGenericWriter(Schema, stream, /*leave open*/ true, Codec.Null)) using (var seqWriter = new SequentialWriter <object>(writer, 24)) { RecordSchema parameterSchema = (RecordSchema)AvroSerializer.CreateGeneric(Schema).WriterSchema; AvroRecord ar = new AvroRecord(parameterSchema); ar["p1"] = "dat"; var personSchema = parameterSchema.GetField("p2").TypeSchema; AvroRecord person = new AvroRecord(personSchema); person["Id"] = 5; person["Title"] = "per1"; ar["p2"] = person; seqWriter.Write(ar); seqWriter.Flush(); } stream.Flush(); stream.Seek(0, SeekOrigin.Begin); var reader = new ODataAvroParameterReader(this.CreateODataInputContext(stream), operation); Assert.AreEqual(ODataParameterReaderState.Start, reader.State); Assert.IsTrue(reader.Read()); Assert.AreEqual(ODataParameterReaderState.Value, reader.State); Assert.AreEqual("p1", reader.Name); Assert.AreEqual("dat", reader.Value); Assert.IsTrue(reader.Read()); Assert.AreEqual(ODataParameterReaderState.Entry, reader.State); Assert.AreEqual("p2", reader.Name); var ew = reader.CreateEntryReader(); Assert.AreEqual(ODataReaderState.Start, ew.State); Assert.IsTrue(ew.Read()); Assert.AreEqual(ODataReaderState.EntryStart, ew.State); Assert.IsTrue(ew.Read()); Assert.AreEqual(ODataReaderState.EntryEnd, ew.State); var entry = ew.Item as ODataEntry; Assert.IsFalse(ew.Read()); Assert.AreEqual(ODataReaderState.Completed, ew.State); Assert.IsNotNull(entry); var properties = entry.Properties.ToList(); Assert.AreEqual(2, properties.Count); Assert.AreEqual("Id", properties[0].Name); Assert.AreEqual(5, properties[0].Value); Assert.AreEqual("Title", properties[1].Name); Assert.AreEqual("per1", properties[1].Value); Assert.IsFalse(reader.Read()); Assert.AreEqual(ODataParameterReaderState.Completed, reader.State); }
static int Main(string[] args) { bool showHelp = false; bool breakWrongParameterUsage = false; string connectionString = string.Empty; string sqlQuery = string.Empty; string sqlQueryFile = string.Empty; string outputFile = string.Empty; string format = string.Empty; string schema = string.Empty; string schemaFile = string.Empty; var p = new OptionSet() { { "c|connection=", "the {SQL CONNECTION STRING} for the SQL Server to connect with.", v => connectionString = v }, { "q|query=", "the {SQL QUERY} to be used.", v => sqlQuery = v }, { "qf|queryFile=", "the {SQL QUERY FILE} to beh used.", v => sqlQueryFile = v }, { "of|outputFile=", "the fiel where to save the export file.", v => outputFile = v }, { "f|format=", "the {FORMAT} to use for the export file.", v => format = v }, { "s|schema=", "the schema; required for the AVRO file format", v => schema = v }, { "sf|schemaFile=", "the schema file; required for the AVRO file format", v => schemaFile = v }, { "h|help", "show this message and exit", v => showHelp = v != null }, }; // parse args List <string> extra; try { extra = p.Parse(args); } catch (OptionException e) { Console.Write($"{ApplicationName}: "); Console.WriteLine(e.Message); Console.WriteLine("Try `sqldataexport --help` for more information."); return(1); // Exit Error } if (showHelp) { ShowHelp(p); return(0); // Exit OK } // validate args if (string.IsNullOrEmpty(connectionString)) { Console.WriteLine("SQL connection string not given"); breakWrongParameterUsage = true; } if (string.IsNullOrEmpty(outputFile)) { Console.WriteLine("Output file not given"); breakWrongParameterUsage = true; } if (!string.IsNullOrEmpty(sqlQueryFile)) { if (!File.Exists(sqlQueryFile)) { Console.WriteLine("SQL query file {0} not found", sqlQueryFile); breakWrongParameterUsage = true; } try { sqlQuery = File.ReadAllText(sqlQueryFile); } catch (Exception e) { Console.WriteLine("Error while loading SQL query file: " + e.Message); return(1); // Exit Error } } if (string.IsNullOrEmpty(sqlQuery)) { Console.WriteLine("SQL query not given or empty"); } switch (format.ToLower()) { case "avro": if (!string.IsNullOrEmpty(schemaFile)) { if (!File.Exists(schemaFile)) { Console.WriteLine("Schema file {0} not found", schemaFile); breakWrongParameterUsage = true; } try { schema = File.ReadAllText(schemaFile); } catch (Exception e) { Console.WriteLine("Error while loading schema file: " + e.Message); return(1); // Exit Error } } if (string.IsNullOrEmpty(schema)) { Console.WriteLine("Schema not given or empty"); breakWrongParameterUsage = true; } break; case "orc": throw new NotImplementedException("https://github.com/ddrinka/ApacheOrcDotNet/blob/master/src/ApacheOrcDotNet.WriterTest.App/Program.cs"); break; default: Console.WriteLine("Invalid format {0}", format); breakWrongParameterUsage = true; break; } if (breakWrongParameterUsage) { Console.WriteLine($"Try `{ApplicationName} --help` for more information."); return(1); // Exit Error } // run actual application var connection = new SqlConnection(connectionString); var command = new SqlCommand(sqlQuery, connection); ReportStatus("Open connection ...", true); connection.Open(); try { ReportStatus("Execute SQL command ..."); var dataReader = command.ExecuteReader(); if (dataReader.Read()) { ReportStatus("Process rows ..."); using (var file = File.Create(outputFile)) { // see also: https://code.msdn.microsoft.com/Serialize-data-with-the-86055923/sourcecode?fileId=111532&pathId=931764448 var serializer = AvroSerializer.CreateGeneric(schema); var rootSchema = serializer.WriterSchema; var container = AvroContainer.CreateGenericWriter(schema, file, Codec.Null); var schemaFields = GetSchemaFields(schema); using (var wr = new SequentialWriter <object>(container, 24)) { int rowCount = 0; do { var record = new AvroRecord(rootSchema); for (int i = 0; i < dataReader.FieldCount; i++) { object newValue = dataReader.IsDBNull(i) ? null : dataReader.GetValue(i); dataReader.GetFieldType(i); record[i] = CastToAvroValue(newValue, schemaFields[i]); } wr.Write(record); if (++rowCount % statusReportPerRows == 0) { ReportStatus($"Process rows ({rowCount:G}) ..."); } } while (dataReader.Read()); ReportStatus($"Processed {rowCount:G} rows"); } } } } catch (Exception e) { Console.Write("Application Exception: "); Console.WriteLine(e.Message); return(1); // Exit Error } finally { connection.Close(); } Console.WriteLine("Done"); return(0); // Exit OK }
public void TestReadAvroAsODataEmptyFeed() { string Schema = @" {""type"":""array"", ""items"": { ""type"":""record"", ""name"":""TestNS.Person"", ""fields"": [ { ""name"":""Id"", ""type"":""int"" }, { ""name"":""Title"", ""type"":""string"" }, ] } }"; using (var stream = new MemoryStream()) { using (var writer = AvroContainer.CreateGenericWriter(Schema, stream, /*leave open*/true, Codec.Null)) using (var seqWriter = new SequentialWriter<object>(writer, 24)) { seqWriter.Write(new AvroRecord[] { }); } stream.Seek(0, SeekOrigin.Begin); var avroReader = new ODataAvroReader(this.CreateODataInputContext(stream), true); Assert.AreEqual(ODataReaderState.Start, avroReader.State); Assert.IsTrue(avroReader.Read()); Assert.AreEqual(ODataReaderState.FeedStart, avroReader.State); Assert.IsTrue(avroReader.Read()); Assert.AreEqual(ODataReaderState.FeedEnd, avroReader.State); Assert.IsFalse(avroReader.Read()); Assert.AreEqual(ODataReaderState.Completed, avroReader.State); } }
public static void SerializeDeserializeObjectUsingReflectionStream() { string blobName = "aaa/avrotest/test008"; CloudBlobClient client = new Microsoft.WindowsAzure.Storage.Blob.CloudBlobClient( new Uri("http://hanzstorage.blob.core.windows.net"), new Microsoft.WindowsAzure.Storage.Auth.StorageCredentials( "hanzstorage", "w9TEpvGTusvFlGAdCoWdDrwqLzy6er0Zm5YKdDD0YTkQdOj3WufeVrgd2c8q8amLR0o6xD0tBChcIIA+DCgxXA==" )); CloudBlobContainer container = client.GetContainerReference("hanzhdi"); CloudBlockBlob blockBlob = container.GetBlockBlobReference(blobName); foreach (var md in blockBlob.Metadata) { Console.WriteLine("{0} {1}", md.Key, md.Value); } Console.WriteLine("Serializing Sample Data Set USING REFLECTION\n"); AvroBlobAppender.AvroBlobAppenderWriter <MyDataType> writer = new AvroBlobAppender.AvroBlobAppenderWriter <MyDataType>(blockBlob, false, AvroSerializer.Create <MyDataType>(), Codec.Null); Microsoft.Hadoop.Avro.Container.SequentialWriter <MyDataType> sequentialWriter = new SequentialWriter <MyDataType>(writer, 10000); List <MyDataType> myDataList = MyDataType.GenerateData(555, 10); foreach (var myData in myDataList) { sequentialWriter.Write(myData); } sequentialWriter.Flush(); sequentialWriter.Dispose(); #region commented code //blockBlob.DownloadToFile(blobName, FileMode.Create); //using (Stream stream = File.OpenRead(blobName)) //{ // Microsoft.Hadoop.Avro.Container.SequentialReader<MyDataType> reader = // new Microsoft.Hadoop.Avro.Container.SequentialReader<MyDataType>(AvroContainer.CreateReader<MyDataType>(stream)); // List<MyDataType> actuals = reader.Objects.ToList(); // Console.WriteLine("Number of objects: {0}", actuals.Count); // for (int i = 0; i < actuals.Count; i++) // { // var actual = actuals[i]; // MyDataType exp = null; // switch (i) // { // case 0: // exp = expected; // break; // case 1: // exp = expected2; // break; // default: // Console.WriteLine("No expected for object {0}", i); // continue; // } // Console.WriteLine("Result of Data Set Identity Comparison is {0}", Program.Equal(exp, actual)); // } //} #endregion }
public void TestReadAvroAsODataFeed() { const string Schema = @" {""type"":""array"", ""items"": { ""type"":""record"", ""name"":""TestNS.Person"", ""fields"": [ { ""name"":""Id"", ""type"":""int"" }, { ""name"":""Title"", ""type"":""string"" }, ] } }"; var serializer = AvroSerializer.CreateGeneric(Schema); using (var stream = new MemoryStream()) { var arraySchema = (ArraySchema)serializer.WriterSchema; var recordSchema = arraySchema.ItemSchema; var rec1 = new AvroRecord(recordSchema); rec1["Id"] = 1; rec1["Title"] = "s1"; var rec2 = new AvroRecord(recordSchema); rec2["Id"] = 2; rec2["Title"] = "s2"; var array = new[] { rec1, rec2 }; using (var writer = AvroContainer.CreateGenericWriter(Schema, stream, /*leave open*/ true, Codec.Null)) using (var seqWriter = new SequentialWriter <object>(writer, 24)) { seqWriter.Write(array); } stream.Seek(0, SeekOrigin.Begin); var avroReader = new ODataAvroReader(this.CreateODataInputContext(stream), true); Assert.AreEqual(ODataReaderState.Start, avroReader.State); Assert.IsTrue(avroReader.Read()); Assert.AreEqual(ODataReaderState.FeedStart, avroReader.State); Assert.IsTrue(avroReader.Read()); // Entry 1 Assert.AreEqual(ODataReaderState.EntryStart, avroReader.State); Assert.IsTrue(avroReader.Read()); Assert.AreEqual(ODataReaderState.EntryEnd, avroReader.State); var entry = avroReader.Item as ODataEntry; Assert.IsNotNull(entry); var properties = entry.Properties.ToList(); Assert.AreEqual(2, properties.Count); Assert.AreEqual("Id", properties[0].Name); Assert.AreEqual(1, properties[0].Value); Assert.AreEqual("Title", properties[1].Name); Assert.AreEqual("s1", properties[1].Value); // Entry 2 Assert.IsTrue(avroReader.Read()); Assert.AreEqual(ODataReaderState.EntryStart, avroReader.State); Assert.IsTrue(avroReader.Read()); Assert.AreEqual(ODataReaderState.EntryEnd, avroReader.State); entry = avroReader.Item as ODataEntry; Assert.IsNotNull(entry); properties = entry.Properties.ToList(); Assert.AreEqual(2, properties.Count); Assert.AreEqual("Id", properties[0].Name); Assert.AreEqual(2, properties[0].Value); Assert.AreEqual("Title", properties[1].Name); Assert.AreEqual("s2", properties[1].Value); Assert.IsTrue(avroReader.Read()); Assert.AreEqual(ODataReaderState.FeedEnd, avroReader.State); Assert.IsFalse(avroReader.Read()); Assert.AreEqual(ODataReaderState.Completed, avroReader.State); } }
public void TestReadParameterWithFeed() { MemoryStream ms = new MemoryStream(); using (var writer = AvroContainer.CreateWriter <GetMaxIdParameter>(ms, true, new AvroSerializerSettings(), Codec.Null)) using (var seqWriter = new SequentialWriter <GetMaxIdParameter>(writer, 24)) { seqWriter.Write(new GetMaxIdParameter { Products = new List <Product>() { product0, product1 } }); } ms.Seek(0, SeekOrigin.Begin); var result = new Dictionary <string, object>(); using (var omr = TestHelper.CreateMessageReader(ms, "avro/binary", AvroMediaTypeResolver.Instance, new EdmModel())) { var reader = omr.CreateODataParameterReader(GetMaxId); while (reader.Read()) { switch (reader.State) { case ODataParameterReaderState.Value: result.Add(reader.Name, reader.Value); break; case ODataParameterReaderState.Entry: var entryReader = reader.CreateEntryReader(); while (entryReader.Read()) { if (entryReader.State == ODataReaderState.EntryEnd) { result.Add(reader.Name, entryReader.Item); break; } } break; case ODataParameterReaderState.Feed: var feedReader = reader.CreateFeedReader(); IList <ODataEntry> entryList = new List <ODataEntry>(); while (feedReader.Read()) { if (feedReader.State == ODataReaderState.EntryEnd) { entryList.Add((ODataEntry)feedReader.Item); } } result.Add(reader.Name, entryList); break; } } } Assert.AreEqual(1, result.Count); var feed = result["Products"] as IList <ODataEntry>; Assert.IsNotNull(feed); Assert.AreEqual(2, feed.Count); Assert.IsTrue(TestHelper.EntryEqual(entry0, feed[0])); Assert.IsTrue(TestHelper.EntryEqual(entry1, feed[1])); }
public void TestReadAvroAsODataFeed() { const string Schema = @" {""type"":""array"", ""items"": { ""type"":""record"", ""name"":""TestNS.Person"", ""fields"": [ { ""name"":""Id"", ""type"":""int"" }, { ""name"":""Title"", ""type"":""string"" }, ] } }"; var serializer = AvroSerializer.CreateGeneric(Schema); using (var stream = new MemoryStream()) { var arraySchema = (ArraySchema)serializer.WriterSchema; var recordSchema = arraySchema.ItemSchema; var rec1 = new AvroRecord(recordSchema); rec1["Id"] = 1; rec1["Title"] = "s1"; var rec2 = new AvroRecord(recordSchema); rec2["Id"] = 2; rec2["Title"] = "s2"; var array = new[] { rec1, rec2 }; using (var writer = AvroContainer.CreateGenericWriter(Schema, stream, /*leave open*/true, Codec.Null)) using (var seqWriter = new SequentialWriter<object>(writer, 24)) { seqWriter.Write(array); } stream.Seek(0, SeekOrigin.Begin); var avroReader = new ODataAvroReader(this.CreateODataInputContext(stream), true); Assert.AreEqual(ODataReaderState.Start, avroReader.State); Assert.IsTrue(avroReader.Read()); Assert.AreEqual(ODataReaderState.FeedStart, avroReader.State); Assert.IsTrue(avroReader.Read()); // Entry 1 Assert.AreEqual(ODataReaderState.EntryStart, avroReader.State); Assert.IsTrue(avroReader.Read()); Assert.AreEqual(ODataReaderState.EntryEnd, avroReader.State); var entry = avroReader.Item as ODataEntry; Assert.IsNotNull(entry); var properties = entry.Properties.ToList(); Assert.AreEqual(2, properties.Count); Assert.AreEqual("Id", properties[0].Name); Assert.AreEqual(1, properties[0].Value); Assert.AreEqual("Title", properties[1].Name); Assert.AreEqual("s1", properties[1].Value); // Entry 2 Assert.IsTrue(avroReader.Read()); Assert.AreEqual(ODataReaderState.EntryStart, avroReader.State); Assert.IsTrue(avroReader.Read()); Assert.AreEqual(ODataReaderState.EntryEnd, avroReader.State); entry = avroReader.Item as ODataEntry; Assert.IsNotNull(entry); properties = entry.Properties.ToList(); Assert.AreEqual(2, properties.Count); Assert.AreEqual("Id", properties[0].Name); Assert.AreEqual(2, properties[0].Value); Assert.AreEqual("Title", properties[1].Name); Assert.AreEqual("s2", properties[1].Value); Assert.IsTrue(avroReader.Read()); Assert.AreEqual(ODataReaderState.FeedEnd, avroReader.State); Assert.IsFalse(avroReader.Read()); Assert.AreEqual(ODataReaderState.Completed, avroReader.State); } }
/// <summary> /// Reads CSV file data and converts records to instances of Stock class then serializes them using Reflection. /// Stock type definition is created from JSON schema using Microsoft Avro Library Code Generation utility /// </summary> /// <param name="inputFilePath">The input file path.</param> /// <param name="outputStream">The output stream.</param> private void SerializeCsv(string inputFilePath, Stream outputStream) { try { // Create an Avro writer using Null Codec var writer = AvroContainer.CreateWriter<Stock>( outputStream, new AvroSerializerSettings { Resolver = new AvroDataContractResolver(true) }, Codec.Null); // Create a new sequential writer and use it to write Avro file using a block length of 100 using (var seq = new SequentialWriter<Stock>(writer, 100)) { foreach (var stock in this.ReadCsv(inputFilePath)) { seq.Write(stock); } } } catch (Exception e) { ReportError("Error while creating Avro file(s) from CSV file(s)" + e); } }
public void TestReadFeed() { MemoryStream ms = new MemoryStream(); IAvroWriter<Product[]> writer = null; try { writer = AvroContainer.CreateWriter<Product[]>(ms, true, new AvroSerializerSettings(), Codec.Null); using (var seqWriter = new SequentialWriter<Product[]>(writer, 24)) { seqWriter.Write(new[] { product1, product2, product3 }); } } finally { if (writer != null) writer.Dispose(); } ms.Seek(0, SeekOrigin.Begin); var entries = new List<ODataEntry>(); using (var omr = TestHelper.CreateMessageReader(ms, "avro/binary", AvroMediaTypeResolver.Instance)) { var reader = omr.CreateODataFeedReader(); while (reader.Read()) { if (reader.State == ODataReaderState.EntryEnd) { entries.Add((ODataEntry)reader.Item); } } } Assert.AreEqual(3, entries.Count); Assert.IsTrue(TestHelper.EntryEqual(entry1, entries[0])); Assert.IsTrue(TestHelper.EntryEqual(entry2, entries[1])); Assert.IsTrue(TestHelper.EntryEqual(entry3, entries[2])); }
public void TestReadEntry() { MemoryStream ms = new MemoryStream(); IAvroWriter<Product> writer = null; try { writer = AvroContainer.CreateWriter<Product>(ms, true, new AvroSerializerSettings(), Codec.Null); using (var seqWriter = new SequentialWriter<Product>(writer, 24)) { seqWriter.Write(product0); } } finally { if (writer != null) writer.Dispose(); } ms.Seek(0, SeekOrigin.Begin); ODataEntry entry = null; using (var omr = TestHelper.CreateMessageReader(ms, "avro/binary", AvroMediaTypeResolver.Instance)) { var reader = omr.CreateODataEntryReader(); while (reader.Read()) { if (reader.State == ODataReaderState.EntryEnd) { entry = (ODataEntry)reader.Item; } } } Assert.IsTrue(TestHelper.EntryEqual(entry0, entry)); }
public void TestReadCollection() { MemoryStream ms = new MemoryStream(); using (var writer = AvroContainer.CreateWriter<string[]>(ms, true, new AvroSerializerSettings(), Codec.Null)) using (var seqWriter = new SequentialWriter<string[]>(writer, 24)) { seqWriter.Write(new[] { "p9t", "cjk如" }); } ms.Seek(0, SeekOrigin.Begin); var values = new List<string>(); using (var omr = TestHelper.CreateMessageReader(ms, "avro/binary", AvroMediaTypeResolver.Instance)) { var reader = omr.CreateODataCollectionReader(); while (reader.Read()) { if (reader.State == ODataCollectionReaderState.Value) { values.Add((string)reader.Item); } } } Assert.AreEqual(2, values.Count); Assert.AreEqual("p9t", values[0]); Assert.AreEqual("cjk如", values[1]); }