public bool Export()
{
using (var buffer = new MemoryStream())
{
//Data is compressed using the Deflate codec.
using (var w = AvroContainer.CreateWriter <Models.Claim>(buffer, Codec.Deflate))
{
using (var writer = new SequentialWriter <Models.Claim>(w, 24))
{
// Serialize the data to stream by using the sequential writer
values.ToList().ForEach(writer.Write);
}
}
//Save stream to file
Console.WriteLine("Saving serialized data to file...");
if (!WriteFile(buffer))
{
Console.WriteLine("Error during file operation. Quitting method");
}
else
{
IsImported = true;
}
}
return(IsImported);
}
public void ReadAvroAsError()
{
const string Schema = @"{
""type"":""record"",
""name"":""OData.Error"",
""fields"":
[
{ ""name"":""ErrorCode"", ""type"":""string"" },
{ ""name"":""Message"" , ""type"":""string"" },
]
}";
var stream = new MemoryStream();
using (var writer = AvroContainer.CreateGenericWriter(Schema, stream, /*leave open*/ true, Codec.Null))
using (var seqWriter = new SequentialWriter <object>(writer, 24))
{
RecordSchema parameterSchema = (RecordSchema)AvroSerializer.CreateGeneric(Schema).WriterSchema;
AvroRecord ar = new AvroRecord(parameterSchema);
ar["ErrorCode"] = "e1";
ar["Message"] = "m1";
seqWriter.Write(ar);
seqWriter.Flush();
}
stream.Flush();
stream.Seek(0, SeekOrigin.Begin);
var error = this.CreateODataInputContext(stream).ReadError();
Assert.AreEqual("e1", error.ErrorCode);
Assert.AreEqual("m1", error.Message);
}
public TypeSchema UpdateSchema(object value, IEdmType edmType, bool collection = false)
{
TypeSchema schema = null;
if (edmType != null)
{
try
{
schema = ODataAvroSchemaGen.GetSchemaFromModel(edmType);
}
catch (ApplicationException) { }
}
if (schema == null)
{
if (value == null)
{
return null;
}
schema = ODataAvroSchemaGen.GetSchema(value);
}
TypeSchema single = AvroSerializer.CreateGeneric(schema.ToString()).WriterSchema;
if (collection)
{
schema = ODataAvroSchemaGen.GetArraySchema(schema);
single = ((ArraySchema)AvroSerializer.CreateGeneric(schema.ToString()).WriterSchema).ItemSchema;
}
var writer = AvroContainer.CreateGenericWriter(schema.ToString(), stream, true, Codec.Null);
this.seqWriter = new SequentialWriter<object>(writer, 16);
return single;
}
private static async Task PutBlockAsync(String blockId, CloudBlockBlob blob, int start, int end, CancellationToken token)
{
var initialMessages = Enumerable
.Range(start, end)
.Select(x => new MyDataType(x));
using (var buffer = new MemoryStream())
{
using (var w = AvroContainer.CreateWriter <MyDataType>(buffer, Codec.Null))
{
using (var writer = new SequentialWriter <MyDataType>(w, 24))
{
initialMessages
.ToList()
.ForEach(writer.Write);
}
}
await buffer.FlushAsync(token);
//var blockId = "AAAAA";
buffer.Seek(0, SeekOrigin.Begin);
await blob.PutBlockAsync(blockId, buffer, string.Empty, token);
//await blob.PutBlockAsync(blockId, new MemoryStream(Encoding.Default.GetBytes("Hello! " + start.ToString() + " - " + end.ToString() + "\r\n")), string.Empty, token);
//await blob.PutBlockAsync(blockId, buffer, null);
//await blob.PutBlockAsync(blockId, new MemoryStream(Encoding.Default.GetBytes("Hello! " + start.ToString() + " - " + end.ToString() + "\r\n")), null);
}
}
/// <summary>
/// Reads CSV file data and converts records to instances of Stock class then serializes them using Reflection.
/// Stock type definition is created from JSON schema using Microsoft Avro Library Code Generation utility
/// </summary>
/// <param name="inputFilePath">The input file path.</param>
/// <param name="outputStream">The output stream.</param>
private void SerializeCsv(string inputFilePath, Stream outputStream)
{
try
{
// Create an Avro writer using Null Codec
var writer = AvroContainer.CreateWriter <Stock>(
outputStream,
new AvroSerializerSettings {
Resolver = new AvroDataContractResolver(true)
},
Codec.Null);
// Create a new sequential writer and use it to write Avro file using a block length of 100
using (var seq = new SequentialWriter <Stock>(writer, 100))
{
foreach (var stock in this.ReadCsv(inputFilePath))
{
seq.Write(stock);
}
}
}
catch (Exception e)
{
ReportError("Error while creating Avro file(s) from CSV file(s)" + e);
}
}
static void Main(string[] args)
{
SampleData obj = new SampleData();
using (FileStream xmlStream = new FileStream(@"SampleData.xml", FileMode.Open))
{
using (XmlReader xmlReader = XmlReader.Create(xmlStream))
{
obj.ReadXml(xmlReader);
}
}
// hi ahmed
var serializer = new AvroSerializer(typeof(Person));
StreamWriter schemafs = new StreamWriter(File.Create(@"SampleData.avsc"));
schemafs.Write(serializer.Schema.ToString());
schemafs.Close();
var resultStream = new FileStream(@"SampleData.avro", FileMode.OpenOrCreate);
var writer = new SequentialWriter <Person>(resultStream, serializer, Codec.Create("null"), 24);
int i;
foreach (DataRow item in obj.Tables["Person"].Rows)
{
var p = new Person {
First_Name = item["Name"].ToString(), Street = item["Street"].ToString(), City = item["City"].ToString(), State = item["State"].ToString()
};
writer.Write(p);
}
writer.Flush();
}
public static Tuple <byte[], byte[]> CreateAvroHeaderAndBlocks <TSchema>(List <TSchema> dataList, string schema, string codec)
{
byte[] avroHeaderAndBlocksByteArray = null;
byte[] syncMarker = new byte[16];
new Random().NextBytes(syncMarker);
var objContainerHeader = new ObjectContainerHeader(syncMarker)
{
CodecName = codec,
Schema = schema
};
using (var buffer = new MemoryStream())
{
// TODO refactor the above AvroBlobAppenderWriter class to expose the syncMarker
AvroBlobAppender.AvroBlobAppenderWriter <TSchema> avroWriter = new AvroBlobAppender.AvroBlobAppenderWriter <TSchema>(null, false, AvroSerializer.Create <TSchema>(), Codec.Null);
syncMarker = avroWriter.syncMarker;
//using (var avroWriter = AvroContainer.CreateWriter<TSchema>(buffer, Codec.Null))
{
using (var sequentialWriter = new SequentialWriter <TSchema>(avroWriter, 24))
{
dataList.ToList().ForEach(sequentialWriter.Write);
}
}
buffer.Seek(0, SeekOrigin.Begin);
avroHeaderAndBlocksByteArray = buffer.ToArray();
}
return(new Tuple <byte[], byte[]>(avroHeaderAndBlocksByteArray, syncMarker));
}
public void SequentialWriter_MicrosoftWriterApacherReaderOfDictionary()
{
var expected = new List <ContainingDictionaryClass <string, string> >();
for (var i = 0; i < 7; i++)
{
expected.Add(ContainingDictionaryClass <string, string> .Create(
new Dictionary <string, string>
{
{ "testkey" + i, "testvalue" + i }
}));
}
var w = AvroContainer.CreateWriter <ContainingDictionaryClass <string, string> >(this.resultStream, Codec.Deflate);
using (var writer = new SequentialWriter <ContainingDictionaryClass <string, string> >(w, 2))
{
expected.ForEach(writer.Write);
}
this.resultStream.Seek(0, SeekOrigin.Begin);
var reader = DataFileReader <GenericRecord> .OpenReader(this.resultStream);
var actual = new List <GenericRecord>(reader);
Assert.AreEqual(expected.Count, actual.Count);
for (var i = 0; i < expected.Count; ++i)
{
var actualValue = actual[i]["Property"] as Dictionary <string, object>;
Assert.IsNotNull(actualValue);
Assert.AreEqual(actualValue["testkey" + i] as string, expected[i].Property["testkey" + i]);
}
}
public void SequentialReaderWriter_Reset()
{
var expected = new List <ClassOfListOfGuid>();
for (var i = 0; i < 7; i++)
{
expected.Add(ClassOfListOfGuid.Create(true));
}
var w = AvroContainer.CreateWriter <ClassOfListOfGuid>(this.resultStream, Codec.Deflate);
using (var writer = new SequentialWriter <ClassOfListOfGuid>(w, 3))
{
expected.ForEach(writer.Write);
}
this.resultStream.Seek(0, SeekOrigin.Begin);
var r = AvroContainer.CreateReader <ClassOfListOfGuid>(this.resultStream);
using (var reader = new SequentialReader <ClassOfListOfGuid>(r))
{
Assert.IsTrue(expected.SequenceEqual(reader.Objects));
var enumerator = (IEnumerator)reader.Objects.GetEnumerator();
Assert.IsFalse(enumerator.MoveNext());
}
}
public void SequentialWriter_MicrosoftWriterApacheReader()
{
var expected = new List <ClassOfInt>();
for (var i = 0; i < 7; i++)
{
expected.Add(ClassOfInt.Create(true));
}
var w = AvroContainer.CreateWriter <ClassOfInt>(this.resultStream, Codec.Deflate);
using (var writer = new SequentialWriter <ClassOfInt>(w, 2))
{
expected.ForEach(writer.Write);
}
this.resultStream.Seek(0, SeekOrigin.Begin);
var reader = DataFileReader <GenericRecord> .OpenReader(this.resultStream);
var actual = new List <GenericRecord>(reader);
for (var i = 0; i < expected.Count; ++i)
{
Assert.AreEqual(expected[i].PrimitiveInt, actual[i]["PrimitiveInt"]);
}
}
public void TestReadCollection()
{
MemoryStream ms = new MemoryStream();
using (var writer = AvroContainer.CreateWriter <string[]>(ms, true, new AvroSerializerSettings(), Codec.Null))
using (var seqWriter = new SequentialWriter <string[]>(writer, 24))
{
seqWriter.Write(new[] { "p9t", "cjk如" });
}
ms.Seek(0, SeekOrigin.Begin);
var values = new List <string>();
using (var omr = TestHelper.CreateMessageReader(ms, "avro/binary", AvroMediaTypeResolver.Instance))
{
var reader = omr.CreateODataCollectionReader();
while (reader.Read())
{
if (reader.State == ODataCollectionReaderState.Value)
{
values.Add((string)reader.Item);
}
}
}
Assert.AreEqual(2, values.Count);
Assert.AreEqual("p9t", values[0]);
Assert.AreEqual("cjk如", values[1]);
}
public void SequentialReaderWriter_SerializeHugeObject()
{
var single = new SimpleFlatClass
{
StringField = new string('a', 16254),
ByteArrayField = Encoding.ASCII.GetBytes(new string('b', 65666)),
ZeroByteArrayField = Encoding.ASCII.GetBytes(new string('c', 128344))
};
var expected = new List <SimpleFlatClass>
{
single,
single,
single,
};
var w = AvroContainer.CreateWriter <SimpleFlatClass>(this.resultStream, new AvroSerializerSettings {
Resolver = new AvroDataContractResolver(true)
}, Codec.Null);
using (var writer = new SequentialWriter <SimpleFlatClass>(w, 24))
{
expected.ForEach(writer.Write);
}
this.resultStream.Seek(0, SeekOrigin.Begin);
var r = AvroContainer.CreateReader <SimpleFlatClass>(this.resultStream, true, new AvroSerializerSettings {
Resolver = new AvroDataContractResolver(true)
}, new CodecFactory());
using (var reader = new SequentialReader <SimpleFlatClass>(r))
{
Assert.IsTrue(expected.SequenceEqual(reader.Objects));
}
}
public void SequentialReaderWriter_CreateReaderForSchemaWithNullableField()
{
var expected = new ClassWithNullableIntField {
NullableIntField = 10
};
using (var writer = AvroContainer.CreateWriter <ClassWithNullableIntField>(this.resultStream, Codec.Deflate))
{
using (var swriter = new SequentialWriter <ClassWithNullableIntField>(writer, 2))
{
swriter.Write(expected);
swriter.Write(expected);
swriter.Write(expected);
swriter.Write(expected);
}
}
this.resultStream.Seek(0, SeekOrigin.Begin);
using (var reader = AvroContainer.CreateReader <ClassWithNullableIntField>(this.resultStream))
{
using (var sreader = new SequentialReader <ClassWithNullableIntField>(reader))
{
foreach (var actual in sreader.Objects)
{
Assert.AreEqual(expected.NullableIntField, actual.NullableIntField);
}
}
}
}
public void TestReadInvalidAvroInput()
{
string Schema = @"
{""type"":""array"",
""items"":
{
""type"":""record"",
""name"":""TestNS.Person"",
""fields"":
[
{ ""name"":""Id"", ""type"":""int"" },
{ ""name"":""Title"", ""type"":""string"" },
]
}
}";
using (var stream = new MemoryStream())
{
using (var writer = AvroContainer.CreateGenericWriter(Schema, stream, /*leave open*/ true, Codec.Null))
using (var seqWriter = new SequentialWriter <object>(writer, 24))
{
}
stream.Seek(0, SeekOrigin.Begin);
var avroReader = new ODataAvroReader(this.CreateODataInputContext(stream), true);
Assert.AreEqual(ODataReaderState.Start, avroReader.State);
Assert.IsFalse(avroReader.Read());
Assert.AreEqual(ODataReaderState.Exception, avroReader.State);
}
}
public void TestReadAvroAsCollection()
{
const string Schema = @"{""type"":""array"", ""items"":""long""}";
var stream = new MemoryStream();
using (var writer = AvroContainer.CreateGenericWriter(Schema, stream, /*leave open*/ true, Codec.Null))
using (var seqWriter = new SequentialWriter <object>(writer, 24))
{
// Serialize the data to stream using the sequential writer
seqWriter.Write(new[] { 6L, 8 });
seqWriter.Flush();
}
stream.Flush();
stream.Seek(0, SeekOrigin.Begin);
var reader = new ODataAvroCollectionReader(this.CreateODataInputContext(stream));
Assert.AreEqual(ODataCollectionReaderState.Start, reader.State);
Assert.IsTrue(reader.Read());
Assert.AreEqual(ODataCollectionReaderState.CollectionStart, reader.State);
Assert.IsTrue(reader.Read());
Assert.AreEqual(ODataCollectionReaderState.Value, reader.State);
Assert.AreEqual(6L, (long)reader.Item);
Assert.IsTrue(reader.Read());
Assert.AreEqual(ODataCollectionReaderState.Value, reader.State);
Assert.IsTrue(reader.Read());
Assert.AreEqual(ODataCollectionReaderState.CollectionEnd, reader.State);
Assert.IsFalse(reader.Read());
Assert.AreEqual(ODataCollectionReaderState.Completed, reader.State);
}
public static void Main(string[] args)
{
string line = Environment.NewLine;
string fileName = "Messages.avro";
string filePath = null;
if (Environment.NewLine.Contains("\r"))
{
filePath = new DirectoryInfo(".") + @"\" + fileName;
}
else
{
filePath = new DirectoryInfo(".") + @"/" + fileName;
}
List <TestMsg> msgs = new List <TestMsg>();
msgs.Add(new AvroTestApp.TestMsg(1, 189.12));
msgs.Add(new AvroTestApp.TestMsg(2, 345.94));
using (var dataStream = new FileStream(filePath, FileMode.Create))
{
using (var avroWriter = AvroContainer.CreateWriter <TestMsg>(dataStream, Codec.Deflate))
{
using (var seqWriter = new SequentialWriter <TestMsg>(avroWriter, msgs.Count))
{
msgs.ForEach(seqWriter.Write);
}
}
dataStream.Dispose();
}
}
public void SequentialWriter_AddMetadataWithNullValue()
{
var writer = AvroContainer.CreateWriter <ClassOfInt>(this.resultStream, Codec.Null);
using (var sequentialWriter = new SequentialWriter <ClassOfInt>(writer, 2))
{
sequentialWriter.AddMetadata("Key", null);
}
}
public void SequentialWriter_AddMetadataWithNullKey()
{
var writer = AvroContainer.CreateWriter <ClassOfInt>(this.resultStream, Codec.Null);
using (var sequentialWriter = new SequentialWriter <ClassOfInt>(writer, 2))
{
sequentialWriter.AddMetadata(null, Utilities.GetRandom <byte[]>(false));
}
}
public void SequentialGenericWritingReading_RecursiveRecord()
{
const string StringSchema = @"{
""type"":""record"",
""name"":""Microsoft.Hadoop.Avro.Tests.Recursive"",
""fields"":[
{""name"":""IntField"",""type"":""int""},
{""name"":""RecursiveField"",""type"":[
""null"",
""Microsoft.Hadoop.Avro.Tests.Recursive""
]
}
]}";
using (var stream = new MemoryStream())
{
var serializer = AvroSerializer.CreateGeneric(StringSchema);
using (var streamWriter = AvroContainer.CreateGenericWriter(StringSchema, stream, Codec.Null))
{
using (var writer = new SequentialWriter <object>(streamWriter, 24))
{
var expected = new List <AvroRecord>();
var random = new Random(93);
for (int i = 0; i < 10; i++)
{
dynamic record = new AvroRecord(serializer.WriterSchema);
record.IntField = random.Next();
record.RecursiveField =
new AvroRecord(
((serializer.ReaderSchema as RecordSchema).GetField("RecursiveField").TypeSchema as UnionSchema).Schemas[1]);
record.RecursiveField.IntField = random.Next();
record.RecursiveField.RecursiveField = null;
expected.Add(record);
}
expected.ForEach(writer.Write);
writer.Flush();
stream.Seek(0, SeekOrigin.Begin);
var streamReader = AvroContainer.CreateReader <Recursive>(stream, true, this.dataContractSettings, new CodecFactory());
using (var reader = new SequentialReader <Recursive>(streamReader))
{
var j = 0;
foreach (var avroRecord in reader.Objects)
{
Assert.Equal(expected[j]["IntField"], avroRecord.IntField);
Assert.Equal(((dynamic)expected[j]["RecursiveField"])["IntField"], avroRecord.RecursiveField.IntField);
Assert.Equal(
((dynamic)expected[j++]["RecursiveField"])["RecursiveField"], avroRecord.RecursiveField.RecursiveField);
}
}
}
}
}
}
public void SequentialWriter_AddMetadataWithNullValue()
{
Assert.Throws <ArgumentNullException>(() =>
{
var writer = AvroContainer.CreateWriter <ClassOfInt>(this.resultStream, Codec.Null);
using (var sequentialWriter = new SequentialWriter <ClassOfInt>(writer, 2))
{
sequentialWriter.AddMetadata("Key", null);
}
}
);
}
public void TestSerializeDeserializeUsingSequentialContainers()
{
//CreateDeserializerOnly a new AvroSerializer instance and specify a custom serialization strategy AvroDataContractResolver
//for handling only properties attributed with DataContract/DateMember.
var settings = new AvroSerializerSettings();
//CreateDeserializerOnly a new buffer
using (var buffer = new MemoryStream())
{
//CreateDeserializerOnly sample data
var testData = new List <SensorData>
{
new SensorData {
Value = new byte[] { 1, 2, 3, 4, 5 }, Position = new Location {
Room = 243, Floor = 1
}
},
new SensorData {
Value = new byte[] { 6, 7, 8, 9 }, Position = new Location {
Room = 244, Floor = 1
}
}
};
//CreateDeserializerOnly a SequentialWriter instance for type SensorData which can serialize a sequence of SensorData objects to stream
using (var w = AvroContainer.CreateWriter <SensorData>(buffer, settings, Codec.Null))
{
using (var writer = new SequentialWriter <SensorData>(w, 24))
{
// Serialize the data to stream using the sequential writer
testData.ForEach(writer.Write);
}
}
//Prepare the stream for deserializing the data
buffer.Seek(0, SeekOrigin.Begin);
//CreateDeserializerOnly a SequentialReader for type SensorData which will derserialize all serialized objects from the given stream
//It allows iterating over the deserialized objects because it implements IEnumerable<T> interface
using (var reader = new SequentialReader <SensorData>(
AvroContainer.CreateReader <SensorData>(buffer, true, settings, new CodecFactory())))
{
var results = reader.Objects;
//Finally, verify that deserialized data matches the original ones
var pairs = testData.Zip(results, (serialized, deserialized) => new { expected = serialized, actual = deserialized });
foreach (var pair in pairs)
{
Assert.True(this.Equal(pair.expected, pair.actual));
}
}
}
}
public override void WriteAvroFile(Stream stream)
{
using (IAvroWriter <object> streamWriter = AvroContainer.CreateGenericWriter(this.GetSchema().ToString(), stream, this.Codec))
{
using (var sequentialWriter = new SequentialWriter <object>(streamWriter, AvroIntegrationWithHiveConfigurations.AvroBlockSize))
{
foreach (LargeClass record in this.Expected)
{
sequentialWriter.Write(record.ToAvroRecord(this.GetSchema()));
}
}
}
}
public void SequentialGenericWritingReading_SimpleRecord()
{
const string StringSchema = @"{
""type"":""record"",
""name"":""Microsoft.Hadoop.Avro.Tests.ClassOfInt"",
""fields"":
[
{
""name"":""PrimitiveInt"",
""type"":""int""
}
]
}";
using (var stream = new MemoryStream())
{
var serializer = AvroSerializer.CreateGeneric(StringSchema);
using (var streamWriter = AvroContainer.CreateGenericWriter(StringSchema, stream, Codec.Null))
{
using (var writer = new SequentialWriter<object>(streamWriter, 24))
{
var expected = new List<AvroRecord>();
var random = new Random(113);
for (int i = 0; i < 10; i++)
{
dynamic record = new AvroRecord(serializer.WriterSchema);
record.PrimitiveInt = random.Next();
expected.Add(record);
}
expected.ForEach(writer.Write);
writer.Flush();
stream.Seek(0, SeekOrigin.Begin);
var streamReader = AvroContainer.CreateReader<ClassOfInt>(stream, true, this.dataContractSettings, new CodecFactory());
using (var reader = new SequentialReader<ClassOfInt>(streamReader))
{
var j = 0;
foreach (var avroRecord in reader.Objects)
{
Assert.AreEqual(expected[j++]["PrimitiveInt"], avroRecord.PrimitiveInt);
}
}
}
}
}
}
public void SequentialGenericWritingReading_SimpleRecord()
{
const string StringSchema = @"{
""type"":""record"",
""name"":""Microsoft.Hadoop.Avro.Tests.ClassOfInt"",
""fields"":
[
{
""name"":""PrimitiveInt"",
""type"":""int""
}
]
}";
using (var stream = new MemoryStream())
{
var serializer = AvroSerializer.CreateGeneric(StringSchema);
using (var streamWriter = AvroContainer.CreateGenericWriter(StringSchema, stream, Codec.Null))
{
using (var writer = new SequentialWriter <object>(streamWriter, 24))
{
var expected = new List <AvroRecord>();
var random = new Random(113);
for (int i = 0; i < 10; i++)
{
dynamic record = new AvroRecord(serializer.WriterSchema);
record.PrimitiveInt = random.Next();
expected.Add(record);
}
expected.ForEach(writer.Write);
writer.Flush();
stream.Seek(0, SeekOrigin.Begin);
var streamReader = AvroContainer.CreateReader <ClassOfInt>(stream, true, this.dataContractSettings, new CodecFactory());
using (var reader = new SequentialReader <ClassOfInt>(streamReader))
{
var j = 0;
foreach (var avroRecord in reader.Objects)
{
Assert.Equal(expected[j++]["PrimitiveInt"], avroRecord.PrimitiveInt);
}
}
}
}
}
}
public void Test_Serialization(bool withSchema)
{
var schema = Resources.SalesRecord;
var avroFile = Path.ChangeExtension(Path.GetTempFileName(), "avro");
using (var fs = new FileStream(avroFile, FileMode.Create))
using (var writer = AvroContainer.CreateGenericWriter(schema, fs, Codec.Deflate))
using (var sequentialWriter = new SequentialWriter <object>(writer, 1000))
using (var sr = new StreamReader("1000 Sales Records.csv"))
{
sr.ReadLine();
while (!sr.EndOfStream)
{
var fields = sr.ReadLine()?.Split(',');
var salesRecord = new SalesRecord
{
Region = fields[0],
Country = fields[1],
ItemType = fields[2],
SalesChannel = Enum.Parse <SalesChannelType>(fields[3]),
OrderPriority = Enum.Parse <OrderPriorityType>(fields[4]),
OrderDate = DateTime.Parse(fields[5]),
OrderId = int.Parse(fields[6]),
ShipDate = DateTime.Parse(fields[7]),
UnitsSold = int.Parse(fields[8]),
UnitPrice = double.Parse(fields[9]),
UnitCost = double.Parse(fields[10]),
TotalRevenue = double.Parse(fields[11]),
TotalCost = double.Parse(fields[12]),
TotalProfit = double.Parse(fields[13])
};
sequentialWriter.Write(withSchema ? salesRecord.ToAvroRecord(schema) : salesRecord.ToAvroRecord());
}
}
var salesRecords = new List <SalesRecord>();
using (var fs = new FileStream(avroFile, FileMode.Open))
using (var reader = AvroContainer.CreateGenericReader(fs))
using (var sequentialReader = new SequentialReader <object>(reader))
{
salesRecords.AddRange(sequentialReader.Objects.Cast <AvroRecord>()
.Select(o => withSchema ? o.FromAvroRecord <SalesRecord>(schema) : o.FromAvroRecord <SalesRecord>()));
}
File.Delete(avroFile);
}
public override void WriteAvroFile(Stream stream)
{
IAvroWriter <LargeClass> avroWriter = AvroContainer.CreateWriter <LargeClass>(
stream,
new AvroSerializerSettings {
UsePosixTime = true
},
this.Codec);
using (var writer = new SequentialWriter <LargeClass>(avroWriter, AvroIntegrationWithHiveConfigurations.AvroBlockSize))
{
foreach (LargeClass record in this.Expected)
{
writer.Write(record);
}
}
}
public void TestAvroSerializeUsingObjectContainersGenericRecord()
{
const string schema = @"{
""type"":""record"",
""name"":""Microsoft.Hadoop.Avro.Specifications.SensorData"",
""fields"":
[
{
""name"":""Location"",
""type"":
{
""type"":""record"",
""name"":""Microsoft.Hadoop.Avro.Specifications.Location"",
""fields"":
[
{ ""name"":""Floor"", ""type"":""int"" },
{ ""name"":""Room"", ""type"":""int"" }
]
}
},
{ ""name"":""Value"", ""type"":""bytes"" }
]
}";
//Create a generic serializer based on the schema
var serializer = AvroSerializer.CreateGeneric(schema);
var rootSchema = serializer.WriterSchema as RecordSchema;
dynamic sensorData = CreateDynamicData(rootSchema);
PerformanceHarness.Test(() =>
{
using (var buffer = new MemoryStream())
{
using (var avroWriter = AvroContainer.CreateGenericWriter(schema, buffer, Codec.Null))
{
using (var writer = new SequentialWriter<object>(avroWriter, 24))
{
writer.Write(sensorData);
}
}
}
}, "Avro serialization through generic records", 10000);
}
public static void UseAvroPushToS3Bucket(this SampleContext context)
{
context.PushToS3Bucket = (int numOfFiles, int numOfRecords) =>
{
//Get compression method
var codec = Codec.Null;
if (!String.IsNullOrEmpty(context.CompressionMethod))
{
if (context.CompressionMethod.ToLower() == "deflate")
{
codec = Codec.Deflate;
}
}
using (var client = new SampleS3Client(context.Format, context.S3BucketName,
context.S3BucketPath, Amazon.RegionEndpoint.USEast1))
{
for (int i = 0; i < numOfFiles; i++)
{
DateTime randDateTime = SampleData.RandDate();
var testData = new List <LogEntry>();
testData.AddRange(SampleData.GetBunchOfData(numOfRecords, randDateTime));
using (MemoryStream buffer = new MemoryStream())
{
using (var w = AvroContainer.CreateWriter <LogEntry>(buffer, codec))
{
using (var writer = new SequentialWriter <LogEntry>(w, 24))
{
testData.ForEach(writer.Write);
}
}
//Objects are push sync. to keep the order.
client.PutObject(buffer, randDateTime);
}
SampleContext.ClearConsoleLine();
Console.Write($"\r{(i + 1f) / (float)numOfFiles,6:P2}");
}
}
};
}
public void TestReadParameter()
{
MemoryStream ms = new MemoryStream();
using (var writer = AvroContainer.CreateWriter <AddProductParameter>(ms, true, new AvroSerializerSettings(), Codec.Null))
using (var seqWriter = new SequentialWriter <AddProductParameter>(writer, 24))
{
seqWriter.Write(new AddProductParameter {
Location = address0, Product = product0
});
}
ms.Seek(0, SeekOrigin.Begin);
var result = new Dictionary <string, object>();
using (var omr = TestHelper.CreateMessageReader(ms, "avro/binary", AvroMediaTypeResolver.Instance, new EdmModel()))
{
var reader = omr.CreateODataParameterReader(AddProduct);
while (reader.Read())
{
switch (reader.State)
{
case ODataParameterReaderState.Value:
result.Add(reader.Name, reader.Value);
break;
case ODataParameterReaderState.Entry:
var entryReader = reader.CreateEntryReader();
while (entryReader.Read())
{
if (entryReader.State == ODataReaderState.EntryEnd)
{
result.Add(reader.Name, entryReader.Item);
break;
}
}
break;
}
}
}
Assert.AreEqual(2, result.Count);
Assert.IsTrue(TestHelper.ComplexValueEqual(complexValue0, (ODataComplexValue)result["Location"]));
Assert.IsTrue(TestHelper.EntryEqual(entry0, (ODataEntry)result["Product"]));
}
public void TestReadAvroAsPrimitiveCollection()
{
const string Schema = @"{""type"":""array"", ""items"":""long""}";
var stream = new MemoryStream();
using (var writer = AvroContainer.CreateGenericWriter(Schema, stream, /*leave open*/ true, Codec.Null))
using (var seqWriter = new SequentialWriter <object>(writer, 24))
{
// Serialize the data to stream using the sequential writer
seqWriter.Write(new[] { 6L, 8 });
seqWriter.Flush();
}
stream.Flush();
stream.Seek(0, SeekOrigin.Begin);
var property = this.CreateODataInputContext(stream).ReadProperty(null, null);
}
public override Stream GetContentStream(IEnumerable <ExpandoObject> records)
{
var result = new MemoryStream();
// We get a JSON schema for the passed type. Normally, the AvroSerializer expects a type passed that has DataContract/DataMember attributes, but I don't want users of this utility
// to have to modify their POCOs in any way, hence building a schema here and proceeding with that.
string schema = GetJsonSchema();
var serializer = AvroSerializer.CreateGeneric(schema);
var rootSchema = serializer.WriterSchema as RecordSchema;
// We'll write the Avro content to a memory stream
using (var interim = new MemoryStream())
{
// Avro serializer with deflate
using (var avroWriter = AvroContainer.CreateGenericWriter(schema, interim, Codec.Deflate))
{
using (var seqWriter = new SequentialWriter <object>(avroWriter, SYNCNUM))
{
foreach (var record in records)
{
IDictionary <string, object> recordProperties = record as IDictionary <string, object>;
dynamic avroRecord = new AvroRecord(rootSchema);
foreach (KeyValuePair <string, object> recordKVP in recordProperties)
{
avroRecord[recordKVP.Key] = recordKVP.Value;
}
seqWriter.Write(avroRecord);
}
seqWriter.Flush();
interim.Seek(0, SeekOrigin.Begin);
interim.CopyTo(result);
}
}
}
return(result);
}
public Stream Compress(IEnumerable <T> documents)
{
var buffer = new MemoryStream();
var settings = new AvroSerializerSettings {
GenerateSerializer = true
};
using (var w = AvroContainer.CreateWriter <T>(buffer, settings, Codec.Deflate))
using (var writer = new SequentialWriter <T>(w, 24))
{
// Serialize the data to stream by using the sequential writer
documents.ToList().ForEach(writer.Write);
}
buffer.Seek(0, SeekOrigin.Begin);
return(buffer);
}
public void TestAvroSerializationThroughReflection()
{
SensorData sensorData = CreateTestData();
PerformanceHarness.Test(() =>
{
using (var buffer = new MemoryStream())
{
using (var avroWriter = AvroContainer.CreateWriter<SensorData>(buffer, Codec.Deflate))
{
using (var writer = new SequentialWriter<SensorData>(avroWriter, 24))
{
writer.Write(sensorData);
}
}
}
}, "Avro serialization through reflection", 10000);
}
internal void Dispose <T>()
{
if (_sw != null)
{
RaiseEndWrite(_sw);
}
else if (_avroWriter != null)
{
RaiseEndWrite(_avroWriter);
}
if (Configuration.UseAvroSerializer)
{
if (IsDynamicType)
{
IAvroSerializer <Dictionary <string, object> > avroSerializer = _avroSerializer as IAvroSerializer <Dictionary <string, object> >;
}
else
{
IAvroSerializer <T> avroSerializer = _avroSerializer as IAvroSerializer <T>;
}
}
else
{
if (IsDynamicType)
{
SequentialWriter <Dictionary <string, object> > avroWriter = _avroWriter as SequentialWriter <Dictionary <string, object> >;
if (avroWriter != null)
{
avroWriter.Dispose();
}
}
else
{
SequentialWriter <T> avroWriter = _avroWriter as SequentialWriter <T>;
if (avroWriter != null)
{
avroWriter.Dispose();
}
}
}
}
/// <summary>
/// Serialize a ClassHierarchy into a file
/// </summary>
/// <param name="c"></param>
/// <param name="fileName"></param>
public void ToFile(IClassHierarchy c, string fileName)
{
var avronNodeData = ToAvroNode(c);
using (var buffer = new MemoryStream())
{
using (var w = AvroContainer.CreateWriter<AvroNode>(buffer, Codec.Null))
{
using (var writer = new SequentialWriter<AvroNode>(w, 24))
{
writer.Write(avronNodeData);
}
}
if (!WriteFile(buffer, fileName))
{
var e = new ApplicationException("Error during file operation. Quitting method: " + fileName);
Utilities.Diagnostics.Exceptions.Throw(e, LOGGER);
}
}
}
public void SequentialReaderWriter_SerializeThreeObjects()
{
var expected = new List<ClassOfInt>
{
ClassOfInt.Create(true),
ClassOfInt.Create(true),
ClassOfInt.Create(true),
};
var w = AvroContainer.CreateWriter<ClassOfInt>(this.resultStream, Codec.Null);
using (var writer = new SequentialWriter<ClassOfInt>(w, 24))
{
expected.ForEach(writer.Write);
}
this.resultStream.Seek(0, SeekOrigin.Begin);
var r = AvroContainer.CreateReader<ClassOfInt>(this.resultStream);
using (var reader = new SequentialReader<ClassOfInt>(r))
{
Assert.IsTrue(expected.SequenceEqual(reader.Objects));
}
}
public void SequentialWriter_AddMetadataWithNullKey()
{
var writer = AvroContainer.CreateWriter<ClassOfInt>(this.resultStream, Codec.Null);
using (var sequentialWriter = new SequentialWriter<ClassOfInt>(writer, 2))
{
sequentialWriter.AddMetadata(null, Utilities.GetRandom<byte[]>(false));
}
}
public void SequentialWriter_MicrosoftWriterApacherReaderOfDictionary()
{
var expected = new List<ContainingDictionaryClass<string, string>>();
for (var i = 0; i < 7; i++)
{
expected.Add(ContainingDictionaryClass<string, string>.Create(
new Dictionary<string, string>
{
{ "testkey" + i, "testvalue" + i }
}));
}
var w = AvroContainer.CreateWriter<ContainingDictionaryClass<string, string>>(this.resultStream, Codec.Deflate);
using (var writer = new SequentialWriter<ContainingDictionaryClass<string, string>>(w, 2))
{
expected.ForEach(writer.Write);
}
this.resultStream.Seek(0, SeekOrigin.Begin);
var reader = DataFileReader<GenericRecord>.OpenReader(this.resultStream);
var actual = new List<GenericRecord>(reader);
Assert.AreEqual(expected.Count, actual.Count);
for (var i = 0; i < expected.Count; ++i)
{
var actualValue = actual[i]["Property"] as Dictionary<string, object>;
Assert.IsNotNull(actualValue);
Assert.AreEqual(actualValue["testkey" + i] as string, expected[i].Property["testkey" + i]);
}
}
public void SequentialWriter_MicrosoftWriterApacherReaderOfNestedType()
{
var expected = new List<NestedClass>();
for (var i = 0; i < 7; i++)
{
expected.Add(NestedClass.Create(true));
}
var w = AvroContainer.CreateWriter<NestedClass>(this.resultStream, new AvroSerializerSettings { Resolver = new AvroDataContractResolver(true) }, Codec.Deflate);
using (var writer = new SequentialWriter<NestedClass>(w, 2))
{
expected.ForEach(writer.Write);
}
this.resultStream.Seek(0, SeekOrigin.Begin);
var reader = DataFileReader<GenericRecord>.OpenReader(this.resultStream);
var actual = new List<GenericRecord>(reader);
for (var i = 0; i < expected.Count; ++i)
{
Assert.AreEqual(expected[i].PrimitiveInt, actual[i]["PrimitiveInt"]);
if (expected[i].ClassOfIntReference == null)
{
Assert.IsNull(actual[i]["ClassOfIntReference"]);
}
else
{
Assert.IsNotNull(actual[i]["ClassOfIntReference"] as GenericRecord);
Assert.AreEqual(expected[i].ClassOfIntReference.PrimitiveInt, (actual[i]["ClassOfIntReference"] as GenericRecord)["PrimitiveInt"]);
}
}
}
public void SequentialWriter_MicrosoftWriterApacheReader()
{
var expected = new List<ClassOfInt>();
for (var i = 0; i < 7; i++)
{
expected.Add(ClassOfInt.Create(true));
}
var w = AvroContainer.CreateWriter<ClassOfInt>(this.resultStream, Codec.Deflate);
using (var writer = new SequentialWriter<ClassOfInt>(w, 2))
{
expected.ForEach(writer.Write);
}
this.resultStream.Seek(0, SeekOrigin.Begin);
var reader = DataFileReader<GenericRecord>.OpenReader(this.resultStream);
var actual = new List<GenericRecord>(reader);
for (var i = 0; i < expected.Count; ++i)
{
Assert.AreEqual(expected[i].PrimitiveInt, actual[i]["PrimitiveInt"]);
}
}
public void TestReadParameter()
{
MemoryStream ms = new MemoryStream();
using (var writer = AvroContainer.CreateWriter<AddProductParameter>(ms, true, new AvroSerializerSettings(), Codec.Null))
using (var seqWriter = new SequentialWriter<AddProductParameter>(writer, 24))
{
seqWriter.Write(new AddProductParameter { Location = address0, Product = product0 });
}
ms.Seek(0, SeekOrigin.Begin);
var result = new Dictionary<string, object>();
using (var omr = TestHelper.CreateMessageReader(ms, "avro/binary", AvroMediaTypeResolver.Instance, new EdmModel()))
{
var reader = omr.CreateODataParameterReader(AddProduct);
while (reader.Read())
{
switch (reader.State)
{
case ODataParameterReaderState.Value:
result.Add(reader.Name, reader.Value);
break;
case ODataParameterReaderState.Entry:
var entryReader = reader.CreateEntryReader();
while (entryReader.Read())
{
if (entryReader.State == ODataReaderState.EntryEnd)
{
result.Add(reader.Name, entryReader.Item);
break;
}
}
break;
}
}
}
Assert.AreEqual(2, result.Count);
Assert.IsTrue(TestHelper.ComplexValueEqual(complexValue0, (ODataComplexValue)result["Location"]));
Assert.IsTrue(TestHelper.EntryEqual(entry0, (ODataEntry)result["Product"]));
}
public void ReadAvroAsParameter()
{
EdmEntityType personType = new EdmEntityType("TestNS", "Person");
personType.AddStructuralProperty("Id", EdmPrimitiveTypeKind.Int32);
personType.AddStructuralProperty("Title", EdmPrimitiveTypeKind.String);
var operation = new EdmAction("NS", "op1", null);
operation.AddParameter("p1", EdmCoreModel.Instance.GetString(false));
operation.AddParameter("p2", new EdmEntityTypeReference(personType, false));
const string Schema = @"{
""type"":""record"",
""name"":""NS.op1Parameter"",
""fields"":
[
{ ""name"":""p1"", ""type"":""string"" },
{ ""name"":""p2"", ""type"":
{""type"":""record"",
""name"":""TestNS.Person"",
""fields"":
[
{ ""name"":""Id"", ""type"":""int"" },
{ ""name"":""Title"", ""type"":""string"" },
]
}
}
]
}";
var stream = new MemoryStream();
using (var writer = AvroContainer.CreateGenericWriter(Schema, stream, /*leave open*/true, Codec.Null))
using (var seqWriter = new SequentialWriter<object>(writer, 24))
{
RecordSchema parameterSchema = (RecordSchema)AvroSerializer.CreateGeneric(Schema).WriterSchema;
AvroRecord ar = new AvroRecord(parameterSchema);
ar["p1"] = "dat";
var personSchema = parameterSchema.GetField("p2").TypeSchema;
AvroRecord person = new AvroRecord(personSchema);
person["Id"] = 5;
person["Title"] = "per1";
ar["p2"] = person;
seqWriter.Write(ar);
seqWriter.Flush();
}
stream.Flush();
stream.Seek(0, SeekOrigin.Begin);
var reader = new ODataAvroParameterReader(this.CreateODataInputContext(stream), operation);
Assert.AreEqual(ODataParameterReaderState.Start, reader.State);
Assert.IsTrue(reader.Read());
Assert.AreEqual(ODataParameterReaderState.Value, reader.State);
Assert.AreEqual("p1", reader.Name);
Assert.AreEqual("dat", reader.Value);
Assert.IsTrue(reader.Read());
Assert.AreEqual(ODataParameterReaderState.Entry, reader.State);
Assert.AreEqual("p2", reader.Name);
var ew = reader.CreateEntryReader();
Assert.AreEqual(ODataReaderState.Start, ew.State);
Assert.IsTrue(ew.Read());
Assert.AreEqual(ODataReaderState.EntryStart, ew.State);
Assert.IsTrue(ew.Read());
Assert.AreEqual(ODataReaderState.EntryEnd, ew.State);
var entry = ew.Item as ODataEntry;
Assert.IsFalse(ew.Read());
Assert.AreEqual(ODataReaderState.Completed, ew.State);
Assert.IsNotNull(entry);
var properties = entry.Properties.ToList();
Assert.AreEqual(2, properties.Count);
Assert.AreEqual("Id", properties[0].Name);
Assert.AreEqual(5, properties[0].Value);
Assert.AreEqual("Title", properties[1].Name);
Assert.AreEqual("per1", properties[1].Value);
Assert.IsFalse(reader.Read());
Assert.AreEqual(ODataParameterReaderState.Completed, reader.State);
}
public void TestReadFeed()
{
MemoryStream ms = new MemoryStream();
IAvroWriter<Product[]> writer = null;
try
{
writer = AvroContainer.CreateWriter<Product[]>(ms, true, new AvroSerializerSettings(), Codec.Null);
using (var seqWriter = new SequentialWriter<Product[]>(writer, 24))
{
seqWriter.Write(new[] { product1, product2, product3 });
}
}
finally
{
if (writer != null) writer.Dispose();
}
ms.Seek(0, SeekOrigin.Begin);
var entries = new List<ODataEntry>();
using (var omr = TestHelper.CreateMessageReader(ms, "avro/binary", AvroMediaTypeResolver.Instance))
{
var reader = omr.CreateODataFeedReader();
while (reader.Read())
{
if (reader.State == ODataReaderState.EntryEnd)
{
entries.Add((ODataEntry)reader.Item);
}
}
}
Assert.AreEqual(3, entries.Count);
Assert.IsTrue(TestHelper.EntryEqual(entry1, entries[0]));
Assert.IsTrue(TestHelper.EntryEqual(entry2, entries[1]));
Assert.IsTrue(TestHelper.EntryEqual(entry3, entries[2]));
}
public void ToFile(IConfiguration c, string fileName)
{
var avronConfigurationData = ToAvroConfiguration(c);
using (var buffer = new MemoryStream())
{
using (var w = AvroContainer.CreateWriter<AvroConfiguration>(buffer, Codec.Null))
{
using (var writer = new SequentialWriter<AvroConfiguration>(w, 24))
{
// Serialize the data to stream using the sequential writer
writer.Write(avronConfigurationData);
}
}
if (!WriteFile(buffer, fileName))
{
var e = new ApplicationException("Error during file operation. Quitting method: " + fileName);
Org.Apache.REEF.Utilities.Diagnostics.Exceptions.Throw(e, LOGGER);
}
}
}
static void SerializeReadings()
{
try
{
//Serialize GPS data to file
Console.WriteLine("Saving GPS data...");
using (var buffer = new FileStream(gpsFile, FileMode.Create))
{
//Serialize a sequence of GpsReading objects to stream
//Data will be compressed using Deflate codec
using (var w = AvroContainer.CreateWriter<GpsReading>(buffer, Codec.Deflate))
{
using (var writer = new SequentialWriter<GpsReading>(w, 24))
{
// Serialize the data to stream using the sequential writer
GpsReadings.ForEach(writer.Write);
}
}
buffer.Close();
}
//Serialize Engine data to file
Console.WriteLine("Saving engine data...");
using (var buffer = new FileStream(engineFile, FileMode.Create))
{
//Serialize a sequence of EngineReading objects to stream
//Data will be compressed using Deflate codec
using (var w = AvroContainer.CreateWriter<EngineReading>(buffer, Codec.Deflate))
{
using (var writer = new SequentialWriter<EngineReading>(w, 24))
{
// Serialize the data to stream using the sequential writer
EngineReadings.ForEach(writer.Write);
}
}
buffer.Close();
}
//Serialize Brake data to file
Console.WriteLine("Saving brake data...");
using (var buffer = new FileStream(brakeFile, FileMode.Create))
{
//Serialize a sequence of BrakeReading objects to stream
//Data will be compressed using Deflate codec
using (var w = AvroContainer.CreateWriter<BrakeReading>(buffer, Codec.Deflate))
{
using (var writer = new SequentialWriter<BrakeReading>(w, 24))
{
// Serialize the data to stream using the sequential writer
BrakeReadings.ForEach(writer.Write);
}
}
buffer.Close();
}
}
catch (Exception ex)
{
Console.WriteLine("An error occurred while serializing the data");
throw (ex);
}
}
public void ReadAvroAsError()
{
const string Schema = @"{
""type"":""record"",
""name"":""OData.Error"",
""fields"":
[
{ ""name"":""ErrorCode"", ""type"":""string"" },
{ ""name"":""Message"" , ""type"":""string"" },
]
}";
var stream = new MemoryStream();
using (var writer = AvroContainer.CreateGenericWriter(Schema, stream, /*leave open*/true, Codec.Null))
using (var seqWriter = new SequentialWriter<object>(writer, 24))
{
RecordSchema parameterSchema = (RecordSchema)AvroSerializer.CreateGeneric(Schema).WriterSchema;
AvroRecord ar = new AvroRecord(parameterSchema);
ar["ErrorCode"] = "e1";
ar["Message"] = "m1";
seqWriter.Write(ar);
seqWriter.Flush();
}
stream.Flush();
stream.Seek(0, SeekOrigin.Begin);
var error = this.CreateODataInputContext(stream).ReadError();
Assert.AreEqual("e1", error.ErrorCode);
Assert.AreEqual("m1", error.Message);
}
public void TestReadError()
{
MemoryStream ms = new MemoryStream();
using (var writer = AvroContainer.CreateWriter<Error>(ms, true, new AvroSerializerSettings(), Codec.Null))
using (var seqWriter = new SequentialWriter<Error>(writer, 24))
{
seqWriter.Write(new Error { ErrorCode = "500", Message = "Internal Error" });
}
ms.Seek(0, SeekOrigin.Begin);
ODataError error = null;
using (var omr = TestHelper.CreateMessageReader(ms, "avro/binary", AvroMediaTypeResolver.Instance, new EdmModel(), true))
{
error = omr.ReadError();
}
Assert.IsNotNull(error);
Assert.AreEqual("500", error.ErrorCode);
Assert.AreEqual("Internal Error", error.Message);
}
public void TestReadParameterWithFeed()
{
MemoryStream ms = new MemoryStream();
using (var writer = AvroContainer.CreateWriter<GetMaxIdParameter>(ms, true, new AvroSerializerSettings(), Codec.Null))
using (var seqWriter = new SequentialWriter<GetMaxIdParameter>(writer, 24))
{
seqWriter.Write(new GetMaxIdParameter { Products = new List<Product>() { product0, product1 } });
}
ms.Seek(0, SeekOrigin.Begin);
var result = new Dictionary<string, object>();
using (var omr = TestHelper.CreateMessageReader(ms, "avro/binary", AvroMediaTypeResolver.Instance, new EdmModel()))
{
var reader = omr.CreateODataParameterReader(GetMaxId);
while (reader.Read())
{
switch (reader.State)
{
case ODataParameterReaderState.Value:
result.Add(reader.Name, reader.Value);
break;
case ODataParameterReaderState.Entry:
var entryReader = reader.CreateEntryReader();
while (entryReader.Read())
{
if (entryReader.State == ODataReaderState.EntryEnd)
{
result.Add(reader.Name, entryReader.Item);
break;
}
}
break;
case ODataParameterReaderState.Feed:
var feedReader = reader.CreateFeedReader();
IList<ODataEntry> entryList = new List<ODataEntry>();
while (feedReader.Read())
{
if (feedReader.State == ODataReaderState.EntryEnd)
{
entryList.Add((ODataEntry)feedReader.Item);
}
}
result.Add(reader.Name, entryList);
break;
}
}
}
Assert.AreEqual(1, result.Count);
var feed = result["Products"] as IList<ODataEntry>;
Assert.IsNotNull(feed);
Assert.AreEqual(2, feed.Count);
Assert.IsTrue(TestHelper.EntryEqual(entry0, feed[0]));
Assert.IsTrue(TestHelper.EntryEqual(entry1, feed[1]));
}
public void SequentialWriter_AddMetadataWithNullValue()
{
var writer = AvroContainer.CreateWriter<ClassOfInt>(this.resultStream, Codec.Null);
using (var sequentialWriter = new SequentialWriter<ClassOfInt>(writer, 2))
{
sequentialWriter.AddMetadata("Key", null);
}
}
//Serializes and deserializes sample data set using Generic Record and Avro Object Container Files
//Serialized data is not compressed
//
//This sample uses Memory Stream for all operations related to serialization, deserialization and
//Object Container manipulation, though File Stream could be easily used.
public void SerializeDeserializeUsingObjectContainersGenericRecord()
{
Console.WriteLine("SERIALIZATION USING GENERIC RECORD AND AVRO OBJECT CONTAINER FILES\n");
//Path for Avro Object Container File
string path = "AvroSampleGenericRecordNullCodec.avro";
Console.WriteLine("Defining the Schema and creating Sample Data Set...");
//Define the schema in JSON
const string Schema = @"{
""type"":""record"",
""name"":""Microsoft.Hadoop.Avro.Specifications.SensorData"",
""fields"":
[
{
""name"":""Location"",
""type"":
{
""type"":""record"",
""name"":""Microsoft.Hadoop.Avro.Specifications.Location"",
""fields"":
[
{ ""name"":""Floor"", ""type"":""int"" },
{ ""name"":""Room"", ""type"":""int"" }
]
}
},
{ ""name"":""Value"", ""type"":""bytes"" }
]
}";
//Create a generic serializer based on the schema
var serializer = AvroSerializer.CreateGeneric(Schema);
var rootSchema = serializer.WriterSchema as RecordSchema;
//Create a generic record to represent the data
var testData = new List<AvroRecord>();
dynamic expected1 = new AvroRecord(rootSchema);
dynamic location1 = new AvroRecord(rootSchema.GetField("Location").TypeSchema);
location1.Floor = 1;
location1.Room = 243;
expected1.Location = location1;
expected1.Value = new byte[] { 1, 2, 3, 4, 5 };
testData.Add(expected1);
dynamic expected2 = new AvroRecord(rootSchema);
dynamic location2 = new AvroRecord(rootSchema.GetField("Location").TypeSchema);
location2.Floor = 1;
location2.Room = 244;
expected2.Location = location2;
expected2.Value = new byte[] { 6, 7, 8, 9 };
testData.Add(expected2);
//Serializing and saving data to file
//Create a MemoryStream buffer
using (var buffer = new MemoryStream())
{
Console.WriteLine("Serializing Sample Data Set...");
//Create a SequentialWriter instance for type SensorData which can serialize a sequence of SensorData objects to stream
//Data will not be compressed (Null compression codec)
using (var writer = AvroContainer.CreateGenericWriter(Schema, buffer, Codec.Null))
{
using (var streamWriter = new SequentialWriter<object>(writer, 24))
{
// Serialize the data to stream using the sequential writer
testData.ForEach(streamWriter.Write);
}
}
Console.WriteLine("Saving serialized data to file...");
//Save stream to file
if (!WriteFile(buffer, path))
{
Console.WriteLine("Error during file operation. Quitting method");
return;
}
}
//Reading and deserializng the data
//Create a Memory Stream buffer
using (var buffer = new MemoryStream())
{
Console.WriteLine("Reading data from file...");
//Reading data from Object Container File
if (!ReadFile(buffer, path))
{
Console.WriteLine("Error during file operation. Quitting method");
return;
}
Console.WriteLine("Deserializing Sample Data Set...");
//Prepare the stream for deserializing the data
buffer.Seek(0, SeekOrigin.Begin);
//Create a SequentialReader for type SensorData which will derserialize all serialized objects from the given stream
//It allows iterating over the deserialized objects because it implements IEnumerable<T> interface
using (var reader = AvroContainer.CreateGenericReader(buffer))
{
using (var streamReader = new SequentialReader<object>(reader))
{
var results = streamReader.Objects;
Console.WriteLine("Comparing Initial and Deserialized Data Sets...");
//Finally, verify the results
var pairs = testData.Zip(results, (serialized, deserialized) => new { expected = (dynamic)serialized, actual = (dynamic)deserialized });
int count = 1;
foreach (var pair in pairs)
{
bool isEqual = pair.expected.Location.Floor.Equals(pair.actual.Location.Floor);
isEqual = isEqual && pair.expected.Location.Room.Equals(pair.actual.Location.Room);
isEqual = isEqual && ((byte[])pair.expected.Value).SequenceEqual((byte[])pair.actual.Value);
Console.WriteLine("For Pair {0} result of Data Set Identity Comparison is {1}", count, isEqual.ToString());
count++;
}
}
}
}
//Delete the file
RemoveFile(path);
}
public void SequentialReaderWriter_CreateReaderForSchemaWithNullableField()
{
var expected = new ClassWithNullableIntField { NullableIntField = 10 };
using (var writer = AvroContainer.CreateWriter<ClassWithNullableIntField>(this.resultStream, Codec.Deflate))
{
using (var swriter = new SequentialWriter<ClassWithNullableIntField>(writer, 2))
{
swriter.Write(expected);
swriter.Write(expected);
swriter.Write(expected);
swriter.Write(expected);
}
}
this.resultStream.Seek(0, SeekOrigin.Begin);
using (var reader = AvroContainer.CreateReader<ClassWithNullableIntField>(this.resultStream))
{
using (var sreader = new SequentialReader<ClassWithNullableIntField>(reader))
{
foreach (var actual in sreader.Objects)
{
Assert.AreEqual(expected.NullableIntField, actual.NullableIntField);
}
}
}
}
public void TestReadEntry()
{
MemoryStream ms = new MemoryStream();
IAvroWriter<Product> writer = null;
try
{
writer = AvroContainer.CreateWriter<Product>(ms, true, new AvroSerializerSettings(), Codec.Null);
using (var seqWriter = new SequentialWriter<Product>(writer, 24))
{
seqWriter.Write(product0);
}
}
finally
{
if (writer != null) writer.Dispose();
}
ms.Seek(0, SeekOrigin.Begin);
ODataEntry entry = null;
using (var omr = TestHelper.CreateMessageReader(ms, "avro/binary", AvroMediaTypeResolver.Instance))
{
var reader = omr.CreateODataEntryReader();
while (reader.Read())
{
if (reader.State == ODataReaderState.EntryEnd)
{
entry = (ODataEntry)reader.Item;
}
}
}
Assert.IsTrue(TestHelper.EntryEqual(entry0, entry));
}
public void SequentialReaderWriter_SerializeHugeObject()
{
var single = new SimpleFlatClass
{
StringField = new string('a', 16254),
ByteArrayField = Encoding.ASCII.GetBytes(new string('b', 65666)),
ZeroByteArrayField = Encoding.ASCII.GetBytes(new string('c', 128344))
};
var expected = new List<SimpleFlatClass>
{
single,
single,
single,
};
var w = AvroContainer.CreateWriter<SimpleFlatClass>(this.resultStream, new AvroSerializerSettings { Resolver = new AvroDataContractResolver(true) }, Codec.Null);
using (var writer = new SequentialWriter<SimpleFlatClass>(w, 24))
{
expected.ForEach(writer.Write);
}
this.resultStream.Seek(0, SeekOrigin.Begin);
var r = AvroContainer.CreateReader<SimpleFlatClass>(this.resultStream, true, new AvroSerializerSettings { Resolver = new AvroDataContractResolver(true) }, new CodecFactory());
using (var reader = new SequentialReader<SimpleFlatClass>(r))
{
Assert.IsTrue(expected.SequenceEqual(reader.Objects));
}
}
public void SequentialReaderWriter_Reset()
{
var expected = new List<ClassOfListOfGuid>();
for (var i = 0; i < 7; i++)
{
expected.Add(ClassOfListOfGuid.Create(true));
}
var w = AvroContainer.CreateWriter<ClassOfListOfGuid>(this.resultStream, Codec.Deflate);
using (var writer = new SequentialWriter<ClassOfListOfGuid>(w, 3))
{
expected.ForEach(writer.Write);
}
this.resultStream.Seek(0, SeekOrigin.Begin);
var r = AvroContainer.CreateReader<ClassOfListOfGuid>(this.resultStream);
using (var reader = new SequentialReader<ClassOfListOfGuid>(r))
{
Assert.IsTrue(expected.SequenceEqual(reader.Objects));
var enumerator = (IEnumerator)reader.Objects.GetEnumerator();
Assert.IsFalse(enumerator.MoveNext());
}
}
/// <summary>
/// Reads CSV file data and converts records to instances of Stock class then serializes them using Reflection.
/// Stock type definition is created from JSON schema using Microsoft Avro Library Code Generation utility
/// </summary>
/// <param name="inputFilePath">The input file path.</param>
/// <param name="outputStream">The output stream.</param>
private void SerializeCsv(string inputFilePath, Stream outputStream)
{
try
{
// Create an Avro writer using Null Codec
var writer = AvroContainer.CreateWriter<Stock>(
outputStream,
new AvroSerializerSettings { Resolver = new AvroDataContractResolver(true) },
Codec.Null);
// Create a new sequential writer and use it to write Avro file using a block length of 100
using (var seq = new SequentialWriter<Stock>(writer, 100))
{
foreach (var stock in this.ReadCsv(inputFilePath))
{
seq.Write(stock);
}
}
}
catch (Exception e)
{
ReportError("Error while creating Avro file(s) from CSV file(s)" + e);
}
}
public void TestReadCollection()
{
MemoryStream ms = new MemoryStream();
using (var writer = AvroContainer.CreateWriter<string[]>(ms, true, new AvroSerializerSettings(), Codec.Null))
using (var seqWriter = new SequentialWriter<string[]>(writer, 24))
{
seqWriter.Write(new[] { "p9t", "cjk如" });
}
ms.Seek(0, SeekOrigin.Begin);
var values = new List<string>();
using (var omr = TestHelper.CreateMessageReader(ms, "avro/binary", AvroMediaTypeResolver.Instance))
{
var reader = omr.CreateODataCollectionReader();
while (reader.Read())
{
if (reader.State == ODataCollectionReaderState.Value)
{
values.Add((string)reader.Item);
}
}
}
Assert.AreEqual(2, values.Count);
Assert.AreEqual("p9t", values[0]);
Assert.AreEqual("cjk如", values[1]);
}
//Serializes and deserializes sample data set using Reflection and Avro Object Container Files
//Serialized data is compressed with Deflate codec
//
//This sample uses Memory Stream for all operations related to serialization, deserialization and
//Object Container manipulation, though File Stream could be easily used.
public void SerializeDeserializeUsingObjectContainersReflection()
{
Console.WriteLine("SERIALIZATION USING REFLECTION AND AVRO OBJECT CONTAINER FILES\n");
//Path for Avro Object Container File
string path = "AvroSampleReflectionDeflate.avro";
//Create a data set using sample Class and struct
var testData = new List<SensorData>
{
new SensorData { Value = new byte[] { 1, 2, 3, 4, 5 }, Position = new Location { Room = 243, Floor = 1 } },
new SensorData { Value = new byte[] { 6, 7, 8, 9 }, Position = new Location { Room = 244, Floor = 1 } }
};
//Serializing and saving data to file
//Creating a Memory Stream buffer
using (var buffer = new MemoryStream())
{
Console.WriteLine("Serializing Sample Data Set...");
//Create a SequentialWriter instance for type SensorData which can serialize a sequence of SensorData objects to stream
//Data will be compressed using Deflate codec
using (var w = AvroContainer.CreateWriter<SensorData>(buffer, Codec.Deflate))
{
using (var writer = new SequentialWriter<SensorData>(w, 24))
{
// Serialize the data to stream using the sequential writer
testData.ForEach(writer.Write);
}
}
//Save stream to file
Console.WriteLine("Saving serialized data to file...");
if (!WriteFile(buffer, path))
{
Console.WriteLine("Error during file operation. Quitting method");
return;
}
}
//Reading and deserializing data
//Creating a Memory Stream buffer
using (var buffer = new MemoryStream())
{
Console.WriteLine("Reading data from file...");
//Reading data from Object Container File
if (!ReadFile(buffer, path))
{
Console.WriteLine("Error during file operation. Quitting method");
return;
}
Console.WriteLine("Deserializing Sample Data Set...");
//Prepare the stream for deserializing the data
buffer.Seek(0, SeekOrigin.Begin);
//Create a SequentialReader for type SensorData which will derserialize all serialized objects from the given stream
//It allows iterating over the deserialized objects because it implements IEnumerable<T> interface
using (var reader = new SequentialReader<SensorData>(
AvroContainer.CreateReader<SensorData>(buffer, true)))
{
var results = reader.Objects;
//Finally, verify that deserialized data matches the original one
Console.WriteLine("Comparing Initial and Deserialized Data Sets...");
int count = 1;
var pairs = testData.Zip(results, (serialized, deserialized) => new { expected = serialized, actual = deserialized });
foreach (var pair in pairs)
{
bool isEqual = this.Equal(pair.expected, pair.actual);
Console.WriteLine("For Pair {0} result of Data Set Identity Comparison is {1}", count, isEqual.ToString());
count++;
}
}
}
//Delete the file
RemoveFile(path);
}
//Serializes and deserializes sample data set using Reflection and Avro Object Container Files
//Serialized data is compressed with the Custom compression codec (Deflate of .NET Framework 4.5)
//
//This sample uses Memory Stream for all operations related to serialization, deserialization and
//Object Container manipulation, though File Stream could be easily used.
public void SerializeDeserializeUsingObjectContainersReflectionCustomCodec()
{
Console.WriteLine("SERIALIZATION USING REFLECTION, AVRO OBJECT CONTAINER FILES AND CUSTOM CODEC\n");
//Path for Avro Object Container File
string path = "AvroSampleReflectionDeflate45.avro";
//Create a data set using sample Class and struct
var testData = new List<SensorData>
{
new SensorData { Value = new byte[] { 1, 2, 3, 4, 5 }, Position = new Location { Room = 243, Floor = 1 } },
new SensorData { Value = new byte[] { 6, 7, 8, 9 }, Position = new Location { Room = 244, Floor = 1 } }
};
//Serializing and saving data to file
//Creating a Memory Stream buffer
using (var buffer = new MemoryStream())
{
Console.WriteLine("Serializing Sample Data Set...");
//Create a SequentialWriter instance for type SensorData which can serialize a sequence of SensorData objects to stream
//Here the custom Codec is introduced. For convenience the next commented code line shows how to use built-in Deflate.
//Note, that because the sample deals with different IMPLEMENTATIONS of Deflate, built-in and custom codecs are interchangeable
//in read-write operations
//using (var w = AvroContainer.CreateWriter<SensorData>(buffer, Codec.Deflate))
using (var w = AvroContainer.CreateWriter<SensorData>(buffer, new DeflateCodec45()))
{
using (var writer = new SequentialWriter<SensorData>(w, 24))
{
// Serialize the data to stream using the sequential writer
testData.ForEach(writer.Write);
}
}
//Save stream to file
Console.WriteLine("Saving serialized data to file...");
if (!WriteFile(buffer, path))
{
Console.WriteLine("Error during file operation. Quitting method");
return;
}
}
//Reading and deserializing data
//Creating a Memory Stream buffer
using (var buffer = new MemoryStream())
{
Console.WriteLine("Reading data from file...");
//Reading data from Object Container File
if (!ReadFile(buffer, path))
{
Console.WriteLine("Error during file operation. Quitting method");
return;
}
Console.WriteLine("Deserializing Sample Data Set...");
//Prepare the stream for deserializing the data
buffer.Seek(0, SeekOrigin.Begin);
//Because of SequentialReader<T> constructor signature an AvroSerializerSettings instance is required
//when Codec Factory is explicitly specified
//You may comment the line below if you want to use built-in Deflate (see next comment)
AvroSerializerSettings settings = new AvroSerializerSettings();
//Create a SequentialReader for type SensorData which will derserialize all serialized objects from the given stream
//It allows iterating over the deserialized objects because it implements IEnumerable<T> interface
//Here the custom Codec Factory is introduced.
//For convenience the next commented code line shows how to use built-in Deflate
//(no explicit Codec Factory parameter is required in this case).
//Note, that because the sample deals with different IMPLEMENTATIONS of Deflate, built-in and custom codecs are interchangeable
//in read-write operations
//using (var reader = new SequentialReader<SensorData>(AvroContainer.CreateReader<SensorData>(buffer, true)))
using (var reader = new SequentialReader<SensorData>(
AvroContainer.CreateReader<SensorData>(buffer, true, settings, new CodecFactoryDeflate45())))
{
var results = reader.Objects;
//Finally, verify that deserialized data matches the original one
Console.WriteLine("Comparing Initial and Deserialized Data Sets...");
bool isEqual;
int count = 1;
var pairs = testData.Zip(results, (serialized, deserialized) => new { expected = serialized, actual = deserialized });
foreach (var pair in pairs)
{
isEqual = this.Equal(pair.expected, pair.actual);
Console.WriteLine("For Pair {0} result of Data Set Identity Comparison is {1}", count, isEqual.ToString());
count++;
}
}
}
//Delete the file
RemoveFile(path);
}
public void SequentialReaderWriter_SyncAfterDeflateCodec()
{
var expected = new List<ClassOfInt>();
for (var i = 0; i < 7; i++)
{
expected.Add(ClassOfInt.Create(true));
}
var w = AvroContainer.CreateWriter<ClassOfInt>(this.resultStream, Codec.Deflate);
using (var writer = new SequentialWriter<ClassOfInt>(w, 2))
{
expected.ForEach(writer.Write);
}
this.resultStream.Seek(0, SeekOrigin.Begin);
var r = AvroContainer.CreateReader<ClassOfInt>(this.resultStream);
using (var reader = new SequentialReader<ClassOfInt>(r))
{
Assert.IsTrue(expected.SequenceEqual(reader.Objects));
}
}
public void TestReadAvroAsODataFeed()
{
const string Schema = @"
{""type"":""array"",
""items"":
{
""type"":""record"",
""name"":""TestNS.Person"",
""fields"":
[
{ ""name"":""Id"", ""type"":""int"" },
{ ""name"":""Title"", ""type"":""string"" },
]
}
}";
var serializer = AvroSerializer.CreateGeneric(Schema);
using (var stream = new MemoryStream())
{
var arraySchema = (ArraySchema)serializer.WriterSchema;
var recordSchema = arraySchema.ItemSchema;
var rec1 = new AvroRecord(recordSchema);
rec1["Id"] = 1;
rec1["Title"] = "s1";
var rec2 = new AvroRecord(recordSchema);
rec2["Id"] = 2;
rec2["Title"] = "s2";
var array = new[] { rec1, rec2 };
using (var writer = AvroContainer.CreateGenericWriter(Schema, stream, /*leave open*/true, Codec.Null))
using (var seqWriter = new SequentialWriter<object>(writer, 24))
{
seqWriter.Write(array);
}
stream.Seek(0, SeekOrigin.Begin);
var avroReader = new ODataAvroReader(this.CreateODataInputContext(stream), true);
Assert.AreEqual(ODataReaderState.Start, avroReader.State);
Assert.IsTrue(avroReader.Read());
Assert.AreEqual(ODataReaderState.FeedStart, avroReader.State);
Assert.IsTrue(avroReader.Read());
// Entry 1
Assert.AreEqual(ODataReaderState.EntryStart, avroReader.State);
Assert.IsTrue(avroReader.Read());
Assert.AreEqual(ODataReaderState.EntryEnd, avroReader.State);
var entry = avroReader.Item as ODataEntry;
Assert.IsNotNull(entry);
var properties = entry.Properties.ToList();
Assert.AreEqual(2, properties.Count);
Assert.AreEqual("Id", properties[0].Name);
Assert.AreEqual(1, properties[0].Value);
Assert.AreEqual("Title", properties[1].Name);
Assert.AreEqual("s1", properties[1].Value);
// Entry 2
Assert.IsTrue(avroReader.Read());
Assert.AreEqual(ODataReaderState.EntryStart, avroReader.State);
Assert.IsTrue(avroReader.Read());
Assert.AreEqual(ODataReaderState.EntryEnd, avroReader.State);
entry = avroReader.Item as ODataEntry;
Assert.IsNotNull(entry);
properties = entry.Properties.ToList();
Assert.AreEqual(2, properties.Count);
Assert.AreEqual("Id", properties[0].Name);
Assert.AreEqual(2, properties[0].Value);
Assert.AreEqual("Title", properties[1].Name);
Assert.AreEqual("s2", properties[1].Value);
Assert.IsTrue(avroReader.Read());
Assert.AreEqual(ODataReaderState.FeedEnd, avroReader.State);
Assert.IsFalse(avroReader.Read());
Assert.AreEqual(ODataReaderState.Completed, avroReader.State);
}
}
