public void SequentialReaderWriter_CreateReaderForSchemaWithNullableField()
{
var expected = new ClassWithNullableIntField {
NullableIntField = 10
};
using (var writer = AvroContainer.CreateWriter <ClassWithNullableIntField>(this.resultStream, Codec.Deflate))
{
using (var swriter = new SequentialWriter <ClassWithNullableIntField>(writer, 2))
{
swriter.Write(expected);
swriter.Write(expected);
swriter.Write(expected);
swriter.Write(expected);
}
}
this.resultStream.Seek(0, SeekOrigin.Begin);
using (var reader = AvroContainer.CreateReader <ClassWithNullableIntField>(this.resultStream))
{
using (var sreader = new SequentialReader <ClassWithNullableIntField>(reader))
{
foreach (var actual in sreader.Objects)
{
Assert.AreEqual(expected.NullableIntField, actual.NullableIntField);
}
}
}
}
static void Main(string[] args)
{
SampleData obj = new SampleData();
using (FileStream xmlStream = new FileStream(@"SampleData.xml", FileMode.Open))
{
using (XmlReader xmlReader = XmlReader.Create(xmlStream))
{
obj.ReadXml(xmlReader);
}
}
// hi ahmed
var serializer = new AvroSerializer(typeof(Person));
StreamWriter schemafs = new StreamWriter(File.Create(@"SampleData.avsc"));
schemafs.Write(serializer.Schema.ToString());
schemafs.Close();
var resultStream = new FileStream(@"SampleData.avro", FileMode.OpenOrCreate);
var writer = new SequentialWriter <Person>(resultStream, serializer, Codec.Create("null"), 24);
int i;
foreach (DataRow item in obj.Tables["Person"].Rows)
{
var p = new Person {
First_Name = item["Name"].ToString(), Street = item["Street"].ToString(), City = item["City"].ToString(), State = item["State"].ToString()
};
writer.Write(p);
}
writer.Flush();
}
public void TestReadCollection()
{
MemoryStream ms = new MemoryStream();
using (var writer = AvroContainer.CreateWriter <string[]>(ms, true, new AvroSerializerSettings(), Codec.Null))
using (var seqWriter = new SequentialWriter <string[]>(writer, 24))
{
seqWriter.Write(new[] { "p9t", "cjk如" });
}
ms.Seek(0, SeekOrigin.Begin);
var values = new List <string>();
using (var omr = TestHelper.CreateMessageReader(ms, "avro/binary", AvroMediaTypeResolver.Instance))
{
var reader = omr.CreateODataCollectionReader();
while (reader.Read())
{
if (reader.State == ODataCollectionReaderState.Value)
{
values.Add((string)reader.Item);
}
}
}
Assert.AreEqual(2, values.Count);
Assert.AreEqual("p9t", values[0]);
Assert.AreEqual("cjk如", values[1]);
}
public void ReadAvroAsError()
{
const string Schema = @"{
""type"":""record"",
""name"":""OData.Error"",
""fields"":
[
{ ""name"":""ErrorCode"", ""type"":""string"" },
{ ""name"":""Message"" , ""type"":""string"" },
]
}";
var stream = new MemoryStream();
using (var writer = AvroContainer.CreateGenericWriter(Schema, stream, /*leave open*/ true, Codec.Null))
using (var seqWriter = new SequentialWriter <object>(writer, 24))
{
RecordSchema parameterSchema = (RecordSchema)AvroSerializer.CreateGeneric(Schema).WriterSchema;
AvroRecord ar = new AvroRecord(parameterSchema);
ar["ErrorCode"] = "e1";
ar["Message"] = "m1";
seqWriter.Write(ar);
seqWriter.Flush();
}
stream.Flush();
stream.Seek(0, SeekOrigin.Begin);
var error = this.CreateODataInputContext(stream).ReadError();
Assert.AreEqual("e1", error.ErrorCode);
Assert.AreEqual("m1", error.Message);
}
public void TestReadAvroAsCollection()
{
const string Schema = @"{""type"":""array"", ""items"":""long""}";
var stream = new MemoryStream();
using (var writer = AvroContainer.CreateGenericWriter(Schema, stream, /*leave open*/ true, Codec.Null))
using (var seqWriter = new SequentialWriter <object>(writer, 24))
{
// Serialize the data to stream using the sequential writer
seqWriter.Write(new[] { 6L, 8 });
seqWriter.Flush();
}
stream.Flush();
stream.Seek(0, SeekOrigin.Begin);
var reader = new ODataAvroCollectionReader(this.CreateODataInputContext(stream));
Assert.AreEqual(ODataCollectionReaderState.Start, reader.State);
Assert.IsTrue(reader.Read());
Assert.AreEqual(ODataCollectionReaderState.CollectionStart, reader.State);
Assert.IsTrue(reader.Read());
Assert.AreEqual(ODataCollectionReaderState.Value, reader.State);
Assert.AreEqual(6L, (long)reader.Item);
Assert.IsTrue(reader.Read());
Assert.AreEqual(ODataCollectionReaderState.Value, reader.State);
Assert.IsTrue(reader.Read());
Assert.AreEqual(ODataCollectionReaderState.CollectionEnd, reader.State);
Assert.IsFalse(reader.Read());
Assert.AreEqual(ODataCollectionReaderState.Completed, reader.State);
}
/// <summary>
/// Reads CSV file data and converts records to instances of Stock class then serializes them using Reflection.
/// Stock type definition is created from JSON schema using Microsoft Avro Library Code Generation utility
/// </summary>
/// <param name="inputFilePath">The input file path.</param>
/// <param name="outputStream">The output stream.</param>
private void SerializeCsv(string inputFilePath, Stream outputStream)
{
try
{
// Create an Avro writer using Null Codec
var writer = AvroContainer.CreateWriter <Stock>(
outputStream,
new AvroSerializerSettings {
Resolver = new AvroDataContractResolver(true)
},
Codec.Null);
// Create a new sequential writer and use it to write Avro file using a block length of 100
using (var seq = new SequentialWriter <Stock>(writer, 100))
{
foreach (var stock in this.ReadCsv(inputFilePath))
{
seq.Write(stock);
}
}
}
catch (Exception e)
{
ReportError("Error while creating Avro file(s) from CSV file(s)" + e);
}
}
public override void WriteAvroFile(Stream stream)
{
using (IAvroWriter <object> streamWriter = AvroContainer.CreateGenericWriter(this.GetSchema().ToString(), stream, this.Codec))
{
using (var sequentialWriter = new SequentialWriter <object>(streamWriter, AvroIntegrationWithHiveConfigurations.AvroBlockSize))
{
foreach (LargeClass record in this.Expected)
{
sequentialWriter.Write(record.ToAvroRecord(this.GetSchema()));
}
}
}
}
public void Test_Serialization(bool withSchema)
{
var schema = Resources.SalesRecord;
var avroFile = Path.ChangeExtension(Path.GetTempFileName(), "avro");
using (var fs = new FileStream(avroFile, FileMode.Create))
using (var writer = AvroContainer.CreateGenericWriter(schema, fs, Codec.Deflate))
using (var sequentialWriter = new SequentialWriter <object>(writer, 1000))
using (var sr = new StreamReader("1000 Sales Records.csv"))
{
sr.ReadLine();
while (!sr.EndOfStream)
{
var fields = sr.ReadLine()?.Split(',');
var salesRecord = new SalesRecord
{
Region = fields[0],
Country = fields[1],
ItemType = fields[2],
SalesChannel = Enum.Parse <SalesChannelType>(fields[3]),
OrderPriority = Enum.Parse <OrderPriorityType>(fields[4]),
OrderDate = DateTime.Parse(fields[5]),
OrderId = int.Parse(fields[6]),
ShipDate = DateTime.Parse(fields[7]),
UnitsSold = int.Parse(fields[8]),
UnitPrice = double.Parse(fields[9]),
UnitCost = double.Parse(fields[10]),
TotalRevenue = double.Parse(fields[11]),
TotalCost = double.Parse(fields[12]),
TotalProfit = double.Parse(fields[13])
};
sequentialWriter.Write(withSchema ? salesRecord.ToAvroRecord(schema) : salesRecord.ToAvroRecord());
}
}
var salesRecords = new List <SalesRecord>();
using (var fs = new FileStream(avroFile, FileMode.Open))
using (var reader = AvroContainer.CreateGenericReader(fs))
using (var sequentialReader = new SequentialReader <object>(reader))
{
salesRecords.AddRange(sequentialReader.Objects.Cast <AvroRecord>()
.Select(o => withSchema ? o.FromAvroRecord <SalesRecord>(schema) : o.FromAvroRecord <SalesRecord>()));
}
File.Delete(avroFile);
}
public override void WriteAvroFile(Stream stream)
{
IAvroWriter <LargeClass> avroWriter = AvroContainer.CreateWriter <LargeClass>(
stream,
new AvroSerializerSettings {
UsePosixTime = true
},
this.Codec);
using (var writer = new SequentialWriter <LargeClass>(avroWriter, AvroIntegrationWithHiveConfigurations.AvroBlockSize))
{
foreach (LargeClass record in this.Expected)
{
writer.Write(record);
}
}
}
public void TestReadParameter()
{
MemoryStream ms = new MemoryStream();
using (var writer = AvroContainer.CreateWriter <AddProductParameter>(ms, true, new AvroSerializerSettings(), Codec.Null))
using (var seqWriter = new SequentialWriter <AddProductParameter>(writer, 24))
{
seqWriter.Write(new AddProductParameter {
Location = address0, Product = product0
});
}
ms.Seek(0, SeekOrigin.Begin);
var result = new Dictionary <string, object>();
using (var omr = TestHelper.CreateMessageReader(ms, "avro/binary", AvroMediaTypeResolver.Instance, new EdmModel()))
{
var reader = omr.CreateODataParameterReader(AddProduct);
while (reader.Read())
{
switch (reader.State)
{
case ODataParameterReaderState.Value:
result.Add(reader.Name, reader.Value);
break;
case ODataParameterReaderState.Entry:
var entryReader = reader.CreateEntryReader();
while (entryReader.Read())
{
if (entryReader.State == ODataReaderState.EntryEnd)
{
result.Add(reader.Name, entryReader.Item);
break;
}
}
break;
}
}
}
Assert.AreEqual(2, result.Count);
Assert.IsTrue(TestHelper.ComplexValueEqual(complexValue0, (ODataComplexValue)result["Location"]));
Assert.IsTrue(TestHelper.EntryEqual(entry0, (ODataEntry)result["Product"]));
}
public void TestAvroSerializeUsingObjectContainersGenericRecord()
{
const string schema = @"{
""type"":""record"",
""name"":""Microsoft.Hadoop.Avro.Specifications.SensorData"",
""fields"":
[
{
""name"":""Location"",
""type"":
{
""type"":""record"",
""name"":""Microsoft.Hadoop.Avro.Specifications.Location"",
""fields"":
[
{ ""name"":""Floor"", ""type"":""int"" },
{ ""name"":""Room"", ""type"":""int"" }
]
}
},
{ ""name"":""Value"", ""type"":""bytes"" }
]
}";
//Create a generic serializer based on the schema
var serializer = AvroSerializer.CreateGeneric(schema);
var rootSchema = serializer.WriterSchema as RecordSchema;
dynamic sensorData = CreateDynamicData(rootSchema);
PerformanceHarness.Test(() =>
{
using (var buffer = new MemoryStream())
{
using (var avroWriter = AvroContainer.CreateGenericWriter(schema, buffer, Codec.Null))
{
using (var writer = new SequentialWriter<object>(avroWriter, 24))
{
writer.Write(sensorData);
}
}
}
}, "Avro serialization through generic records", 10000);
}
public void TestReadAvroAsPrimitiveCollection()
{
const string Schema = @"{""type"":""array"", ""items"":""long""}";
var stream = new MemoryStream();
using (var writer = AvroContainer.CreateGenericWriter(Schema, stream, /*leave open*/ true, Codec.Null))
using (var seqWriter = new SequentialWriter <object>(writer, 24))
{
// Serialize the data to stream using the sequential writer
seqWriter.Write(new[] { 6L, 8 });
seqWriter.Flush();
}
stream.Flush();
stream.Seek(0, SeekOrigin.Begin);
var property = this.CreateODataInputContext(stream).ReadProperty(null, null);
}
public void TestAvroSerializationThroughReflection()
{
SensorData sensorData = CreateTestData();
PerformanceHarness.Test(() =>
{
using (var buffer = new MemoryStream())
{
using (var avroWriter = AvroContainer.CreateWriter<SensorData>(buffer, Codec.Deflate))
{
using (var writer = new SequentialWriter<SensorData>(avroWriter, 24))
{
writer.Write(sensorData);
}
}
}
}, "Avro serialization through reflection", 10000);
}
public override Stream GetContentStream(IEnumerable <ExpandoObject> records)
{
var result = new MemoryStream();
// We get a JSON schema for the passed type. Normally, the AvroSerializer expects a type passed that has DataContract/DataMember attributes, but I don't want users of this utility
// to have to modify their POCOs in any way, hence building a schema here and proceeding with that.
string schema = GetJsonSchema();
var serializer = AvroSerializer.CreateGeneric(schema);
var rootSchema = serializer.WriterSchema as RecordSchema;
// We'll write the Avro content to a memory stream
using (var interim = new MemoryStream())
{
// Avro serializer with deflate
using (var avroWriter = AvroContainer.CreateGenericWriter(schema, interim, Codec.Deflate))
{
using (var seqWriter = new SequentialWriter <object>(avroWriter, SYNCNUM))
{
foreach (var record in records)
{
IDictionary <string, object> recordProperties = record as IDictionary <string, object>;
dynamic avroRecord = new AvroRecord(rootSchema);
foreach (KeyValuePair <string, object> recordKVP in recordProperties)
{
avroRecord[recordKVP.Key] = recordKVP.Value;
}
seqWriter.Write(avroRecord);
}
seqWriter.Flush();
interim.Seek(0, SeekOrigin.Begin);
interim.CopyTo(result);
}
}
}
return(result);
}
public void TestReadFeed()
{
MemoryStream ms = new MemoryStream();
IAvroWriter <Product[]> writer = null;
try
{
writer = AvroContainer.CreateWriter <Product[]>(ms, true, new AvroSerializerSettings(), Codec.Null);
using (var seqWriter = new SequentialWriter <Product[]>(writer, 24))
{
seqWriter.Write(new[] { product1, product2, product3 });
}
}
finally
{
if (writer != null)
{
writer.Dispose();
}
}
ms.Seek(0, SeekOrigin.Begin);
var entries = new List <ODataEntry>();
using (var omr = TestHelper.CreateMessageReader(ms, "avro/binary", AvroMediaTypeResolver.Instance))
{
var reader = omr.CreateODataFeedReader();
while (reader.Read())
{
if (reader.State == ODataReaderState.EntryEnd)
{
entries.Add((ODataEntry)reader.Item);
}
}
}
Assert.AreEqual(3, entries.Count);
Assert.IsTrue(TestHelper.EntryEqual(entry1, entries[0]));
Assert.IsTrue(TestHelper.EntryEqual(entry2, entries[1]));
Assert.IsTrue(TestHelper.EntryEqual(entry3, entries[2]));
}
/// <summary>
/// Serialize a ClassHierarchy into a file
/// </summary>
/// <param name="c"></param>
/// <param name="fileName"></param>
public void ToFile(IClassHierarchy c, string fileName)
{
var avronNodeData = ToAvroNode(c);
using (var buffer = new MemoryStream())
{
using (var w = AvroContainer.CreateWriter<AvroNode>(buffer, Codec.Null))
{
using (var writer = new SequentialWriter<AvroNode>(w, 24))
{
writer.Write(avronNodeData);
}
}
if (!WriteFile(buffer, fileName))
{
var e = new ApplicationException("Error during file operation. Quitting method: " + fileName);
Utilities.Diagnostics.Exceptions.Throw(e, LOGGER);
}
}
}
/// <summary>
/// Serialize a ClassHierarchy into a file
/// </summary>
/// <param name="c"></param>
/// <param name="fileName"></param>
public void ToFile(IClassHierarchy c, string fileName)
{
var avroNodeData = ToAvroNode(c);
using (var buffer = new MemoryStream())
{
using (var w = AvroContainer.CreateWriter <AvroNode>(buffer, Codec.Null))
{
using (var writer = new SequentialWriter <AvroNode>(w, 24))
{
writer.Write(avroNodeData);
}
}
if (!WriteFile(buffer, fileName))
{
var e = new TangApplicationException("Error during file operation. Quitting method: " + fileName);
Utilities.Diagnostics.Exceptions.Throw(e, LOGGER);
}
}
}
public void ToFile(IConfiguration c, string fileName)
{
var avronConfigurationData = ToAvroConfiguration(c);
using (var buffer = new MemoryStream())
{
using (var w = AvroContainer.CreateWriter <AvroConfiguration>(buffer, Codec.Null))
{
using (var writer = new SequentialWriter <AvroConfiguration>(w, 24))
{
// Serialize the data to stream using the sequential writer
writer.Write(avronConfigurationData);
}
}
if (!WriteFile(buffer, fileName))
{
var e = new ApplicationException("Error during file operation. Quitting method: " + fileName);
Org.Apache.REEF.Utilities.Diagnostics.Exceptions.Throw(e, LOGGER);
}
}
}
public override void Output(IRow row, IUnstructuredWriter output)
{
// First Row
if (this.writer == null)
{
this.writer = new SequentialWriter <object>(AvroContainer.CreateGenericWriter(this.avroSchema, output.BaseStream, Codec.Deflate), 24);
this.serializer = AvroSerializer.CreateGeneric(this.avroSchema);
}
dynamic data = new AvroRecord(this.serializer.WriterSchema);
data.station = row.Get <string>(row.Schema[0].Name);
data.datekey = row.Get <int?>(row.Schema[1].Name);
data.element = row.Get <string>(row.Schema[2].Name);
data.value = (double?)row.Get <decimal?>(row.Schema[3].Name);
data.mflag = row.Get <string>(row.Schema[4].Name);
data.qflag = row.Get <string>(row.Schema[5].Name);
data.sflag = row.Get <string>(row.Schema[6].Name);
data.timekey = row.Get <int?>(row.Schema[7].Name);
writer.Write(data);
}
public void TestReadEntry()
{
MemoryStream ms = new MemoryStream();
IAvroWriter <Product> writer = null;
try
{
writer = AvroContainer.CreateWriter <Product>(ms, true, new AvroSerializerSettings(), Codec.Null);
using (var seqWriter = new SequentialWriter <Product>(writer, 24))
{
seqWriter.Write(product0);
}
}
finally
{
if (writer != null)
{
writer.Dispose();
}
}
ms.Seek(0, SeekOrigin.Begin);
ODataEntry entry = null;
using (var omr = TestHelper.CreateMessageReader(ms, "avro/binary", AvroMediaTypeResolver.Instance))
{
var reader = omr.CreateODataEntryReader();
while (reader.Read())
{
if (reader.State == ODataReaderState.EntryEnd)
{
entry = (ODataEntry)reader.Item;
}
}
}
Assert.IsTrue(TestHelper.EntryEqual(entry0, entry));
}
public void TestReadAvroAsODataEmptyFeed()
{
string Schema = @"
{""type"":""array"",
""items"":
{
""type"":""record"",
""name"":""TestNS.Person"",
""fields"":
[
{ ""name"":""Id"", ""type"":""int"" },
{ ""name"":""Title"", ""type"":""string"" },
]
}
}";
using (var stream = new MemoryStream())
{
using (var writer = AvroContainer.CreateGenericWriter(Schema, stream, /*leave open*/ true, Codec.Null))
using (var seqWriter = new SequentialWriter <object>(writer, 24))
{
seqWriter.Write(new AvroRecord[] { });
}
stream.Seek(0, SeekOrigin.Begin);
var avroReader = new ODataAvroReader(this.CreateODataInputContext(stream), true);
Assert.AreEqual(ODataReaderState.Start, avroReader.State);
Assert.IsTrue(avroReader.Read());
Assert.AreEqual(ODataReaderState.FeedStart, avroReader.State);
Assert.IsTrue(avroReader.Read());
Assert.AreEqual(ODataReaderState.FeedEnd, avroReader.State);
Assert.IsFalse(avroReader.Read());
Assert.AreEqual(ODataReaderState.Completed, avroReader.State);
}
}
public void TestReadError()
{
MemoryStream ms = new MemoryStream();
using (var writer = AvroContainer.CreateWriter <Error>(ms, true, new AvroSerializerSettings(), Codec.Null))
using (var seqWriter = new SequentialWriter <Error>(writer, 24))
{
seqWriter.Write(new Error {
ErrorCode = "500", Message = "Internal Error"
});
}
ms.Seek(0, SeekOrigin.Begin);
ODataError error = null;
using (var omr = TestHelper.CreateMessageReader(ms, "avro/binary", AvroMediaTypeResolver.Instance, new EdmModel(), true))
{
error = omr.ReadError();
}
Assert.IsNotNull(error);
Assert.AreEqual("500", error.ErrorCode);
Assert.AreEqual("Internal Error", error.Message);
}
public void TestReadParameter()
{
MemoryStream ms = new MemoryStream();
using (var writer = AvroContainer.CreateWriter<AddProductParameter>(ms, true, new AvroSerializerSettings(), Codec.Null))
using (var seqWriter = new SequentialWriter<AddProductParameter>(writer, 24))
{
seqWriter.Write(new AddProductParameter { Location = address0, Product = product0 });
}
ms.Seek(0, SeekOrigin.Begin);
var result = new Dictionary<string, object>();
using (var omr = TestHelper.CreateMessageReader(ms, "avro/binary", AvroMediaTypeResolver.Instance, new EdmModel()))
{
var reader = omr.CreateODataParameterReader(AddProduct);
while (reader.Read())
{
switch (reader.State)
{
case ODataParameterReaderState.Value:
result.Add(reader.Name, reader.Value);
break;
case ODataParameterReaderState.Entry:
var entryReader = reader.CreateEntryReader();
while (entryReader.Read())
{
if (entryReader.State == ODataReaderState.EntryEnd)
{
result.Add(reader.Name, entryReader.Item);
break;
}
}
break;
}
}
}
Assert.AreEqual(2, result.Count);
Assert.IsTrue(TestHelper.ComplexValueEqual(complexValue0, (ODataComplexValue)result["Location"]));
Assert.IsTrue(TestHelper.EntryEqual(entry0, (ODataEntry)result["Product"]));
}
public void ReadAvroAsParameter()
{
EdmEntityType personType = new EdmEntityType("TestNS", "Person");
personType.AddStructuralProperty("Id", EdmPrimitiveTypeKind.Int32);
personType.AddStructuralProperty("Title", EdmPrimitiveTypeKind.String);
var operation = new EdmAction("NS", "op1", null);
operation.AddParameter("p1", EdmCoreModel.Instance.GetString(false));
operation.AddParameter("p2", new EdmEntityTypeReference(personType, false));
const string Schema = @"{
""type"":""record"",
""name"":""NS.op1Parameter"",
""fields"":
[
{ ""name"":""p1"", ""type"":""string"" },
{ ""name"":""p2"", ""type"":
{""type"":""record"",
""name"":""TestNS.Person"",
""fields"":
[
{ ""name"":""Id"", ""type"":""int"" },
{ ""name"":""Title"", ""type"":""string"" },
]
}
}
]
}";
var stream = new MemoryStream();
using (var writer = AvroContainer.CreateGenericWriter(Schema, stream, /*leave open*/true, Codec.Null))
using (var seqWriter = new SequentialWriter<object>(writer, 24))
{
RecordSchema parameterSchema = (RecordSchema)AvroSerializer.CreateGeneric(Schema).WriterSchema;
AvroRecord ar = new AvroRecord(parameterSchema);
ar["p1"] = "dat";
var personSchema = parameterSchema.GetField("p2").TypeSchema;
AvroRecord person = new AvroRecord(personSchema);
person["Id"] = 5;
person["Title"] = "per1";
ar["p2"] = person;
seqWriter.Write(ar);
seqWriter.Flush();
}
stream.Flush();
stream.Seek(0, SeekOrigin.Begin);
var reader = new ODataAvroParameterReader(this.CreateODataInputContext(stream), operation);
Assert.AreEqual(ODataParameterReaderState.Start, reader.State);
Assert.IsTrue(reader.Read());
Assert.AreEqual(ODataParameterReaderState.Value, reader.State);
Assert.AreEqual("p1", reader.Name);
Assert.AreEqual("dat", reader.Value);
Assert.IsTrue(reader.Read());
Assert.AreEqual(ODataParameterReaderState.Entry, reader.State);
Assert.AreEqual("p2", reader.Name);
var ew = reader.CreateEntryReader();
Assert.AreEqual(ODataReaderState.Start, ew.State);
Assert.IsTrue(ew.Read());
Assert.AreEqual(ODataReaderState.EntryStart, ew.State);
Assert.IsTrue(ew.Read());
Assert.AreEqual(ODataReaderState.EntryEnd, ew.State);
var entry = ew.Item as ODataEntry;
Assert.IsFalse(ew.Read());
Assert.AreEqual(ODataReaderState.Completed, ew.State);
Assert.IsNotNull(entry);
var properties = entry.Properties.ToList();
Assert.AreEqual(2, properties.Count);
Assert.AreEqual("Id", properties[0].Name);
Assert.AreEqual(5, properties[0].Value);
Assert.AreEqual("Title", properties[1].Name);
Assert.AreEqual("per1", properties[1].Value);
Assert.IsFalse(reader.Read());
Assert.AreEqual(ODataParameterReaderState.Completed, reader.State);
}
public IEnumerable <T> WriteTo <T>(object writer, IEnumerable <object> records, Func <object, bool> predicate = null)
{
_sw = writer;
IAvroWriter <T> sw1 = writer as IAvroWriter <T>;
ChoGuard.ArgumentNotNull(sw1, "AvroWriter");
using (SequentialWriter <T> sw = new SequentialWriter <T>(sw1, 24))
{
if (records == null)
{
yield break;
}
if (!BeginWrite.Value)
{
yield break;
}
CultureInfo prevCultureInfo = System.Threading.Thread.CurrentThread.CurrentCulture;
System.Threading.Thread.CurrentThread.CurrentCulture = Configuration.Culture;
object recOutput = null;
var recEnum = records.GetEnumerator();
try
{
object notNullRecord = GetFirstNotNullRecord(recEnum);
if (notNullRecord == null)
{
yield break;
}
foreach (object record in GetRecords(recEnum))
{
_index++;
if (TraceSwitch.TraceVerbose)
{
if (record is IChoETLNameableObject)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Writing [{0}] object...".FormatString(((IChoETLNameableObject)record).Name));
}
else
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Writing [{0}] object...".FormatString(_index));
}
}
recOutput = record;
if (record != null)
{
if (predicate == null || predicate(record))
{
if (!RaiseBeforeRecordWrite(record, _index, ref recOutput))
{
yield break;
}
if (recOutput == null)
{
continue;
}
else if (!(recOutput is T))
{
continue;
}
try
{
if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.ObjectLevel) == ChoObjectValidationMode.ObjectLevel)
{
record.DoObjectLevelValidation(Configuration, Configuration.AvroRecordFieldConfigurations);
}
if (recOutput is T)
{
sw.Write((T)recOutput);
if (!RaiseAfterRecordWrite(record, _index, recOutput))
{
yield break;
}
}
}
catch (Exception ex)
{
ChoETLFramework.HandleException(ref ex);
if (Configuration.ErrorMode == ChoErrorMode.IgnoreAndContinue)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceError, "Error [{0}] found. Ignoring record...".FormatString(ex.Message));
}
else if (Configuration.ErrorMode == ChoErrorMode.ReportAndContinue)
{
if (!RaiseRecordWriteError(record, _index, recOutput, ex))
{
throw;
}
else
{
//ChoETLFramework.WriteLog(TraceSwitch.TraceError, "Error [{0}] found. Ignoring record...".FormatString(ex.Message));
}
}
else
{
throw;
}
}
}
}
yield return((T)recOutput);
if (Configuration.NotifyAfter > 0 && _index % Configuration.NotifyAfter == 0)
{
if (RaisedRowsWritten(_index))
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Abort requested.");
yield break;
}
}
}
}
finally
{
System.Threading.Thread.CurrentThread.CurrentCulture = prevCultureInfo;
}
}
}
public void ReadAvroAsError()
{
const string Schema = @"{
""type"":""record"",
""name"":""OData.Error"",
""fields"":
[
{ ""name"":""ErrorCode"", ""type"":""string"" },
{ ""name"":""Message"" , ""type"":""string"" },
]
}";
var stream = new MemoryStream();
using (var writer = AvroContainer.CreateGenericWriter(Schema, stream, /*leave open*/true, Codec.Null))
using (var seqWriter = new SequentialWriter<object>(writer, 24))
{
RecordSchema parameterSchema = (RecordSchema)AvroSerializer.CreateGeneric(Schema).WriterSchema;
AvroRecord ar = new AvroRecord(parameterSchema);
ar["ErrorCode"] = "e1";
ar["Message"] = "m1";
seqWriter.Write(ar);
seqWriter.Flush();
}
stream.Flush();
stream.Seek(0, SeekOrigin.Begin);
var error = this.CreateODataInputContext(stream).ReadError();
Assert.AreEqual("e1", error.ErrorCode);
Assert.AreEqual("m1", error.Message);
}
public void SequentialReaderWriter_CreateReaderForSchemaWithNullableField()
{
var expected = new ClassWithNullableIntField { NullableIntField = 10 };
using (var writer = AvroContainer.CreateWriter<ClassWithNullableIntField>(this.resultStream, Codec.Deflate))
{
using (var swriter = new SequentialWriter<ClassWithNullableIntField>(writer, 2))
{
swriter.Write(expected);
swriter.Write(expected);
swriter.Write(expected);
swriter.Write(expected);
}
}
this.resultStream.Seek(0, SeekOrigin.Begin);
using (var reader = AvroContainer.CreateReader<ClassWithNullableIntField>(this.resultStream))
{
using (var sreader = new SequentialReader<ClassWithNullableIntField>(reader))
{
foreach (var actual in sreader.Objects)
{
Assert.AreEqual(expected.NullableIntField, actual.NullableIntField);
}
}
}
}
public IEnumerable <T> WriteTo <T>(object writer, IEnumerable <object> records, Func <object, bool> predicate = null)
{
Configuration.Init();
if (records == null)
{
yield break;
}
var recEnum = records.GetEnumerator();
object notNullRecord = GetFirstNotNullRecord(recEnum);
if (notNullRecord == null)
{
yield break;
}
DiscoverKnownTypes(notNullRecord);
StreamWriter sw = null;
if (writer is Lazy <StreamWriter> )
{
var lsw = writer as Lazy <StreamWriter>;
ChoGuard.ArgumentNotNull(lsw, "StreamWriter");
_sw = sw = lsw.Value;
if (!Configuration.UseAvroSerializer)
{
if (_avroWriter == null)
{
_avroWriter = new SequentialWriter <T>(CreateAvroWriter <T>(sw), Configuration.SyncNumberOfObjects);
}
}
else
{
_avroSerializer = CreateAvroSerializer <T>();
}
}
else
{
_avroWriter = writer as IAvroWriter <T>;
if (_avroWriter == null)
{
throw new ChoParserException("Missing valid writer object passed.");
}
}
if (!BeginWrite.Value)
{
yield break;
}
CultureInfo prevCultureInfo = System.Threading.Thread.CurrentThread.CurrentCulture;
System.Threading.Thread.CurrentThread.CurrentCulture = Configuration.Culture;
object recOutput = null;
try
{
foreach (object record in GetRecords(recEnum))
{
_index++;
if (TraceSwitch.TraceVerbose)
{
if (record is IChoETLNameableObject)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Writing [{0}] object...".FormatString(((IChoETLNameableObject)record).Name));
}
else
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Writing [{0}] object...".FormatString(_index));
}
}
recOutput = record;
if (record != null)
{
if (predicate == null || predicate(record))
{
if (!RaiseBeforeRecordWrite(record, _index, ref recOutput))
{
yield break;
}
if (recOutput == null)
{
continue;
}
else if (!(recOutput is T))
{
continue;
}
try
{
if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.ObjectLevel) == ChoObjectValidationMode.ObjectLevel)
{
record.DoObjectLevelValidation(Configuration, Configuration.AvroRecordFieldConfigurations);
}
if (recOutput is T)
{
if (recOutput is ChoDynamicObject)
{
recOutput = new Dictionary <string, object>((ChoDynamicObject)recOutput);
}
if (_sw != null)
{
if (Configuration.UseAvroSerializer)
{
IAvroSerializer <T> avroSerializer = _avroSerializer as IAvroSerializer <T>;
avroSerializer.Serialize(sw.BaseStream, (T)(recOutput as object));
}
else
{
SequentialWriter <T> avroWriter = _avroWriter as SequentialWriter <T>;
avroWriter.Write((T)(recOutput as object));
}
}
else
{
SequentialWriter <T> avroWriter = _avroWriter as SequentialWriter <T>;
avroWriter.Write((T)(recOutput as object));
}
if (!RaiseAfterRecordWrite(record, _index, recOutput))
{
yield break;
}
}
}
catch (Exception ex)
{
ChoETLFramework.HandleException(ref ex);
if (Configuration.ErrorMode == ChoErrorMode.IgnoreAndContinue)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceError, "Error [{0}] found. Ignoring record...".FormatString(ex.Message));
}
else if (Configuration.ErrorMode == ChoErrorMode.ReportAndContinue)
{
if (!RaiseRecordWriteError(record, _index, recOutput, ex))
{
throw;
}
else
{
//ChoETLFramework.WriteLog(TraceSwitch.TraceError, "Error [{0}] found. Ignoring record...".FormatString(ex.Message));
}
}
else
{
throw;
}
}
}
}
yield return((T)recOutput);
if (Configuration.NotifyAfter > 0 && _index % Configuration.NotifyAfter == 0)
{
if (RaisedRowsWritten(_index))
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Abort requested.");
yield break;
}
}
}
}
finally
{
System.Threading.Thread.CurrentThread.CurrentCulture = prevCultureInfo;
}
}
public void ToFile(IConfiguration c, string fileName)
{
var avronConfigurationData = ToAvroConfiguration(c);
using (var buffer = new MemoryStream())
{
using (var w = AvroContainer.CreateWriter<AvroConfiguration>(buffer, Codec.Null))
{
using (var writer = new SequentialWriter<AvroConfiguration>(w, 24))
{
// Serialize the data to stream using the sequential writer
writer.Write(avronConfigurationData);
}
}
if (!WriteFile(buffer, fileName))
{
var e = new ApplicationException("Error during file operation. Quitting method: " + fileName);
Org.Apache.REEF.Utilities.Diagnostics.Exceptions.Throw(e, LOGGER);
}
}
}
public void TestReadAvroAsODataEntry()
{
const string Schema = @"
{
""type"":""record"",
""name"":""TestNS.Person"",
""fields"":
[
{ ""name"":""Id"", ""type"":""int"" },
{ ""name"":""Title"", ""type"":""string"" },
{ ""name"":""Address"", ""type"":{
""name"":""TestNS.Address"",
""type"":""record"",
""fields"":[
{ ""name"":""ZipCode"", ""type"":""long"" },
]
}
},
]
}";
var serializer = AvroSerializer.CreateGeneric(Schema);
using (var stream = new MemoryStream())
{
var expected = new AvroRecord(serializer.WriterSchema);
expected["Id"] = -5;
expected["Title"] = "set";
var cpxSchema = ((RecordSchema)serializer.WriterSchema).GetField("Address").TypeSchema;
var cpx = new AvroRecord(cpxSchema);
cpx["ZipCode"] = 5L;
expected["Address"] = cpx;
using (var writer = AvroContainer.CreateGenericWriter(Schema, stream, /*leave open*/true, Codec.Null))
{
using (var streamWriter = new SequentialWriter<object>(writer, 24))
{
// Serialize the data to stream using the sequential writer
streamWriter.Write(expected);
}
}
stream.Seek(0, SeekOrigin.Begin);
var avroReader = new ODataAvroReader(this.CreateODataInputContext(stream), false);
Assert.AreEqual(ODataReaderState.Start, avroReader.State);
Assert.IsTrue(avroReader.Read());
Assert.AreEqual(ODataReaderState.EntryStart, avroReader.State);
Assert.IsTrue(avroReader.Read());
Assert.AreEqual(ODataReaderState.EntryEnd, avroReader.State);
var entry = avroReader.Item as ODataEntry;
Assert.IsNotNull(entry);
Assert.AreEqual("TestNS.Person", entry.TypeName);
var properties = entry.Properties.ToList();
Assert.AreEqual(3, properties.Count);
Assert.AreEqual("Id", properties[0].Name);
Assert.AreEqual(-5, properties[0].Value);
Assert.AreEqual("Title", properties[1].Name);
Assert.AreEqual("set", properties[1].Value);
Assert.AreEqual("Address", properties[2].Name);
var address = properties[2].Value as ODataComplexValue;
Assert.IsNotNull(address);
Assert.AreEqual("TestNS.Address", address.TypeName);
var zip = address.Properties.Single();
Assert.AreEqual(5L, zip.Value);
Assert.IsFalse(avroReader.Read());
Assert.AreEqual(ODataReaderState.Completed, avroReader.State);
}
}
public void TestReadAvroAsPrimitiveCollection()
{
const string Schema = @"{""type"":""array"", ""items"":""long""}";
var stream = new MemoryStream();
using (var writer = AvroContainer.CreateGenericWriter(Schema, stream, /*leave open*/true, Codec.Null))
using (var seqWriter = new SequentialWriter<object>(writer, 24))
{
// Serialize the data to stream using the sequential writer
seqWriter.Write(new[] { 6L, 8 });
seqWriter.Flush();
}
stream.Flush();
stream.Seek(0, SeekOrigin.Begin);
var property = this.CreateODataInputContext(stream).ReadProperty(null, null);
}
public void TestReadAvroAsODataEntry()
{
const string Schema = @"
{
""type"":""record"",
""name"":""TestNS.Person"",
""fields"":
[
{ ""name"":""Id"", ""type"":""int"" },
{ ""name"":""Title"", ""type"":""string"" },
{ ""name"":""Address"", ""type"":{
""name"":""TestNS.Address"",
""type"":""record"",
""fields"":[
{ ""name"":""ZipCode"", ""type"":""long"" },
]
}
},
]
}";
var serializer = AvroSerializer.CreateGeneric(Schema);
using (var stream = new MemoryStream())
{
var expected = new AvroRecord(serializer.WriterSchema);
expected["Id"] = -5;
expected["Title"] = "set";
var cpxSchema = ((RecordSchema)serializer.WriterSchema).GetField("Address").TypeSchema;
var cpx = new AvroRecord(cpxSchema);
cpx["ZipCode"] = 5L;
expected["Address"] = cpx;
using (var writer = AvroContainer.CreateGenericWriter(Schema, stream, /*leave open*/ true, Codec.Null))
{
using (var streamWriter = new SequentialWriter <object>(writer, 24))
{
// Serialize the data to stream using the sequential writer
streamWriter.Write(expected);
}
}
stream.Seek(0, SeekOrigin.Begin);
var avroReader = new ODataAvroReader(this.CreateODataInputContext(stream), false);
Assert.AreEqual(ODataReaderState.Start, avroReader.State);
Assert.IsTrue(avroReader.Read());
Assert.AreEqual(ODataReaderState.EntryStart, avroReader.State);
Assert.IsTrue(avroReader.Read());
Assert.AreEqual(ODataReaderState.EntryEnd, avroReader.State);
var entry = avroReader.Item as ODataEntry;
Assert.IsNotNull(entry);
Assert.AreEqual("TestNS.Person", entry.TypeName);
var properties = entry.Properties.ToList();
Assert.AreEqual(3, properties.Count);
Assert.AreEqual("Id", properties[0].Name);
Assert.AreEqual(-5, properties[0].Value);
Assert.AreEqual("Title", properties[1].Name);
Assert.AreEqual("set", properties[1].Value);
Assert.AreEqual("Address", properties[2].Name);
var address = properties[2].Value as ODataComplexValue;
Assert.IsNotNull(address);
Assert.AreEqual("TestNS.Address", address.TypeName);
var zip = address.Properties.Single();
Assert.AreEqual(5L, zip.Value);
Assert.IsFalse(avroReader.Read());
Assert.AreEqual(ODataReaderState.Completed, avroReader.State);
}
}
public void Test_ComplexType_Serialization_To_Stream()
{
dynamic instance = new ShapeBasket
{
Shapes = new List <IShape>
{
// the schema for Circle allows for Names and Radius to be null
new Circle(),
new Circle {
Name = "Red Dot", Radius = 15, Color = BasicColor.Red
},
new Square {
Name = "Blue Square", Width = 20, Color = BasicColor.Blue
},
new Triangle
{
Name = "Bermuda Triangle", SideA = 10, SideB = 20, SideC = 30, Color = BasicColor.Indigo
},
new StrangeShape
{
Name = "Blob",
ChildShape = new StrangeShape
{
Name = "Child Blob", ChildShape = new Square {
Name = "Child Square", Width = 50
}
}
}
}
};
var schema = Resources.ShapeBasket_v1_0;
var avroFile = Path.GetTempFileName();
// serialize
using (var fs = new FileStream(avroFile, FileMode.Create))
{
using var writer = AvroContainer.CreateGenericWriter(schema, fs, Codec.Deflate);
using var sequentialWriter = new SequentialWriter <object>(writer, 1);
sequentialWriter.Write(((object)instance).ToAvroRecord(schema));
}
dynamic target;
// deserialize
using (var fs = new FileStream(avroFile, FileMode.Open))
{
using var reader = AvroContainer.CreateGenericReader(fs);
using var sequentialReader = new SequentialReader <object>(reader);
target = sequentialReader.Objects.Cast <AvroRecord>().Select(r => r.FromAvroRecord <ShapeBasket>())
.FirstOrDefault();
}
File.Delete(avroFile);
Assert.AreEqual(instance.Shapes[0].Name, target.Shapes[0].Name);
Assert.AreEqual(instance.Shapes[1].Name, target.Shapes[1].Name);
Assert.AreEqual(instance.Shapes[1].Radius, target.Shapes[1].Radius);
Assert.AreEqual(instance.Shapes[1].Color, target.Shapes[1].Color);
Assert.AreEqual(instance.Shapes[2].Name, target.Shapes[2].Name);
Assert.AreEqual(instance.Shapes[2].Width, target.Shapes[2].Width);
Assert.AreEqual(instance.Shapes[2].Color, target.Shapes[2].Color);
Assert.AreEqual(instance.Shapes[3].Name, target.Shapes[3].Name);
Assert.AreEqual(instance.Shapes[3].SideA, target.Shapes[3].SideA);
Assert.AreEqual(instance.Shapes[3].SideB, target.Shapes[3].SideB);
Assert.AreEqual(instance.Shapes[3].SideC, target.Shapes[3].SideC);
Assert.AreEqual(instance.Shapes[3].Color, target.Shapes[3].Color);
Assert.AreEqual(instance.Shapes[4].Name, target.Shapes[4].Name);
Assert.AreEqual(instance.Shapes[4].ChildShape.Name, target.Shapes[4].ChildShape.Name);
Assert.AreEqual(instance.Shapes[4].ChildShape.ChildShape.Name, target.Shapes[4].ChildShape.ChildShape.Name);
Assert.AreEqual(instance.Shapes[4].ChildShape.ChildShape.Width,
target.Shapes[4].ChildShape.ChildShape.Width);
}
public void TestReadAvroAsCollection()
{
const string Schema = @"{""type"":""array"", ""items"":""long""}";
var stream = new MemoryStream();
using (var writer = AvroContainer.CreateGenericWriter(Schema, stream, /*leave open*/true, Codec.Null))
using (var seqWriter = new SequentialWriter<object>(writer, 24))
{
// Serialize the data to stream using the sequential writer
seqWriter.Write(new[] { 6L, 8 });
seqWriter.Flush();
}
stream.Flush();
stream.Seek(0, SeekOrigin.Begin);
var reader = new ODataAvroCollectionReader(this.CreateODataInputContext(stream));
Assert.AreEqual(ODataCollectionReaderState.Start, reader.State);
Assert.IsTrue(reader.Read());
Assert.AreEqual(ODataCollectionReaderState.CollectionStart, reader.State);
Assert.IsTrue(reader.Read());
Assert.AreEqual(ODataCollectionReaderState.Value, reader.State);
Assert.AreEqual(6L, (long)reader.Item);
Assert.IsTrue(reader.Read());
Assert.AreEqual(ODataCollectionReaderState.Value, reader.State);
Assert.IsTrue(reader.Read());
Assert.AreEqual(ODataCollectionReaderState.CollectionEnd, reader.State);
Assert.IsFalse(reader.Read());
Assert.AreEqual(ODataCollectionReaderState.Completed, reader.State);
}
static void Main(string[] args)
{
//Read data from files
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(CloudConfigurationManager.GetSetting("storageAccountName"));
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
CloudBlobContainer container = blobClient.GetContainerReference(CloudConfigurationManager.GetSetting("eventHubName"));
var blobInputInitialPath = CloudConfigurationManager.GetSetting("eventHubName") + "/" + CloudConfigurationManager.GetSetting("eventHubName") + "/";
var blobOutputInitialPath = "outputdata/";
var dt = DateTime.UtcNow.AddHours(-1);
var dtStr = dt.ToString("yyyy/MM/dd/HH/");
var ehPartitions = 8;
MemoryStream stream = new MemoryStream();
MemoryStream outStream = new MemoryStream();
var outputSchemaJson = "{\"type\":\"record\",\"name\":\"EventDataOut\",\"namespace\":\"Microsoft.ServiceBus.Messaging\",\"fields\":[{\"name\":\"Offset\",\"type\":\"string\"},{\"name\":\"EnqueuedTimeUtc\",\"type\":\"string\"}, {\"name\":\"Body\",\"type\":[\"null\",\"bytes\"]}]}";
var outputSchema = Microsoft.Hadoop.Avro.Schema.TypeSchema.Create(outputSchemaJson);
var writer = AvroContainer.CreateGenericWriter(outputSchemaJson, outStream, Codec.Null);
for (int i = 0; i < ehPartitions; i++)
{
var blobPath = blobInputInitialPath + i.ToString() + "/" + dtStr;
foreach (IListBlobItem item in container.ListBlobs(blobPath, true))
{
CloudBlockBlob cbb = (CloudBlockBlob)item;
if (cbb.Properties.Length > 200)
{
Console.Out.WriteLine("Downloading File: " + cbb.Name);
stream = new MemoryStream();
cbb.DownloadToStream(stream);
//Setting stream position to zero so we can read from the start
stream.Position = 0;
var reader = AvroContainer.CreateGenericReader(stream); //Reading the stream
AvroRecord avroRecord = new AvroRecord(outputSchema); //Creating a new Avro Record to be written
while (reader.MoveNext())
{
var streamWriter = new SequentialWriter <object>(writer, reader.Current.Objects.Count());
var testCount = reader.Current.Objects.Count();
Console.Out.WriteLine("Reading downloaded Objects. Count: " + testCount);
foreach (dynamic record in reader.Current.Objects)
{
var eventData = new EventData(record);
var eventDataOut = new EventDataOut(eventData);
dynamic outRecord = new AvroRecord(outputSchema);
outRecord.Offset = eventData.Offset;
outRecord.EnqueuedTimeUtc = eventData.EnqueuedTimeUtc.ToString();
outRecord.Body = eventData.Body;
streamWriter.Write(outRecord); //Writing the Avro Record
}
}
}
}
}
//Test to Read the Avro File we just wrote
outStream.Position = 0;
var testreader = AvroContainer.CreateGenericReader(outStream);
while (testreader.MoveNext())
{
var testCount = testreader.Current.Objects.Count();
Console.Out.WriteLine("Reading Written Objects. Count: " + testCount);
foreach (dynamic record in testreader.Current.Objects)
{
var eventData = new EventDataOut(record);
Console.Out.WriteLine("Test - EventData Body: " + Encoding.UTF8.GetString(eventData.Body));
}
}
outStream.Position = 0;
var outputBlob = container.GetBlockBlobReference(blobOutputInitialPath + dtStr + "output.avro");
outputBlob.UploadFromStream(outStream);
Console.Out.WriteLine("Test Done");
}
public void TestReadParameterWithFeed()
{
MemoryStream ms = new MemoryStream();
using (var writer = AvroContainer.CreateWriter<GetMaxIdParameter>(ms, true, new AvroSerializerSettings(), Codec.Null))
using (var seqWriter = new SequentialWriter<GetMaxIdParameter>(writer, 24))
{
seqWriter.Write(new GetMaxIdParameter { Products = new List<Product>() { product0, product1 } });
}
ms.Seek(0, SeekOrigin.Begin);
var result = new Dictionary<string, object>();
using (var omr = TestHelper.CreateMessageReader(ms, "avro/binary", AvroMediaTypeResolver.Instance, new EdmModel()))
{
var reader = omr.CreateODataParameterReader(GetMaxId);
while (reader.Read())
{
switch (reader.State)
{
case ODataParameterReaderState.Value:
result.Add(reader.Name, reader.Value);
break;
case ODataParameterReaderState.Entry:
var entryReader = reader.CreateEntryReader();
while (entryReader.Read())
{
if (entryReader.State == ODataReaderState.EntryEnd)
{
result.Add(reader.Name, entryReader.Item);
break;
}
}
break;
case ODataParameterReaderState.Feed:
var feedReader = reader.CreateFeedReader();
IList<ODataEntry> entryList = new List<ODataEntry>();
while (feedReader.Read())
{
if (feedReader.State == ODataReaderState.EntryEnd)
{
entryList.Add((ODataEntry)feedReader.Item);
}
}
result.Add(reader.Name, entryList);
break;
}
}
}
Assert.AreEqual(1, result.Count);
var feed = result["Products"] as IList<ODataEntry>;
Assert.IsNotNull(feed);
Assert.AreEqual(2, feed.Count);
Assert.IsTrue(TestHelper.EntryEqual(entry0, feed[0]));
Assert.IsTrue(TestHelper.EntryEqual(entry1, feed[1]));
}
public void TestReadError()
{
MemoryStream ms = new MemoryStream();
using (var writer = AvroContainer.CreateWriter<Error>(ms, true, new AvroSerializerSettings(), Codec.Null))
using (var seqWriter = new SequentialWriter<Error>(writer, 24))
{
seqWriter.Write(new Error { ErrorCode = "500", Message = "Internal Error" });
}
ms.Seek(0, SeekOrigin.Begin);
ODataError error = null;
using (var omr = TestHelper.CreateMessageReader(ms, "avro/binary", AvroMediaTypeResolver.Instance, new EdmModel(), true))
{
error = omr.ReadError();
}
Assert.IsNotNull(error);
Assert.AreEqual("500", error.ErrorCode);
Assert.AreEqual("Internal Error", error.Message);
}
public void ReadAvroAsParameter()
{
EdmEntityType personType = new EdmEntityType("TestNS", "Person");
personType.AddStructuralProperty("Id", EdmPrimitiveTypeKind.Int32);
personType.AddStructuralProperty("Title", EdmPrimitiveTypeKind.String);
var operation = new EdmAction("NS", "op1", null);
operation.AddParameter("p1", EdmCoreModel.Instance.GetString(false));
operation.AddParameter("p2", new EdmEntityTypeReference(personType, false));
const string Schema = @"{
""type"":""record"",
""name"":""NS.op1Parameter"",
""fields"":
[
{ ""name"":""p1"", ""type"":""string"" },
{ ""name"":""p2"", ""type"":
{""type"":""record"",
""name"":""TestNS.Person"",
""fields"":
[
{ ""name"":""Id"", ""type"":""int"" },
{ ""name"":""Title"", ""type"":""string"" },
]
}
}
]
}";
var stream = new MemoryStream();
using (var writer = AvroContainer.CreateGenericWriter(Schema, stream, /*leave open*/ true, Codec.Null))
using (var seqWriter = new SequentialWriter <object>(writer, 24))
{
RecordSchema parameterSchema = (RecordSchema)AvroSerializer.CreateGeneric(Schema).WriterSchema;
AvroRecord ar = new AvroRecord(parameterSchema);
ar["p1"] = "dat";
var personSchema = parameterSchema.GetField("p2").TypeSchema;
AvroRecord person = new AvroRecord(personSchema);
person["Id"] = 5;
person["Title"] = "per1";
ar["p2"] = person;
seqWriter.Write(ar);
seqWriter.Flush();
}
stream.Flush();
stream.Seek(0, SeekOrigin.Begin);
var reader = new ODataAvroParameterReader(this.CreateODataInputContext(stream), operation);
Assert.AreEqual(ODataParameterReaderState.Start, reader.State);
Assert.IsTrue(reader.Read());
Assert.AreEqual(ODataParameterReaderState.Value, reader.State);
Assert.AreEqual("p1", reader.Name);
Assert.AreEqual("dat", reader.Value);
Assert.IsTrue(reader.Read());
Assert.AreEqual(ODataParameterReaderState.Entry, reader.State);
Assert.AreEqual("p2", reader.Name);
var ew = reader.CreateEntryReader();
Assert.AreEqual(ODataReaderState.Start, ew.State);
Assert.IsTrue(ew.Read());
Assert.AreEqual(ODataReaderState.EntryStart, ew.State);
Assert.IsTrue(ew.Read());
Assert.AreEqual(ODataReaderState.EntryEnd, ew.State);
var entry = ew.Item as ODataEntry;
Assert.IsFalse(ew.Read());
Assert.AreEqual(ODataReaderState.Completed, ew.State);
Assert.IsNotNull(entry);
var properties = entry.Properties.ToList();
Assert.AreEqual(2, properties.Count);
Assert.AreEqual("Id", properties[0].Name);
Assert.AreEqual(5, properties[0].Value);
Assert.AreEqual("Title", properties[1].Name);
Assert.AreEqual("per1", properties[1].Value);
Assert.IsFalse(reader.Read());
Assert.AreEqual(ODataParameterReaderState.Completed, reader.State);
}
static int Main(string[] args)
{
bool showHelp = false;
bool breakWrongParameterUsage = false;
string connectionString = string.Empty;
string sqlQuery = string.Empty;
string sqlQueryFile = string.Empty;
string outputFile = string.Empty;
string format = string.Empty;
string schema = string.Empty;
string schemaFile = string.Empty;
var p = new OptionSet()
{
{ "c|connection=", "the {SQL CONNECTION STRING} for the SQL Server to connect with.", v => connectionString = v },
{ "q|query=", "the {SQL QUERY} to be used.", v => sqlQuery = v },
{ "qf|queryFile=", "the {SQL QUERY FILE} to beh used.", v => sqlQueryFile = v },
{ "of|outputFile=", "the fiel where to save the export file.", v => outputFile = v },
{ "f|format=", "the {FORMAT} to use for the export file.", v => format = v },
{ "s|schema=", "the schema; required for the AVRO file format", v => schema = v },
{ "sf|schemaFile=", "the schema file; required for the AVRO file format", v => schemaFile = v },
{ "h|help", "show this message and exit", v => showHelp = v != null },
};
// parse args
List <string> extra;
try
{
extra = p.Parse(args);
}
catch (OptionException e)
{
Console.Write($"{ApplicationName}: ");
Console.WriteLine(e.Message);
Console.WriteLine("Try `sqldataexport --help` for more information.");
return(1); // Exit Error
}
if (showHelp)
{
ShowHelp(p);
return(0); // Exit OK
}
// validate args
if (string.IsNullOrEmpty(connectionString))
{
Console.WriteLine("SQL connection string not given");
breakWrongParameterUsage = true;
}
if (string.IsNullOrEmpty(outputFile))
{
Console.WriteLine("Output file not given");
breakWrongParameterUsage = true;
}
if (!string.IsNullOrEmpty(sqlQueryFile))
{
if (!File.Exists(sqlQueryFile))
{
Console.WriteLine("SQL query file {0} not found", sqlQueryFile);
breakWrongParameterUsage = true;
}
try
{
sqlQuery = File.ReadAllText(sqlQueryFile);
}
catch (Exception e)
{
Console.WriteLine("Error while loading SQL query file: " + e.Message);
return(1); // Exit Error
}
}
if (string.IsNullOrEmpty(sqlQuery))
{
Console.WriteLine("SQL query not given or empty");
}
switch (format.ToLower())
{
case "avro":
if (!string.IsNullOrEmpty(schemaFile))
{
if (!File.Exists(schemaFile))
{
Console.WriteLine("Schema file {0} not found", schemaFile);
breakWrongParameterUsage = true;
}
try
{
schema = File.ReadAllText(schemaFile);
}
catch (Exception e)
{
Console.WriteLine("Error while loading schema file: " + e.Message);
return(1); // Exit Error
}
}
if (string.IsNullOrEmpty(schema))
{
Console.WriteLine("Schema not given or empty");
breakWrongParameterUsage = true;
}
break;
case "orc":
throw new NotImplementedException("https://github.com/ddrinka/ApacheOrcDotNet/blob/master/src/ApacheOrcDotNet.WriterTest.App/Program.cs");
break;
default:
Console.WriteLine("Invalid format {0}", format);
breakWrongParameterUsage = true;
break;
}
if (breakWrongParameterUsage)
{
Console.WriteLine($"Try `{ApplicationName} --help` for more information.");
return(1); // Exit Error
}
// run actual application
var connection = new SqlConnection(connectionString);
var command = new SqlCommand(sqlQuery, connection);
ReportStatus("Open connection ...", true);
connection.Open();
try
{
ReportStatus("Execute SQL command ...");
var dataReader = command.ExecuteReader();
if (dataReader.Read())
{
ReportStatus("Process rows ...");
using (var file = File.Create(outputFile))
{
// see also: https://code.msdn.microsoft.com/Serialize-data-with-the-86055923/sourcecode?fileId=111532&pathId=931764448
var serializer = AvroSerializer.CreateGeneric(schema);
var rootSchema = serializer.WriterSchema;
var container = AvroContainer.CreateGenericWriter(schema, file, Codec.Null);
var schemaFields = GetSchemaFields(schema);
using (var wr = new SequentialWriter <object>(container, 24))
{
int rowCount = 0;
do
{
var record = new AvroRecord(rootSchema);
for (int i = 0; i < dataReader.FieldCount; i++)
{
object newValue = dataReader.IsDBNull(i) ? null : dataReader.GetValue(i);
dataReader.GetFieldType(i);
record[i] = CastToAvroValue(newValue, schemaFields[i]);
}
wr.Write(record);
if (++rowCount % statusReportPerRows == 0)
{
ReportStatus($"Process rows ({rowCount:G}) ...");
}
} while (dataReader.Read());
ReportStatus($"Processed {rowCount:G} rows");
}
}
}
}
catch (Exception e)
{
Console.Write("Application Exception: ");
Console.WriteLine(e.Message);
return(1); // Exit Error
}
finally
{
connection.Close();
}
Console.WriteLine("Done");
return(0); // Exit OK
}
public void TestReadAvroAsODataEmptyFeed()
{
string Schema = @"
{""type"":""array"",
""items"":
{
""type"":""record"",
""name"":""TestNS.Person"",
""fields"":
[
{ ""name"":""Id"", ""type"":""int"" },
{ ""name"":""Title"", ""type"":""string"" },
]
}
}";
using (var stream = new MemoryStream())
{
using (var writer = AvroContainer.CreateGenericWriter(Schema, stream, /*leave open*/true, Codec.Null))
using (var seqWriter = new SequentialWriter<object>(writer, 24))
{
seqWriter.Write(new AvroRecord[] { });
}
stream.Seek(0, SeekOrigin.Begin);
var avroReader = new ODataAvroReader(this.CreateODataInputContext(stream), true);
Assert.AreEqual(ODataReaderState.Start, avroReader.State);
Assert.IsTrue(avroReader.Read());
Assert.AreEqual(ODataReaderState.FeedStart, avroReader.State);
Assert.IsTrue(avroReader.Read());
Assert.AreEqual(ODataReaderState.FeedEnd, avroReader.State);
Assert.IsFalse(avroReader.Read());
Assert.AreEqual(ODataReaderState.Completed, avroReader.State);
}
}
public static void SerializeDeserializeObjectUsingReflectionStream()
{
string blobName = "aaa/avrotest/test008";
CloudBlobClient client = new Microsoft.WindowsAzure.Storage.Blob.CloudBlobClient(
new Uri("http://hanzstorage.blob.core.windows.net"),
new Microsoft.WindowsAzure.Storage.Auth.StorageCredentials(
"hanzstorage",
"w9TEpvGTusvFlGAdCoWdDrwqLzy6er0Zm5YKdDD0YTkQdOj3WufeVrgd2c8q8amLR0o6xD0tBChcIIA+DCgxXA=="
));
CloudBlobContainer container = client.GetContainerReference("hanzhdi");
CloudBlockBlob blockBlob = container.GetBlockBlobReference(blobName);
foreach (var md in blockBlob.Metadata)
{
Console.WriteLine("{0} {1}", md.Key, md.Value);
}
Console.WriteLine("Serializing Sample Data Set USING REFLECTION\n");
AvroBlobAppender.AvroBlobAppenderWriter <MyDataType> writer =
new AvroBlobAppender.AvroBlobAppenderWriter <MyDataType>(blockBlob, false, AvroSerializer.Create <MyDataType>(), Codec.Null);
Microsoft.Hadoop.Avro.Container.SequentialWriter <MyDataType> sequentialWriter =
new SequentialWriter <MyDataType>(writer, 10000);
List <MyDataType> myDataList = MyDataType.GenerateData(555, 10);
foreach (var myData in myDataList)
{
sequentialWriter.Write(myData);
}
sequentialWriter.Flush();
sequentialWriter.Dispose();
#region commented code
//blockBlob.DownloadToFile(blobName, FileMode.Create);
//using (Stream stream = File.OpenRead(blobName))
//{
// Microsoft.Hadoop.Avro.Container.SequentialReader<MyDataType> reader =
// new Microsoft.Hadoop.Avro.Container.SequentialReader<MyDataType>(AvroContainer.CreateReader<MyDataType>(stream));
// List<MyDataType> actuals = reader.Objects.ToList();
// Console.WriteLine("Number of objects: {0}", actuals.Count);
// for (int i = 0; i < actuals.Count; i++)
// {
// var actual = actuals[i];
// MyDataType exp = null;
// switch (i)
// {
// case 0:
// exp = expected;
// break;
// case 1:
// exp = expected2;
// break;
// default:
// Console.WriteLine("No expected for object {0}", i);
// continue;
// }
// Console.WriteLine("Result of Data Set Identity Comparison is {0}", Program.Equal(exp, actual));
// }
//}
#endregion
}
public void TestReadAvroAsODataFeed()
{
const string Schema = @"
{""type"":""array"",
""items"":
{
""type"":""record"",
""name"":""TestNS.Person"",
""fields"":
[
{ ""name"":""Id"", ""type"":""int"" },
{ ""name"":""Title"", ""type"":""string"" },
]
}
}";
var serializer = AvroSerializer.CreateGeneric(Schema);
using (var stream = new MemoryStream())
{
var arraySchema = (ArraySchema)serializer.WriterSchema;
var recordSchema = arraySchema.ItemSchema;
var rec1 = new AvroRecord(recordSchema);
rec1["Id"] = 1;
rec1["Title"] = "s1";
var rec2 = new AvroRecord(recordSchema);
rec2["Id"] = 2;
rec2["Title"] = "s2";
var array = new[] { rec1, rec2 };
using (var writer = AvroContainer.CreateGenericWriter(Schema, stream, /*leave open*/ true, Codec.Null))
using (var seqWriter = new SequentialWriter <object>(writer, 24))
{
seqWriter.Write(array);
}
stream.Seek(0, SeekOrigin.Begin);
var avroReader = new ODataAvroReader(this.CreateODataInputContext(stream), true);
Assert.AreEqual(ODataReaderState.Start, avroReader.State);
Assert.IsTrue(avroReader.Read());
Assert.AreEqual(ODataReaderState.FeedStart, avroReader.State);
Assert.IsTrue(avroReader.Read());
// Entry 1
Assert.AreEqual(ODataReaderState.EntryStart, avroReader.State);
Assert.IsTrue(avroReader.Read());
Assert.AreEqual(ODataReaderState.EntryEnd, avroReader.State);
var entry = avroReader.Item as ODataEntry;
Assert.IsNotNull(entry);
var properties = entry.Properties.ToList();
Assert.AreEqual(2, properties.Count);
Assert.AreEqual("Id", properties[0].Name);
Assert.AreEqual(1, properties[0].Value);
Assert.AreEqual("Title", properties[1].Name);
Assert.AreEqual("s1", properties[1].Value);
// Entry 2
Assert.IsTrue(avroReader.Read());
Assert.AreEqual(ODataReaderState.EntryStart, avroReader.State);
Assert.IsTrue(avroReader.Read());
Assert.AreEqual(ODataReaderState.EntryEnd, avroReader.State);
entry = avroReader.Item as ODataEntry;
Assert.IsNotNull(entry);
properties = entry.Properties.ToList();
Assert.AreEqual(2, properties.Count);
Assert.AreEqual("Id", properties[0].Name);
Assert.AreEqual(2, properties[0].Value);
Assert.AreEqual("Title", properties[1].Name);
Assert.AreEqual("s2", properties[1].Value);
Assert.IsTrue(avroReader.Read());
Assert.AreEqual(ODataReaderState.FeedEnd, avroReader.State);
Assert.IsFalse(avroReader.Read());
Assert.AreEqual(ODataReaderState.Completed, avroReader.State);
}
}
public void TestReadParameterWithFeed()
{
MemoryStream ms = new MemoryStream();
using (var writer = AvroContainer.CreateWriter <GetMaxIdParameter>(ms, true, new AvroSerializerSettings(), Codec.Null))
using (var seqWriter = new SequentialWriter <GetMaxIdParameter>(writer, 24))
{
seqWriter.Write(new GetMaxIdParameter {
Products = new List <Product>()
{
product0, product1
}
});
}
ms.Seek(0, SeekOrigin.Begin);
var result = new Dictionary <string, object>();
using (var omr = TestHelper.CreateMessageReader(ms, "avro/binary", AvroMediaTypeResolver.Instance, new EdmModel()))
{
var reader = omr.CreateODataParameterReader(GetMaxId);
while (reader.Read())
{
switch (reader.State)
{
case ODataParameterReaderState.Value:
result.Add(reader.Name, reader.Value);
break;
case ODataParameterReaderState.Entry:
var entryReader = reader.CreateEntryReader();
while (entryReader.Read())
{
if (entryReader.State == ODataReaderState.EntryEnd)
{
result.Add(reader.Name, entryReader.Item);
break;
}
}
break;
case ODataParameterReaderState.Feed:
var feedReader = reader.CreateFeedReader();
IList <ODataEntry> entryList = new List <ODataEntry>();
while (feedReader.Read())
{
if (feedReader.State == ODataReaderState.EntryEnd)
{
entryList.Add((ODataEntry)feedReader.Item);
}
}
result.Add(reader.Name, entryList);
break;
}
}
}
Assert.AreEqual(1, result.Count);
var feed = result["Products"] as IList <ODataEntry>;
Assert.IsNotNull(feed);
Assert.AreEqual(2, feed.Count);
Assert.IsTrue(TestHelper.EntryEqual(entry0, feed[0]));
Assert.IsTrue(TestHelper.EntryEqual(entry1, feed[1]));
}
public void TestReadAvroAsODataFeed()
{
const string Schema = @"
{""type"":""array"",
""items"":
{
""type"":""record"",
""name"":""TestNS.Person"",
""fields"":
[
{ ""name"":""Id"", ""type"":""int"" },
{ ""name"":""Title"", ""type"":""string"" },
]
}
}";
var serializer = AvroSerializer.CreateGeneric(Schema);
using (var stream = new MemoryStream())
{
var arraySchema = (ArraySchema)serializer.WriterSchema;
var recordSchema = arraySchema.ItemSchema;
var rec1 = new AvroRecord(recordSchema);
rec1["Id"] = 1;
rec1["Title"] = "s1";
var rec2 = new AvroRecord(recordSchema);
rec2["Id"] = 2;
rec2["Title"] = "s2";
var array = new[] { rec1, rec2 };
using (var writer = AvroContainer.CreateGenericWriter(Schema, stream, /*leave open*/true, Codec.Null))
using (var seqWriter = new SequentialWriter<object>(writer, 24))
{
seqWriter.Write(array);
}
stream.Seek(0, SeekOrigin.Begin);
var avroReader = new ODataAvroReader(this.CreateODataInputContext(stream), true);
Assert.AreEqual(ODataReaderState.Start, avroReader.State);
Assert.IsTrue(avroReader.Read());
Assert.AreEqual(ODataReaderState.FeedStart, avroReader.State);
Assert.IsTrue(avroReader.Read());
// Entry 1
Assert.AreEqual(ODataReaderState.EntryStart, avroReader.State);
Assert.IsTrue(avroReader.Read());
Assert.AreEqual(ODataReaderState.EntryEnd, avroReader.State);
var entry = avroReader.Item as ODataEntry;
Assert.IsNotNull(entry);
var properties = entry.Properties.ToList();
Assert.AreEqual(2, properties.Count);
Assert.AreEqual("Id", properties[0].Name);
Assert.AreEqual(1, properties[0].Value);
Assert.AreEqual("Title", properties[1].Name);
Assert.AreEqual("s1", properties[1].Value);
// Entry 2
Assert.IsTrue(avroReader.Read());
Assert.AreEqual(ODataReaderState.EntryStart, avroReader.State);
Assert.IsTrue(avroReader.Read());
Assert.AreEqual(ODataReaderState.EntryEnd, avroReader.State);
entry = avroReader.Item as ODataEntry;
Assert.IsNotNull(entry);
properties = entry.Properties.ToList();
Assert.AreEqual(2, properties.Count);
Assert.AreEqual("Id", properties[0].Name);
Assert.AreEqual(2, properties[0].Value);
Assert.AreEqual("Title", properties[1].Name);
Assert.AreEqual("s2", properties[1].Value);
Assert.IsTrue(avroReader.Read());
Assert.AreEqual(ODataReaderState.FeedEnd, avroReader.State);
Assert.IsFalse(avroReader.Read());
Assert.AreEqual(ODataReaderState.Completed, avroReader.State);
}
}
/// <summary>
/// Reads CSV file data and converts records to instances of Stock class then serializes them using Reflection.
/// Stock type definition is created from JSON schema using Microsoft Avro Library Code Generation utility
/// </summary>
/// <param name="inputFilePath">The input file path.</param>
/// <param name="outputStream">The output stream.</param>
private void SerializeCsv(string inputFilePath, Stream outputStream)
{
try
{
// Create an Avro writer using Null Codec
var writer = AvroContainer.CreateWriter<Stock>(
outputStream,
new AvroSerializerSettings { Resolver = new AvroDataContractResolver(true) },
Codec.Null);
// Create a new sequential writer and use it to write Avro file using a block length of 100
using (var seq = new SequentialWriter<Stock>(writer, 100))
{
foreach (var stock in this.ReadCsv(inputFilePath))
{
seq.Write(stock);
}
}
}
catch (Exception e)
{
ReportError("Error while creating Avro file(s) from CSV file(s)" + e);
}
}
public void TestReadFeed()
{
MemoryStream ms = new MemoryStream();
IAvroWriter<Product[]> writer = null;
try
{
writer = AvroContainer.CreateWriter<Product[]>(ms, true, new AvroSerializerSettings(), Codec.Null);
using (var seqWriter = new SequentialWriter<Product[]>(writer, 24))
{
seqWriter.Write(new[] { product1, product2, product3 });
}
}
finally
{
if (writer != null) writer.Dispose();
}
ms.Seek(0, SeekOrigin.Begin);
var entries = new List<ODataEntry>();
using (var omr = TestHelper.CreateMessageReader(ms, "avro/binary", AvroMediaTypeResolver.Instance))
{
var reader = omr.CreateODataFeedReader();
while (reader.Read())
{
if (reader.State == ODataReaderState.EntryEnd)
{
entries.Add((ODataEntry)reader.Item);
}
}
}
Assert.AreEqual(3, entries.Count);
Assert.IsTrue(TestHelper.EntryEqual(entry1, entries[0]));
Assert.IsTrue(TestHelper.EntryEqual(entry2, entries[1]));
Assert.IsTrue(TestHelper.EntryEqual(entry3, entries[2]));
}
public void TestReadEntry()
{
MemoryStream ms = new MemoryStream();
IAvroWriter<Product> writer = null;
try
{
writer = AvroContainer.CreateWriter<Product>(ms, true, new AvroSerializerSettings(), Codec.Null);
using (var seqWriter = new SequentialWriter<Product>(writer, 24))
{
seqWriter.Write(product0);
}
}
finally
{
if (writer != null) writer.Dispose();
}
ms.Seek(0, SeekOrigin.Begin);
ODataEntry entry = null;
using (var omr = TestHelper.CreateMessageReader(ms, "avro/binary", AvroMediaTypeResolver.Instance))
{
var reader = omr.CreateODataEntryReader();
while (reader.Read())
{
if (reader.State == ODataReaderState.EntryEnd)
{
entry = (ODataEntry)reader.Item;
}
}
}
Assert.IsTrue(TestHelper.EntryEqual(entry0, entry));
}
public void TestReadCollection()
{
MemoryStream ms = new MemoryStream();
using (var writer = AvroContainer.CreateWriter<string[]>(ms, true, new AvroSerializerSettings(), Codec.Null))
using (var seqWriter = new SequentialWriter<string[]>(writer, 24))
{
seqWriter.Write(new[] { "p9t", "cjk如" });
}
ms.Seek(0, SeekOrigin.Begin);
var values = new List<string>();
using (var omr = TestHelper.CreateMessageReader(ms, "avro/binary", AvroMediaTypeResolver.Instance))
{
var reader = omr.CreateODataCollectionReader();
while (reader.Read())
{
if (reader.State == ODataCollectionReaderState.Value)
{
values.Add((string)reader.Item);
}
}
}
Assert.AreEqual(2, values.Count);
Assert.AreEqual("p9t", values[0]);
Assert.AreEqual("cjk如", values[1]);
}
