private static GifImageDirectory ReadImageBlock(SequentialReader reader)
{
var imageDirectory = new GifImageDirectory();
imageDirectory.Set(GifImageDirectory.TagLeft, reader.GetUInt16());
imageDirectory.Set(GifImageDirectory.TagTop, reader.GetUInt16());
imageDirectory.Set(GifImageDirectory.TagWidth, reader.GetUInt16());
imageDirectory.Set(GifImageDirectory.TagHeight, reader.GetUInt16());
var flags = reader.GetByte();
var hasColorTable = (flags >> 7) != 0;
var isInterlaced = (flags & 0x40) != 0;
imageDirectory.Set(GifImageDirectory.TagHasLocalColourTable, hasColorTable);
imageDirectory.Set(GifImageDirectory.TagIsInterlaced, isInterlaced);
if (hasColorTable)
{
var isColorTableSorted = (flags & 0x20) != 0;
imageDirectory.Set(GifImageDirectory.TagIsColorTableSorted, isColorTableSorted);
var bitsPerPixel = (flags & 0x7) + 1;
imageDirectory.Set(GifImageDirectory.TagLocalColourTableBitsPerPixel, bitsPerPixel);
// skip color table
reader.Skip(3 * (2 << (flags & 0x7)));
}
// skip "LZW Minimum Code Size" byte
reader.GetByte();
return imageDirectory;
}
public AdobeJpegDirectory Extract([NotNull] SequentialReader reader)
{
reader = reader.WithByteOrder(isMotorolaByteOrder: false);
var directory = new AdobeJpegDirectory();
try
{
if (reader.GetString(JpegSegmentPreamble.Length, Encoding.UTF8) != JpegSegmentPreamble)
{
directory.AddError("Invalid Adobe JPEG data header.");
return(directory);
}
directory.Set(AdobeJpegDirectory.TagDctEncodeVersion, reader.GetUInt16());
directory.Set(AdobeJpegDirectory.TagApp14Flags0, reader.GetUInt16());
directory.Set(AdobeJpegDirectory.TagApp14Flags1, reader.GetUInt16());
directory.Set(AdobeJpegDirectory.TagColorTransform, reader.GetSByte());
}
catch (IOException ex)
{
directory.AddError("IO exception processing data: " + ex.Message);
}
return(directory);
}
public ColorInformationBox(BoxLocation location, SequentialReader sr)
: base(location)
{
ColorType = sr.GetUInt32();
switch (ColorType)
{
case NclxTag:
{
ColorPrimaries = sr.GetUInt16();
TransferCharacteristics = sr.GetUInt16();
MatrixCharacteristics = sr.GetUInt16();
FullRangeFlag = (sr.GetByte() & 128) == 128;
IccProfile = _emptyByteArray;
break;
}
case RICCTag:
case ProfTag:
{
IccProfile = ReadRemainingData(sr);
break;
}
default:
{
IccProfile = _emptyByteArray;
break;
}
}
}
static Box ParseBox(BoxLocation location, SequentialReader sr)
{
return(location.Type switch
{
BoxTypes.FTypTag => new FileTypeBox(location, sr),
BoxTypes.MetaTag => new MetaBox(location, sr),
BoxTypes.HdlrTag => new HandlerBox(location, sr),
BoxTypes.DinfTag => new DataInformationBox(location, sr),
BoxTypes.DrefTag => new DataReferenceBox(location, sr),
BoxTypes.UrlTag => new DataEntryLocationBox(location, sr, hasName: false),
BoxTypes.UrnTag => new DataEntryLocationBox(location, sr, hasName: true),
BoxTypes.PitmTag => new PrimaryItemBox(location, sr),
BoxTypes.Iinftag => new ItemInformationBox(location, sr),
BoxTypes.InfeTag => new ItemInfoEntryBox(location, sr),
BoxTypes.IrefTag => new ItemReferenceBox(location, sr),
BoxTypes.IprpTag => new ItemPropertyBox(location, sr),
BoxTypes.IpcoTag => new ItemPropertyContainerBox(location, sr),
BoxTypes.IspeTag => new ImageSpatialExtentsBox(location, sr),
BoxTypes.HvcCTag => new DecoderConfigurationBox(location, sr),
BoxTypes.ColrTag => new ColorInformationBox(location, sr),
BoxTypes.IrotTag => new ImageRotationBox(location, sr),
BoxTypes.PixiTag => new PixelInformationBox(location, sr),
BoxTypes.IdatTag => new ItemDataBox(location, sr),
BoxTypes.IlocTag => new ItemLocationBox(location, sr),
BoxTypes.IpmaTag => new ItemPropertyAssociationBox(location, sr),
_ => new Box(location)
});
public void WritePropertyAsAvroTest()
{
ODataProperty prop = new ODataProperty()
{
Name = "prop1",
Value = 589.901f
};
MemoryStream ms = new MemoryStream();
this.CreateOutputContext(ms).WriteProperty(prop);
ms.Flush();
ms.Seek(0, SeekOrigin.Begin);
IEnumerable <object> results = null;
using (var reader = AvroContainer.CreateGenericReader(ms))
using (var seqReader = new SequentialReader <object>(reader))
{
results = seqReader.Objects;
}
var records = results.Cast <float>().ToList();
Assert.AreEqual(1, records.Count());
Assert.IsTrue(TestHelper.FloatEqual(589.901f, records[0]));
}
public DecoderConfigurationBox(BoxLocation location, SequentialReader sr) : base(location)
{
var bitReader = new BitReader(sr);
ConfigurationVersion = bitReader.GetByte(8);
GeneralProfileSpace = bitReader.GetByte(2);
GeneralTierTag = bitReader.GetByte(1);
GeneralProfileIdc = bitReader.GetByte(5);
GeneralProfileCompatibilityFlags = bitReader.GetUInt32(32);
for (int i = 0; i < 6; i++)
{
GeneralConstraintIndicatorFlags[i] = bitReader.GetByte(8);
}
GeneralLevelIdc = bitReader.GetByte(8);
bitReader.GetUInt32(4); // reserved should be all 1's
MinSpacialSegmentationIdc = bitReader.GetUInt16(12);
bitReader.GetUInt32(6); // reserved should be all 1's
ParallelismType = bitReader.GetByte(2);
bitReader.GetUInt32(6); // reserved should be all 1's
ChromaFormat = bitReader.GetByte(2);
bitReader.GetUInt32(5); // reserved should be all 1's
BitDepthLumaMinus8 = bitReader.GetByte(3);
bitReader.GetUInt32(5); // reserved should be all 1's
BitDepthChromaMinus8 = bitReader.GetByte(3);
AvgFrameRate = bitReader.GetUInt16(16);
ConstantFrameRate = bitReader.GetByte(2);
NumTemporalLayers = bitReader.GetByte(3);
TemporalIdNested = bitReader.GetByte(1);
LengthSizeMinus1 = bitReader.GetByte(2);
}
public void SequentialReaderWriter_CreateReaderForSchemaWithNullableField()
{
var expected = new ClassWithNullableIntField {
NullableIntField = 10
};
using (var writer = AvroContainer.CreateWriter <ClassWithNullableIntField>(this.resultStream, Codec.Deflate))
{
using (var swriter = new SequentialWriter <ClassWithNullableIntField>(writer, 2))
{
swriter.Write(expected);
swriter.Write(expected);
swriter.Write(expected);
swriter.Write(expected);
}
}
this.resultStream.Seek(0, SeekOrigin.Begin);
using (var reader = AvroContainer.CreateReader <ClassWithNullableIntField>(this.resultStream))
{
using (var sreader = new SequentialReader <ClassWithNullableIntField>(reader))
{
foreach (var actual in sreader.Objects)
{
Assert.AreEqual(expected.NullableIntField, actual.NullableIntField);
}
}
}
}
public void SequentialReaderWriter_Reset()
{
var expected = new List <ClassOfListOfGuid>();
for (var i = 0; i < 7; i++)
{
expected.Add(ClassOfListOfGuid.Create(true));
}
var w = AvroContainer.CreateWriter <ClassOfListOfGuid>(this.resultStream, Codec.Deflate);
using (var writer = new SequentialWriter <ClassOfListOfGuid>(w, 3))
{
expected.ForEach(writer.Write);
}
this.resultStream.Seek(0, SeekOrigin.Begin);
var r = AvroContainer.CreateReader <ClassOfListOfGuid>(this.resultStream);
using (var reader = new SequentialReader <ClassOfListOfGuid>(r))
{
Assert.IsTrue(expected.SequenceEqual(reader.Objects));
var enumerator = (IEnumerator)reader.Objects.GetEnumerator();
Assert.IsFalse(enumerator.MoveNext());
}
}
/// <summary>Processes a RIFF data sequence.</summary>
/// <param name="reader">The <see cref="SequentialReader"/> from which the data should be read.</param>
/// <param name="handler">The <see cref="IRiffHandler"/> that will coordinate processing and accept read values.</param>
/// <exception cref="RiffProcessingException">An error occurred during the processing of RIFF data that could not be ignored or recovered from.</exception>
/// <exception cref="System.IO.IOException">an error occurred while accessing the required data</exception>
public void ProcessRiff([NotNull] SequentialReader reader, [NotNull] IRiffHandler handler)
{
// RIFF files are always little-endian
reader = reader.WithByteOrder(isMotorolaByteOrder: false);
// PROCESS FILE HEADER
var fileFourCc = reader.GetString(4, Encoding.ASCII);
if (fileFourCc != "RIFF")
{
throw new RiffProcessingException("Invalid RIFF header: " + fileFourCc);
}
// The total size of the chunks that follow plus 4 bytes for the 'WEBP' or 'AVI ' FourCC
int fileSize = reader.GetInt32();
int sizeLeft = fileSize;
string identifier = reader.GetString(4, Encoding.ASCII);
sizeLeft -= 4;
if (!handler.ShouldAcceptRiffIdentifier(identifier))
{
return;
}
ProcessChunks(reader, sizeLeft, handler);
}
public void TestWriteCollection()
{
MemoryStream ms = new MemoryStream();
using (var omw = TestHelper.CreateMessageWriter(ms, "avro/binary", AvroMediaTypeResolver.Instance))
{
var cw = omw.CreateODataCollectionWriter();
cw.WriteStart(new ODataCollectionStart());
cw.WriteStart(new ODataCollectionStart());
cw.WriteItem("s0");
cw.WriteItem("s1");
cw.WriteEnd();
}
ms.Seek(0, SeekOrigin.Begin);
IEnumerable <string[]> results = null;
using (var reader = AvroContainer.CreateReader <string[]>(ms))
using (var seqReader = new SequentialReader <string[]>(reader))
{
results = seqReader.Objects;
}
var records = results.ToList();
Assert.AreEqual(1, records.Count());
Assert.AreEqual("s0", records[0][0]);
Assert.AreEqual("s1", records[0][1]);
}
public void TestWriteError()
{
ODataError odataError = new ODataError()
{
ErrorCode = "404",
Message = "Not Found",
};
MemoryStream ms = new MemoryStream();
using (var omw = TestHelper.CreateMessageWriter(ms, "avro/binary", AvroMediaTypeResolver.Instance))
{
omw.WriteError(odataError, false);
}
ms.Seek(0, SeekOrigin.Begin);
Error error;
IAvroReader <Error> reader = null;
using (reader = AvroContainer.CreateReader <Error>(ms))
using (var seqReader = new SequentialReader <Error>(reader))
{
error = seqReader.Objects.First();
}
Assert.IsNotNull(error);
Assert.AreEqual("404", error.ErrorCode);
Assert.AreEqual("Not Found", error.Message);
}
public void WriteErrorAsAvroTest()
{
ODataError error = new ODataError()
{
ErrorCode = "32",
Message = "msg1",
};
MemoryStream ms = new MemoryStream();
this.CreateOutputContext(ms).WriteError(error, false);
ms.Flush();
ms.Seek(0, SeekOrigin.Begin);
IEnumerable <object> results = null;
using (var reader = AvroContainer.CreateGenericReader(ms))
using (var seqReader = new SequentialReader <object>(reader))
{
results = seqReader.Objects;
}
var records = results.Cast <AvroRecord>().ToList();
Assert.AreEqual(1, records.Count());
dynamic err = records[0];
Assert.AreEqual("32", err.ErrorCode);
Assert.AreEqual("msg1", err.Message);
}
public PcxDirectory Extract([NotNull] SequentialReader reader)
{
reader = reader.WithByteOrder(isMotorolaByteOrder: false);
var directory = new PcxDirectory();
try
{
var identifier = reader.GetSByte();
if (identifier != 0x0A)
{
throw new ImageProcessingException("Invalid PCX identifier byte");
}
directory.Set(PcxDirectory.TagVersion, reader.GetSByte());
var encoding = reader.GetSByte();
if (encoding != 0x01)
{
throw new ImageProcessingException("Invalid PCX encoding byte");
}
directory.Set(PcxDirectory.TagBitsPerPixel, reader.GetByte());
directory.Set(PcxDirectory.TagXMin, reader.GetUInt16());
directory.Set(PcxDirectory.TagYMin, reader.GetUInt16());
directory.Set(PcxDirectory.TagXMax, reader.GetUInt16());
directory.Set(PcxDirectory.TagYMax, reader.GetUInt16());
directory.Set(PcxDirectory.TagHorizontalDpi, reader.GetUInt16());
directory.Set(PcxDirectory.TagVerticalDpi, reader.GetUInt16());
directory.Set(PcxDirectory.TagPalette, reader.GetBytes(48));
reader.Skip(1);
directory.Set(PcxDirectory.TagColorPlanes, reader.GetByte());
directory.Set(PcxDirectory.TagBytesPerLine, reader.GetUInt16());
var paletteType = reader.GetUInt16();
if (paletteType != 0)
{
directory.Set(PcxDirectory.TagPaletteType, paletteType);
}
var hScrSize = reader.GetUInt16();
if (hScrSize != 0)
{
directory.Set(PcxDirectory.TagHScrSize, hScrSize);
}
var vScrSize = reader.GetUInt16();
if (vScrSize != 0)
{
directory.Set(PcxDirectory.TagVScrSize, vScrSize);
}
}
catch (Exception ex)
{
directory.AddError("Exception reading PCX file metadata: " + ex.Message);
}
return(directory);
}
public static void Populate(WavFormatDirectory directory, SequentialReader reader, int chunkSize)
{
directory.Set(TagFormat, reader.GetUInt16());
directory.Set(TagChannels, reader.GetUInt16());
directory.Set(TagSamplesPerSec, reader.GetUInt32());
directory.Set(TagBytesPerSec, reader.GetUInt32());
directory.Set(TagBlockAlign, reader.GetUInt16());
directory.Set(TagBitsPerSample, reader.GetUInt16());
if (chunkSize > 16)
{
var exSize = reader.GetUInt16();
if (exSize > 0)
{
if (exSize > chunkSize - 16)
{
directory.AddError("Invalid chunk 'fmt ' extension size");
}
else
{
PopulateEx(directory, reader, exSize);
}
}
}
}
public void WriteCollectionAsAvroTest()
{
MemoryStream ms = new MemoryStream();
ODataAvroCollectionWriter ocw = new ODataAvroCollectionWriter(this.CreateOutputContext(ms), null);
ocw.WriteStartAsync(new ODataCollectionStart()).Wait();
ocw.WriteItemAsync(1).Wait();
ocw.WriteItemAsync(3).Wait();
ocw.WriteEndAsync().Wait();
ocw.FlushAsync().Wait();
ms.Seek(0, SeekOrigin.Begin);
IEnumerable <int[]> results = null;
using (var reader = AvroContainer.CreateReader <int[]>(ms))
using (var seqReader = new SequentialReader <int[]>(reader))
{
results = seqReader.Objects;
}
var records = results.ToList();
Assert.AreEqual(1, records.Count());
Assert.AreEqual(1, records[0][0]);
Assert.AreEqual(3, records[0][1]);
}
public void SequentialReaderWriter_SerializeHugeObject()
{
var single = new SimpleFlatClass
{
StringField = new string('a', 16254),
ByteArrayField = Encoding.ASCII.GetBytes(new string('b', 65666)),
ZeroByteArrayField = Encoding.ASCII.GetBytes(new string('c', 128344))
};
var expected = new List <SimpleFlatClass>
{
single,
single,
single,
};
var w = AvroContainer.CreateWriter <SimpleFlatClass>(this.resultStream, new AvroSerializerSettings {
Resolver = new AvroDataContractResolver(true)
}, Codec.Null);
using (var writer = new SequentialWriter <SimpleFlatClass>(w, 24))
{
expected.ForEach(writer.Write);
}
this.resultStream.Seek(0, SeekOrigin.Begin);
var r = AvroContainer.CreateReader <SimpleFlatClass>(this.resultStream, true, new AvroSerializerSettings {
Resolver = new AvroDataContractResolver(true)
}, new CodecFactory());
using (var reader = new SequentialReader <SimpleFlatClass>(r))
{
Assert.IsTrue(expected.SequenceEqual(reader.Objects));
}
}
public AvroConfiguration AvroDeserializeFromFile(string fileName)
{
AvroConfiguration avroConf = null;
try
{
using (var buffer = new MemoryStream())
{
if (!ReadFile(buffer, fileName))
{
var e = new ApplicationException("Error during file operation. Quitting method : " + fileName);
Org.Apache.REEF.Utilities.Diagnostics.Exceptions.Throw(e, LOGGER);
}
buffer.Seek(0, SeekOrigin.Begin);
using (var reader = new SequentialReader <AvroConfiguration>(AvroContainer.CreateReader <AvroConfiguration>(buffer, true)))
{
var results = reader.Objects;
if (results != null)
{
avroConf = (AvroConfiguration)results.First();
}
}
}
}
catch (SerializationException ex)
{
Org.Apache.REEF.Utilities.Diagnostics.Exceptions.Caught(ex, Level.Error, LOGGER);
var e = new ApplicationException("Cannot deserialize the file: " + fileName, ex);
Org.Apache.REEF.Utilities.Diagnostics.Exceptions.Throw(e, LOGGER);
}
return(avroConf);
}
public override IEnumerable <IRow> Extract(IUnstructuredReader input, IUpdatableRow output)
{
if (input.Length == 0)
{
yield break;
}
var serializer = AvroSerializer.CreateGeneric(avroSchema);
using (var genericReader = AvroContainer.CreateGenericReader(input.BaseStream))
{
using (var reader = new SequentialReader <dynamic>(genericReader))
{
foreach (var obj in reader.Objects)
{
foreach (var column in output.Schema)
{
output.Set(column.Name, obj[column.Name]);
}
yield return(output.AsReadOnly());
}
}
}
}
public void Extract(SequentialReader reader, HuffmanTablesDirectory directory)
{
try
{
while (reader.Available() > 0)
{
byte header = reader.GetByte();
HuffmanTableClass tableClass = HuffmanTable.TypeOf((header & 0xF0) >> 4);
int tableDestinationId = header & 0xF;
byte[] lBytes = GetBytes(reader, 16);
int vCount = 0;
foreach (byte b in lBytes)
{
vCount += (b & 0xFF);
}
byte[] vBytes = GetBytes(reader, vCount);
directory.AddTable(new HuffmanTable(tableClass, tableDestinationId, lBytes, vBytes));
}
}
catch (IOException me)
{
directory.AddError(me.ToString());
}
}
public void WritePrimitiveCollectionPropertyAsAvroTest()
{
var expected = new object[] { 3, 4 };
var value = new ODataCollectionValue {
Items = expected
};
ODataProperty prop = new ODataProperty
{
Name = "prop1",
Value = value
};
MemoryStream ms = new MemoryStream();
this.CreateOutputContext(ms).WriteProperty(prop);
ms.Flush();
ms.Seek(0, SeekOrigin.Begin);
IEnumerable <object> results = null;
using (var reader = AvroContainer.CreateGenericReader(ms))
using (var seqReader = new SequentialReader <object>(reader))
{
results = seqReader.Objects;
}
var records = results.Cast <object[]>().ToList();
Assert.AreEqual(1, records.Count());
Assert.IsTrue(expected.SequenceEqual(records[0]));
}
/// <summary>
/// Main method that parses all comments and then distributes data extraction among other methods that parse the
/// rest of file and store encountered data in metadata(if there exists an entry in EpsDirectory
/// for the found data). Reads until a begin data/binary comment is found or reader's estimated
/// available data has run out (or AI09 End Private Data). Will extract data from normal EPS comments, Photoshop, ICC, and XMP.
/// </summary>
/// <param name="directory"></param>
/// <param name="directories">list to add directory to and extracted data</param>
/// <param name="reader"></param>
private void Extract(EpsDirectory directory, List <Directory> directories, SequentialReader reader)
{
var line = new StringBuilder();
while (true)
{
line.Length = 0;
// Read the next line, excluding any trailing newline character
// Note that for Windows-style line endings ("\r\n") the outer loop will be run a second time with an empty
// string, which is fine.
while (true)
{
char c = (char)reader.GetByte();
if (c == '\r' || c == '\n')
{
break;
}
line.Append(c);
}
// Stop when we hit a line that is not a comment
if (line.Length != 0 && line[0] != '%')
{
break;
}
string name;
// ':' signifies there is an associated keyword (should be put in directory)
// otherwise, the name could be a marker
int colonIndex = line.IndexOf(':');
if (colonIndex != -1)
{
name = line.ToString(0, colonIndex).Trim();
var value = line.ToString(colonIndex + 1, line.Length - (colonIndex + 1)).Trim();
AddToDirectory(directory, name, value);
}
else
{
name = line.ToString().Trim();
}
// Some comments will both have a value and signify a new block to follow
if (name.Equals("%BeginPhotoshop"))
{
ExtractPhotoshopData(directories, reader);
}
else if (name.Equals("%%BeginICCProfile"))
{
ExtractIccData(directories, reader);
}
else if (name.Equals("%begin_xml_packet"))
{
ExtractXmpData(directories, reader);
}
}
}
/// <summary>
/// Decodes a commented hex section, and uses <see cref="PhotoshopReader"/> to decode the resulting data.
/// </summary>
private static void ExtractPhotoshopData(List <Directory> directories, SequentialReader reader)
{
var buffer = DecodeHexCommentBlock(reader);
if (buffer != null)
{
directories.AddRange(new PhotoshopReader().Extract(new SequentialByteArrayReader(buffer), buffer.Length));
}
}
/// <summary>
/// Decodes a commented hex section, and uses <see cref="IccReader"/> to decode the resulting data.
/// </summary>
private static void ExtractIccData(List <Directory> directories, SequentialReader reader)
{
var buffer = DecodeHexCommentBlock(reader);
if (buffer != null)
{
directories.Add(new IccReader().Extract(new ByteArrayReader(buffer)));
}
}
public void WriteOperationParametersAsAvroTest()
{
var operation = new EdmAction("NS", "op1", null);
operation.AddParameter("p1", EdmCoreModel.Instance.GetString(false));
operation.AddParameter("p2", new EdmEntityTypeReference(TestEntityType, false));
MemoryStream ms = new MemoryStream();
var context = this.CreateOutputContext(ms);
var opw = new ODataAvroParameterWriter(context, operation);
{
opw.WriteStart();
opw.WriteValue("p1", "dat");
{
var ew = opw.CreateResourceWriter("p2");
ew.WriteStart(entry0);
ew.WriteStart(nestedResource0);
ew.WriteStart(complex0);
ew.WriteEnd();
ew.WriteEnd();
ew.WriteEnd();
ew.Flush();
}
opw.WriteEnd();
opw.Flush();
}
ms.Flush();
ms.Seek(0, SeekOrigin.Begin);
IEnumerable <object> results = null;
using (var reader = AvroContainer.CreateGenericReader(ms))
using (var seqReader = new SequentialReader <object>(reader))
{
results = seqReader.Objects;
}
dynamic record = results.Cast <AvroRecord>().Single();
Assert.AreEqual("dat", record.p1);
dynamic p2 = record.p2;
Assert.AreEqual(true, p2.TBoolean);
Assert.AreEqual(32, p2.TInt32);
var col = p2.TCollection as object[];
Assert.IsNotNull(col);
Assert.IsTrue(longCollection0.SequenceEqual(col));
dynamic cpx = p2.TComplex as AvroRecord;
Assert.IsNotNull(cpx);
Assert.IsTrue(binary0.SequenceEqual((byte[])cpx.TBinary));
Assert.AreEqual("iamstr", cpx.TString);
}
public void SequentialGenericWritingReading_RecursiveRecord()
{
const string StringSchema = @"{
""type"":""record"",
""name"":""Microsoft.Hadoop.Avro.Tests.Recursive"",
""fields"":[
{""name"":""IntField"",""type"":""int""},
{""name"":""RecursiveField"",""type"":[
""null"",
""Microsoft.Hadoop.Avro.Tests.Recursive""
]
}
]}";
using (var stream = new MemoryStream())
{
var serializer = AvroSerializer.CreateGeneric(StringSchema);
using (var streamWriter = AvroContainer.CreateGenericWriter(StringSchema, stream, Codec.Null))
{
using (var writer = new SequentialWriter <object>(streamWriter, 24))
{
var expected = new List <AvroRecord>();
var random = new Random(93);
for (int i = 0; i < 10; i++)
{
dynamic record = new AvroRecord(serializer.WriterSchema);
record.IntField = random.Next();
record.RecursiveField =
new AvroRecord(
((serializer.ReaderSchema as RecordSchema).GetField("RecursiveField").TypeSchema as UnionSchema).Schemas[1]);
record.RecursiveField.IntField = random.Next();
record.RecursiveField.RecursiveField = null;
expected.Add(record);
}
expected.ForEach(writer.Write);
writer.Flush();
stream.Seek(0, SeekOrigin.Begin);
var streamReader = AvroContainer.CreateReader <Recursive>(stream, true, this.dataContractSettings, new CodecFactory());
using (var reader = new SequentialReader <Recursive>(streamReader))
{
var j = 0;
foreach (var avroRecord in reader.Objects)
{
Assert.Equal(expected[j]["IntField"], avroRecord.IntField);
Assert.Equal(((dynamic)expected[j]["RecursiveField"])["IntField"], avroRecord.RecursiveField.IntField);
Assert.Equal(
((dynamic)expected[j++]["RecursiveField"])["RecursiveField"], avroRecord.RecursiveField.RecursiveField);
}
}
}
}
}
}
public HandlerBox(BoxLocation location, SequentialReader reader)
: base(location, reader)
{
reader.GetUInt32(); // should be Zero
HandlerType = reader.GetUInt32();
reader.GetUInt32(); // should be Zero
reader.GetUInt32(); // should be Zero
reader.GetUInt32(); // should be Zero
TrackType = reader.GetString((int)reader.BytesRemainingInBox(location), Encoding.UTF8);
}
public static string Get4ccString([NotNull] this SequentialReader reader)
{
var sb = new StringBuilder(4);
sb.Append((char)reader.GetByte());
sb.Append((char)reader.GetByte());
sb.Append((char)reader.GetByte());
sb.Append((char)reader.GetByte());
return(sb.ToString());
}
public SingleItemTypeReferenceBox(BoxLocation loc, SequentialReader sr, byte parentVersion) : base(loc)
{
FromItemId = parentVersion == 0 ? sr.GetUInt16() : sr.GetUInt32();
ReferenceCount = sr.GetUInt16();
ToItemIds = new uint[ReferenceCount];
for (int i = 0; i < ReferenceCount; i++)
{
ToItemIds[i] = parentVersion == 0 ? sr.GetUInt16() : sr.GetUInt32();
}
}
private IEnumerable <T> ReadObjects <T>(IAvroReader <T> sr, Func <object, bool?> filterFunc = null)
{
using (var streamReader = new SequentialReader <T>(sr))
{
foreach (var item in streamReader.Objects)
{
yield return(item);
}
}
}
public DuckyDirectory Extract(SequentialReader reader)
{
var directory = new DuckyDirectory();
try
{
while (true)
{
var tag = reader.GetUInt16();
// End of Segment is marked with zero
if (tag == 0)
{
break;
}
var length = reader.GetUInt16();
switch (tag)
{
case DuckyDirectory.TagQuality:
{
if (length != 4)
{
directory.AddError("Unexpected length for the quality tag");
return(directory);
}
directory.Set(tag, reader.GetUInt32());
break;
}
case DuckyDirectory.TagComment:
case DuckyDirectory.TagCopyright:
{
reader.Skip(4);
directory.Set(tag, reader.GetString(length - 4, Encoding.BigEndianUnicode));
break;
}
default:
{
// Unexpected tag
directory.Set(tag, reader.GetBytes(length));
break;
}
}
}
}
catch (IOException e)
{
directory.AddError(e.Message);
}
return(directory);
}
public void SequentialGenericWritingReading_SimpleRecord()
{
const string StringSchema = @"{
""type"":""record"",
""name"":""Microsoft.Hadoop.Avro.Tests.ClassOfInt"",
""fields"":
[
{
""name"":""PrimitiveInt"",
""type"":""int""
}
]
}";
using (var stream = new MemoryStream())
{
var serializer = AvroSerializer.CreateGeneric(StringSchema);
using (var streamWriter = AvroContainer.CreateGenericWriter(StringSchema, stream, Codec.Null))
{
using (var writer = new SequentialWriter<object>(streamWriter, 24))
{
var expected = new List<AvroRecord>();
var random = new Random(113);
for (int i = 0; i < 10; i++)
{
dynamic record = new AvroRecord(serializer.WriterSchema);
record.PrimitiveInt = random.Next();
expected.Add(record);
}
expected.ForEach(writer.Write);
writer.Flush();
stream.Seek(0, SeekOrigin.Begin);
var streamReader = AvroContainer.CreateReader<ClassOfInt>(stream, true, this.dataContractSettings, new CodecFactory());
using (var reader = new SequentialReader<ClassOfInt>(streamReader))
{
var j = 0;
foreach (var avroRecord in reader.Objects)
{
Assert.AreEqual(expected[j++]["PrimitiveInt"], avroRecord.PrimitiveInt);
}
}
}
}
}
}
public override IEnumerable<IRow> Extract(IUnstructuredReader input, IUpdatableRow output)
{
var serializer = AvroSerializer.CreateGeneric(avroSchema);
using (var genericReader = AvroContainer.CreateGenericReader(input.BaseStream))
{
using (var reader = new SequentialReader<dynamic>(genericReader))
{
foreach (var obj in reader.Objects)
{
foreach (var column in output.Schema)
{
output.Set(column.Name, obj[column.Name]);
}
yield return output.AsReadOnly();
}
}
}
}
public void SequentialReaderWriter_SerializeThreeObjects()
{
var expected = new List<ClassOfInt>
{
ClassOfInt.Create(true),
ClassOfInt.Create(true),
ClassOfInt.Create(true),
};
var w = AvroContainer.CreateWriter<ClassOfInt>(this.resultStream, Codec.Null);
using (var writer = new SequentialWriter<ClassOfInt>(w, 24))
{
expected.ForEach(writer.Write);
}
this.resultStream.Seek(0, SeekOrigin.Begin);
var r = AvroContainer.CreateReader<ClassOfInt>(this.resultStream);
using (var reader = new SequentialReader<ClassOfInt>(r))
{
Assert.IsTrue(expected.SequenceEqual(reader.Objects));
}
}
public void SequentialReaderWriter_SyncAfterDeflateCodec()
{
var expected = new List<ClassOfInt>();
for (var i = 0; i < 7; i++)
{
expected.Add(ClassOfInt.Create(true));
}
var w = AvroContainer.CreateWriter<ClassOfInt>(this.resultStream, Codec.Deflate);
using (var writer = new SequentialWriter<ClassOfInt>(w, 2))
{
expected.ForEach(writer.Write);
}
this.resultStream.Seek(0, SeekOrigin.Begin);
var r = AvroContainer.CreateReader<ClassOfInt>(this.resultStream);
using (var reader = new SequentialReader<ClassOfInt>(r))
{
Assert.IsTrue(expected.SequenceEqual(reader.Objects));
}
}
//Serializes and deserializes sample data set using Reflection and Avro Object Container Files
//Serialized data is compressed with Deflate codec
//
//This sample uses Memory Stream for all operations related to serialization, deserialization and
//Object Container manipulation, though File Stream could be easily used.
public void SerializeDeserializeUsingObjectContainersReflection()
{
Console.WriteLine("SERIALIZATION USING REFLECTION AND AVRO OBJECT CONTAINER FILES\n");
//Path for Avro Object Container File
string path = "AvroSampleReflectionDeflate.avro";
//Create a data set using sample Class and struct
var testData = new List<SensorData>
{
new SensorData { Value = new byte[] { 1, 2, 3, 4, 5 }, Position = new Location { Room = 243, Floor = 1 } },
new SensorData { Value = new byte[] { 6, 7, 8, 9 }, Position = new Location { Room = 244, Floor = 1 } }
};
//Serializing and saving data to file
//Creating a Memory Stream buffer
using (var buffer = new MemoryStream())
{
Console.WriteLine("Serializing Sample Data Set...");
//Create a SequentialWriter instance for type SensorData which can serialize a sequence of SensorData objects to stream
//Data will be compressed using Deflate codec
using (var w = AvroContainer.CreateWriter<SensorData>(buffer, Codec.Deflate))
{
using (var writer = new SequentialWriter<SensorData>(w, 24))
{
// Serialize the data to stream using the sequential writer
testData.ForEach(writer.Write);
}
}
//Save stream to file
Console.WriteLine("Saving serialized data to file...");
if (!WriteFile(buffer, path))
{
Console.WriteLine("Error during file operation. Quitting method");
return;
}
}
//Reading and deserializing data
//Creating a Memory Stream buffer
using (var buffer = new MemoryStream())
{
Console.WriteLine("Reading data from file...");
//Reading data from Object Container File
if (!ReadFile(buffer, path))
{
Console.WriteLine("Error during file operation. Quitting method");
return;
}
Console.WriteLine("Deserializing Sample Data Set...");
//Prepare the stream for deserializing the data
buffer.Seek(0, SeekOrigin.Begin);
//Create a SequentialReader for type SensorData which will derserialize all serialized objects from the given stream
//It allows iterating over the deserialized objects because it implements IEnumerable<T> interface
using (var reader = new SequentialReader<SensorData>(
AvroContainer.CreateReader<SensorData>(buffer, true)))
{
var results = reader.Objects;
//Finally, verify that deserialized data matches the original one
Console.WriteLine("Comparing Initial and Deserialized Data Sets...");
int count = 1;
var pairs = testData.Zip(results, (serialized, deserialized) => new { expected = serialized, actual = deserialized });
foreach (var pair in pairs)
{
bool isEqual = this.Equal(pair.expected, pair.actual);
Console.WriteLine("For Pair {0} result of Data Set Identity Comparison is {1}", count, isEqual.ToString());
count++;
}
}
}
//Delete the file
RemoveFile(path);
}
//Serializes and deserializes sample data set using Generic Record and Avro Object Container Files
//Serialized data is not compressed
//
//This sample uses Memory Stream for all operations related to serialization, deserialization and
//Object Container manipulation, though File Stream could be easily used.
public void SerializeDeserializeUsingObjectContainersGenericRecord()
{
Console.WriteLine("SERIALIZATION USING GENERIC RECORD AND AVRO OBJECT CONTAINER FILES\n");
//Path for Avro Object Container File
string path = "AvroSampleGenericRecordNullCodec.avro";
Console.WriteLine("Defining the Schema and creating Sample Data Set...");
//Define the schema in JSON
const string Schema = @"{
""type"":""record"",
""name"":""Microsoft.Hadoop.Avro.Specifications.SensorData"",
""fields"":
[
{
""name"":""Location"",
""type"":
{
""type"":""record"",
""name"":""Microsoft.Hadoop.Avro.Specifications.Location"",
""fields"":
[
{ ""name"":""Floor"", ""type"":""int"" },
{ ""name"":""Room"", ""type"":""int"" }
]
}
},
{ ""name"":""Value"", ""type"":""bytes"" }
]
}";
//Create a generic serializer based on the schema
var serializer = AvroSerializer.CreateGeneric(Schema);
var rootSchema = serializer.WriterSchema as RecordSchema;
//Create a generic record to represent the data
var testData = new List<AvroRecord>();
dynamic expected1 = new AvroRecord(rootSchema);
dynamic location1 = new AvroRecord(rootSchema.GetField("Location").TypeSchema);
location1.Floor = 1;
location1.Room = 243;
expected1.Location = location1;
expected1.Value = new byte[] { 1, 2, 3, 4, 5 };
testData.Add(expected1);
dynamic expected2 = new AvroRecord(rootSchema);
dynamic location2 = new AvroRecord(rootSchema.GetField("Location").TypeSchema);
location2.Floor = 1;
location2.Room = 244;
expected2.Location = location2;
expected2.Value = new byte[] { 6, 7, 8, 9 };
testData.Add(expected2);
//Serializing and saving data to file
//Create a MemoryStream buffer
using (var buffer = new MemoryStream())
{
Console.WriteLine("Serializing Sample Data Set...");
//Create a SequentialWriter instance for type SensorData which can serialize a sequence of SensorData objects to stream
//Data will not be compressed (Null compression codec)
using (var writer = AvroContainer.CreateGenericWriter(Schema, buffer, Codec.Null))
{
using (var streamWriter = new SequentialWriter<object>(writer, 24))
{
// Serialize the data to stream using the sequential writer
testData.ForEach(streamWriter.Write);
}
}
Console.WriteLine("Saving serialized data to file...");
//Save stream to file
if (!WriteFile(buffer, path))
{
Console.WriteLine("Error during file operation. Quitting method");
return;
}
}
//Reading and deserializng the data
//Create a Memory Stream buffer
using (var buffer = new MemoryStream())
{
Console.WriteLine("Reading data from file...");
//Reading data from Object Container File
if (!ReadFile(buffer, path))
{
Console.WriteLine("Error during file operation. Quitting method");
return;
}
Console.WriteLine("Deserializing Sample Data Set...");
//Prepare the stream for deserializing the data
buffer.Seek(0, SeekOrigin.Begin);
//Create a SequentialReader for type SensorData which will derserialize all serialized objects from the given stream
//It allows iterating over the deserialized objects because it implements IEnumerable<T> interface
using (var reader = AvroContainer.CreateGenericReader(buffer))
{
using (var streamReader = new SequentialReader<object>(reader))
{
var results = streamReader.Objects;
Console.WriteLine("Comparing Initial and Deserialized Data Sets...");
//Finally, verify the results
var pairs = testData.Zip(results, (serialized, deserialized) => new { expected = (dynamic)serialized, actual = (dynamic)deserialized });
int count = 1;
foreach (var pair in pairs)
{
bool isEqual = pair.expected.Location.Floor.Equals(pair.actual.Location.Floor);
isEqual = isEqual && pair.expected.Location.Room.Equals(pair.actual.Location.Room);
isEqual = isEqual && ((byte[])pair.expected.Value).SequenceEqual((byte[])pair.actual.Value);
Console.WriteLine("For Pair {0} result of Data Set Identity Comparison is {1}", count, isEqual.ToString());
count++;
}
}
}
}
//Delete the file
RemoveFile(path);
}
public void TestWriteError()
{
ODataError odataError = new ODataError()
{
ErrorCode = "404",
Message = "Not Found",
};
MemoryStream ms = new MemoryStream();
using (var omw = TestHelper.CreateMessageWriter(ms, "avro/binary", AvroMediaTypeResolver.Instance))
{
omw.WriteError(odataError, false);
}
ms.Seek(0, SeekOrigin.Begin);
Error error;
IAvroReader<Error> reader = null;
using (reader = AvroContainer.CreateReader<Error>(ms))
using (var seqReader = new SequentialReader<Error>(reader))
{
error = seqReader.Objects.First();
}
Assert.IsNotNull(error);
Assert.AreEqual("404", error.ErrorCode);
Assert.AreEqual("Not Found", error.Message);
}
public void WriteEntryAsAvroTest()
{
MemoryStream ms = new MemoryStream();
var ctx = this.CreateOutputContext(ms);
ODataAvroWriter aw = new ODataAvroWriter(ctx, value => ctx.AvroWriter.Write(value), ctx.AvroWriter.UpdateSchema(null, TestEntityType), false);
aw.WriteStart(entry0);
aw.WriteEnd();
aw.Flush();
ms.Seek(0, SeekOrigin.Begin);
IEnumerable<object> results;
IAvroReader<object> reader = null;
try
{
reader = AvroContainer.CreateGenericReader(ms);
using (var seqReader = new SequentialReader<object>(reader))
{
reader = null;
results = seqReader.Objects;
}
}
finally
{
if (reader != null) reader.Dispose();
}
dynamic record = results.Single() as AvroRecord;
Assert.IsNotNull(record);
Assert.AreEqual(true, record.TBoolean);
Assert.AreEqual(32, record.TInt32);
var col = record.TCollection as object[];
Assert.IsNotNull(col);
Assert.IsTrue(longCollection0.SequenceEqual(col.Cast<long>()));
dynamic cpx = record.TComplex as AvroRecord;
Assert.IsNotNull(cpx);
Assert.IsTrue(binary0.SequenceEqual((byte[])cpx.TBinary));
Assert.AreEqual("iamstr", cpx.TString);
}
public void TestSerializeDeserializeUsingSequentialContainers()
{
//CreateDeserializerOnly a new AvroSerializer instance and specify a custom serialization strategy AvroDataContractResolver
//for handling only properties attributed with DataContract/DateMember.
var settings = new AvroSerializerSettings();
//CreateDeserializerOnly a new buffer
using (var buffer = new MemoryStream())
{
//CreateDeserializerOnly sample data
var testData = new List<SensorData>
{
new SensorData { Value = new byte[] { 1, 2, 3, 4, 5 }, Position = new Location { Room = 243, Floor = 1 } },
new SensorData { Value = new byte[] { 6, 7, 8, 9 }, Position = new Location { Room = 244, Floor = 1 } }
};
//CreateDeserializerOnly a SequentialWriter instance for type SensorData which can serialize a sequence of SensorData objects to stream
using (var w = AvroContainer.CreateWriter<SensorData>(buffer, settings, Codec.Null))
{
using (var writer = new SequentialWriter<SensorData>(w, 24))
{
// Serialize the data to stream using the sequential writer
testData.ForEach(writer.Write);
}
}
//Prepare the stream for deserializing the data
buffer.Seek(0, SeekOrigin.Begin);
//CreateDeserializerOnly a SequentialReader for type SensorData which will derserialize all serialized objects from the given stream
//It allows iterating over the deserialized objects because it implements IEnumerable<T> interface
using (var reader = new SequentialReader<SensorData>(
AvroContainer.CreateReader<SensorData>(buffer, true, settings, new CodecFactory())))
{
var results = reader.Objects;
//Finally, verify that deserialized data matches the original ones
var pairs = testData.Zip(results, (serialized, deserialized) => new { expected = serialized, actual = deserialized });
foreach (var pair in pairs)
{
Assert.IsTrue(this.Equal(pair.expected, pair.actual));
}
}
}
}
private AvroNode AvroDeserializeFromFile(string fileName)
{
AvroNode avroNode = null;
try
{
using (var buffer = new MemoryStream())
{
if (!ReadFile(buffer, fileName))
{
var e =
new ApplicationException("Error during file operation. Quitting method : " + fileName);
Utilities.Diagnostics.Exceptions.Throw(e, LOGGER);
}
buffer.Seek(0, SeekOrigin.Begin);
using (
var reader =
new SequentialReader<AvroNode>(AvroContainer.CreateReader<AvroNode>(buffer, true)))
{
var results = reader.Objects;
if (results != null)
{
avroNode = (AvroNode)results.First();
}
}
}
}
catch (SerializationException ex)
{
Utilities.Diagnostics.Exceptions.Caught(ex, Level.Error, LOGGER);
var e = new ApplicationException("Cannot deserialize the file: " + fileName, ex);
Utilities.Diagnostics.Exceptions.Throw(e, LOGGER);
}
return avroNode;
}
public void WriteCollectionAsAvroTest()
{
MemoryStream ms = new MemoryStream();
ODataAvroCollectionWriter ocw = new ODataAvroCollectionWriter(this.CreateOutputContext(ms), null);
ocw.WriteStartAsync(new ODataCollectionStart()).Wait();
ocw.WriteItemAsync(1).Wait();
ocw.WriteItemAsync(3).Wait();
ocw.WriteEndAsync().Wait();
ocw.FlushAsync().Wait();
ms.Seek(0, SeekOrigin.Begin);
IEnumerable<int[]> results = null;
using (var reader = AvroContainer.CreateReader<int[]>(ms))
using (var seqReader = new SequentialReader<int[]>(reader))
{
results = seqReader.Objects;
}
var records = results.ToList();
Assert.AreEqual(1, records.Count());
Assert.AreEqual(1, records[0][0]);
Assert.AreEqual(3, records[0][1]);
}
public void WritePropertyAsAvroTest()
{
ODataProperty prop = new ODataProperty()
{
Name = "prop1",
Value = 589.901f
};
MemoryStream ms = new MemoryStream();
this.CreateOutputContext(ms).WriteProperty(prop);
ms.Flush();
ms.Seek(0, SeekOrigin.Begin);
IEnumerable<object> results = null;
using (var reader = AvroContainer.CreateGenericReader(ms))
using (var seqReader = new SequentialReader<object>(reader))
{
results = seqReader.Objects;
}
var records = results.Cast<float>().ToList();
Assert.AreEqual(1, records.Count());
Assert.IsTrue(TestHelper.FloatEqual(589.901f, records[0]));
}
//Serializes and deserializes sample data set using Reflection and Avro Object Container Files
//Serialized data is compressed with the Custom compression codec (Deflate of .NET Framework 4.5)
//
//This sample uses Memory Stream for all operations related to serialization, deserialization and
//Object Container manipulation, though File Stream could be easily used.
public void SerializeDeserializeUsingObjectContainersReflectionCustomCodec()
{
Console.WriteLine("SERIALIZATION USING REFLECTION, AVRO OBJECT CONTAINER FILES AND CUSTOM CODEC\n");
//Path for Avro Object Container File
string path = "AvroSampleReflectionDeflate45.avro";
//Create a data set using sample Class and struct
var testData = new List<SensorData>
{
new SensorData { Value = new byte[] { 1, 2, 3, 4, 5 }, Position = new Location { Room = 243, Floor = 1 } },
new SensorData { Value = new byte[] { 6, 7, 8, 9 }, Position = new Location { Room = 244, Floor = 1 } }
};
//Serializing and saving data to file
//Creating a Memory Stream buffer
using (var buffer = new MemoryStream())
{
Console.WriteLine("Serializing Sample Data Set...");
//Create a SequentialWriter instance for type SensorData which can serialize a sequence of SensorData objects to stream
//Here the custom Codec is introduced. For convenience the next commented code line shows how to use built-in Deflate.
//Note, that because the sample deals with different IMPLEMENTATIONS of Deflate, built-in and custom codecs are interchangeable
//in read-write operations
//using (var w = AvroContainer.CreateWriter<SensorData>(buffer, Codec.Deflate))
using (var w = AvroContainer.CreateWriter<SensorData>(buffer, new DeflateCodec45()))
{
using (var writer = new SequentialWriter<SensorData>(w, 24))
{
// Serialize the data to stream using the sequential writer
testData.ForEach(writer.Write);
}
}
//Save stream to file
Console.WriteLine("Saving serialized data to file...");
if (!WriteFile(buffer, path))
{
Console.WriteLine("Error during file operation. Quitting method");
return;
}
}
//Reading and deserializing data
//Creating a Memory Stream buffer
using (var buffer = new MemoryStream())
{
Console.WriteLine("Reading data from file...");
//Reading data from Object Container File
if (!ReadFile(buffer, path))
{
Console.WriteLine("Error during file operation. Quitting method");
return;
}
Console.WriteLine("Deserializing Sample Data Set...");
//Prepare the stream for deserializing the data
buffer.Seek(0, SeekOrigin.Begin);
//Because of SequentialReader<T> constructor signature an AvroSerializerSettings instance is required
//when Codec Factory is explicitly specified
//You may comment the line below if you want to use built-in Deflate (see next comment)
AvroSerializerSettings settings = new AvroSerializerSettings();
//Create a SequentialReader for type SensorData which will derserialize all serialized objects from the given stream
//It allows iterating over the deserialized objects because it implements IEnumerable<T> interface
//Here the custom Codec Factory is introduced.
//For convenience the next commented code line shows how to use built-in Deflate
//(no explicit Codec Factory parameter is required in this case).
//Note, that because the sample deals with different IMPLEMENTATIONS of Deflate, built-in and custom codecs are interchangeable
//in read-write operations
//using (var reader = new SequentialReader<SensorData>(AvroContainer.CreateReader<SensorData>(buffer, true)))
using (var reader = new SequentialReader<SensorData>(
AvroContainer.CreateReader<SensorData>(buffer, true, settings, new CodecFactoryDeflate45())))
{
var results = reader.Objects;
//Finally, verify that deserialized data matches the original one
Console.WriteLine("Comparing Initial and Deserialized Data Sets...");
bool isEqual;
int count = 1;
var pairs = testData.Zip(results, (serialized, deserialized) => new { expected = serialized, actual = deserialized });
foreach (var pair in pairs)
{
isEqual = this.Equal(pair.expected, pair.actual);
Console.WriteLine("For Pair {0} result of Data Set Identity Comparison is {1}", count, isEqual.ToString());
count++;
}
}
}
//Delete the file
RemoveFile(path);
}
public void WriteErrorAsAvroTest()
{
ODataError error = new ODataError()
{
ErrorCode = "32",
Message = "msg1",
};
MemoryStream ms = new MemoryStream();
this.CreateOutputContext(ms).WriteError(error, false);
ms.Flush();
ms.Seek(0, SeekOrigin.Begin);
IEnumerable<object> results = null;
using (var reader = AvroContainer.CreateGenericReader(ms))
using (var seqReader = new SequentialReader<object>(reader))
{
results = seqReader.Objects;
}
var records = results.Cast<AvroRecord>().ToList();
Assert.AreEqual(1, records.Count());
dynamic err = records[0];
Assert.AreEqual("32", err.ErrorCode);
Assert.AreEqual("msg1", err.Message);
}
public void TestWriteEntry()
{
MemoryStream ms = new MemoryStream();
using (var omw = TestHelper.CreateMessageWriter(ms, "avro/binary", AvroMediaTypeResolver.Instance))
{
var entryWriter = omw.CreateODataEntryWriter(null, EntryType);
entryWriter.WriteStart(entry0);
entryWriter.WriteEnd();
entryWriter.Flush();
}
ms.Seek(0, SeekOrigin.Begin);
Product prd;
IAvroReader<Product> reader = null;
try
{
reader = AvroContainer.CreateReader<Product>(ms);
using (var seqReader = new SequentialReader<Product>(reader))
{
reader = null;
prd = seqReader.Objects.First();
}
}
finally
{
if (reader != null) reader.Dispose();
}
Assert.AreEqual(product0, prd);
}
public void TestWriteFeed()
{
MemoryStream ms = new MemoryStream();
using (var omw = TestHelper.CreateMessageWriter(ms, "avro/binary", AvroMediaTypeResolver.Instance))
{
var entryWriter = omw.CreateODataFeedWriter(null, EntryType);
entryWriter.WriteStart(new ODataFeed());
entryWriter.WriteStart(entry0);
entryWriter.WriteEnd();
entryWriter.WriteStart(entry1);
entryWriter.WriteEnd();
entryWriter.WriteStart(entry2);
entryWriter.WriteEnd();
entryWriter.WriteStart(entry3);
entryWriter.WriteEnd();
entryWriter.WriteEnd();
entryWriter.Flush();
}
ms.Seek(0, SeekOrigin.Begin);
Product[] products;
IAvroReader<Product[]> reader = null;
try
{
reader = AvroContainer.CreateReader<Product[]>(ms);
using (var seqReader = new SequentialReader<Product[]>(reader))
{
reader = null;
products = seqReader.Objects.ToList().Single();
}
}
finally
{
if (reader != null) reader.Dispose();
}
Assert.AreEqual(product0, products[0]);
Assert.AreEqual(product1, products[1]);
Assert.AreEqual(product2, products[2]);
Assert.AreEqual(product3, products[3]);
}
public void TestWriteCollection()
{
MemoryStream ms = new MemoryStream();
using (var omw = TestHelper.CreateMessageWriter(ms, "avro/binary", AvroMediaTypeResolver.Instance))
{
var cw = omw.CreateODataCollectionWriter();
cw.WriteStart(new ODataCollectionStart());
cw.WriteStart(new ODataCollectionStart());
cw.WriteItem("s0");
cw.WriteItem("s1");
cw.WriteEnd();
}
ms.Seek(0, SeekOrigin.Begin);
IEnumerable<string[]> results = null;
using (var reader = AvroContainer.CreateReader<string[]>(ms))
using (var seqReader = new SequentialReader<string[]>(reader))
{
results = seqReader.Objects;
}
var records = results.ToList();
Assert.AreEqual(1, records.Count());
Assert.AreEqual("s0", records[0][0]);
Assert.AreEqual("s1", records[0][1]);
}
public void TestWriteProperty()
{
ODataProperty prop = new ODataProperty()
{
Name = "prop1",
Value = complexValue0
};
MemoryStream ms = new MemoryStream();
using (var omw = TestHelper.CreateMessageWriter(ms, "avro/binary", AvroMediaTypeResolver.Instance))
{
omw.WriteProperty(prop);
}
ms.Seek(0, SeekOrigin.Begin);
Address addr;
IAvroReader<Address> reader = null;
try
{
reader = AvroContainer.CreateReader<Address>(ms);
using (var seqReader = new SequentialReader<Address>(reader))
{
reader = null;
addr = seqReader.Objects.First();
}
}
finally
{
if (reader != null) reader.Dispose();
}
Assert.AreEqual(address0, addr);
}
public AvroConfiguration AvroDeserializeFromFile(string fileName)
{
AvroConfiguration avroConf = null;
try
{
using (var buffer = new MemoryStream())
{
if (!ReadFile(buffer, fileName))
{
var e = new ApplicationException("Error during file operation. Quitting method : " + fileName);
Org.Apache.REEF.Utilities.Diagnostics.Exceptions.Throw(e, LOGGER);
}
buffer.Seek(0, SeekOrigin.Begin);
//AvroSerializerSettings settings = new AvroSerializerSettings();
//settings.Resolver = new AvroConfigurationResolver();
//using (var reader = new SequentialReader<AvroConfiguration>(AvroContainer.CreateReader<AvroConfiguration>(buffer, true, settings, new CodecFactory())))
using (var reader = new SequentialReader<AvroConfiguration>(AvroContainer.CreateReader<AvroConfiguration>(buffer, true)))
{
var results = reader.Objects;
if (results != null)
{
avroConf = (AvroConfiguration)results.First();
}
}
}
}
catch (SerializationException ex)
{
Org.Apache.REEF.Utilities.Diagnostics.Exceptions.Caught(ex, Level.Error, LOGGER);
var e = new ApplicationException("Cannot deserialize the file: " + fileName, ex);
Org.Apache.REEF.Utilities.Diagnostics.Exceptions.Throw(e, LOGGER);
}
return avroConf;
}
public void WriteOperationParametersAsAvroTest()
{
var operation = new EdmAction("NS", "op1", null);
operation.AddParameter("p1", EdmCoreModel.Instance.GetString(false));
operation.AddParameter("p2", new EdmEntityTypeReference(TestEntityType, false));
MemoryStream ms = new MemoryStream();
var context = this.CreateOutputContext(ms);
var opw = new ODataAvroParameterWriter(context, operation);
{
opw.WriteStart();
opw.WriteValue("p1", "dat");
{
var ew = opw.CreateEntryWriter("p2");
ew.WriteStart(entry0);
ew.WriteEnd();
ew.Flush();
}
opw.WriteEnd();
opw.Flush();
}
ms.Flush();
ms.Seek(0, SeekOrigin.Begin);
IEnumerable<object> results = null;
using (var reader = AvroContainer.CreateGenericReader(ms))
using (var seqReader = new SequentialReader<object>(reader))
{
results = seqReader.Objects;
}
dynamic record = results.Cast<AvroRecord>().Single();
Assert.AreEqual("dat", record.p1);
dynamic p2 = record.p2;
Assert.AreEqual(true, p2.TBoolean);
Assert.AreEqual(32, p2.TInt32);
var col = p2.TCollection as object[];
Assert.IsNotNull(col);
Assert.IsTrue(longCollection0.SequenceEqual(col.Cast<long>()));
dynamic cpx = p2.TComplex as AvroRecord;
Assert.IsNotNull(cpx);
Assert.IsTrue(binary0.SequenceEqual((byte[])cpx.TBinary));
Assert.AreEqual("iamstr", cpx.TString);
}
public void SequentialReaderWriter_Reset()
{
var expected = new List<ClassOfListOfGuid>();
for (var i = 0; i < 7; i++)
{
expected.Add(ClassOfListOfGuid.Create(true));
}
var w = AvroContainer.CreateWriter<ClassOfListOfGuid>(this.resultStream, Codec.Deflate);
using (var writer = new SequentialWriter<ClassOfListOfGuid>(w, 3))
{
expected.ForEach(writer.Write);
}
this.resultStream.Seek(0, SeekOrigin.Begin);
var r = AvroContainer.CreateReader<ClassOfListOfGuid>(this.resultStream);
using (var reader = new SequentialReader<ClassOfListOfGuid>(r))
{
Assert.IsTrue(expected.SequenceEqual(reader.Objects));
var enumerator = (IEnumerator)reader.Objects.GetEnumerator();
Assert.IsFalse(enumerator.MoveNext());
}
}
public void WritePrimitiveCollectionPropertyAsAvroTest()
{
var expected = new[] { 3, 4 };
var value = new ODataCollectionValue { Items = expected };
ODataProperty prop = new ODataProperty
{
Name = "prop1",
Value = value
};
MemoryStream ms = new MemoryStream();
this.CreateOutputContext(ms).WriteProperty(prop);
ms.Flush();
ms.Seek(0, SeekOrigin.Begin);
IEnumerable<object> results = null;
using (var reader = AvroContainer.CreateGenericReader(ms))
using (var seqReader = new SequentialReader<object>(reader))
{
results = seqReader.Objects;
}
var records = results.Cast<object[]>().ToList();
Assert.AreEqual(1, records.Count());
Assert.IsTrue(expected.SequenceEqual(records[0].Cast<int>()));
}
public void SequentialReader_ApacheWriterMicrosoftReader()
{
var serializer = AvroSerializer.Create<ClassOfInt>(new AvroSerializerSettings { Resolver = new AvroDataContractResolver(true) });
var schema = ApacheAvro.Schema.Parse(serializer.WriterSchema.ToString()) as ApacheAvro.UnionSchema;
Assert.IsNotNull(schema);
var recordSchema = schema.Schemas[1] as ApacheAvro.RecordSchema;
Assert.IsNotNull(recordSchema);
var expected = new List<GenericRecord>();
for (var i = 0; i < 7; i++)
{
var record = new GenericRecord(recordSchema);
record.Add("PrimitiveInt", ClassOfInt.Create(true).PrimitiveInt);
expected.Add(record);
}
var datumWriter = new GenericWriter<GenericRecord>(schema);
var writer = DataFileWriter<GenericRecord>.OpenWriter(datumWriter, this.resultStream);
writer.WriteHeader();
foreach (var obj in expected)
{
writer.Append(obj);
}
writer.Flush();
this.resultStream.Seek(0, SeekOrigin.Begin);
var r = AvroContainer.CreateReader<ClassOfInt>(this.resultStream, true, new AvroSerializerSettings { Resolver = new AvroDataContractResolver(true) }, new CodecFactory());
using (var reader = new SequentialReader<ClassOfInt>(r))
{
var actual = reader.Objects.ToList();
Assert.AreEqual(expected.Count, actual.Count);
for (var i = 0; i < expected.Count; ++i)
{
Assert.AreEqual(expected[i]["PrimitiveInt"], actual[i].PrimitiveInt);
}
}
}
public void WriteFeedAsAvroTest()
{
ODataEntry entry1 = new ODataEntry
{
Properties = new[]
{
new ODataProperty {Name = "TBoolean", Value = true,},
new ODataProperty {Name = "TInt32", Value = 32,},
},
TypeName = "NS.SimpleEntry"
};
ODataEntry entry2 = new ODataEntry
{
Properties = new[]
{
new ODataProperty {Name = "TBoolean", Value = false,},
new ODataProperty {Name = "TInt32", Value = 325,},
},
TypeName = "NS.SimpleEntry"
};
MemoryStream ms = new MemoryStream();
var ctx = this.CreateOutputContext(ms);
ODataAvroWriter aw = new ODataAvroWriter(ctx, value => ctx.AvroWriter.Write(value), null, true);
aw.WriteStart(new ODataFeed());
aw.WriteStart(entry1);
aw.WriteEnd();
aw.WriteStart(entry2);
aw.WriteEnd();
aw.WriteEnd();
aw.Flush();
ms.Seek(0, SeekOrigin.Begin);
IEnumerable<object> results;
IAvroReader<object> reader = null;
try
{
reader = AvroContainer.CreateGenericReader(ms);
using (var seqReader = new SequentialReader<object>(reader))
{
reader = null;
results = seqReader.Objects;
}
}
finally
{
if (reader != null) reader.Dispose();
}
var records = results.Cast<object[]>().Single();
Assert.AreEqual(2, records.Count());
dynamic record = records[0];
Assert.AreEqual(true, record.TBoolean);
Assert.AreEqual(32, record.TInt32);
record = records[1];
Assert.AreEqual(false, record.TBoolean);
Assert.AreEqual(325, record.TInt32);
}
public void SequentialReaderWriter_CreateReaderForSchemaWithNullableField()
{
var expected = new ClassWithNullableIntField { NullableIntField = 10 };
using (var writer = AvroContainer.CreateWriter<ClassWithNullableIntField>(this.resultStream, Codec.Deflate))
{
using (var swriter = new SequentialWriter<ClassWithNullableIntField>(writer, 2))
{
swriter.Write(expected);
swriter.Write(expected);
swriter.Write(expected);
swriter.Write(expected);
}
}
this.resultStream.Seek(0, SeekOrigin.Begin);
using (var reader = AvroContainer.CreateReader<ClassWithNullableIntField>(this.resultStream))
{
using (var sreader = new SequentialReader<ClassWithNullableIntField>(reader))
{
foreach (var actual in sreader.Objects)
{
Assert.AreEqual(expected.NullableIntField, actual.NullableIntField);
}
}
}
}
public void SequentialGenericWritingReading_RecursiveRecord()
{
const string StringSchema = @"{
""type"":""record"",
""name"":""Microsoft.Hadoop.Avro.Tests.Recursive"",
""fields"":[
{""name"":""IntField"",""type"":""int""},
{""name"":""RecursiveField"",""type"":[
""null"",
""Microsoft.Hadoop.Avro.Tests.Recursive""
]
}
]}";
using (var stream = new MemoryStream())
{
var serializer = AvroSerializer.CreateGeneric(StringSchema);
using (var streamWriter = AvroContainer.CreateGenericWriter(StringSchema, stream, Codec.Null))
{
using (var writer = new SequentialWriter<object>(streamWriter, 24))
{
var expected = new List<AvroRecord>();
var random = new Random(93);
for (int i = 0; i < 10; i++)
{
dynamic record = new AvroRecord(serializer.WriterSchema);
record.IntField = random.Next();
record.RecursiveField =
new AvroRecord(
((serializer.ReaderSchema as RecordSchema).GetField("RecursiveField").TypeSchema as UnionSchema).Schemas[1]);
record.RecursiveField.IntField = random.Next();
record.RecursiveField.RecursiveField = null;
expected.Add(record);
}
expected.ForEach(writer.Write);
writer.Flush();
stream.Seek(0, SeekOrigin.Begin);
var streamReader = AvroContainer.CreateReader<Recursive>(stream, true, this.dataContractSettings, new CodecFactory());
using (var reader = new SequentialReader<Recursive>(streamReader))
{
var j = 0;
foreach (var avroRecord in reader.Objects)
{
Assert.AreEqual(expected[j]["IntField"], avroRecord.IntField);
Assert.AreEqual(((dynamic)expected[j]["RecursiveField"])["IntField"], avroRecord.RecursiveField.IntField);
Assert.AreEqual(
((dynamic)expected[j++]["RecursiveField"])["RecursiveField"], avroRecord.RecursiveField.RecursiveField);
}
}
}
}
}
}
public void SequentialReaderWriter_SerializeHugeObject()
{
var single = new SimpleFlatClass
{
StringField = new string('a', 16254),
ByteArrayField = Encoding.ASCII.GetBytes(new string('b', 65666)),
ZeroByteArrayField = Encoding.ASCII.GetBytes(new string('c', 128344))
};
var expected = new List<SimpleFlatClass>
{
single,
single,
single,
};
var w = AvroContainer.CreateWriter<SimpleFlatClass>(this.resultStream, new AvroSerializerSettings { Resolver = new AvroDataContractResolver(true) }, Codec.Null);
using (var writer = new SequentialWriter<SimpleFlatClass>(w, 24))
{
expected.ForEach(writer.Write);
}
this.resultStream.Seek(0, SeekOrigin.Begin);
var r = AvroContainer.CreateReader<SimpleFlatClass>(this.resultStream, true, new AvroSerializerSettings { Resolver = new AvroDataContractResolver(true) }, new CodecFactory());
using (var reader = new SequentialReader<SimpleFlatClass>(r))
{
Assert.IsTrue(expected.SequenceEqual(reader.Objects));
}
}
public void TestWriteParameter()
{
MemoryStream ms = new MemoryStream();
using (var omw = TestHelper.CreateMessageWriter(ms, "avro/binary", AvroMediaTypeResolver.Instance, new EdmModel(), false))
{
var opw = omw.CreateODataParameterWriter(AddProduct);
var ew = opw.CreateEntryWriter("Product");
ew.WriteStart(entry0);
ew.WriteEnd();
ew.Flush();
opw.WriteValue("Location", complexValue0);
opw.WriteEnd();
opw.Flush();
}
ms.Seek(0, SeekOrigin.Begin);
AddProductParameter parameter;
IAvroReader<AddProductParameter> reader = null;
try
{
reader = AvroContainer.CreateReader<AddProductParameter>(ms);
using (var seqReader = new SequentialReader<AddProductParameter>(reader))
{
reader = null;
parameter = seqReader.Objects.First();
}
}
finally
{
if (reader != null) reader.Dispose();
}
Assert.IsNotNull(parameter);
Assert.AreEqual(product0, parameter.Product);
Assert.AreEqual(address0, parameter.Location);
}
public void TestWriteParameterWithFeed()
{
MemoryStream ms = new MemoryStream();
using (var omw = TestHelper.CreateMessageWriter(ms, "avro/binary", AvroMediaTypeResolver.Instance, new EdmModel(), false))
{
var opw = omw.CreateODataParameterWriter(GetMaxId);
var ew = opw.CreateFeedWriter("Products");
ew.WriteStart(new ODataFeed());
ew.WriteStart(entry0);
ew.WriteEnd();
ew.WriteStart(entry1);
ew.WriteEnd();
ew.WriteEnd();
ew.Flush();
opw.WriteEnd();
opw.Flush();
}
ms.Seek(0, SeekOrigin.Begin);
GetMaxIdParameter parameter;
IAvroReader<GetMaxIdParameter> reader = null;
try
{
reader = AvroContainer.CreateReader<GetMaxIdParameter>(ms);
using (var seqReader = new SequentialReader<GetMaxIdParameter>(reader))
{
reader = null;
parameter = seqReader.Objects.First();
}
}
finally
{
if (reader != null) reader.Dispose();
}
Assert.IsNotNull(parameter);
var products = parameter.Products as IList<Product>;
Assert.IsNotNull(products);
Assert.AreEqual(product0, products[0]);
Assert.AreEqual(product1, products[1]);
}
