static void Main(string[] args)
{
SampleData obj = new SampleData();
using (FileStream xmlStream = new FileStream(@"SampleData.xml", FileMode.Open))
{
using (XmlReader xmlReader = XmlReader.Create(xmlStream))
{
obj.ReadXml(xmlReader);
}
}
// hi ahmed
var serializer = new AvroSerializer(typeof(Person));
StreamWriter schemafs = new StreamWriter(File.Create(@"SampleData.avsc"));
schemafs.Write(serializer.Schema.ToString());
schemafs.Close();
var resultStream = new FileStream(@"SampleData.avro", FileMode.OpenOrCreate);
var writer = new SequentialWriter <Person>(resultStream, serializer, Codec.Create("null"), 24);
int i;
foreach (DataRow item in obj.Tables["Person"].Rows)
{
var p = new Person {
First_Name = item["Name"].ToString(), Street = item["Street"].ToString(), City = item["City"].ToString(), State = item["State"].ToString()
};
writer.Write(p);
}
writer.Flush();
}
public void ReadAvroAsError()
{
const string Schema = @"{
""type"":""record"",
""name"":""OData.Error"",
""fields"":
[
{ ""name"":""ErrorCode"", ""type"":""string"" },
{ ""name"":""Message"" , ""type"":""string"" },
]
}";
var stream = new MemoryStream();
using (var writer = AvroContainer.CreateGenericWriter(Schema, stream, /*leave open*/ true, Codec.Null))
using (var seqWriter = new SequentialWriter <object>(writer, 24))
{
RecordSchema parameterSchema = (RecordSchema)AvroSerializer.CreateGeneric(Schema).WriterSchema;
AvroRecord ar = new AvroRecord(parameterSchema);
ar["ErrorCode"] = "e1";
ar["Message"] = "m1";
seqWriter.Write(ar);
seqWriter.Flush();
}
stream.Flush();
stream.Seek(0, SeekOrigin.Begin);
var error = this.CreateODataInputContext(stream).ReadError();
Assert.AreEqual("e1", error.ErrorCode);
Assert.AreEqual("m1", error.Message);
}
public void TestReadAvroAsCollection()
{
const string Schema = @"{""type"":""array"", ""items"":""long""}";
var stream = new MemoryStream();
using (var writer = AvroContainer.CreateGenericWriter(Schema, stream, /*leave open*/ true, Codec.Null))
using (var seqWriter = new SequentialWriter <object>(writer, 24))
{
// Serialize the data to stream using the sequential writer
seqWriter.Write(new[] { 6L, 8 });
seqWriter.Flush();
}
stream.Flush();
stream.Seek(0, SeekOrigin.Begin);
var reader = new ODataAvroCollectionReader(this.CreateODataInputContext(stream));
Assert.AreEqual(ODataCollectionReaderState.Start, reader.State);
Assert.IsTrue(reader.Read());
Assert.AreEqual(ODataCollectionReaderState.CollectionStart, reader.State);
Assert.IsTrue(reader.Read());
Assert.AreEqual(ODataCollectionReaderState.Value, reader.State);
Assert.AreEqual(6L, (long)reader.Item);
Assert.IsTrue(reader.Read());
Assert.AreEqual(ODataCollectionReaderState.Value, reader.State);
Assert.IsTrue(reader.Read());
Assert.AreEqual(ODataCollectionReaderState.CollectionEnd, reader.State);
Assert.IsFalse(reader.Read());
Assert.AreEqual(ODataCollectionReaderState.Completed, reader.State);
}
public void SequentialGenericWritingReading_RecursiveRecord()
{
const string StringSchema = @"{
""type"":""record"",
""name"":""Microsoft.Hadoop.Avro.Tests.Recursive"",
""fields"":[
{""name"":""IntField"",""type"":""int""},
{""name"":""RecursiveField"",""type"":[
""null"",
""Microsoft.Hadoop.Avro.Tests.Recursive""
]
}
]}";
using (var stream = new MemoryStream())
{
var serializer = AvroSerializer.CreateGeneric(StringSchema);
using (var streamWriter = AvroContainer.CreateGenericWriter(StringSchema, stream, Codec.Null))
{
using (var writer = new SequentialWriter <object>(streamWriter, 24))
{
var expected = new List <AvroRecord>();
var random = new Random(93);
for (int i = 0; i < 10; i++)
{
dynamic record = new AvroRecord(serializer.WriterSchema);
record.IntField = random.Next();
record.RecursiveField =
new AvroRecord(
((serializer.ReaderSchema as RecordSchema).GetField("RecursiveField").TypeSchema as UnionSchema).Schemas[1]);
record.RecursiveField.IntField = random.Next();
record.RecursiveField.RecursiveField = null;
expected.Add(record);
}
expected.ForEach(writer.Write);
writer.Flush();
stream.Seek(0, SeekOrigin.Begin);
var streamReader = AvroContainer.CreateReader <Recursive>(stream, true, this.dataContractSettings, new CodecFactory());
using (var reader = new SequentialReader <Recursive>(streamReader))
{
var j = 0;
foreach (var avroRecord in reader.Objects)
{
Assert.Equal(expected[j]["IntField"], avroRecord.IntField);
Assert.Equal(((dynamic)expected[j]["RecursiveField"])["IntField"], avroRecord.RecursiveField.IntField);
Assert.Equal(
((dynamic)expected[j++]["RecursiveField"])["RecursiveField"], avroRecord.RecursiveField.RecursiveField);
}
}
}
}
}
}
public void SequentialGenericWritingReading_SimpleRecord()
{
const string StringSchema = @"{
""type"":""record"",
""name"":""Microsoft.Hadoop.Avro.Tests.ClassOfInt"",
""fields"":
[
{
""name"":""PrimitiveInt"",
""type"":""int""
}
]
}";
using (var stream = new MemoryStream())
{
var serializer = AvroSerializer.CreateGeneric(StringSchema);
using (var streamWriter = AvroContainer.CreateGenericWriter(StringSchema, stream, Codec.Null))
{
using (var writer = new SequentialWriter <object>(streamWriter, 24))
{
var expected = new List <AvroRecord>();
var random = new Random(113);
for (int i = 0; i < 10; i++)
{
dynamic record = new AvroRecord(serializer.WriterSchema);
record.PrimitiveInt = random.Next();
expected.Add(record);
}
expected.ForEach(writer.Write);
writer.Flush();
stream.Seek(0, SeekOrigin.Begin);
var streamReader = AvroContainer.CreateReader <ClassOfInt>(stream, true, this.dataContractSettings, new CodecFactory());
using (var reader = new SequentialReader <ClassOfInt>(streamReader))
{
var j = 0;
foreach (var avroRecord in reader.Objects)
{
Assert.Equal(expected[j++]["PrimitiveInt"], avroRecord.PrimitiveInt);
}
}
}
}
}
}
public void SequentialGenericWritingReading_SimpleRecord()
{
const string StringSchema = @"{
""type"":""record"",
""name"":""Microsoft.Hadoop.Avro.Tests.ClassOfInt"",
""fields"":
[
{
""name"":""PrimitiveInt"",
""type"":""int""
}
]
}";
using (var stream = new MemoryStream())
{
var serializer = AvroSerializer.CreateGeneric(StringSchema);
using (var streamWriter = AvroContainer.CreateGenericWriter(StringSchema, stream, Codec.Null))
{
using (var writer = new SequentialWriter<object>(streamWriter, 24))
{
var expected = new List<AvroRecord>();
var random = new Random(113);
for (int i = 0; i < 10; i++)
{
dynamic record = new AvroRecord(serializer.WriterSchema);
record.PrimitiveInt = random.Next();
expected.Add(record);
}
expected.ForEach(writer.Write);
writer.Flush();
stream.Seek(0, SeekOrigin.Begin);
var streamReader = AvroContainer.CreateReader<ClassOfInt>(stream, true, this.dataContractSettings, new CodecFactory());
using (var reader = new SequentialReader<ClassOfInt>(streamReader))
{
var j = 0;
foreach (var avroRecord in reader.Objects)
{
Assert.AreEqual(expected[j++]["PrimitiveInt"], avroRecord.PrimitiveInt);
}
}
}
}
}
}
public void TestReadAvroAsPrimitiveCollection()
{
const string Schema = @"{""type"":""array"", ""items"":""long""}";
var stream = new MemoryStream();
using (var writer = AvroContainer.CreateGenericWriter(Schema, stream, /*leave open*/ true, Codec.Null))
using (var seqWriter = new SequentialWriter <object>(writer, 24))
{
// Serialize the data to stream using the sequential writer
seqWriter.Write(new[] { 6L, 8 });
seqWriter.Flush();
}
stream.Flush();
stream.Seek(0, SeekOrigin.Begin);
var property = this.CreateODataInputContext(stream).ReadProperty(null, null);
}
public override Stream GetContentStream(IEnumerable <ExpandoObject> records)
{
var result = new MemoryStream();
// We get a JSON schema for the passed type. Normally, the AvroSerializer expects a type passed that has DataContract/DataMember attributes, but I don't want users of this utility
// to have to modify their POCOs in any way, hence building a schema here and proceeding with that.
string schema = GetJsonSchema();
var serializer = AvroSerializer.CreateGeneric(schema);
var rootSchema = serializer.WriterSchema as RecordSchema;
// We'll write the Avro content to a memory stream
using (var interim = new MemoryStream())
{
// Avro serializer with deflate
using (var avroWriter = AvroContainer.CreateGenericWriter(schema, interim, Codec.Deflate))
{
using (var seqWriter = new SequentialWriter <object>(avroWriter, SYNCNUM))
{
foreach (var record in records)
{
IDictionary <string, object> recordProperties = record as IDictionary <string, object>;
dynamic avroRecord = new AvroRecord(rootSchema);
foreach (KeyValuePair <string, object> recordKVP in recordProperties)
{
avroRecord[recordKVP.Key] = recordKVP.Value;
}
seqWriter.Write(avroRecord);
}
seqWriter.Flush();
interim.Seek(0, SeekOrigin.Begin);
interim.CopyTo(result);
}
}
}
return(result);
}
public void ReadAvroAsParameter()
{
EdmEntityType personType = new EdmEntityType("TestNS", "Person");
personType.AddStructuralProperty("Id", EdmPrimitiveTypeKind.Int32);
personType.AddStructuralProperty("Title", EdmPrimitiveTypeKind.String);
var operation = new EdmAction("NS", "op1", null);
operation.AddParameter("p1", EdmCoreModel.Instance.GetString(false));
operation.AddParameter("p2", new EdmEntityTypeReference(personType, false));
const string Schema = @"{
""type"":""record"",
""name"":""NS.op1Parameter"",
""fields"":
[
{ ""name"":""p1"", ""type"":""string"" },
{ ""name"":""p2"", ""type"":
{""type"":""record"",
""name"":""TestNS.Person"",
""fields"":
[
{ ""name"":""Id"", ""type"":""int"" },
{ ""name"":""Title"", ""type"":""string"" },
]
}
}
]
}";
var stream = new MemoryStream();
using (var writer = AvroContainer.CreateGenericWriter(Schema, stream, /*leave open*/ true, Codec.Null))
using (var seqWriter = new SequentialWriter <object>(writer, 24))
{
RecordSchema parameterSchema = (RecordSchema)AvroSerializer.CreateGeneric(Schema).WriterSchema;
AvroRecord ar = new AvroRecord(parameterSchema);
ar["p1"] = "dat";
var personSchema = parameterSchema.GetField("p2").TypeSchema;
AvroRecord person = new AvroRecord(personSchema);
person["Id"] = 5;
person["Title"] = "per1";
ar["p2"] = person;
seqWriter.Write(ar);
seqWriter.Flush();
}
stream.Flush();
stream.Seek(0, SeekOrigin.Begin);
var reader = new ODataAvroParameterReader(this.CreateODataInputContext(stream), operation);
Assert.AreEqual(ODataParameterReaderState.Start, reader.State);
Assert.IsTrue(reader.Read());
Assert.AreEqual(ODataParameterReaderState.Value, reader.State);
Assert.AreEqual("p1", reader.Name);
Assert.AreEqual("dat", reader.Value);
Assert.IsTrue(reader.Read());
Assert.AreEqual(ODataParameterReaderState.Entry, reader.State);
Assert.AreEqual("p2", reader.Name);
var ew = reader.CreateEntryReader();
Assert.AreEqual(ODataReaderState.Start, ew.State);
Assert.IsTrue(ew.Read());
Assert.AreEqual(ODataReaderState.EntryStart, ew.State);
Assert.IsTrue(ew.Read());
Assert.AreEqual(ODataReaderState.EntryEnd, ew.State);
var entry = ew.Item as ODataEntry;
Assert.IsFalse(ew.Read());
Assert.AreEqual(ODataReaderState.Completed, ew.State);
Assert.IsNotNull(entry);
var properties = entry.Properties.ToList();
Assert.AreEqual(2, properties.Count);
Assert.AreEqual("Id", properties[0].Name);
Assert.AreEqual(5, properties[0].Value);
Assert.AreEqual("Title", properties[1].Name);
Assert.AreEqual("per1", properties[1].Value);
Assert.IsFalse(reader.Read());
Assert.AreEqual(ODataParameterReaderState.Completed, reader.State);
}
public void ReadAvroAsError()
{
const string Schema = @"{
""type"":""record"",
""name"":""OData.Error"",
""fields"":
[
{ ""name"":""ErrorCode"", ""type"":""string"" },
{ ""name"":""Message"" , ""type"":""string"" },
]
}";
var stream = new MemoryStream();
using (var writer = AvroContainer.CreateGenericWriter(Schema, stream, /*leave open*/true, Codec.Null))
using (var seqWriter = new SequentialWriter<object>(writer, 24))
{
RecordSchema parameterSchema = (RecordSchema)AvroSerializer.CreateGeneric(Schema).WriterSchema;
AvroRecord ar = new AvroRecord(parameterSchema);
ar["ErrorCode"] = "e1";
ar["Message"] = "m1";
seqWriter.Write(ar);
seqWriter.Flush();
}
stream.Flush();
stream.Seek(0, SeekOrigin.Begin);
var error = this.CreateODataInputContext(stream).ReadError();
Assert.AreEqual("e1", error.ErrorCode);
Assert.AreEqual("m1", error.Message);
}
public void ReadAvroAsParameter()
{
EdmEntityType personType = new EdmEntityType("TestNS", "Person");
personType.AddStructuralProperty("Id", EdmPrimitiveTypeKind.Int32);
personType.AddStructuralProperty("Title", EdmPrimitiveTypeKind.String);
var operation = new EdmAction("NS", "op1", null);
operation.AddParameter("p1", EdmCoreModel.Instance.GetString(false));
operation.AddParameter("p2", new EdmEntityTypeReference(personType, false));
const string Schema = @"{
""type"":""record"",
""name"":""NS.op1Parameter"",
""fields"":
[
{ ""name"":""p1"", ""type"":""string"" },
{ ""name"":""p2"", ""type"":
{""type"":""record"",
""name"":""TestNS.Person"",
""fields"":
[
{ ""name"":""Id"", ""type"":""int"" },
{ ""name"":""Title"", ""type"":""string"" },
]
}
}
]
}";
var stream = new MemoryStream();
using (var writer = AvroContainer.CreateGenericWriter(Schema, stream, /*leave open*/true, Codec.Null))
using (var seqWriter = new SequentialWriter<object>(writer, 24))
{
RecordSchema parameterSchema = (RecordSchema)AvroSerializer.CreateGeneric(Schema).WriterSchema;
AvroRecord ar = new AvroRecord(parameterSchema);
ar["p1"] = "dat";
var personSchema = parameterSchema.GetField("p2").TypeSchema;
AvroRecord person = new AvroRecord(personSchema);
person["Id"] = 5;
person["Title"] = "per1";
ar["p2"] = person;
seqWriter.Write(ar);
seqWriter.Flush();
}
stream.Flush();
stream.Seek(0, SeekOrigin.Begin);
var reader = new ODataAvroParameterReader(this.CreateODataInputContext(stream), operation);
Assert.AreEqual(ODataParameterReaderState.Start, reader.State);
Assert.IsTrue(reader.Read());
Assert.AreEqual(ODataParameterReaderState.Value, reader.State);
Assert.AreEqual("p1", reader.Name);
Assert.AreEqual("dat", reader.Value);
Assert.IsTrue(reader.Read());
Assert.AreEqual(ODataParameterReaderState.Entry, reader.State);
Assert.AreEqual("p2", reader.Name);
var ew = reader.CreateEntryReader();
Assert.AreEqual(ODataReaderState.Start, ew.State);
Assert.IsTrue(ew.Read());
Assert.AreEqual(ODataReaderState.EntryStart, ew.State);
Assert.IsTrue(ew.Read());
Assert.AreEqual(ODataReaderState.EntryEnd, ew.State);
var entry = ew.Item as ODataEntry;
Assert.IsFalse(ew.Read());
Assert.AreEqual(ODataReaderState.Completed, ew.State);
Assert.IsNotNull(entry);
var properties = entry.Properties.ToList();
Assert.AreEqual(2, properties.Count);
Assert.AreEqual("Id", properties[0].Name);
Assert.AreEqual(5, properties[0].Value);
Assert.AreEqual("Title", properties[1].Name);
Assert.AreEqual("per1", properties[1].Value);
Assert.IsFalse(reader.Read());
Assert.AreEqual(ODataParameterReaderState.Completed, reader.State);
}
public void TestReadAvroAsCollection()
{
const string Schema = @"{""type"":""array"", ""items"":""long""}";
var stream = new MemoryStream();
using (var writer = AvroContainer.CreateGenericWriter(Schema, stream, /*leave open*/true, Codec.Null))
using (var seqWriter = new SequentialWriter<object>(writer, 24))
{
// Serialize the data to stream using the sequential writer
seqWriter.Write(new[] { 6L, 8 });
seqWriter.Flush();
}
stream.Flush();
stream.Seek(0, SeekOrigin.Begin);
var reader = new ODataAvroCollectionReader(this.CreateODataInputContext(stream));
Assert.AreEqual(ODataCollectionReaderState.Start, reader.State);
Assert.IsTrue(reader.Read());
Assert.AreEqual(ODataCollectionReaderState.CollectionStart, reader.State);
Assert.IsTrue(reader.Read());
Assert.AreEqual(ODataCollectionReaderState.Value, reader.State);
Assert.AreEqual(6L, (long)reader.Item);
Assert.IsTrue(reader.Read());
Assert.AreEqual(ODataCollectionReaderState.Value, reader.State);
Assert.IsTrue(reader.Read());
Assert.AreEqual(ODataCollectionReaderState.CollectionEnd, reader.State);
Assert.IsFalse(reader.Read());
Assert.AreEqual(ODataCollectionReaderState.Completed, reader.State);
}
public void TestReadAvroAsPrimitiveCollection()
{
const string Schema = @"{""type"":""array"", ""items"":""long""}";
var stream = new MemoryStream();
using (var writer = AvroContainer.CreateGenericWriter(Schema, stream, /*leave open*/true, Codec.Null))
using (var seqWriter = new SequentialWriter<object>(writer, 24))
{
// Serialize the data to stream using the sequential writer
seqWriter.Write(new[] { 6L, 8 });
seqWriter.Flush();
}
stream.Flush();
stream.Seek(0, SeekOrigin.Begin);
var property = this.CreateODataInputContext(stream).ReadProperty(null, null);
}
public void SequentialGenericWritingReading_RecursiveRecord()
{
const string StringSchema = @"{
""type"":""record"",
""name"":""Microsoft.Hadoop.Avro.Tests.Recursive"",
""fields"":[
{""name"":""IntField"",""type"":""int""},
{""name"":""RecursiveField"",""type"":[
""null"",
""Microsoft.Hadoop.Avro.Tests.Recursive""
]
}
]}";
using (var stream = new MemoryStream())
{
var serializer = AvroSerializer.CreateGeneric(StringSchema);
using (var streamWriter = AvroContainer.CreateGenericWriter(StringSchema, stream, Codec.Null))
{
using (var writer = new SequentialWriter<object>(streamWriter, 24))
{
var expected = new List<AvroRecord>();
var random = new Random(93);
for (int i = 0; i < 10; i++)
{
dynamic record = new AvroRecord(serializer.WriterSchema);
record.IntField = random.Next();
record.RecursiveField =
new AvroRecord(
((serializer.ReaderSchema as RecordSchema).GetField("RecursiveField").TypeSchema as UnionSchema).Schemas[1]);
record.RecursiveField.IntField = random.Next();
record.RecursiveField.RecursiveField = null;
expected.Add(record);
}
expected.ForEach(writer.Write);
writer.Flush();
stream.Seek(0, SeekOrigin.Begin);
var streamReader = AvroContainer.CreateReader<Recursive>(stream, true, this.dataContractSettings, new CodecFactory());
using (var reader = new SequentialReader<Recursive>(streamReader))
{
var j = 0;
foreach (var avroRecord in reader.Objects)
{
Assert.AreEqual(expected[j]["IntField"], avroRecord.IntField);
Assert.AreEqual(((dynamic)expected[j]["RecursiveField"])["IntField"], avroRecord.RecursiveField.IntField);
Assert.AreEqual(
((dynamic)expected[j++]["RecursiveField"])["RecursiveField"], avroRecord.RecursiveField.RecursiveField);
}
}
}
}
}
}
public static void SerializeDeserializeObjectUsingReflectionStream()
{
string blobName = "aaa/avrotest/test008";
CloudBlobClient client = new Microsoft.WindowsAzure.Storage.Blob.CloudBlobClient(
new Uri("http://hanzstorage.blob.core.windows.net"),
new Microsoft.WindowsAzure.Storage.Auth.StorageCredentials(
"hanzstorage",
"w9TEpvGTusvFlGAdCoWdDrwqLzy6er0Zm5YKdDD0YTkQdOj3WufeVrgd2c8q8amLR0o6xD0tBChcIIA+DCgxXA=="
));
CloudBlobContainer container = client.GetContainerReference("hanzhdi");
CloudBlockBlob blockBlob = container.GetBlockBlobReference(blobName);
foreach (var md in blockBlob.Metadata)
{
Console.WriteLine("{0} {1}", md.Key, md.Value);
}
Console.WriteLine("Serializing Sample Data Set USING REFLECTION\n");
AvroBlobAppender.AvroBlobAppenderWriter <MyDataType> writer =
new AvroBlobAppender.AvroBlobAppenderWriter <MyDataType>(blockBlob, false, AvroSerializer.Create <MyDataType>(), Codec.Null);
Microsoft.Hadoop.Avro.Container.SequentialWriter <MyDataType> sequentialWriter =
new SequentialWriter <MyDataType>(writer, 10000);
List <MyDataType> myDataList = MyDataType.GenerateData(555, 10);
foreach (var myData in myDataList)
{
sequentialWriter.Write(myData);
}
sequentialWriter.Flush();
sequentialWriter.Dispose();
#region commented code
//blockBlob.DownloadToFile(blobName, FileMode.Create);
//using (Stream stream = File.OpenRead(blobName))
//{
// Microsoft.Hadoop.Avro.Container.SequentialReader<MyDataType> reader =
// new Microsoft.Hadoop.Avro.Container.SequentialReader<MyDataType>(AvroContainer.CreateReader<MyDataType>(stream));
// List<MyDataType> actuals = reader.Objects.ToList();
// Console.WriteLine("Number of objects: {0}", actuals.Count);
// for (int i = 0; i < actuals.Count; i++)
// {
// var actual = actuals[i];
// MyDataType exp = null;
// switch (i)
// {
// case 0:
// exp = expected;
// break;
// case 1:
// exp = expected2;
// break;
// default:
// Console.WriteLine("No expected for object {0}", i);
// continue;
// }
// Console.WriteLine("Result of Data Set Identity Comparison is {0}", Program.Equal(exp, actual));
// }
//}
#endregion
}
