public void TestSerializeDeserializeObject()
{
//CreateDeserializerOnly a new AvroSerializer instance and specify a custom serialization strategy AvroDataContractResolver
//for serializing only properties attributed with DataContract/DateMember
var settings = new AvroSerializerSettings();
var avroSerializer = AvroSerializer.Create <SensorData>(settings);
//CreateDeserializerOnly a new buffer
using (var buffer = new MemoryStream())
{
//CreateDeserializerOnly sample data
var expected = new SensorData {
Value = new byte[] { 1, 2, 3, 4, 5 }, Position = new Location {
Room = 243, Floor = 1
}
};
//Serialize the data to the specified stream
avroSerializer.Serialize(buffer, expected);
//Prepare the stream for deserializing the data
buffer.Seek(0, SeekOrigin.Begin);
//Derserialize data from the stream and cast it to the same type used for serialization
var actual = avroSerializer.Deserialize(buffer);
//Finally, verify that deserialized data matches the original ones
Assert.True(this.Equal(expected, actual));
}
}
public void JsonSchemaBuilder_BuildEnumSchemaUsingPublicMemberContractResolverWithCSharpNulls()
{
const string Expected =
"{" +
"\"type\":\"enum\"," +
"\"name\":\"Microsoft.Hadoop.Avro.Tests.RandomEnumeration\"," +
"\"symbols\":" + "[" +
"\"Value0\"," +
"\"Value1\"," +
"\"Value2\"," +
"\"Value3\"," +
"\"Value4\"," +
"\"Value5\"," +
"\"Value6\"," +
"\"Value7\"," +
"\"Value8\"," +
"\"Value9\"" +
"]}";
var serializer =
AvroSerializer.Create <Utilities.RandomEnumeration>(
new AvroSerializerSettings {
Resolver = new AvroPublicMemberContractResolver(true)
});
Assert.AreEqual(serializer.WriterSchema, serializer.ReaderSchema);
Assert.AreEqual(Expected, serializer.ReaderSchema.ToString());
}
/// <summary>
/// Generate and store the metadata necessary to serialze and deserialize a specific message type.
/// </summary>
/// <typeparam name="TMessage">The class type of the message being registered.</typeparam>
internal void Register <TMessage>()
{
Logr.Log(Level.Info, "Registering message type: {0} {1}",
typeof(TMessage).FullName, typeof(TMessage).Name);
IAvroSerializer <TMessage> messageSerializer = AvroSerializer.Create <TMessage>();
Serialize serialize = (MemoryStream stream, object message) =>
{
messageSerializer.Serialize(stream, (TMessage)message);
};
serializeMap.Add(typeof(TMessage).Name, serialize);
Deserialize deserialize = (MemoryStream stream, object observer, long sequence) =>
{
TMessage message = messageSerializer.Deserialize(stream);
var msgObserver = observer as IObserver <IMessageInstance <TMessage> >;
if (msgObserver != null)
{
msgObserver.OnNext(new MessageInstance <TMessage>(sequence, message));
}
else
{
Logr.Log(Level.Warning, "Unhandled message received: {0}", message);
}
};
deserializeMap.Add(typeof(TMessage).Name, deserialize);
}
static Serializer()
{
ser = AvroSerializer.Create <T>(new AvroSerializerSettings()
{
UseCache = true
});
}
public void Performance_SimpleFlatClass()
{
var serializationTime = new Stopwatch();
var deserializationTime = new Stopwatch();
var expected = SimpleFlatClass.Create();
var serializer = AvroSerializer.Create <SimpleFlatClass>(new AvroSerializerSettings {
Resolver = new AvroDataContractResolver(true)
});
using (var stream = new MemoryStream())
{
serializationTime.Start();
for (var i = 0; i < NumberOfRuns; i++)
{
serializer.Serialize(stream, expected);
}
serializationTime.Stop();
stream.Seek(0, SeekOrigin.Begin);
deserializationTime.Start();
for (var i = 0; i < NumberOfRuns; i++)
{
var deserialized = serializer.Deserialize(stream);
}
deserializationTime.Stop();
Console.WriteLine(serializationTime.ElapsedTicks);
Console.WriteLine(deserializationTime.ElapsedTicks);
}
}
public void JsonSchemaBuilder_BuildRecordSchemaForClassWithListFieldUsingPublicMemberContractResolverWithCSharpNulls()
{
const string Expected =
"[" +
"\"null\"," +
"{" +
"\"type\":\"record\"," +
"\"name\":\"Microsoft.Hadoop.Avro.Tests.ClassOfListOfGuid\"," +
"\"fields\":[" +
"{" +
"\"name\":\"ListOfGuid\"," +
"\"type\":[" +
"\"null\"," +
"{\"type\":\"array\",\"items\":{\"type\":\"fixed\",\"name\":\"System.Guid\",\"size\":16}}" +
"]" +
"}" +
"]" +
"}" +
"]";
var serializer =
AvroSerializer.Create <ClassOfListOfGuid>(new AvroSerializerSettings {
Resolver = new AvroPublicMemberContractResolver(true)
});
Assert.AreEqual(serializer.WriterSchema, serializer.ReaderSchema);
Assert.AreEqual(Expected, serializer.ReaderSchema.ToString());
}
public void TestGetSchema()
{
var serializer = AvroSerializer.Create <AvroNode>();
var s = serializer.WriterSchema.ToString();
Assert.NotNull(s);
}
public static string GenerateSchema <T>()
{
return(JsonConvert.SerializeObject(
new
{
schema = AvroSerializer.Create <T>().ReaderSchema.ToString()
}));
}
private AvroNode AvroDeserialize(byte[] serializedBytes)
{
var serializer = AvroSerializer.Create <AvroNode>();
using (var stream = new MemoryStream(serializedBytes))
{
return(serializer.Deserialize(stream));
}
}
public static AvroHttpRequest FromBytes(byte[] serializedBytes)
{
var serializer = AvroSerializer.Create <AvroHttpRequest>();
using (var stream = new MemoryStream(serializedBytes))
{
return(serializer.Deserialize(stream));
}
}
/// <summary>
/// Converts a byte array to an object using Avro deserialization.
/// </summary>
/// <param name="data">The byte array to deserialize</param>
/// <returns>The deserialized object</returns>
public static T AvroDeserialize <T>(byte[] data)
{
IAvroSerializer <T> deserializer = AvroSerializer.Create <T>();
using (MemoryStream stream = new MemoryStream(data))
{
return(deserializer.Deserialize(stream));
}
}
/// <summary>
/// Convert an object to byte array using Avro serialization
/// </summary>
/// <param name="obj">The object to serialize</param>
/// <returns>The serialized object in a byte array</returns>
public static byte[] AvroSerialize <T>(T obj)
{
IAvroSerializer <T> serializer = AvroSerializer.Create <T>();
using (MemoryStream stream = new MemoryStream())
{
serializer.Serialize(stream, obj);
return(stream.GetBuffer());
}
}
public void JsonSchemaBuilder_ClassOfGuidWriterSchema()
{
var settings = new AvroSerializerSettings {
Resolver = new AvroDataContractResolver(false)
};
IAvroSerializer <ClassOfGuid> serializer = AvroSerializer.Create <ClassOfGuid>(settings);
string writerSchema = serializer.WriterSchema.ToString();
AvroSerializer.CreateDeserializerOnly <ClassOfGuid>(writerSchema, settings);
}
/// <summary>
/// Serialize AvroHttpRequest object into bytes
/// </summary>
/// <param name="obj">The object.</param>
/// <returns>System.Byte[].</returns>
public static byte[] ToBytes(AvroHttpRequest obj)
{
var serializer = AvroSerializer.Create <AvroHttpRequest>();
using (MemoryStream stream = new MemoryStream())
{
serializer.Serialize(stream, obj);
return(stream.GetBuffer());
}
}
private byte[] AvroSerialize(AvroConfiguration obj)
{
var serializer = AvroSerializer.Create <AvroConfiguration>();
using (MemoryStream stream = new MemoryStream())
{
serializer.Serialize(stream, obj);
return(stream.GetBuffer());
}
}
private byte[] AvroSerialize(AvroNode obj)
{
var serializer = AvroSerializer.Create <AvroNode>();
using (MemoryStream stream = new MemoryStream())
{
serializer.Serialize(stream, obj);
return(stream.ToArray());
}
}
public void Container_ApacheWriterMicrosoftReader()
{
var serializer = AvroSerializer.Create <ClassOfInt>(new AvroSerializerSettings {
Resolver = new AvroDataContractResolver(true)
});
var schema = ApacheAvro.Schema.Parse(serializer.WriterSchema.ToString()) as ApacheAvro.UnionSchema;
Assert.NotNull(schema);
var recordSchema = schema.Schemas[1] as ApacheAvro.RecordSchema;
Assert.NotNull(recordSchema);
var expected = new List <GenericRecord>();
for (var i = 0; i < 7; i++)
{
var record = new GenericRecord(recordSchema);
record.Add("PrimitiveInt", ClassOfInt.Create(true).PrimitiveInt);
expected.Add(record);
}
using (var memoryStream = new MemoryStream())
{
var datumWriter = new GenericWriter <GenericRecord>(schema);
var writer = DataFileWriter <GenericRecord> .OpenWriter(datumWriter, memoryStream);
writer.WriteHeader();
foreach (var obj in expected)
{
writer.Append(obj);
}
writer.Flush();
memoryStream.Seek(0, SeekOrigin.Begin);
var reader = AvroContainer.CreateReader <ClassOfInt>(memoryStream, true, new AvroSerializerSettings {
Resolver = new AvroDataContractResolver(true)
}, new CodecFactory());
var actual = new List <ClassOfInt>();
while (reader.MoveNext())
{
actual.AddRange(reader.Current.Objects);
}
Assert.Equal(expected.Count, actual.Count);
for (var i = 0; i < expected.Count; ++i)
{
Assert.Equal(expected[i]["PrimitiveInt"], actual[i].PrimitiveInt);
}
}
}
public void ReflectionSchemaBuilder_BuildSchemaForUnicodeClassName()
{
var serializer = AvroSerializer.Create <UnicodeClassNameŠŽŒ>(new AvroSerializerSettings {
Resolver = new AvroDataContractResolver(true)
});
var jsonArray = (JArray)JsonConvert.DeserializeObject(serializer.ReaderSchema.ToString());
var obj = jsonArray[1] as JObject;
string qualifiedRecordName = obj.GetValue("name", StringComparison.Ordinal).ToString();
string recordName = qualifiedRecordName.Substring(qualifiedRecordName.LastIndexOf('.') + 1);
Assert.IsTrue(Regex.Match(recordName, @"^[a-zA-Z_]([a-zA-Z0-9_]*)$").Success, @"Avro 1.7.4 spec does not allow unicode characters in names");
}
public void AvroRecord_SerializeRecordWithSByteFields()
{
var schema = AvroSerializer.Create <ClassWithSByteFields>().WriterSchema;
var expected = ClassWithSByteFields.Create();
var serializer = AvroSerializer.CreateGeneric(schema.ToString());
using (var stream = new MemoryStream())
{
serializer.Serialize(stream, expected.ToAvroRecord(schema));
stream.Seek(0, SeekOrigin.Begin);
var result = serializer.Deserialize(stream);
var actual = ClassWithSByteFields.Create((AvroRecord)result);
Assert.Equal(expected, actual);
}
}
private static async Task GenerateSchema(CancellationToken token)
{
var serializer = AvroSerializer.Create <MyDataType>();
var schema = serializer.WriterSchema.ToString();
//var printPretty = schema;
var printPretty = JObject
.Parse(schema)
.ToString(Formatting.Indented);
using (var writer = File.CreateText("avro.schema.json"))
{
await writer.WriteAsync(printPretty);
}
}
public void Should_serialize_deserialize_object()
{
var settings = new AvroSerializerSettings
{
GenerateDeserializer = true,
GenerateSerializer = true,
Resolver = new AvroPublicMemberContractResolver(false),
MaxItemsInSchemaTree = 40,
UseCache = true
};
//var avroSerializer2 = AvroSerializer.Create<ChannelRangeInfo>(settings);
var avroSerializer = AvroSerializer.Create <GetRange>(settings);
var message = new GetRange
{
ChannelRanges = new List <ChannelRangeInfo>
{
new ChannelRangeInfo {
ChannelId = new List <long> {
1, 2, 3
}, Interval = new Etp.v12.Datatypes.Object.IndexInterval {
DepthDatum = "datum", EndIndex = new Etp.v12.Datatypes.IndexValue {
Item = 0.0
}, StartIndex = new Etp.v12.Datatypes.IndexValue {
Item = 0.0
}, Uom = "Uom"
}
}
},
RequestUuid = new Etp.v12.Datatypes.Uuid
{
Value = new byte[16]
}
};
using (var buffer = new MemoryStream())
{
avroSerializer.Serialize(buffer, message);
buffer.Seek(0, SeekOrigin.Begin);
var result = avroSerializer.Deserialize(buffer);
Console.WriteLine(result);
}
}
/// <summary>
/// Creates a writer of a static C# type.
/// </summary>
/// <typeparam name="T">The type of serialized objects.</typeparam>
/// <param name="stream">The stream that will contain the resulting Avro object container.</param>
/// <param name="leaveOpen">If set to <c>true</c> the <paramref name="stream"/> is left open.</param>
/// <param name="settings">The serializer settings.</param>
/// <param name="codec">The codec.</param>
/// <returns> A writer. </returns>
/// <exception cref="System.ArgumentNullException">Thrown when any argument is null.</exception>
public static IAvroWriter <T> CreateWriter <T>(Stream stream, bool leaveOpen, AvroSerializerSettings settings, Codec codec, bool isHeaderWritten = false, byte[] syncMarker = null)
{
if (stream == null)
{
throw new ArgumentNullException("stream");
}
if (settings == null)
{
throw new ArgumentNullException("settings");
}
if (codec == null)
{
throw new ArgumentNullException("codec");
}
return(new StreamWriter <T>(stream, leaveOpen, AvroSerializer.Create <T>(settings), codec, isHeaderWritten, syncMarker));
}
/// <summary>
/// Creates a writer of a static C# type.
/// </summary>
/// <typeparam name="T">The type of serialized objects.</typeparam>
/// <param name="stream">The stream that will contain the resulting Avro object container.</param>
/// <param name="leaveOpen">If set to <c>true</c> the <paramref name="stream"/> is left open.</param>
/// <param name="settings">The serializer settings.</param>
/// <param name="codec">The codec.</param>
/// <returns> A writer. </returns>
/// <exception cref="System.ArgumentNullException">Thrown when any argument is null.</exception>
public static IAvroWriter <T> CreateWriter <T>(Stream stream, bool leaveOpen, AvroSerializerSettings settings, Codec codec)
{
if (stream == null)
{
throw new ArgumentNullException("stream");
}
if (settings == null)
{
throw new ArgumentNullException("settings");
}
if (codec == null)
{
throw new ArgumentNullException("codec");
}
return(new StreamWriter <T>(stream, leaveOpen, AvroSerializer.Create <T>(settings), codec));
}
public static long GetPackedSize <T>(T obj)
{
try
{
var msgPack = AvroSerializer.Create <T>();
using (var mem = new MemoryStream())
{
msgPack.Serialize(mem, obj);
return(mem.Length);
}
}
catch (Exception)
{
return(-1);
}
}
private object CreateAvroSerializer <T>()
{
if (_avroSerializer == null)
{
if (_avroWriter != null && _avroWriter is ChoAvroWriter <T> )
{
_avroSerializer = ((ChoAvroWriter <T>)_avroWriter).AvroSerializer;
}
if (_avroSerializer == null)
{
_avroSerializer = AvroSerializer.Create <T>(Configuration.AvroSerializerSettings);
}
}
return(_avroSerializer);
}
public void JsonSchemaBuilder_BuildRecordSchemaForNestedClassUsingPublicMemberContractResolverWithCSharpNulls()
{
const string Expected =
"[" +
"\"null\"," +
"{" +
"\"type\":\"record\"," +
"\"name\":\"Microsoft.Hadoop.Avro.Tests.NestedClass\"," +
"\"fields\":" +
"[" +
"{" +
"\"name\":\"ClassOfIntReference\"," +
"\"type\":" +
"[" +
"\"null\"," +
"{" +
"\"type\":\"record\"," +
"\"name\":\"Microsoft.Hadoop.Avro.Tests.ClassOfInt\"," +
"\"fields\":" +
"[" +
"{" +
"\"name\":\"PrimitiveInt\"," +
"\"type\":\"int\"" +
"}" +
"]" +
"}" +
"]" +
"}," +
"{" +
"\"name\":\"PrimitiveInt\"," +
"\"type\":\"int\"" +
"}" +
"]" +
"}" +
"]";
var serializer =
AvroSerializer.Create <NestedClass>(new AvroSerializerSettings {
Resolver = new AvroPublicMemberContractResolver(true)
});
Assert.AreEqual(serializer.WriterSchema, serializer.ReaderSchema);
Assert.AreEqual(Expected, serializer.ReaderSchema.ToString());
}
static void Main(string[] args)
{
var schema = Schema.Parse(AvroSerializer.Create <Blog>().WriterSchema.ToString());
var inputs = new List <Blog>
{
new Blog {
BlogId = 101, Name = "Tanaka", Author = "One"
},
new Blog {
BlogId = 201, Name = "Sato", Author = "Two"
},
new Blog {
BlogId = 301, Name = "Suzuki", Author = "Three"
}
};
var writer = new SpecificDatumWriter <Blog>(schema);
using (var fw = DataFileWriter <Blog> .OpenWriter(writer, "./blog.avro"))
{
foreach (var blog in inputs)
{
fw.Append(blog);
}
}
var outputs = new List <Blog>();
using (var fr = DataFileReader <Blog> .OpenReader("./blog.avro"))
{
while (fr.HasNext())
{
outputs.Add(fr.Next());
}
}
foreach (var b in outputs)
{
Console.WriteLine("----- Avro → POCO 変換後 -----");
Console.WriteLine($"{b.BlogId} {b.Name} {b.Author}");
}
}
public override void Output(IRow row, IUnstructuredWriter output)
{
// First Row
if (this.writer == null)
{
//this.writer = new SequentialWriter<object>(AvroContainer.Create<AnonType_WeatherDataSet>(this.avroSchema, output.BaseStream, Codec.Deflate), 24);
this.serializer = AvroSerializer.Create <AnonType_WeatherDataSet>();
}
AnonType_WeatherDataSet data = new AnonType_WeatherDataSet(
row.Get <string>(row.Schema[0].Name),
row.Get <int?>(row.Schema[1].Name),
row.Get <string>(row.Schema[2].Name),
(double?)row.Get <decimal?>(row.Schema[3].Name),
row.Get <string>(row.Schema[4].Name),
row.Get <string>(row.Schema[5].Name),
row.Get <string>(row.Schema[6].Name),
row.Get <int?>(row.Schema[7].Name)
);
serializer.Serialize(output.BaseStream, data);
}
//Serialize and deserialize sample data set represented as an object using Reflection
//No explicit schema definition is required - schema of serialized objects is automatically built
public void SerializeDeserializeObjectUsingReflection()
{
Console.WriteLine("SERIALIZATION USING REFLECTION\n");
Console.WriteLine("Serializing Sample Data Set...");
//Create a new AvroSerializer instance and specify a custom serialization strategy AvroDataContractResolver
//for serializing only properties attributed with DataContract/DateMember
var avroSerializer = AvroSerializer.Create <SensorData>();
//Create a Memory Stream buffer
using (var buffer = new MemoryStream())
{
//Create a data set using sample Class and struct
var expected = new SensorData {
Value = new byte[] { 1, 2, 3, 4, 5 }, Position = new Location {
Room = 243, Floor = 1
}
};
//Serialize the data to the specified stream
avroSerializer.Serialize(buffer, expected);
Console.WriteLine("Deserializing Sample Data Set...");
//Prepare the stream for deserializing the data
buffer.Seek(0, SeekOrigin.Begin);
//Deserialize data from the stream and cast it to the same type used for serialization
var actual = avroSerializer.Deserialize(buffer);
Console.WriteLine("Comparing Initial and Deserialized Data Sets...");
//Finally, verify that deserialized data matches the original one
bool isEqual = this.Equal(expected, actual);
Console.WriteLine("Result of Data Set Identity Comparison is {0}", isEqual);
}
}
private void buttonAvro_Click(object sender, EventArgs e)
{
Run(() =>
{
var records = new[]
{
new AvroRecord <string, Person>
{
PartitionId = GetPartitionId(),
Value = GetPerson()
},
new AvroRecord <string, Person>
{
Value = GetPerson()
}
};
var recordSet = new AvroRecordSet <string, Person>(records)
{
ValueSchema = AvroSerializer.Create <Person>().ReaderSchema.ToString()
};
return(_confluentClient.PublishAsAvroAsync(textBoxTopic.Text, recordSet).Result);
});
}
