public void TestSerializeDeserializeObject() { //CreateDeserializerOnly a new AvroSerializer instance and specify a custom serialization strategy AvroDataContractResolver //for serializing only properties attributed with DataContract/DateMember var settings = new AvroSerializerSettings(); var avroSerializer = AvroSerializer.Create <SensorData>(settings); //CreateDeserializerOnly a new buffer using (var buffer = new MemoryStream()) { //CreateDeserializerOnly sample data var expected = new SensorData { Value = new byte[] { 1, 2, 3, 4, 5 }, Position = new Location { Room = 243, Floor = 1 } }; //Serialize the data to the specified stream avroSerializer.Serialize(buffer, expected); //Prepare the stream for deserializing the data buffer.Seek(0, SeekOrigin.Begin); //Derserialize data from the stream and cast it to the same type used for serialization var actual = avroSerializer.Deserialize(buffer); //Finally, verify that deserialized data matches the original ones Assert.True(this.Equal(expected, actual)); } }
public void JsonSchemaBuilder_BuildEnumSchemaUsingPublicMemberContractResolverWithCSharpNulls() { const string Expected = "{" + "\"type\":\"enum\"," + "\"name\":\"Microsoft.Hadoop.Avro.Tests.RandomEnumeration\"," + "\"symbols\":" + "[" + "\"Value0\"," + "\"Value1\"," + "\"Value2\"," + "\"Value3\"," + "\"Value4\"," + "\"Value5\"," + "\"Value6\"," + "\"Value7\"," + "\"Value8\"," + "\"Value9\"" + "]}"; var serializer = AvroSerializer.Create <Utilities.RandomEnumeration>( new AvroSerializerSettings { Resolver = new AvroPublicMemberContractResolver(true) }); Assert.AreEqual(serializer.WriterSchema, serializer.ReaderSchema); Assert.AreEqual(Expected, serializer.ReaderSchema.ToString()); }
/// <summary> /// Generate and store the metadata necessary to serialze and deserialize a specific message type. /// </summary> /// <typeparam name="TMessage">The class type of the message being registered.</typeparam> internal void Register <TMessage>() { Logr.Log(Level.Info, "Registering message type: {0} {1}", typeof(TMessage).FullName, typeof(TMessage).Name); IAvroSerializer <TMessage> messageSerializer = AvroSerializer.Create <TMessage>(); Serialize serialize = (MemoryStream stream, object message) => { messageSerializer.Serialize(stream, (TMessage)message); }; serializeMap.Add(typeof(TMessage).Name, serialize); Deserialize deserialize = (MemoryStream stream, object observer, long sequence) => { TMessage message = messageSerializer.Deserialize(stream); var msgObserver = observer as IObserver <IMessageInstance <TMessage> >; if (msgObserver != null) { msgObserver.OnNext(new MessageInstance <TMessage>(sequence, message)); } else { Logr.Log(Level.Warning, "Unhandled message received: {0}", message); } }; deserializeMap.Add(typeof(TMessage).Name, deserialize); }
static Serializer() { ser = AvroSerializer.Create <T>(new AvroSerializerSettings() { UseCache = true }); }
public void Performance_SimpleFlatClass() { var serializationTime = new Stopwatch(); var deserializationTime = new Stopwatch(); var expected = SimpleFlatClass.Create(); var serializer = AvroSerializer.Create <SimpleFlatClass>(new AvroSerializerSettings { Resolver = new AvroDataContractResolver(true) }); using (var stream = new MemoryStream()) { serializationTime.Start(); for (var i = 0; i < NumberOfRuns; i++) { serializer.Serialize(stream, expected); } serializationTime.Stop(); stream.Seek(0, SeekOrigin.Begin); deserializationTime.Start(); for (var i = 0; i < NumberOfRuns; i++) { var deserialized = serializer.Deserialize(stream); } deserializationTime.Stop(); Console.WriteLine(serializationTime.ElapsedTicks); Console.WriteLine(deserializationTime.ElapsedTicks); } }
public void JsonSchemaBuilder_BuildRecordSchemaForClassWithListFieldUsingPublicMemberContractResolverWithCSharpNulls() { const string Expected = "[" + "\"null\"," + "{" + "\"type\":\"record\"," + "\"name\":\"Microsoft.Hadoop.Avro.Tests.ClassOfListOfGuid\"," + "\"fields\":[" + "{" + "\"name\":\"ListOfGuid\"," + "\"type\":[" + "\"null\"," + "{\"type\":\"array\",\"items\":{\"type\":\"fixed\",\"name\":\"System.Guid\",\"size\":16}}" + "]" + "}" + "]" + "}" + "]"; var serializer = AvroSerializer.Create <ClassOfListOfGuid>(new AvroSerializerSettings { Resolver = new AvroPublicMemberContractResolver(true) }); Assert.AreEqual(serializer.WriterSchema, serializer.ReaderSchema); Assert.AreEqual(Expected, serializer.ReaderSchema.ToString()); }
public void TestGetSchema() { var serializer = AvroSerializer.Create <AvroNode>(); var s = serializer.WriterSchema.ToString(); Assert.NotNull(s); }
public static string GenerateSchema <T>() { return(JsonConvert.SerializeObject( new { schema = AvroSerializer.Create <T>().ReaderSchema.ToString() })); }
private AvroNode AvroDeserialize(byte[] serializedBytes) { var serializer = AvroSerializer.Create <AvroNode>(); using (var stream = new MemoryStream(serializedBytes)) { return(serializer.Deserialize(stream)); } }
public static AvroHttpRequest FromBytes(byte[] serializedBytes) { var serializer = AvroSerializer.Create <AvroHttpRequest>(); using (var stream = new MemoryStream(serializedBytes)) { return(serializer.Deserialize(stream)); } }
/// <summary> /// Converts a byte array to an object using Avro deserialization. /// </summary> /// <param name="data">The byte array to deserialize</param> /// <returns>The deserialized object</returns> public static T AvroDeserialize <T>(byte[] data) { IAvroSerializer <T> deserializer = AvroSerializer.Create <T>(); using (MemoryStream stream = new MemoryStream(data)) { return(deserializer.Deserialize(stream)); } }
/// <summary> /// Convert an object to byte array using Avro serialization /// </summary> /// <param name="obj">The object to serialize</param> /// <returns>The serialized object in a byte array</returns> public static byte[] AvroSerialize <T>(T obj) { IAvroSerializer <T> serializer = AvroSerializer.Create <T>(); using (MemoryStream stream = new MemoryStream()) { serializer.Serialize(stream, obj); return(stream.GetBuffer()); } }
public void JsonSchemaBuilder_ClassOfGuidWriterSchema() { var settings = new AvroSerializerSettings { Resolver = new AvroDataContractResolver(false) }; IAvroSerializer <ClassOfGuid> serializer = AvroSerializer.Create <ClassOfGuid>(settings); string writerSchema = serializer.WriterSchema.ToString(); AvroSerializer.CreateDeserializerOnly <ClassOfGuid>(writerSchema, settings); }
/// <summary> /// Serialize AvroHttpRequest object into bytes /// </summary> /// <param name="obj">The object.</param> /// <returns>System.Byte[].</returns> public static byte[] ToBytes(AvroHttpRequest obj) { var serializer = AvroSerializer.Create <AvroHttpRequest>(); using (MemoryStream stream = new MemoryStream()) { serializer.Serialize(stream, obj); return(stream.GetBuffer()); } }
private byte[] AvroSerialize(AvroConfiguration obj) { var serializer = AvroSerializer.Create <AvroConfiguration>(); using (MemoryStream stream = new MemoryStream()) { serializer.Serialize(stream, obj); return(stream.GetBuffer()); } }
private byte[] AvroSerialize(AvroNode obj) { var serializer = AvroSerializer.Create <AvroNode>(); using (MemoryStream stream = new MemoryStream()) { serializer.Serialize(stream, obj); return(stream.ToArray()); } }
public void Container_ApacheWriterMicrosoftReader() { var serializer = AvroSerializer.Create <ClassOfInt>(new AvroSerializerSettings { Resolver = new AvroDataContractResolver(true) }); var schema = ApacheAvro.Schema.Parse(serializer.WriterSchema.ToString()) as ApacheAvro.UnionSchema; Assert.NotNull(schema); var recordSchema = schema.Schemas[1] as ApacheAvro.RecordSchema; Assert.NotNull(recordSchema); var expected = new List <GenericRecord>(); for (var i = 0; i < 7; i++) { var record = new GenericRecord(recordSchema); record.Add("PrimitiveInt", ClassOfInt.Create(true).PrimitiveInt); expected.Add(record); } using (var memoryStream = new MemoryStream()) { var datumWriter = new GenericWriter <GenericRecord>(schema); var writer = DataFileWriter <GenericRecord> .OpenWriter(datumWriter, memoryStream); writer.WriteHeader(); foreach (var obj in expected) { writer.Append(obj); } writer.Flush(); memoryStream.Seek(0, SeekOrigin.Begin); var reader = AvroContainer.CreateReader <ClassOfInt>(memoryStream, true, new AvroSerializerSettings { Resolver = new AvroDataContractResolver(true) }, new CodecFactory()); var actual = new List <ClassOfInt>(); while (reader.MoveNext()) { actual.AddRange(reader.Current.Objects); } Assert.Equal(expected.Count, actual.Count); for (var i = 0; i < expected.Count; ++i) { Assert.Equal(expected[i]["PrimitiveInt"], actual[i].PrimitiveInt); } } }
public void ReflectionSchemaBuilder_BuildSchemaForUnicodeClassName() { var serializer = AvroSerializer.Create <UnicodeClassNameŠŽŒ>(new AvroSerializerSettings { Resolver = new AvroDataContractResolver(true) }); var jsonArray = (JArray)JsonConvert.DeserializeObject(serializer.ReaderSchema.ToString()); var obj = jsonArray[1] as JObject; string qualifiedRecordName = obj.GetValue("name", StringComparison.Ordinal).ToString(); string recordName = qualifiedRecordName.Substring(qualifiedRecordName.LastIndexOf('.') + 1); Assert.IsTrue(Regex.Match(recordName, @"^[a-zA-Z_]([a-zA-Z0-9_]*)$").Success, @"Avro 1.7.4 spec does not allow unicode characters in names"); }
public void AvroRecord_SerializeRecordWithSByteFields() { var schema = AvroSerializer.Create <ClassWithSByteFields>().WriterSchema; var expected = ClassWithSByteFields.Create(); var serializer = AvroSerializer.CreateGeneric(schema.ToString()); using (var stream = new MemoryStream()) { serializer.Serialize(stream, expected.ToAvroRecord(schema)); stream.Seek(0, SeekOrigin.Begin); var result = serializer.Deserialize(stream); var actual = ClassWithSByteFields.Create((AvroRecord)result); Assert.Equal(expected, actual); } }
private static async Task GenerateSchema(CancellationToken token) { var serializer = AvroSerializer.Create <MyDataType>(); var schema = serializer.WriterSchema.ToString(); //var printPretty = schema; var printPretty = JObject .Parse(schema) .ToString(Formatting.Indented); using (var writer = File.CreateText("avro.schema.json")) { await writer.WriteAsync(printPretty); } }
public void Should_serialize_deserialize_object() { var settings = new AvroSerializerSettings { GenerateDeserializer = true, GenerateSerializer = true, Resolver = new AvroPublicMemberContractResolver(false), MaxItemsInSchemaTree = 40, UseCache = true }; //var avroSerializer2 = AvroSerializer.Create<ChannelRangeInfo>(settings); var avroSerializer = AvroSerializer.Create <GetRange>(settings); var message = new GetRange { ChannelRanges = new List <ChannelRangeInfo> { new ChannelRangeInfo { ChannelId = new List <long> { 1, 2, 3 }, Interval = new Etp.v12.Datatypes.Object.IndexInterval { DepthDatum = "datum", EndIndex = new Etp.v12.Datatypes.IndexValue { Item = 0.0 }, StartIndex = new Etp.v12.Datatypes.IndexValue { Item = 0.0 }, Uom = "Uom" } } }, RequestUuid = new Etp.v12.Datatypes.Uuid { Value = new byte[16] } }; using (var buffer = new MemoryStream()) { avroSerializer.Serialize(buffer, message); buffer.Seek(0, SeekOrigin.Begin); var result = avroSerializer.Deserialize(buffer); Console.WriteLine(result); } }
/// <summary> /// Creates a writer of a static C# type. /// </summary> /// <typeparam name="T">The type of serialized objects.</typeparam> /// <param name="stream">The stream that will contain the resulting Avro object container.</param> /// <param name="leaveOpen">If set to <c>true</c> the <paramref name="stream"/> is left open.</param> /// <param name="settings">The serializer settings.</param> /// <param name="codec">The codec.</param> /// <returns> A writer. </returns> /// <exception cref="System.ArgumentNullException">Thrown when any argument is null.</exception> public static IAvroWriter <T> CreateWriter <T>(Stream stream, bool leaveOpen, AvroSerializerSettings settings, Codec codec, bool isHeaderWritten = false, byte[] syncMarker = null) { if (stream == null) { throw new ArgumentNullException("stream"); } if (settings == null) { throw new ArgumentNullException("settings"); } if (codec == null) { throw new ArgumentNullException("codec"); } return(new StreamWriter <T>(stream, leaveOpen, AvroSerializer.Create <T>(settings), codec, isHeaderWritten, syncMarker)); }
/// <summary> /// Creates a writer of a static C# type. /// </summary> /// <typeparam name="T">The type of serialized objects.</typeparam> /// <param name="stream">The stream that will contain the resulting Avro object container.</param> /// <param name="leaveOpen">If set to <c>true</c> the <paramref name="stream"/> is left open.</param> /// <param name="settings">The serializer settings.</param> /// <param name="codec">The codec.</param> /// <returns> A writer. </returns> /// <exception cref="System.ArgumentNullException">Thrown when any argument is null.</exception> public static IAvroWriter <T> CreateWriter <T>(Stream stream, bool leaveOpen, AvroSerializerSettings settings, Codec codec) { if (stream == null) { throw new ArgumentNullException("stream"); } if (settings == null) { throw new ArgumentNullException("settings"); } if (codec == null) { throw new ArgumentNullException("codec"); } return(new StreamWriter <T>(stream, leaveOpen, AvroSerializer.Create <T>(settings), codec)); }
public static long GetPackedSize <T>(T obj) { try { var msgPack = AvroSerializer.Create <T>(); using (var mem = new MemoryStream()) { msgPack.Serialize(mem, obj); return(mem.Length); } } catch (Exception) { return(-1); } }
private object CreateAvroSerializer <T>() { if (_avroSerializer == null) { if (_avroWriter != null && _avroWriter is ChoAvroWriter <T> ) { _avroSerializer = ((ChoAvroWriter <T>)_avroWriter).AvroSerializer; } if (_avroSerializer == null) { _avroSerializer = AvroSerializer.Create <T>(Configuration.AvroSerializerSettings); } } return(_avroSerializer); }
public void JsonSchemaBuilder_BuildRecordSchemaForNestedClassUsingPublicMemberContractResolverWithCSharpNulls() { const string Expected = "[" + "\"null\"," + "{" + "\"type\":\"record\"," + "\"name\":\"Microsoft.Hadoop.Avro.Tests.NestedClass\"," + "\"fields\":" + "[" + "{" + "\"name\":\"ClassOfIntReference\"," + "\"type\":" + "[" + "\"null\"," + "{" + "\"type\":\"record\"," + "\"name\":\"Microsoft.Hadoop.Avro.Tests.ClassOfInt\"," + "\"fields\":" + "[" + "{" + "\"name\":\"PrimitiveInt\"," + "\"type\":\"int\"" + "}" + "]" + "}" + "]" + "}," + "{" + "\"name\":\"PrimitiveInt\"," + "\"type\":\"int\"" + "}" + "]" + "}" + "]"; var serializer = AvroSerializer.Create <NestedClass>(new AvroSerializerSettings { Resolver = new AvroPublicMemberContractResolver(true) }); Assert.AreEqual(serializer.WriterSchema, serializer.ReaderSchema); Assert.AreEqual(Expected, serializer.ReaderSchema.ToString()); }
static void Main(string[] args) { var schema = Schema.Parse(AvroSerializer.Create <Blog>().WriterSchema.ToString()); var inputs = new List <Blog> { new Blog { BlogId = 101, Name = "Tanaka", Author = "One" }, new Blog { BlogId = 201, Name = "Sato", Author = "Two" }, new Blog { BlogId = 301, Name = "Suzuki", Author = "Three" } }; var writer = new SpecificDatumWriter <Blog>(schema); using (var fw = DataFileWriter <Blog> .OpenWriter(writer, "./blog.avro")) { foreach (var blog in inputs) { fw.Append(blog); } } var outputs = new List <Blog>(); using (var fr = DataFileReader <Blog> .OpenReader("./blog.avro")) { while (fr.HasNext()) { outputs.Add(fr.Next()); } } foreach (var b in outputs) { Console.WriteLine("----- Avro → POCO 変換後 -----"); Console.WriteLine($"{b.BlogId} {b.Name} {b.Author}"); } }
public override void Output(IRow row, IUnstructuredWriter output) { // First Row if (this.writer == null) { //this.writer = new SequentialWriter<object>(AvroContainer.Create<AnonType_WeatherDataSet>(this.avroSchema, output.BaseStream, Codec.Deflate), 24); this.serializer = AvroSerializer.Create <AnonType_WeatherDataSet>(); } AnonType_WeatherDataSet data = new AnonType_WeatherDataSet( row.Get <string>(row.Schema[0].Name), row.Get <int?>(row.Schema[1].Name), row.Get <string>(row.Schema[2].Name), (double?)row.Get <decimal?>(row.Schema[3].Name), row.Get <string>(row.Schema[4].Name), row.Get <string>(row.Schema[5].Name), row.Get <string>(row.Schema[6].Name), row.Get <int?>(row.Schema[7].Name) ); serializer.Serialize(output.BaseStream, data); }
//Serialize and deserialize sample data set represented as an object using Reflection //No explicit schema definition is required - schema of serialized objects is automatically built public void SerializeDeserializeObjectUsingReflection() { Console.WriteLine("SERIALIZATION USING REFLECTION\n"); Console.WriteLine("Serializing Sample Data Set..."); //Create a new AvroSerializer instance and specify a custom serialization strategy AvroDataContractResolver //for serializing only properties attributed with DataContract/DateMember var avroSerializer = AvroSerializer.Create <SensorData>(); //Create a Memory Stream buffer using (var buffer = new MemoryStream()) { //Create a data set using sample Class and struct var expected = new SensorData { Value = new byte[] { 1, 2, 3, 4, 5 }, Position = new Location { Room = 243, Floor = 1 } }; //Serialize the data to the specified stream avroSerializer.Serialize(buffer, expected); Console.WriteLine("Deserializing Sample Data Set..."); //Prepare the stream for deserializing the data buffer.Seek(0, SeekOrigin.Begin); //Deserialize data from the stream and cast it to the same type used for serialization var actual = avroSerializer.Deserialize(buffer); Console.WriteLine("Comparing Initial and Deserialized Data Sets..."); //Finally, verify that deserialized data matches the original one bool isEqual = this.Equal(expected, actual); Console.WriteLine("Result of Data Set Identity Comparison is {0}", isEqual); } }
private void buttonAvro_Click(object sender, EventArgs e) { Run(() => { var records = new[] { new AvroRecord <string, Person> { PartitionId = GetPartitionId(), Value = GetPerson() }, new AvroRecord <string, Person> { Value = GetPerson() } }; var recordSet = new AvroRecordSet <string, Person>(records) { ValueSchema = AvroSerializer.Create <Person>().ReaderSchema.ToString() }; return(_confluentClient.PublishAsAvroAsync(textBoxTopic.Text, recordSet).Result); }); }