public void Component_PrivatePropertyIsMarkedAsDataMember_ItIsSerialized() { //Arrange long expectedValue = 2137; AttributeClass toSerialize = _fixture.Create <AttributeClass>(); typeof(AttributeClass) .GetFields(BindingFlags.NonPublic | BindingFlags.Instance).FirstOrDefault(f => f.Name.Contains("privateProperty"))? .SetValue(toSerialize, expectedValue); //Act var result = AvroConvert.Serialize(toSerialize); var deserialized = AvroConvert.Deserialize <AttributeClass>(result); //Assert Assert.NotNull(result); Assert.NotNull(deserialized); var resultValue = typeof(AttributeClass) .GetFields(BindingFlags.NonPublic | BindingFlags.Instance).FirstOrDefault(f => f.Name.Contains("privateProperty"))? .GetValue(deserialized); Assert.Equal(expectedValue, resultValue); }
public void Serialize_InputIsList_NoExceptionIsThrown() { //Arrange SomeTestClass someTestClass = new SomeTestClass { objectProperty = new NestedTestClass { justSomeProperty = "spoko", andLongProperty = 2137 }, simpleProperty = 111111 }; SomeTestClass someTestClass2 = new SomeTestClass { objectProperty = new NestedTestClass { justSomeProperty = "loko", andLongProperty = 2137 }, simpleProperty = 2135 }; List <SomeTestClass> someTestClasses = new List <SomeTestClass>(); someTestClasses.Add(someTestClass); someTestClasses.Add(someTestClass2); //Act var result = AvroConvert.Serialize(someTestClasses); //Assert Assert.NotNull(result); }
public void Serialize_InputIsArray_NoExceptionIsThrown() { //Arrange SomeTestClass someTestClass = new SomeTestClass { objectProperty = new NestedTestClass { justSomeProperty = "spoko", andLongProperty = 2137 }, simpleProperty = 111111 }; SomeTestClass dupa2 = new SomeTestClass { objectProperty = new NestedTestClass { justSomeProperty = "loko", andLongProperty = 2137 }, simpleProperty = 2135 }; SomeTestClass[] someTestClasses = new SomeTestClass[2]; someTestClasses[0] = someTestClass; someTestClasses[1] = dupa2; //Act var result = AvroConvert.Serialize(someTestClasses); //Assert Assert.NotNull(result); }
public void AvroProcess(AvroMessage avroMessage) { try { //mapIncomeData(avroMessage); mapIncomeData(); logMappedIncomeData(AvroMsg); byte[] avroObject = AvroConvert.Serialize(AvroMsg); Console.WriteLine("avroObject", avroObject); //sendIncomeDataToKafka(AvroMsg); sendIncomeDataToKafka(avroObject); logEndProcess(result); consumirMensagem(); } catch (Exception ex) { Console.WriteLine("ERR AVRO: " + ex.Message); } }
public void Component_AvroAttributeClass_ResultIsEqualToInput() { //Arrange AttributeClass toSerialize = new AttributeClass { AndAnotherString = "anotherString", NullableIntProperty = 1, NullableIntPropertyWithDefaultValue = null, NullableStringProperty = "nullableString" }; string schema = AvroConvert.GenerateSchema(typeof(AttributeClass)); //Act var result = AvroConvert.SerializeHeadless(toSerialize, schema); var deserialized = AvroConvert.DeserializeHeadless <AttributeClass>(result, schema); //Assert Assert.NotNull(result); Assert.NotNull(deserialized); Assert.Equal(toSerialize.NullableIntProperty, deserialized.NullableIntProperty); Assert.Equal(toSerialize.AndAnotherString, deserialized.AndAnotherString); Assert.Equal(toSerialize.NullableStringProperty, deserialized.NullableStringProperty); Assert.Equal(2137, deserialized.NullableIntPropertyWithDefaultValue); }
public virtual async Task <T> DeserializeAsync(ReadOnlyMemory <byte> data, bool isNull, SerializationContext context) { using (var stream = new MemoryStream(data.ToArray(), false)) { //Confluent Kafka format: //https://docs.confluent.io/current/schema-registry/docs/serializer-formatter.html#wire-format if (stream.ReadByte() != 0x00) { throw new InvalidDataException("Invalid Confluent Kafka data format"); } var bytes = new byte[4]; stream.Read(bytes, 0, bytes.Length); var id = BitConverter.ToInt32(bytes, 0); string schema; if (cache.ContainsKey(id)) { schema = cache[id]; } else { schema = await RegistryClient.GetSchemaAsync(id); cache.AddOrUpdate(id, schema, (key, oldValue) => schema); } var confluentSchema = new ConfluentSchema(schema); var result = AvroConvert.DeserializeHeadless <T>(stream.ToArray(), confluentSchema.SchemaString); return(result); } }
private static async Task <TResponse> SendAvroAsync <TContent, TResponse>(this HttpClient httpClient, string url, HttpMethod httpMethod, TContent content = null) where TContent : class { HttpRequestMessage httpRequest = new HttpRequestMessage(httpMethod, url); if (content != null) { var avroContent = AvroConvert.Serialize(content); HttpContent httpContent = new ByteArrayContent(avroContent); httpContent.Headers.Remove("Content-Type"); httpContent.Headers.ContentType = new MediaTypeHeaderValue("application/avro"); httpRequest.Content = httpContent; } var response = await httpClient.SendAsync(httpRequest); if (response.IsSuccessStatusCode == false) { HandleErrors(); } var responseContent = await response.Content.ReadAsByteArrayAsync(); return(AvroConvert.Deserialize <TResponse>(responseContent)); }
public void SerializeList_ThenDeserialize_ListsAreEqual() { //Arrange SomeTestClass someTestClass = new SomeTestClass { objectProperty = new NestedTestClass { justSomeProperty = "spoko", andLongProperty = 2137 }, simpleProperty = 111111 }; SomeTestClass dupa2 = new SomeTestClass { objectProperty = new NestedTestClass { justSomeProperty = "loko", andLongProperty = 2137 }, simpleProperty = 2135 }; SomeTestClass[] someTestClasses = new SomeTestClass[2]; someTestClasses[0] = someTestClass; someTestClasses[1] = dupa2; //Act var serialized = AvroConvert.Serialize(someTestClasses); var deserialized = AvroConvert.Deserialize <List <SomeTestClass> >(serialized); //Assert Assert.NotNull(deserialized); }
public void Serialize_ByteArray_ResultIsTheSameAsInput() { //Arrange var testObject = new byte[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }; //Act var result = AvroConvert.Serialize(testObject); var deserialized = AvroConvert.Deserialize <byte[]>(result); //Assert Assert.NotNull(result); Assert.NotNull(deserialized); Assert.Equal(testObject, deserialized); }
public void Component_ObjectIsHashSet_ResultIsTheSameAsInput() { //Arrange HashSet <int> hashset = _fixture.Create <HashSet <int> >(); //Act var serialized = AvroConvert.Serialize(hashset); var result = new List <int>(); using (var reader = AvroConvert.OpenDeserializer <int>(new MemoryStream(serialized))) { while (reader.HasNext()) { var item = reader.ReadNext(); result.Add(item); } } //Assert Assert.NotNull(result); Assert.NotNull(serialized); Assert.Equal(hashset, result); }
public void DeserializeByLine_ObjectIsList_ResultIsTheSameAsInput() { //Arrange List <int> list = _fixture.Create <List <int> >(); //Act var serialized = AvroConvert.Serialize(list); var result = new List <int>(); using (var reader = AvroConvert.OpenDeserializer <int>(new MemoryStream(serialized))) { while (reader.HasNext()) { var item = reader.ReadNext(); result.Add(item); } } //Assert Assert.NotNull(result); Assert.NotNull(serialized); Assert.Equal(list, result); }
public void Deserialize_ValidBytes_SetOfPropertiesAreReturned() { //Arrange Dictionary <string, object> result1 = new Dictionary <string, object>(); result1.Add("name", "Alyssa"); result1.Add("favorite_number", 256); result1.Add("favorite_color", null); Dictionary <string, object> result2 = new Dictionary <string, object>(); result2.Add("name", "Ben"); result2.Add("favorite_number", 7); result2.Add("favorite_color", "red"); List <object> expectedResult = new List <object>(); expectedResult.Add(result1); expectedResult.Add(result2); //Act var result = AvroConvert.Deserialize(_avroBytes); //Assert Assert.Equal(expectedResult, result); }
public void DeserializeByLine_ClassWithList_ListsAreEqual() { //Arrange var someTestClasses = _fixture.CreateMany <ClassWithSimpleList>().ToList(); //Act var serialized = AvroConvert.Serialize(someTestClasses); var result = new List <ClassWithSimpleList>(); using (var reader = AvroConvert.OpenDeserializer <ClassWithSimpleList>(new MemoryStream(serialized))) { while (reader.HasNext()) { var item = reader.ReadNext(); result.Add(item); } } //Assert Assert.NotNull(result); Assert.Equal(someTestClasses, result); }
public void Deserialize_SingleItem_ItIsCorrectlyDeserialized() { //Arrange var expectedResult = 1; var serialized = AvroConvert.Serialize(expectedResult); //Act var result = new List <int>(); using (var reader = AvroConvert.OpenDeserializer <int>(new MemoryStream(serialized))) { while (reader.HasNext()) { var item = reader.ReadNext(); result.Add(item); } } //Assert result.Should().HaveCount(1); result.Should().ContainEquivalentOf(1); }
public void DeserializeByLine_FileContainsBlocksOfArrays_ResultCountIsAsExpected() { //Arrange var result = new List <kylosample>(); //Act using (var stream = File.OpenRead("userdata1.avro")) { using (var reader = AvroConvert.OpenDeserializer <kylosample>(stream)) { while (reader.HasNext()) { var item = reader.ReadNext(); result.Add(item); } } } //Assert result.Should().HaveCount(1000); }
public void Component_HeadlessList_ResultIsTheSameAsInput() { //Arrange List <int> list = _fixture.Create <List <int> >(); var schema = AvroConvert.GenerateSchema(typeof(List <int>)); //Act var serialized = AvroConvert.SerializeHeadless(list, schema); var result = new List <int>(); using (var reader = AvroConvert.OpenDeserializer <int>(new MemoryStream(serialized))) { while (reader.HasNext()) { var item = reader.ReadNext(); result.Add(item); } } //Assert Assert.NotNull(result); Assert.NotNull(serialized); Assert.Equal(list, result); }
public static async Task <HttpResponseMessage> PostAsAvro(this HttpClient httpClient, string requestUri, object content) { var body = new ByteArrayContent(AvroConvert.Serialize(content)); body.Headers.ContentType = new MediaTypeHeaderValue(Consts.AvroHeader); return(await httpClient.PostAsync(requestUri, body)); }
public void Deserialize_CustomSchema_OnlyValuesFromCustomSchemaAreReturned() { //Arrange string customSchema = AvroConvert.GenerateSchema(typeof(UserNameClass)); Dictionary <string, object> result1 = new Dictionary <string, object>(); result1.Add("name", "Alyssa"); Dictionary <string, object> result2 = new Dictionary <string, object>(); result2.Add("name", "Ben"); List <object> expectedResult = new List <object>(); expectedResult.Add(result1); expectedResult.Add(result2); //Act var result = AvroConvert.Deserialize(_avroBytes, customSchema); //Assert Assert.Equal(expectedResult, result); }
public void Deserialize_SnappyCodedFile_DataIsDeserialized() { //Arrange List <TwitterModel> expected = new List <TwitterModel>(); expected.Add(new TwitterModel { Timestamp = 1366150681, Tweet = "Rock: Nerf paper, scissors is fine.", Username = "******" }); expected.Add(new TwitterModel { Timestamp = 1366154481, Tweet = "Works as intended. Terran is IMBA.", Username = "******" }); //Act var result = AvroConvert.Deserialize <List <TwitterModel> >(_snappy); //Assert Assert.Equal(expected, result); }
public void Merge_MultipleObject_AllOfThemAreMerged() { //Arrange var users = _fixture.CreateMany <User>(); var avroObjects = users.Select(AvroConvert.Serialize); //Act var result = AvroConvert.Merge <User>(avroObjects); //Assert var deserializedResult = AvroConvert.Deserialize <List <User> >(result); Assert.NotNull(deserializedResult); Assert.Equal(users.Count(), deserializedResult.Count); foreach (var user in users) { var resultUser = deserializedResult.FirstOrDefault(r => r.name == user.name); Assert.NotNull(resultUser); Assert.Equal(user, resultUser); } }
public void GenerateClass_NestedClass_OutputIsEqualToExpected() { //Arrange var schema = AvroConvert.GenerateSchema(typeof(BaseTestClass)); //Act string resultSchema = AvroConvert.GenerateModel(schema); //Assert Assert.Equal( "public class BaseTestClass\r\n" + "{\r\n" + "\tpublic string justSomeProperty { get; set; }\r\n" + "\tpublic long andLongProperty { get; set; }\r\n" + "\tpublic User objectProperty { get; set; }\r\n" + "}\r\n" + "\r\n" + "public class User\r\n" + "{\r\n" + "\tpublic string name { get; set; }\r\n" + "\tpublic int? favorite_number { get; set; }\r\n" + "\tpublic string favorite_color { get; set; }\r\n" + "}\r\n" + "\r\n", resultSchema); }
public void GenerateClass_ClassWithEnum_GeneratedModelIsAsExpected() { //Arrange var schema = AvroConvert.GenerateSchema(typeof(ClassWithEnum)); //Act string resultSchema = AvroConvert.GenerateModel(schema); //Assert Assert.Equal( "public class ClassWithEnum\r\n" + "{\r\n" + "\tpublic TestEnum? EnumProp { get; set; }\r\n" + "}\r\n" + "\r\n" + "public enum TestEnum\r\n" + "{\r\n" + "\ta,\r\n" + "\tbe,\r\n" + "\tca,\r\n" + "\tdlo\r\n" + "}\r\n" + "\r\n", resultSchema); }
public void GenerateClass_SchemaContainsMap_ItIsTranslatedToDictionary() { //Arrange string schema = @" { ""type"": ""record"", ""name"": ""exampleAvro"", ""fields"": [ { ""name"": ""mapdata"", ""type"": ""map"", ""values"": ""int"" } ] }"; //Act string resultClass = AvroConvert.GenerateModel(schema); //Assert Assert.Equal( "public class exampleAvro\r\n" + "{\r\n" + "\tpublic Dictionary<string,int> mapdata { get; set; }\r\n" + "}\r\n" + "\r\n", resultClass); }
public void GenerateClass_SchemaContainsBytes_ItIsTranslatedToByteArray() { //Arrange string schema = @" { ""type"": ""record"", ""name"": ""exampleAvro"", ""fields"": [ { ""name"": ""bdata"", ""type"": ""bytes"" } ] }"; //Act string resultClass = AvroConvert.GenerateModel(schema); //Assert Assert.Equal( "public class exampleAvro\r\n" + "{\r\n" + "\tpublic byte[] bdata { get; set; }\r\n" + "}\r\n" + "\r\n", resultClass); }
public void Deserialize_CustomSchema_OnlyValuesFromCustomSchemaAreReturned() { //Arrange var expectedResult = new List <User>(); expectedResult.Add(new User { name = "Alyssa", favorite_number = 256, favorite_color = null }); expectedResult.Add(new User { name = "Ben", favorite_number = 7, favorite_color = "red" }); //Act var result = AvroConvert.Deserialize <List <User> >(_avroBytes); //Assert Assert.Equal(expectedResult, result); }
public void Serialize_MultiThreadSerialization_NoExceptionIsThrown() { //Arrange VeryComplexClass testClass = _fixture.Create <VeryComplexClass>(); //Act List <Task> listOfTasks = new List <Task>(); for (var counter = 0; counter < 10; counter++) { listOfTasks.Add(Task.Run(() => { for (var iMessagesCntr = 0; iMessagesCntr < 100; iMessagesCntr++) { var result = AvroConvert.Serialize(testClass); var deserialized = AvroConvert.Deserialize <VeryComplexClass>(result); //Assert Assert.Equal(testClass, deserialized); Thread.Sleep(counter ); } })); } Task.WaitAll(listOfTasks.ToArray()); }
public static async Task <T> GetAsAvro <T>(this HttpClient httpClient, string requestUri) { var response = await httpClient.GetByteArrayAsync(requestUri); T result = AvroConvert.Deserialize <T>(response); return(result); }
static void Main(string[] args) { IntegrationTest.Invoke(); var fixture = new Fixture(); var data = fixture.CreateMany <Dataset>(30000).ToArray(); Console.WriteLine("AvroConvert Benchmark - fire!"); Console.WriteLine(""); Console.WriteLine("The Benchmark compares Apache.Avro, AvroConvert (nuget version) and AvroConvert local version"); int noOfRuns = 30; Console.WriteLine($"Number of runs: {noOfRuns}"); string schema = AvroConvert.GenerateSchema(typeof(Dataset[])); //Act List <BenchmarkResult> benchmarkResults = new List <BenchmarkResult>(); for (int i = 0; i < noOfRuns; i++) { benchmarkResults.Add(RunBenchmark(data, schema)); if ((i + 1) % 10 == 0) { Console.WriteLine($"Progress: {i + 1}/{noOfRuns}"); } } BenchmarkResult mean = new BenchmarkResult(); mean.AvroConvertVNextGzipSerializeTime = benchmarkResults.Sum(r => r.AvroConvertVNextGzipSerializeTime) / noOfRuns; mean.AvroConvertVNextGzipDeserializeTime = benchmarkResults.Sum(r => r.AvroConvertVNextGzipDeserializeTime) / noOfRuns; mean.AvroConvertVNextGzipSize = benchmarkResults.Sum(r => r.AvroConvertVNextGzipSize) / noOfRuns; mean.ApacheAvroSerializeTime = benchmarkResults.Sum(r => r.ApacheAvroSerializeTime) / noOfRuns; mean.ApacheAvroDeserializeTime = benchmarkResults.Sum(r => r.ApacheAvroDeserializeTime) / noOfRuns; mean.ApacheAvroSize = benchmarkResults.Sum(r => r.ApacheAvroSize) / noOfRuns; mean.AvroConvertHeadlessSerializeTime = benchmarkResults.Sum(r => r.AvroConvertHeadlessSerializeTime) / noOfRuns; mean.AvroConvertHeadlessDeserializeTime = benchmarkResults.Sum(r => r.AvroConvertHeadlessDeserializeTime) / noOfRuns; mean.AvroConvertHeadlessSize = benchmarkResults.Sum(r => r.AvroConvertHeadlessSize) / noOfRuns; mean.AvroConvertGzipSerializeTime = benchmarkResults.Sum(r => r.AvroConvertGzipSerializeTime) / noOfRuns; mean.AvroConvertGzipDeserializeTime = benchmarkResults.Sum(r => r.AvroConvertGzipDeserializeTime) / noOfRuns; mean.AvroConvertGzipSize = benchmarkResults.Sum(r => r.AvroConvertGzipSize) / noOfRuns; mean.AvroConvertVNextHeadlessSerializeTime = benchmarkResults.Sum(r => r.AvroConvertVNextHeadlessSerializeTime) / noOfRuns; mean.AvroConvertVNextHeadlessDeserializeTime = benchmarkResults.Sum(r => r.AvroConvertVNextHeadlessDeserializeTime) / noOfRuns; mean.AvroConvertVNextSize = benchmarkResults.Sum(r => r.AvroConvertVNextSize) / noOfRuns; ConstructLog(mean); Console.ReadLine(); }
public void GenerateClass_TheyAreGeneratedWithDefaults() { //Arrange string schema = @" { ""type"": ""record"", ""name"": ""Result"", ""fields"": [ { ""name"": ""testString"", ""type"": ""string"", ""default"": ""Default String"" }, { ""name"": ""testBoolean"", ""type"": ""boolean"", ""default"": ""TRUE"" }, { ""name"": ""testInt"", ""type"": ""int"", ""default"": 123 }, { ""name"": ""testLong"", ""type"": ""long"", ""default"": 123 }, { ""name"": ""testFloat"", ""type"": ""float"", ""default"": 1.23 }, { ""name"": ""testDouble"", ""type"": ""double"", ""default"": 1.23 } ] }"; //Act string resultClass = AvroConvert.GenerateModel(schema); //Assert Assert.Equal( "public class Result\r\n" + "{\r\n" + "\tpublic string testString { get; set; } = \"Default String\";\r\n" + "\tpublic bool testBoolean { get; set; } = true;\r\n" + "\tpublic int testInt { get; set; } = 123;\r\n" + "\tpublic long testLong { get; set; } = 123;\r\n" + "\tpublic float testFloat { get; set; } = 1.23;\r\n" + "\tpublic double testDouble { get; set; } = 1.23;\r\n" + "}\r\n" + "\r\n", resultClass); }
public void GenerateClass_TheyAreGeneratedWithDocumentation() { //Arrange string schema = @" { ""type"": ""record"", ""name"": ""Result"", ""fields"": [ { ""name"": ""testString"", ""type"": ""string"", ""doc"": ""This is a doc field"" }, { ""name"": ""testBoolean"", ""type"": ""boolean"" }, { ""name"": ""testInt"", ""type"": ""int"" }, { ""name"": ""testLong"", ""type"": ""long"" }, { ""name"": ""testFloat"", ""type"": ""float"" }, { ""name"": ""testDouble"", ""type"": ""double"" } ] }"; //Act string resultClass = AvroConvert.GenerateModel(schema); //Assert Assert.Equal( "public class Result\r\n" + "{\r\n" + "\t/// <summary>\r\n" + "\t/// This is a doc field\r\n" + "\t/// </summary>\r\n" + "\tpublic string testString { get; set; }\r\n" + "\tpublic bool testBoolean { get; set; }\r\n" + "\tpublic int testInt { get; set; }\r\n" + "\tpublic long testLong { get; set; }\r\n" + "\tpublic float testFloat { get; set; }\r\n" + "\tpublic double testDouble { get; set; }\r\n" + "}\r\n" + "\r\n", resultClass); }