public void Component_PrivatePropertyIsMarkedAsDataMember_ItIsSerialized()
{
//Arrange
long expectedValue = 2137;
AttributeClass toSerialize = _fixture.Create <AttributeClass>();
typeof(AttributeClass)
.GetFields(BindingFlags.NonPublic | BindingFlags.Instance).FirstOrDefault(f => f.Name.Contains("privateProperty"))?
.SetValue(toSerialize, expectedValue);
//Act
var result = AvroConvert.Serialize(toSerialize);
var deserialized = AvroConvert.Deserialize <AttributeClass>(result);
//Assert
Assert.NotNull(result);
Assert.NotNull(deserialized);
var resultValue = typeof(AttributeClass)
.GetFields(BindingFlags.NonPublic | BindingFlags.Instance).FirstOrDefault(f => f.Name.Contains("privateProperty"))?
.GetValue(deserialized);
Assert.Equal(expectedValue, resultValue);
}
public void Serialize_InputIsList_NoExceptionIsThrown()
{
//Arrange
SomeTestClass someTestClass = new SomeTestClass
{
objectProperty = new NestedTestClass
{
justSomeProperty = "spoko",
andLongProperty = 2137
},
simpleProperty = 111111
};
SomeTestClass someTestClass2 = new SomeTestClass
{
objectProperty = new NestedTestClass
{
justSomeProperty = "loko",
andLongProperty = 2137
},
simpleProperty = 2135
};
List <SomeTestClass> someTestClasses = new List <SomeTestClass>();
someTestClasses.Add(someTestClass);
someTestClasses.Add(someTestClass2);
//Act
var result = AvroConvert.Serialize(someTestClasses);
//Assert
Assert.NotNull(result);
}
public void Serialize_InputIsArray_NoExceptionIsThrown()
{
//Arrange
SomeTestClass someTestClass = new SomeTestClass
{
objectProperty = new NestedTestClass
{
justSomeProperty = "spoko",
andLongProperty = 2137
},
simpleProperty = 111111
};
SomeTestClass dupa2 = new SomeTestClass
{
objectProperty = new NestedTestClass
{
justSomeProperty = "loko",
andLongProperty = 2137
},
simpleProperty = 2135
};
SomeTestClass[] someTestClasses = new SomeTestClass[2];
someTestClasses[0] = someTestClass;
someTestClasses[1] = dupa2;
//Act
var result = AvroConvert.Serialize(someTestClasses);
//Assert
Assert.NotNull(result);
}
public void AvroProcess(AvroMessage avroMessage)
{
try
{
//mapIncomeData(avroMessage);
mapIncomeData();
logMappedIncomeData(AvroMsg);
byte[] avroObject = AvroConvert.Serialize(AvroMsg);
Console.WriteLine("avroObject", avroObject);
//sendIncomeDataToKafka(AvroMsg);
sendIncomeDataToKafka(avroObject);
logEndProcess(result);
consumirMensagem();
}
catch (Exception ex)
{
Console.WriteLine("ERR AVRO: " + ex.Message);
}
}
public void Component_AvroAttributeClass_ResultIsEqualToInput()
{
//Arrange
AttributeClass toSerialize = new AttributeClass
{
AndAnotherString = "anotherString",
NullableIntProperty = 1,
NullableIntPropertyWithDefaultValue = null,
NullableStringProperty = "nullableString"
};
string schema = AvroConvert.GenerateSchema(typeof(AttributeClass));
//Act
var result = AvroConvert.SerializeHeadless(toSerialize, schema);
var deserialized = AvroConvert.DeserializeHeadless <AttributeClass>(result, schema);
//Assert
Assert.NotNull(result);
Assert.NotNull(deserialized);
Assert.Equal(toSerialize.NullableIntProperty, deserialized.NullableIntProperty);
Assert.Equal(toSerialize.AndAnotherString, deserialized.AndAnotherString);
Assert.Equal(toSerialize.NullableStringProperty, deserialized.NullableStringProperty);
Assert.Equal(2137, deserialized.NullableIntPropertyWithDefaultValue);
}
public virtual async Task <T> DeserializeAsync(ReadOnlyMemory <byte> data, bool isNull, SerializationContext context)
{
using (var stream = new MemoryStream(data.ToArray(), false))
{
//Confluent Kafka format:
//https://docs.confluent.io/current/schema-registry/docs/serializer-formatter.html#wire-format
if (stream.ReadByte() != 0x00)
{
throw new InvalidDataException("Invalid Confluent Kafka data format");
}
var bytes = new byte[4];
stream.Read(bytes, 0, bytes.Length);
var id = BitConverter.ToInt32(bytes, 0);
string schema;
if (cache.ContainsKey(id))
{
schema = cache[id];
}
else
{
schema = await RegistryClient.GetSchemaAsync(id);
cache.AddOrUpdate(id, schema, (key, oldValue) => schema);
}
var confluentSchema = new ConfluentSchema(schema);
var result = AvroConvert.DeserializeHeadless <T>(stream.ToArray(), confluentSchema.SchemaString);
return(result);
}
}
private static async Task <TResponse> SendAvroAsync <TContent, TResponse>(this HttpClient httpClient, string url, HttpMethod httpMethod, TContent content = null) where TContent : class
{
HttpRequestMessage httpRequest = new HttpRequestMessage(httpMethod, url);
if (content != null)
{
var avroContent = AvroConvert.Serialize(content);
HttpContent httpContent = new ByteArrayContent(avroContent);
httpContent.Headers.Remove("Content-Type");
httpContent.Headers.ContentType = new MediaTypeHeaderValue("application/avro");
httpRequest.Content = httpContent;
}
var response = await httpClient.SendAsync(httpRequest);
if (response.IsSuccessStatusCode == false)
{
HandleErrors();
}
var responseContent = await response.Content.ReadAsByteArrayAsync();
return(AvroConvert.Deserialize <TResponse>(responseContent));
}
public void SerializeList_ThenDeserialize_ListsAreEqual()
{
//Arrange
SomeTestClass someTestClass = new SomeTestClass
{
objectProperty = new NestedTestClass
{
justSomeProperty = "spoko",
andLongProperty = 2137
},
simpleProperty = 111111
};
SomeTestClass dupa2 = new SomeTestClass
{
objectProperty = new NestedTestClass
{
justSomeProperty = "loko",
andLongProperty = 2137
},
simpleProperty = 2135
};
SomeTestClass[] someTestClasses = new SomeTestClass[2];
someTestClasses[0] = someTestClass;
someTestClasses[1] = dupa2;
//Act
var serialized = AvroConvert.Serialize(someTestClasses);
var deserialized = AvroConvert.Deserialize <List <SomeTestClass> >(serialized);
//Assert
Assert.NotNull(deserialized);
}
public void Serialize_ByteArray_ResultIsTheSameAsInput()
{
//Arrange
var testObject = new byte[]
{
0,
1,
2,
3,
4,
5,
6,
7,
8,
9
};
//Act
var result = AvroConvert.Serialize(testObject);
var deserialized = AvroConvert.Deserialize <byte[]>(result);
//Assert
Assert.NotNull(result);
Assert.NotNull(deserialized);
Assert.Equal(testObject, deserialized);
}
public void Component_ObjectIsHashSet_ResultIsTheSameAsInput()
{
//Arrange
HashSet <int> hashset = _fixture.Create <HashSet <int> >();
//Act
var serialized = AvroConvert.Serialize(hashset);
var result = new List <int>();
using (var reader = AvroConvert.OpenDeserializer <int>(new MemoryStream(serialized)))
{
while (reader.HasNext())
{
var item = reader.ReadNext();
result.Add(item);
}
}
//Assert
Assert.NotNull(result);
Assert.NotNull(serialized);
Assert.Equal(hashset, result);
}
public void DeserializeByLine_ObjectIsList_ResultIsTheSameAsInput()
{
//Arrange
List <int> list = _fixture.Create <List <int> >();
//Act
var serialized = AvroConvert.Serialize(list);
var result = new List <int>();
using (var reader = AvroConvert.OpenDeserializer <int>(new MemoryStream(serialized)))
{
while (reader.HasNext())
{
var item = reader.ReadNext();
result.Add(item);
}
}
//Assert
Assert.NotNull(result);
Assert.NotNull(serialized);
Assert.Equal(list, result);
}
public void Deserialize_ValidBytes_SetOfPropertiesAreReturned()
{
//Arrange
Dictionary <string, object> result1 = new Dictionary <string, object>();
result1.Add("name", "Alyssa");
result1.Add("favorite_number", 256);
result1.Add("favorite_color", null);
Dictionary <string, object> result2 = new Dictionary <string, object>();
result2.Add("name", "Ben");
result2.Add("favorite_number", 7);
result2.Add("favorite_color", "red");
List <object> expectedResult = new List <object>();
expectedResult.Add(result1);
expectedResult.Add(result2);
//Act
var result = AvroConvert.Deserialize(_avroBytes);
//Assert
Assert.Equal(expectedResult, result);
}
public void DeserializeByLine_ClassWithList_ListsAreEqual()
{
//Arrange
var someTestClasses = _fixture.CreateMany <ClassWithSimpleList>().ToList();
//Act
var serialized = AvroConvert.Serialize(someTestClasses);
var result = new List <ClassWithSimpleList>();
using (var reader = AvroConvert.OpenDeserializer <ClassWithSimpleList>(new MemoryStream(serialized)))
{
while (reader.HasNext())
{
var item = reader.ReadNext();
result.Add(item);
}
}
//Assert
Assert.NotNull(result);
Assert.Equal(someTestClasses, result);
}
public void Deserialize_SingleItem_ItIsCorrectlyDeserialized()
{
//Arrange
var expectedResult = 1;
var serialized = AvroConvert.Serialize(expectedResult);
//Act
var result = new List <int>();
using (var reader = AvroConvert.OpenDeserializer <int>(new MemoryStream(serialized)))
{
while (reader.HasNext())
{
var item = reader.ReadNext();
result.Add(item);
}
}
//Assert
result.Should().HaveCount(1);
result.Should().ContainEquivalentOf(1);
}
public void DeserializeByLine_FileContainsBlocksOfArrays_ResultCountIsAsExpected()
{
//Arrange
var result = new List <kylosample>();
//Act
using (var stream = File.OpenRead("userdata1.avro"))
{
using (var reader = AvroConvert.OpenDeserializer <kylosample>(stream))
{
while (reader.HasNext())
{
var item = reader.ReadNext();
result.Add(item);
}
}
}
//Assert
result.Should().HaveCount(1000);
}
public void Component_HeadlessList_ResultIsTheSameAsInput()
{
//Arrange
List <int> list = _fixture.Create <List <int> >();
var schema = AvroConvert.GenerateSchema(typeof(List <int>));
//Act
var serialized = AvroConvert.SerializeHeadless(list, schema);
var result = new List <int>();
using (var reader = AvroConvert.OpenDeserializer <int>(new MemoryStream(serialized)))
{
while (reader.HasNext())
{
var item = reader.ReadNext();
result.Add(item);
}
}
//Assert
Assert.NotNull(result);
Assert.NotNull(serialized);
Assert.Equal(list, result);
}
public static async Task <HttpResponseMessage> PostAsAvro(this HttpClient httpClient, string requestUri, object content)
{
var body = new ByteArrayContent(AvroConvert.Serialize(content));
body.Headers.ContentType = new MediaTypeHeaderValue(Consts.AvroHeader);
return(await httpClient.PostAsync(requestUri, body));
}
public void Deserialize_CustomSchema_OnlyValuesFromCustomSchemaAreReturned()
{
//Arrange
string customSchema = AvroConvert.GenerateSchema(typeof(UserNameClass));
Dictionary <string, object> result1 = new Dictionary <string, object>();
result1.Add("name", "Alyssa");
Dictionary <string, object> result2 = new Dictionary <string, object>();
result2.Add("name", "Ben");
List <object> expectedResult = new List <object>();
expectedResult.Add(result1);
expectedResult.Add(result2);
//Act
var result = AvroConvert.Deserialize(_avroBytes, customSchema);
//Assert
Assert.Equal(expectedResult, result);
}
public void Deserialize_SnappyCodedFile_DataIsDeserialized()
{
//Arrange
List <TwitterModel> expected = new List <TwitterModel>();
expected.Add(new TwitterModel
{
Timestamp = 1366150681,
Tweet = "Rock: Nerf paper, scissors is fine.",
Username = "******"
});
expected.Add(new TwitterModel
{
Timestamp = 1366154481,
Tweet = "Works as intended. Terran is IMBA.",
Username = "******"
});
//Act
var result = AvroConvert.Deserialize <List <TwitterModel> >(_snappy);
//Assert
Assert.Equal(expected, result);
}
public void Merge_MultipleObject_AllOfThemAreMerged()
{
//Arrange
var users = _fixture.CreateMany <User>();
var avroObjects = users.Select(AvroConvert.Serialize);
//Act
var result = AvroConvert.Merge <User>(avroObjects);
//Assert
var deserializedResult = AvroConvert.Deserialize <List <User> >(result);
Assert.NotNull(deserializedResult);
Assert.Equal(users.Count(), deserializedResult.Count);
foreach (var user in users)
{
var resultUser = deserializedResult.FirstOrDefault(r => r.name == user.name);
Assert.NotNull(resultUser);
Assert.Equal(user, resultUser);
}
}
public void GenerateClass_NestedClass_OutputIsEqualToExpected()
{
//Arrange
var schema = AvroConvert.GenerateSchema(typeof(BaseTestClass));
//Act
string resultSchema = AvroConvert.GenerateModel(schema);
//Assert
Assert.Equal(
"public class BaseTestClass\r\n" +
"{\r\n" +
"\tpublic string justSomeProperty { get; set; }\r\n" +
"\tpublic long andLongProperty { get; set; }\r\n" +
"\tpublic User objectProperty { get; set; }\r\n" +
"}\r\n" +
"\r\n" +
"public class User\r\n" +
"{\r\n" +
"\tpublic string name { get; set; }\r\n" +
"\tpublic int? favorite_number { get; set; }\r\n" +
"\tpublic string favorite_color { get; set; }\r\n" +
"}\r\n" +
"\r\n",
resultSchema);
}
public void GenerateClass_ClassWithEnum_GeneratedModelIsAsExpected()
{
//Arrange
var schema = AvroConvert.GenerateSchema(typeof(ClassWithEnum));
//Act
string resultSchema = AvroConvert.GenerateModel(schema);
//Assert
Assert.Equal(
"public class ClassWithEnum\r\n" +
"{\r\n" +
"\tpublic TestEnum? EnumProp { get; set; }\r\n" +
"}\r\n" +
"\r\n" +
"public enum TestEnum\r\n" +
"{\r\n" +
"\ta,\r\n" +
"\tbe,\r\n" +
"\tca,\r\n" +
"\tdlo\r\n" +
"}\r\n" +
"\r\n",
resultSchema);
}
public void GenerateClass_SchemaContainsMap_ItIsTranslatedToDictionary()
{
//Arrange
string schema = @"
{
""type"": ""record"",
""name"": ""exampleAvro"",
""fields"": [
{
""name"": ""mapdata"",
""type"": ""map"",
""values"": ""int""
}
]
}";
//Act
string resultClass = AvroConvert.GenerateModel(schema);
//Assert
Assert.Equal(
"public class exampleAvro\r\n" +
"{\r\n" +
"\tpublic Dictionary<string,int> mapdata { get; set; }\r\n" +
"}\r\n" +
"\r\n",
resultClass);
}
public void GenerateClass_SchemaContainsBytes_ItIsTranslatedToByteArray()
{
//Arrange
string schema = @"
{
""type"": ""record"",
""name"": ""exampleAvro"",
""fields"": [
{
""name"": ""bdata"",
""type"": ""bytes""
}
]
}";
//Act
string resultClass = AvroConvert.GenerateModel(schema);
//Assert
Assert.Equal(
"public class exampleAvro\r\n" +
"{\r\n" +
"\tpublic byte[] bdata { get; set; }\r\n" +
"}\r\n" +
"\r\n",
resultClass);
}
public void Deserialize_CustomSchema_OnlyValuesFromCustomSchemaAreReturned()
{
//Arrange
var expectedResult = new List <User>();
expectedResult.Add(new User
{
name = "Alyssa",
favorite_number = 256,
favorite_color = null
});
expectedResult.Add(new User
{
name = "Ben",
favorite_number = 7,
favorite_color = "red"
});
//Act
var result = AvroConvert.Deserialize <List <User> >(_avroBytes);
//Assert
Assert.Equal(expectedResult, result);
}
public void Serialize_MultiThreadSerialization_NoExceptionIsThrown()
{
//Arrange
VeryComplexClass testClass = _fixture.Create <VeryComplexClass>();
//Act
List <Task> listOfTasks = new List <Task>();
for (var counter = 0; counter < 10; counter++)
{
listOfTasks.Add(Task.Run(() =>
{
for (var iMessagesCntr = 0; iMessagesCntr < 100; iMessagesCntr++)
{
var result = AvroConvert.Serialize(testClass);
var deserialized = AvroConvert.Deserialize <VeryComplexClass>(result);
//Assert
Assert.Equal(testClass, deserialized);
Thread.Sleep(counter
);
}
}));
}
Task.WaitAll(listOfTasks.ToArray());
}
public static async Task <T> GetAsAvro <T>(this HttpClient httpClient, string requestUri)
{
var response = await httpClient.GetByteArrayAsync(requestUri);
T result = AvroConvert.Deserialize <T>(response);
return(result);
}
static void Main(string[] args)
{
IntegrationTest.Invoke();
var fixture = new Fixture();
var data = fixture.CreateMany <Dataset>(30000).ToArray();
Console.WriteLine("AvroConvert Benchmark - fire!");
Console.WriteLine("");
Console.WriteLine("The Benchmark compares Apache.Avro, AvroConvert (nuget version) and AvroConvert local version");
int noOfRuns = 30;
Console.WriteLine($"Number of runs: {noOfRuns}");
string schema = AvroConvert.GenerateSchema(typeof(Dataset[]));
//Act
List <BenchmarkResult> benchmarkResults = new List <BenchmarkResult>();
for (int i = 0; i < noOfRuns; i++)
{
benchmarkResults.Add(RunBenchmark(data, schema));
if ((i + 1) % 10 == 0)
{
Console.WriteLine($"Progress: {i + 1}/{noOfRuns}");
}
}
BenchmarkResult mean = new BenchmarkResult();
mean.AvroConvertVNextGzipSerializeTime = benchmarkResults.Sum(r => r.AvroConvertVNextGzipSerializeTime) / noOfRuns;
mean.AvroConvertVNextGzipDeserializeTime = benchmarkResults.Sum(r => r.AvroConvertVNextGzipDeserializeTime) / noOfRuns;
mean.AvroConvertVNextGzipSize = benchmarkResults.Sum(r => r.AvroConvertVNextGzipSize) / noOfRuns;
mean.ApacheAvroSerializeTime = benchmarkResults.Sum(r => r.ApacheAvroSerializeTime) / noOfRuns;
mean.ApacheAvroDeserializeTime = benchmarkResults.Sum(r => r.ApacheAvroDeserializeTime) / noOfRuns;
mean.ApacheAvroSize = benchmarkResults.Sum(r => r.ApacheAvroSize) / noOfRuns;
mean.AvroConvertHeadlessSerializeTime = benchmarkResults.Sum(r => r.AvroConvertHeadlessSerializeTime) / noOfRuns;
mean.AvroConvertHeadlessDeserializeTime = benchmarkResults.Sum(r => r.AvroConvertHeadlessDeserializeTime) / noOfRuns;
mean.AvroConvertHeadlessSize = benchmarkResults.Sum(r => r.AvroConvertHeadlessSize) / noOfRuns;
mean.AvroConvertGzipSerializeTime = benchmarkResults.Sum(r => r.AvroConvertGzipSerializeTime) / noOfRuns;
mean.AvroConvertGzipDeserializeTime = benchmarkResults.Sum(r => r.AvroConvertGzipDeserializeTime) / noOfRuns;
mean.AvroConvertGzipSize = benchmarkResults.Sum(r => r.AvroConvertGzipSize) / noOfRuns;
mean.AvroConvertVNextHeadlessSerializeTime = benchmarkResults.Sum(r => r.AvroConvertVNextHeadlessSerializeTime) / noOfRuns;
mean.AvroConvertVNextHeadlessDeserializeTime = benchmarkResults.Sum(r => r.AvroConvertVNextHeadlessDeserializeTime) / noOfRuns;
mean.AvroConvertVNextSize = benchmarkResults.Sum(r => r.AvroConvertVNextSize) / noOfRuns;
ConstructLog(mean);
Console.ReadLine();
}
public void GenerateClass_TheyAreGeneratedWithDefaults()
{
//Arrange
string schema = @"
{
""type"": ""record"",
""name"": ""Result"",
""fields"": [
{
""name"": ""testString"",
""type"": ""string"",
""default"": ""Default String""
},
{
""name"": ""testBoolean"",
""type"": ""boolean"",
""default"": ""TRUE""
},
{
""name"": ""testInt"",
""type"": ""int"",
""default"": 123
},
{
""name"": ""testLong"",
""type"": ""long"",
""default"": 123
},
{
""name"": ""testFloat"",
""type"": ""float"",
""default"": 1.23
},
{
""name"": ""testDouble"",
""type"": ""double"",
""default"": 1.23
}
]
}";
//Act
string resultClass = AvroConvert.GenerateModel(schema);
//Assert
Assert.Equal(
"public class Result\r\n" +
"{\r\n" +
"\tpublic string testString { get; set; } = \"Default String\";\r\n" +
"\tpublic bool testBoolean { get; set; } = true;\r\n" +
"\tpublic int testInt { get; set; } = 123;\r\n" +
"\tpublic long testLong { get; set; } = 123;\r\n" +
"\tpublic float testFloat { get; set; } = 1.23;\r\n" +
"\tpublic double testDouble { get; set; } = 1.23;\r\n" +
"}\r\n" +
"\r\n",
resultClass);
}
public void GenerateClass_TheyAreGeneratedWithDocumentation()
{
//Arrange
string schema = @"
{
""type"": ""record"",
""name"": ""Result"",
""fields"": [
{
""name"": ""testString"",
""type"": ""string"",
""doc"": ""This is a doc field""
},
{
""name"": ""testBoolean"",
""type"": ""boolean""
},
{
""name"": ""testInt"",
""type"": ""int""
},
{
""name"": ""testLong"",
""type"": ""long""
},
{
""name"": ""testFloat"",
""type"": ""float""
},
{
""name"": ""testDouble"",
""type"": ""double""
}
]
}";
//Act
string resultClass = AvroConvert.GenerateModel(schema);
//Assert
Assert.Equal(
"public class Result\r\n" +
"{\r\n" +
"\t/// <summary>\r\n" +
"\t/// This is a doc field\r\n" +
"\t/// </summary>\r\n" +
"\tpublic string testString { get; set; }\r\n" +
"\tpublic bool testBoolean { get; set; }\r\n" +
"\tpublic int testInt { get; set; }\r\n" +
"\tpublic long testLong { get; set; }\r\n" +
"\tpublic float testFloat { get; set; }\r\n" +
"\tpublic double testDouble { get; set; }\r\n" +
"}\r\n" +
"\r\n",
resultClass);
}
