public static EventTypeAvroHandler Resolve(
ImportService importService,
ConfigurationCommonEventTypeMeta.AvroSettingsConfig avroSettings,
string handlerClass)
{
// Make services that depend on snapshot config entries
EventTypeAvroHandler avroHandler = EventTypeAvroHandlerUnsupported.INSTANCE;
if (avroSettings.IsEnableAvro) {
try {
avroHandler = TypeHelper.Instantiate<EventTypeAvroHandler>(
handlerClass,
importService.ClassForNameProvider);
}
catch (Exception t) {
Log.Debug(
"Avro provider {} not instantiated, not enabling Avro support: {}",
handlerClass,
t.Message);
}
try {
avroHandler.Init(avroSettings, importService);
}
catch (Exception t) {
throw new ConfigurationException("Failed to initialize Esper-Avro: " + t.Message, t);
}
}
return avroHandler;
}
protected override EventTypeAvroHandler MakeEventTypeAvroHandler(
ImportServiceRuntime importServiceRuntime,
ConfigurationCommonEventTypeMeta.AvroSettingsConfig avroSettings,
RuntimeExtensionServices runtimeExt)
{
return EventTypeAvroHandlerFactory.Resolve(
importServiceRuntime, avroSettings,
EventTypeAvroHandlerConstants.RUNTIME_NONHA_HANDLER_IMPL);
}
private Schema Assemble(
object value,
Attribute[] annotations,
ConfigurationCommonEventTypeMeta.AvroSettingsConfig avroSettings,
EventTypeNameResolver eventTypeNameResolver)
{
var fields = new JArray();
AvroSchemaUtil.AssembleField("somefield", value, fields, annotations, avroSettings, eventTypeNameResolver, "stmtname", null);
var schema = SchemaBuilder.Record("myrecord", fields);
return(schema.GetField("somefield").Schema);
}
public void Init(
ConfigurationCommonEventTypeMeta.AvroSettingsConfig avroSettings,
ImportService importService)
{
_avroSettings = avroSettings;
if (avroSettings.TypeRepresentationMapperClass != null) {
_optionalTypeMapper = TypeHelper.Instantiate<TypeRepresentationMapper>(
avroSettings.TypeRepresentationMapperClass,
importService.ClassForNameProvider);
}
if (avroSettings.ObjectValueTypeWidenerFactoryClass != null) {
_optionalWidenerFactory = TypeHelper.Instantiate<ObjectValueTypeWidenerFactory>(
avroSettings.ObjectValueTypeWidenerFactoryClass,
importService.ClassForNameProvider);
}
}
private void AssertTypeArray(
Type componentType,
Schema.Type expectedElementType,
bool unionOfNull,
bool unionOfNullElements,
ConfigurationCommonEventTypeMeta.AvroSettingsConfig avroSettings)
{
var schema = Assemble(
TypeHelper.GetArrayType(componentType),
null,
avroSettings,
EVENT_ADAPTER_SERVICE);
Schema elementSchema;
if (!unionOfNull)
{
var arraySchema = schema.AsArraySchema();
Assert.AreEqual(Schema.Type.Array, schema.Tag);
elementSchema = arraySchema.ItemSchema;
}
else
{
var unionSchema = schema.AsUnionSchema();
Assert.AreEqual(2, unionSchema.Count);
Assert.AreEqual(Schema.Type.Null, unionSchema.Schemas[0].Tag);
Assert.AreEqual(Schema.Type.Array, unionSchema.Schemas[1].Tag);
elementSchema = unionSchema.Schemas[1].AsArraySchema().ItemSchema;
}
// assert element type
if (!unionOfNullElements)
{
Assert.AreEqual(expectedElementType, elementSchema.Tag);
AssertStringNative(elementSchema, avroSettings);
}
else
{
var unionSchema = elementSchema.AsUnionSchema();
Assert.AreEqual(2, unionSchema.Count);
Assert.AreEqual(Schema.Type.Null, unionSchema.Schemas[0].Tag);
Assert.AreEqual(expectedElementType, unionSchema.Schemas[1].Tag);
}
}
private void AssertStringNative(
Schema elementType,
ConfigurationCommonEventTypeMeta.AvroSettingsConfig avroSettings)
{
if (elementType.Tag != Schema.Type.String)
{
return;
}
var prop = elementType.GetProp(AvroConstant.PROP_STRING_KEY);
if (avroSettings.IsEnableNativeString)
{
Assert.AreEqual(prop, AvroConstant.PROP_STRING_VALUE);
}
else
{
Assert.IsNull(prop);
}
}
private void AssertType(
Type clazz,
Schema.Type expected,
bool unionOfNull,
ConfigurationCommonEventTypeMeta.AvroSettingsConfig avroSettings)
{
var schema = Assemble(clazz, null, avroSettings, EVENT_ADAPTER_SERVICE);
if (!unionOfNull)
{
Assert.AreEqual(expected, schema.Tag);
AssertStringNative(schema, avroSettings);
}
else
{
UnionSchema unionSchema = schema.AsUnionSchema();
Assert.AreEqual(Schema.Type.Union, schema.Tag);
Assert.AreEqual(2, unionSchema.Schemas.Count);
Assert.AreEqual(Schema.Type.Null, unionSchema.Schemas[0].Tag);
Assert.AreEqual(expected, unionSchema.Schemas[1].Tag);
}
}
private static Schema AssembleNestedSchema(
MapEventType mapEventType,
ConfigurationCommonEventTypeMeta.AvroSettingsConfig avroSettings,
Attribute[] annotations,
EventTypeNameResolver eventTypeNameResolver,
string statementName,
TypeRepresentationMapper optionalMapper)
{
var fields = new JArray();
foreach (var prop in mapEventType.Types) {
AssembleField(
prop.Key,
prop.Value,
fields,
annotations,
avroSettings,
eventTypeNameResolver,
statementName,
optionalMapper);
}
return SchemaBuilder.Record(mapEventType.Name, fields);
}
public void Init(
ConfigurationCommonEventTypeMeta.AvroSettingsConfig avroSettingsConfig,
ImportService importService)
{
// no action, init is always done
}
protected abstract EventTypeAvroHandler MakeEventTypeAvroHandler(
ImportServiceRuntime importServiceRuntime,
ConfigurationCommonEventTypeMeta.AvroSettingsConfig avroSettings,
RuntimeExtensionServices runtimeExt);
private static void AssembleFieldForCollection(
string propertyName,
object propertyType,
JArray assembler,
ConfigurationCommonEventTypeMeta.AvroSettingsConfig avroSettings,
Type propertyClass,
bool preferNonNull)
{
var componentType = propertyClass.GetIndexType();
var componentTypeBoxed = componentType.GetBoxedType();
var nullableElements = componentType == componentTypeBoxed;
if (componentTypeBoxed == typeof(bool?)) {
AssembleArray(
nullableElements,
ARRAY_OF_REQ_BOOLEAN,
ARRAY_OF_OPT_BOOLEAN,
assembler,
propertyName,
preferNonNull);
}
else if (componentTypeBoxed == typeof(int?)) {
AssembleArray(
nullableElements,
ARRAY_OF_REQ_INT,
ARRAY_OF_OPT_INT,
assembler,
propertyName,
preferNonNull);
}
else if (componentTypeBoxed == typeof(long?)) {
AssembleArray(
nullableElements,
ARRAY_OF_REQ_LONG,
ARRAY_OF_OPT_LONG,
assembler,
propertyName,
preferNonNull);
}
else if (componentTypeBoxed == typeof(float?)) {
AssembleArray(
nullableElements,
ARRAY_OF_REQ_FLOAT,
ARRAY_OF_OPT_FLOAT,
assembler,
propertyName,
preferNonNull);
}
else if (componentTypeBoxed == typeof(double?)) {
AssembleArray(
nullableElements,
ARRAY_OF_REQ_DOUBLE,
ARRAY_OF_OPT_DOUBLE,
assembler,
propertyName,
preferNonNull);
}
else if (componentTypeBoxed == typeof(byte?)) {
AssembleArray(
nullableElements,
ARRAY_OF_REQ_INT,
ARRAY_OF_OPT_INT,
assembler,
propertyName,
preferNonNull);
}
else if (propertyClass == typeof(string[])) {
JObject array;
if (avroSettings.IsEnableNativeString) {
array = TypeBuilder.Array(
TypeBuilder.StringType(
TypeBuilder.Property(
AvroConstant.PROP_STRING_KEY,
AvroConstant.PROP_STRING_VALUE)));
}
else {
array = TypeBuilder.Array(TypeBuilder.StringType());
}
if (preferNonNull) {
assembler.Add(TypeBuilder.Field(propertyName, array));
}
else {
assembler.Add(
TypeBuilder.Field(
propertyName,
TypeBuilder.Union(
TypeBuilder.NullType(),
array)));
}
}
else {
throw MakeEPException(propertyName, propertyType);
}
}
public static void AssembleField(
string propertyName,
object propertyType,
JArray assembler,
Attribute[] annotations,
ConfigurationCommonEventTypeMeta.AvroSettingsConfig avroSettings,
EventTypeNameResolver eventTypeNameResolver,
string statementName,
TypeRepresentationMapper optionalMapper)
{
if (propertyName.Contains(".")) {
throw new EPException(
"Invalid property name as Avro does not allow dot '.' in field names (property '" +
propertyName +
"')");
}
Schema schema = GetAnnotationSchema(propertyName, annotations);
if (schema != null) {
assembler.Add(TypeBuilder.Field(propertyName, schema));
// assembler.Name(propertyName).Type(schema).NoDefault();
return;
}
if (optionalMapper != null && propertyType is Type propertyTypeType) {
var schemaResult = optionalMapper.Map(
new TypeRepresentationMapperContext(propertyTypeType, propertyName, statementName));
if (schemaResult is JToken schemaAsJsonToken) {
assembler.Add(TypeBuilder.Field(propertyName, schemaAsJsonToken));
return;
}
if (schemaResult is Schema schemaFromResult) {
assembler.Add(TypeBuilder.Field(propertyName, schemaFromResult));
// assembler.Name(propertyName).Type(result).NoDefault();
return;
}
}
if (propertyType == null) {
assembler.Add(TypeBuilder.Field(propertyName, TypeBuilder.NullType()));
// assembler.Name(propertyName).Type("null");
}
else if (propertyType is string) {
string propertyTypeName = propertyType.ToString();
bool isArray = EventTypeUtility.IsPropertyArray(propertyTypeName);
if (isArray) {
propertyTypeName = EventTypeUtility.GetPropertyRemoveArray(propertyTypeName);
}
// Add EventType itself as a property
EventType eventType = eventTypeNameResolver.GetTypeByName(propertyTypeName);
if (!(eventType is AvroEventType)) {
throw new EPException(
"Type definition encountered an unexpected property type name '" +
propertyType +
"' for property '" +
propertyName +
"', expected the name of a previously-declared Avro type");
}
schema = ((AvroEventType) eventType).SchemaAvro;
if (!isArray) {
assembler.Add(TypeBuilder.Field(propertyName, schema));
}
else {
assembler.Add(TypeBuilder.Field(propertyName, TypeBuilder.Array(schema)));
}
}
else if (propertyType is EventType) {
var eventType = (EventType) propertyType;
CheckCompatibleType(eventType);
if (eventType is AvroEventType) {
schema = ((AvroEventType) eventType).SchemaAvro;
assembler.Add(TypeBuilder.Field(propertyName, schema));
}
else if (eventType is MapEventType) {
var mapEventType = (MapEventType) eventType;
var nestedSchema = AssembleNestedSchema(
mapEventType,
avroSettings,
annotations,
eventTypeNameResolver,
statementName,
optionalMapper);
assembler.Add(TypeBuilder.Field(propertyName, nestedSchema));
}
else {
throw new IllegalStateException("Unrecognized event type " + eventType);
}
}
else if (propertyType is EventType[]) {
EventType eventType = ((EventType[]) propertyType)[0];
CheckCompatibleType(eventType);
if (eventType is AvroEventType) {
schema = ((AvroEventType) eventType).SchemaAvro;
assembler.Add(TypeBuilder.Field(propertyName, TypeBuilder.Array(schema)));
}
else if (eventType is MapEventType) {
var mapEventType = (MapEventType) eventType;
var nestedSchema = AssembleNestedSchema(
mapEventType,
avroSettings,
annotations,
eventTypeNameResolver,
statementName,
optionalMapper);
assembler.Add(TypeBuilder.Field(propertyName, TypeBuilder.Array(nestedSchema)));
}
else {
throw new IllegalStateException("Unrecognized event type " + eventType);
}
}
else if (propertyType is Type) {
var propertyClass = (Type) propertyType;
var propertyClassBoxed = propertyClass.GetBoxedType();
bool nullable = propertyClass == propertyClassBoxed;
bool preferNonNull = avroSettings.IsEnableSchemaDefaultNonNull;
if (propertyClassBoxed == typeof(bool?)) {
AssemblePrimitive(nullable, REQ_BOOLEAN, OPT_BOOLEAN, assembler, propertyName, preferNonNull);
}
else if (propertyClassBoxed == typeof(int?) || propertyClassBoxed == typeof(byte?)) {
AssemblePrimitive(nullable, REQ_INT, OPT_INT, assembler, propertyName, preferNonNull);
}
else if (propertyClassBoxed == typeof(long?)) {
AssemblePrimitive(nullable, REQ_LONG, OPT_LONG, assembler, propertyName, preferNonNull);
}
else if (propertyClassBoxed == typeof(float?)) {
AssemblePrimitive(nullable, REQ_FLOAT, OPT_FLOAT, assembler, propertyName, preferNonNull);
}
else if (propertyClassBoxed == typeof(double?)) {
AssemblePrimitive(nullable, REQ_DOUBLE, OPT_DOUBLE, assembler, propertyName, preferNonNull);
}
else if (propertyClass == typeof(string)) {
if (avroSettings.IsEnableNativeString) {
if (preferNonNull) {
assembler.Add(
TypeBuilder.Field(
propertyName,
TypeBuilder.Primitive(
"string",
TypeBuilder.Property(
AvroConstant.PROP_STRING_KEY,
AvroConstant.PROP_STRING_VALUE))));
}
else {
assembler.Add(
TypeBuilder.Field(
propertyName,
TypeBuilder.Union(
TypeBuilder.NullType(),
TypeBuilder.StringType(
TypeBuilder.Property(
AvroConstant.PROP_STRING_KEY,
AvroConstant.PROP_STRING_VALUE)))));
}
}
else {
AssemblePrimitive(nullable, REQ_STRING, OPT_STRING, assembler, propertyName, preferNonNull);
}
}
else if (propertyClass == typeof(byte[])) {
if (preferNonNull) {
assembler.Add(TypeBuilder.RequiredBytes(propertyName));
}
else {
assembler.Add(
TypeBuilder.Field(
propertyName,
TypeBuilder.Union(
TypeBuilder.NullType(),
TypeBuilder.BytesType())));
}
}
else if (propertyClass.IsArray) {
var componentType = propertyClass.GetElementType();
var componentTypeBoxed = componentType.GetBoxedType();
var nullableElements = componentType == componentTypeBoxed;
if (componentTypeBoxed == typeof(bool?)) {
AssembleArray(
nullableElements,
ARRAY_OF_REQ_BOOLEAN,
ARRAY_OF_OPT_BOOLEAN,
assembler,
propertyName,
preferNonNull);
}
else if (componentTypeBoxed == typeof(int?)) {
AssembleArray(
nullableElements,
ARRAY_OF_REQ_INT,
ARRAY_OF_OPT_INT,
assembler,
propertyName,
preferNonNull);
}
else if (componentTypeBoxed == typeof(long?)) {
AssembleArray(
nullableElements,
ARRAY_OF_REQ_LONG,
ARRAY_OF_OPT_LONG,
assembler,
propertyName,
preferNonNull);
}
else if (componentTypeBoxed == typeof(float?)) {
AssembleArray(
nullableElements,
ARRAY_OF_REQ_FLOAT,
ARRAY_OF_OPT_FLOAT,
assembler,
propertyName,
preferNonNull);
}
else if (componentTypeBoxed == typeof(double?)) {
AssembleArray(
nullableElements,
ARRAY_OF_REQ_DOUBLE,
ARRAY_OF_OPT_DOUBLE,
assembler,
propertyName,
preferNonNull);
}
else if (componentTypeBoxed == typeof(byte?)) {
AssembleArray(
nullableElements,
ARRAY_OF_REQ_INT,
ARRAY_OF_OPT_INT,
assembler,
propertyName,
preferNonNull);
}
else if (propertyClass == typeof(string[])) {
JObject array;
if (avroSettings.IsEnableNativeString) {
array = TypeBuilder.Array(
TypeBuilder.StringType(
TypeBuilder.Property(
AvroConstant.PROP_STRING_KEY,
AvroConstant.PROP_STRING_VALUE)));
}
else {
array = TypeBuilder.Array(TypeBuilder.StringType());
}
if (preferNonNull) {
assembler.Add(TypeBuilder.Field(propertyName, array));
}
else {
assembler.Add(
TypeBuilder.Field(
propertyName,
TypeBuilder.Union(
TypeBuilder.NullType(),
array)));
}
}
else if (propertyClass.CanUnwrap<object>()) {
}
else {
throw MakeEPException(propertyName, propertyType);
}
}
else if (propertyClass.IsGenericDictionary()) {
JToken value;
if (avroSettings.IsEnableNativeString) {
value = TypeBuilder.StringType(
TypeBuilder.Property(AvroConstant.PROP_STRING_KEY, AvroConstant.PROP_STRING_VALUE));
}
else {
value = TypeBuilder.StringType();
}
if (preferNonNull) {
assembler.Add(TypeBuilder.Field(propertyName, TypeBuilder.Map(value)));
}
else {
assembler.Add(
TypeBuilder.Field(
propertyName,
TypeBuilder.Union(
TypeBuilder.NullType(),
TypeBuilder.Map(value))));
}
}
else if (propertyClass.IsGenericCollection()) {
AssembleFieldForCollection(
propertyName,
propertyType,
assembler,
avroSettings,
propertyClass,
preferNonNull);
}
else {
throw MakeEPException(propertyName, propertyType);
}
}
else {
throw MakeEPException(propertyName, propertyType);
}
}
