/// <summary>
/// Initializes a new instance of the <see cref="FeatureExpression"/> class.
/// </summary>
/// <param name="featureType">The type of the feature.</param>
/// <param name="name">The name of the feature.</param>
/// <param name="valueExpressionFactory">Factory to extract the value for a given feature from the example object (input argument).</param>
/// <param name="valueValidExpressionFactories">Factories to provide validation before invoking the expression created through <see cref="ValueExpressionFactory"/>.</param>
/// <param name="featureExpressionFactory">The expression must create new Feature instances.</param>
/// <param name="namespace">The namespace this feature belongs to.</param>
/// <param name="featureGroup">The feature group this feature belongs to.</param>
/// <param name="enumerize">If true the marshaller enumerates the feature (as in creates a 1-hot encoding).</param>
/// <param name="variableName">The variable name to be used in the generated code.</param>
/// <param name="order">Used to order feature serialization.</param>
/// <param name="addAnchor">True if an anchor element should be added at the beginning of a dense feature array.</param>
/// <param name="stringProcessing">Configures string pre-processing for this feature.</param>
/// <param name="overrideSerializeMethod">An optional method overriding the otherwise auto-resolved serialization method.</param>
/// <param name="dictify">True if a dictionary should be build for this feature.</param>
/// <param name="parent">The parent feature expression.</param>
public FeatureExpression(Type featureType,
string name,
Func<Expression, Expression> valueExpressionFactory,
List<Func<Expression, Expression>> valueValidExpressionFactories = null,
NewFeatureExpressionDelegate featureExpressionFactory = null,
string @namespace = null,
char? featureGroup = null,
bool enumerize = false,
string variableName = null,
int? order = null,
bool addAnchor = false,
StringProcessing stringProcessing = StringProcessing.Split,
MethodInfo overrideSerializeMethod = null,
bool? dictify = null,
FeatureExpression parent = null)
{
if (featureType == null)
throw new ArgumentNullException("featureType");
if (valueExpressionFactory == null)
throw new ArgumentNullException("valueExpressionFactory");
Contract.EndContractBlock();
if(featureType.IsGenericType &&
featureType.GetGenericTypeDefinition() == typeof(Nullable<>))
{
this.IsNullable = true;
this.FeatureType = featureType.GetGenericArguments()[0];
}
else
{
this.IsNullable = false;
this.FeatureType = featureType;
}
this.Name = name;
this.ValueExpressionFactory = valueExpressionFactory;
this.ValueValidExpressionFactories = valueValidExpressionFactories;
this.FeatureExpressionFactory = featureExpressionFactory;
this.Namespace = @namespace;
this.FeatureGroup = featureGroup;
this.Enumerize = enumerize;
this.VariableName = variableName ?? name;
this.Order = order ?? 1;
this.AddAnchor = addAnchor;
this.Dictify = dictify ?? false;
this.StringProcessing = stringProcessing;
this.OverrideSerializeMethod = overrideSerializeMethod;
this.Dictify = dictify ?? false;
this.Parent = parent;
this.DenseFeatureValueElementType = InspectionHelper.GetEnumerableElementType(featureType);
if (!InspectionHelper.IsNumericType(this.DenseFeatureValueElementType))
this.DenseFeatureValueElementType = null;
}
/// <summary>
/// Instantiate the meta information object such as <see cref="PreHashedFeature"/>
/// for a given feature.
/// </summary>
/// <param name="featureInternal">The feature.</param>
/// <param name="namespace">The namespace.</param>
/// <returns>The "new" expression for the meta information object.</returns>
private Expression CreateFeature(FeatureExpressionInternal featureInternal, Expression @namespace)
{
FeatureExpression feature = featureInternal.Source;
var metaFeatureType = featureInternal.MarshalMethod.MetaFeatureType;
if (metaFeatureType.IsGenericType && metaFeatureType.GetGenericTypeDefinition() == typeof(EnumerizedFeature <>))
{
// preemptively calculate all hashes for each enum value
var featureParameter = Expression.Parameter(metaFeatureType);
var valueParameter = Expression.Parameter(feature.FeatureType);
var body = new List <Expression>();
var hashVariables = new List <ParameterExpression>();
foreach (var value in Enum.GetValues(feature.FeatureType))
{
var hashVar = Expression.Variable(typeof(UInt64));
hashVariables.Add(hashVar);
// CODE hashVar = feature.FeatureHashInternal(value);
body.Add(Expression.Assign(hashVar,
Expression.Call(featureParameter,
metaFeatureType.GetMethod("FeatureHashInternal"),
Expression.Constant(value))));
}
var cases = Enum.GetValues(feature.FeatureType)
.Cast <object>()
.Zip(hashVariables, (value, hash) => Expression.SwitchCase(
hash,
Expression.Constant(value, feature.FeatureType)))
.ToArray();
// expand the switch(value) { case enum1: return hash1; .... }
var hashSwitch = Expression.Switch(valueParameter,
Expression.Block(Expression.Throw(Expression.New(typeof(NotSupportedException))), Expression.Constant((UInt64)0, typeof(UInt64))),
cases);
// CODE return value => switch(value) { .... }
body.Add(Expression.Lambda(hashSwitch, valueParameter));
return(CreateNew(
metaFeatureType,
this.vwParameter,
@namespace,
Expression.Constant(feature.Name, typeof(string)),
Expression.Constant(feature.AddAnchor),
Expression.Constant(feature.Dictify),
Expression.Lambda(Expression.Block(hashVariables, body), featureParameter)));
}
else if (metaFeatureType == typeof(PreHashedFeature))
{
// CODE new PreHashedFeature(vw, namespace, "Name", "AddAnchor");
return(CreateNew(
typeof(PreHashedFeature),
this.vwParameter,
@namespace,
Expression.Constant(feature.Name, typeof(string)),
Expression.Constant(feature.AddAnchor),
Expression.Constant(feature.Dictify)));
}
else
{
// CODE new Feature("Name", ...)
return(CreateNew(
metaFeatureType,
Expression.Constant(feature.Name, typeof(string)),
Expression.Constant(feature.AddAnchor),
Expression.Constant(feature.Dictify)));
}
}
private MarshalMethod ResolveFeatureMarshalMethod(FeatureExpression feature)
{
if (feature.OverrideSerializeMethod != null)
{
return(new MarshalMethod
{
Method = feature.OverrideSerializeMethod,
MetaFeatureType = feature.OverrideSerializeMethod.GetParameters().Select(p => p.ParameterType).First(t => typeof(Feature).IsAssignableFrom(t))
});
}
string methodName;
Type[] metaFeatureTypeCandidates;
if (feature.FeatureType == typeof(string))
{
switch (feature.StringProcessing)
{
case StringProcessing.Escape:
methodName = "MarshalFeatureStringEscape";
break;
case StringProcessing.EscapeAndIncludeName:
methodName = "MarshalFeatureStringEscapeAndIncludeName";
break;
case StringProcessing.Split:
methodName = "MarshalFeatureStringSplit";
break;
default:
throw new ArgumentException("feature.StringProcessing is not supported: " + feature.StringProcessing);
}
metaFeatureTypeCandidates = new [] { typeof(Feature) };
}
else if (feature.FeatureType.IsEnum)
{
methodName = "MarshalEnumFeature";
metaFeatureTypeCandidates = new [] { typeof(EnumerizedFeature <>).MakeGenericType(feature.FeatureType) };
}
else if (feature.Enumerize)
{
methodName = "MarshalEnumerizeFeature";
metaFeatureTypeCandidates = new [] { typeof(Feature) };
}
else
{
// probe for PreHashedFeature marshal method, than fallback
methodName = "MarshalFeature";
metaFeatureTypeCandidates = new [] { typeof(PreHashedFeature), typeof(Feature) };
}
// remove Nullable<> from feature type
var featureType = feature.FeatureType;
if (featureType.IsGenericType &&
featureType.GetGenericTypeDefinition() == typeof(Nullable <>))
{
featureType = featureType.GetGenericArguments()[0];
}
foreach (var metaFeatureType in metaFeatureTypeCandidates)
{
// find visitor.MarshalFeature(VowpalWabbitMarshallingContext context, Namespace ns, <PreHashedFeature|Feature> feature, <valueType> value)
var method = this.marshallerTypes
.Select(visitor => ReflectionHelper.FindMethod(
visitor,
methodName,
typeof(VowpalWabbitMarshalContext),
typeof(Namespace),
metaFeatureType,
featureType))
.FirstOrDefault(m => m != null);
if (method != null)
{
return(new MarshalMethod
{
Method = method,
MetaFeatureType = metaFeatureType
});
}
}
return(null);
}
/// <summary>
/// Initializes a new instance of the <see cref="FeatureExpression"/> class.
/// </summary>
/// <param name="featureType">The type of the feature.</param>
/// <param name="name">The name of the feature.</param>
/// <param name="valueExpressionFactory">Factory to extract the value for a given feature from the example object (input argument).</param>
/// <param name="valueValidExpressionFactories">Factories to provide validation before invoking the expression created through <see cref="ValueExpressionFactory"/>.</param>
/// <param name="featureExpressionFactory">The expression must create new Feature instances.</param>
/// <param name="namespace">The namespace this feature belongs to.</param>
/// <param name="featureGroup">The feature group this feature belongs to.</param>
/// <param name="enumerize">If true the marshaller enumerates the feature (as in creates a 1-hot encoding).</param>
/// <param name="variableName">The variable name to be used in the generated code.</param>
/// <param name="order">Used to order feature serialization.</param>
/// <param name="addAnchor">True if an anchor element should be added at the beginning of a dense feature array.</param>
/// <param name="stringProcessing">Configures string pre-processing for this feature.</param>
/// <param name="overrideSerializeMethod">An optional method overriding the otherwise auto-resolved serialization method.</param>
/// <param name="dictify">True if a dictionary should be build for this feature.</param>
/// <param name="parent">The parent feature expression.</param>
public FeatureExpression(Type featureType,
string name,
Func <Expression, Expression> valueExpressionFactory,
List <Func <Expression, Expression> > valueValidExpressionFactories = null,
NewFeatureExpressionDelegate featureExpressionFactory = null,
string @namespace = null,
char?featureGroup = null,
bool enumerize = false,
string variableName = null,
int?order = null,
bool addAnchor = false,
StringProcessing stringProcessing = StringProcessing.Split,
MethodInfo overrideSerializeMethod = null,
bool?dictify = null,
FeatureExpression parent = null)
{
if (featureType == null)
{
throw new ArgumentNullException("featureType");
}
if (valueExpressionFactory == null)
{
throw new ArgumentNullException("valueExpressionFactory");
}
Contract.EndContractBlock();
if (featureType.IsGenericType &&
featureType.GetGenericTypeDefinition() == typeof(Nullable <>))
{
this.IsNullable = true;
this.FeatureType = featureType.GetGenericArguments()[0];
}
else
{
this.IsNullable = false;
this.FeatureType = featureType;
}
this.Name = name;
this.ValueExpressionFactory = valueExpressionFactory;
this.ValueValidExpressionFactories = valueValidExpressionFactories;
this.FeatureExpressionFactory = featureExpressionFactory;
this.Namespace = @namespace;
this.FeatureGroup = featureGroup;
this.Enumerize = enumerize;
this.VariableName = variableName ?? name;
this.Order = order ?? 1;
this.AddAnchor = addAnchor;
this.Dictify = dictify ?? false;
this.StringProcessing = stringProcessing;
this.OverrideSerializeMethod = overrideSerializeMethod;
this.Dictify = dictify ?? false;
this.Parent = parent;
this.DenseFeatureValueElementType = InspectionHelper.GetEnumerableElementType(featureType);
if (!InspectionHelper.IsNumericType(this.DenseFeatureValueElementType))
{
this.DenseFeatureValueElementType = null;
}
}
public VowpalWabbitMultiExampleSerializerCompilerImpl(VowpalWabbitSettings settings, Schema schema, FeatureExpression multiFeature)
{
Contract.Requires(settings != null);
Contract.Requires(schema != null);
Contract.Requires(multiFeature != null);
var nonMultiFeatures = schema.Features.Where(fe => fe != multiFeature).ToList();
this.sharedSerializerCompiler = nonMultiFeatures.Count == 0 ? null :
new VowpalWabbitSingleExampleSerializerCompiler <TExample>(
new Schema {
Features = nonMultiFeatures
},
settings == null ? null : settings.CustomFeaturizer,
!settings.EnableStringExampleGeneration);
this.adfSerializerComputer = new VowpalWabbitSingleExampleSerializerCompiler <TActionDependentFeature>(
settings.TypeInspector.CreateSchema(settings, typeof(TActionDependentFeature)),
settings == null ? null : settings.CustomFeaturizer,
!settings.EnableStringExampleGeneration);
var exampleParameter = Expression.Parameter(typeof(TExample), "example");
// CODE condition1 && condition2 && condition3 ...
var condition = multiFeature.ValueValidExpressionFactories
.Skip(1)
.Aggregate(
multiFeature.ValueValidExpressionFactories.First()(exampleParameter),
(cond, factory) => Expression.AndAlso(cond, factory(exampleParameter)));
var multiExpression = multiFeature.ValueExpressionFactory(exampleParameter);
// CODE example => (IEnumerable<TActionDependentFeature>)(example._multi != null ? example._multi : null)
var expr = Expression.Lambda <Func <TExample, IEnumerable <TActionDependentFeature> > >(
Expression.Condition(
condition,
multiExpression,
Expression.Constant(null, multiExpression.Type),
typeof(IEnumerable <TActionDependentFeature>)),
exampleParameter);
this.adfAccessor = (Func <TExample, IEnumerable <TActionDependentFeature> >)expr.CompileToFunc();
}
private static Schema CreateSchema(
FeatureExpression parent,
Type type,
string parentNamespace,
char?parentFeatureGroup,
bool?parentDictify,
Func <Expression, Expression> valueExpressionFactory,
Stack <Func <Expression, Expression> > valueValidExpressionFactories,
Func <PropertyInfo, FeatureAttribute, bool> featurePropertyPredicate,
Func <PropertyInfo, LabelAttribute, bool> labelPropertyPredicate)
{
var props = type.GetProperties(BindingFlags.Instance | BindingFlags.GetProperty | BindingFlags.Public);
var localFeatures = (from p in props
let declaredAttr = (FeatureAttribute)p.GetCustomAttributes(typeof(FeatureAttribute), true).FirstOrDefault()
where featurePropertyPredicate(p, declaredAttr)
let attr = declaredAttr ?? new FeatureAttribute()
select new FeatureExpression(
featureType: p.PropertyType,
name: attr.Name ?? p.Name,
// CODE example.Property
valueExpressionFactory: valueExpression => Expression.Property(valueExpressionFactory(valueExpression), p),
// @Reverse: make sure conditions are specified in the right order
valueValidExpressionFactories: valueValidExpressionFactories.Reverse().ToList(),
@namespace: attr.Namespace ?? parentNamespace,
featureGroup: attr.InternalFeatureGroup ?? parentFeatureGroup,
enumerize: attr.Enumerize,
variableName: p.Name,
order: attr.Order,
addAnchor: attr.AddAnchor,
stringProcessing: attr.StringProcessing,
dictify: attr.InternalDictify ?? parentDictify,
parent: parent)
).ToList();
var localLabels = from p in props
let declaredAttr = (LabelAttribute)p.GetCustomAttributes(typeof(LabelAttribute), true).FirstOrDefault()
where labelPropertyPredicate(p, declaredAttr) || typeof(ILabel).IsAssignableFrom(p.PropertyType)
let attr = declaredAttr ?? new LabelAttribute()
let labelType = p.PropertyType
where typeof(ILabel).IsAssignableFrom(labelType) || p.PropertyType == typeof(string)
select new LabelExpression
{
Name = p.Name,
LabelType = p.PropertyType,
// CODE example.Property
ValueExpressionFactory = valueExpression => Expression.Property(valueExpressionFactory(valueExpression), p),
// @Reverse: make sure conditions are specified in the right order
ValueValidExpressionFactories = valueValidExpressionFactories.Reverse().ToList()
};
// Recurse
var schemas = localFeatures
.Select(f =>
{
// CODE example.Prop1.Prop2 != null
valueValidExpressionFactories.Push(valueExpression => Expression.NotEqual(f.ValueExpressionFactory(valueExpression), Expression.Constant(null)));
var subSchema = CreateSchema(f, f.FeatureType, f.Namespace, f.FeatureGroup, f.Dictify, f.ValueExpressionFactory, valueValidExpressionFactories, featurePropertyPredicate, labelPropertyPredicate);
valueValidExpressionFactories.Pop();
return(subSchema);
})
.ToList();
return(new Schema
{
Features = localFeatures.Union(schemas.SelectMany(s => s.Features)).ToList(),
Label = localLabels.Union(schemas.Select(s => s.Label)).FirstOrDefault(l => l != null)
});
}
private MarshalMethod ResolveFeatureMarshalMethod(FeatureExpression feature)
{
if (feature.OverrideSerializeMethod != null)
{
return(new MarshalMethod
{
Method = feature.OverrideSerializeMethod,
MetaFeatureType = feature.OverrideSerializeMethod.GetParameters().Select(p => p.ParameterType).First(t => typeof(Feature).IsAssignableFrom(t))
});
}
string methodName;
Type[] metaFeatureTypeCandidates;
if (feature.FeatureType == typeof(string))
{
switch (feature.StringProcessing)
{
case StringProcessing.Escape:
methodName = "MarshalFeatureStringEscape";
break;
case StringProcessing.EscapeAndIncludeName:
methodName = "MarshalFeatureStringEscapeAndIncludeName";
break;
case StringProcessing.Split:
methodName = "MarshalFeatureStringSplit";
break;
default:
throw new ArgumentException("feature.StringProcessing is not supported: " + feature.StringProcessing);
}
metaFeatureTypeCandidates = new [] { typeof(Feature) };
}
else if (feature.FeatureType.IsEnum)
{
methodName = "MarshalEnumFeature";
metaFeatureTypeCandidates = new [] { typeof(EnumerizedFeature <>).MakeGenericType(feature.FeatureType) };
}
else if (feature.Enumerize)
{
methodName = "MarshalEnumerizeFeature";
metaFeatureTypeCandidates = new [] { typeof(Feature) };
}
else
{
// probe for PreHashedFeature marshal method, than fallback
methodName = "MarshalFeature";
metaFeatureTypeCandidates = new [] { typeof(PreHashedFeature), typeof(Feature) };
}
// remove Nullable<> from feature type
var featureType = feature.FeatureType;
if (featureType.IsGenericType &&
featureType.GetGenericTypeDefinition() == typeof(Nullable <>))
{
featureType = featureType.GetGenericArguments()[0];
}
var method = ResolveFeatureMarshalMethod("MarshalNamespace", metaFeatureTypeCandidates, featureType, isNamespace: true);
if (method == null)
{
method = ResolveFeatureMarshalMethod(methodName, metaFeatureTypeCandidates, featureType, isNamespace: false);
}
return(method);
}