internal static LambdaNode ParseAndBindLambda(IHostEnvironment env, string expression, int ivec, DataViewType[] inputTypes, out int[] perm)
{
perm = Utils.GetIdentityPermutation(inputTypes.Length);
if (ivec >= 0)
{
if (ivec > 0)
{
perm[0] = ivec;
for (int i = 1; i <= ivec; i++)
{
perm[i] = i - 1;
}
}
}
CharCursor chars = new CharCursor(expression);
var node = LambdaParser.Parse(out List <Error> errors, out List <int> lineMap, chars, perm, inputTypes);
if (Utils.Size(errors) > 0)
{
throw env.ExceptParam(nameof(expression), $"parsing failed: {errors[0].GetMessage()}");
}
using (var ch = env.Start("LabmdaBinder.Run"))
LambdaBinder.Run(env, ref errors, node, msg => ch.Error(msg));
if (Utils.Size(errors) > 0)
{
throw env.ExceptParam(nameof(expression), $"binding failed: {errors[0].GetMessage()}");
}
return(node);
}
public static IDataView Create <TSrc>(IHostEnvironment env, string name, IDataView input,
string src, DataViewType typeSrc, InPredicate <TSrc> predicate)
{
Contracts.CheckValue(env, nameof(env));
env.CheckNonEmpty(name, nameof(name));
env.CheckValue(input, nameof(input));
env.CheckNonEmpty(src, nameof(src));
env.CheckValue(typeSrc, nameof(typeSrc));
env.CheckValue(predicate, nameof(predicate));
if (typeSrc.RawType != typeof(TSrc))
{
throw env.ExceptParam(nameof(predicate),
"The source column type '{0}' doesn't match the input type of the predicate", typeSrc);
}
int colSrc;
bool tmp = input.Schema.TryGetColumnIndex(src, out colSrc);
if (!tmp)
{
throw env.ExceptParam(nameof(src), "The input data doesn't have a column named '{0}'", src);
}
var typeOrig = input.Schema[colSrc].Type;
// REVIEW: Ideally this should support vector-type conversion. It currently doesn't.
bool ident;
Delegate conv;
if (typeOrig.SameSizeAndItemType(typeSrc))
{
ident = true;
conv = null;
}
else if (!Conversions.DefaultInstance.TryGetStandardConversion(typeOrig, typeSrc, out conv, out ident))
{
throw env.ExceptParam(nameof(predicate),
"The type of column '{0}', '{1}', cannot be converted to the input type of the predicate '{2}'",
src, typeOrig, typeSrc);
}
IDataView impl;
if (ident)
{
impl = new Impl <TSrc, TSrc>(env, name, input, colSrc, predicate);
}
else
{
Func <IHostEnvironment, string, IDataView, int,
InPredicate <int>, ValueMapper <int, int>, Impl <int, int> > del = CreateImpl <int, int>;
var meth = del.GetMethodInfo().GetGenericMethodDefinition()
.MakeGenericMethod(typeOrig.RawType, typeof(TSrc));
impl = (IDataView)meth.Invoke(null, new object[] { env, name, input, colSrc, predicate, conv });
}
return(new OpaqueDataView(impl));
}
private static IMamlEvaluator GetEvaluator(IHostEnvironment env, MacroUtils.TrainerKinds kind)
{
switch (kind)
{
case MacroUtils.TrainerKinds.SignatureBinaryClassifierTrainer:
return(new BinaryClassifierMamlEvaluator(env, new BinaryClassifierMamlEvaluator.Arguments()));
case MacroUtils.TrainerKinds.SignatureMultiClassClassifierTrainer:
return(new MultiClassMamlEvaluator(env, new MultiClassMamlEvaluator.Arguments()));
case MacroUtils.TrainerKinds.SignatureRegressorTrainer:
return(new RegressionMamlEvaluator(env, new RegressionMamlEvaluator.Arguments()));
case MacroUtils.TrainerKinds.SignatureRankerTrainer:
return(new RankerMamlEvaluator(env, new RankerMamlEvaluator.Arguments()));
case MacroUtils.TrainerKinds.SignatureAnomalyDetectorTrainer:
return(new AnomalyDetectionMamlEvaluator(env, new AnomalyDetectionMamlEvaluator.Arguments()));
case MacroUtils.TrainerKinds.SignatureClusteringTrainer:
return(new ClusteringMamlEvaluator(env, new ClusteringMamlEvaluator.Arguments()));
case MacroUtils.TrainerKinds.SignatureMultiOutputRegressorTrainer:
return(new MultiOutputRegressionMamlEvaluator(env, new MultiOutputRegressionMamlEvaluator.Arguments()));
default:
throw env.ExceptParam(nameof(kind), $"Trainer kind {kind} does not have an evaluator");
}
}
/// <summary>
/// This is the constructor called for the initial wrapping.
/// </summary>
public Bound(IHostEnvironment env, ISchemaBoundRowMapper mapper, VectorType type, ValueGetter <VBuffer <T> > getter,
string metadataKind, Func <ISchemaBoundMapper, ColumnType, bool> canWrap)
{
Contracts.CheckValue(env, nameof(env));
_host = env.Register(LoaderSignature);
_host.CheckValue(mapper, nameof(mapper));
_host.CheckValue(type, nameof(type));
_host.CheckValue(getter, nameof(getter));
_host.CheckNonEmpty(metadataKind, nameof(metadataKind));
_host.CheckValueOrNull(canWrap);
_mapper = mapper;
int scoreIdx;
bool result = mapper.Schema.TryGetColumnIndex(MetadataUtils.Const.ScoreValueKind.Score, out scoreIdx);
if (!result)
{
throw env.ExceptParam(nameof(mapper), "Mapper did not have a '{0}' column", MetadataUtils.Const.ScoreValueKind.Score);
}
_labelNameType = type;
_labelNameGetter = getter;
_metadataKind = metadataKind;
_outSchema = new SchemaImpl(mapper.Schema, scoreIdx, _labelNameType, _labelNameGetter, _metadataKind);
_canWrap = canWrap;
}
public static double ExtractValueFromIdv(IHostEnvironment env, IDataView result, string columnName)
{
Contracts.CheckValue(env, nameof(env));
env.CheckValue(result, nameof(result));
env.CheckNonEmpty(columnName, nameof(columnName));
double outputValue = 0;
var schema = result.Schema;
if (!schema.TryGetColumnIndex(columnName, out var metricCol))
{
throw env.ExceptParam(nameof(columnName), $"Schema does not contain column: {columnName}");
}
using (var cursor = result.GetRowCursor(col => col == metricCol))
{
var getter = cursor.GetGetter <double>(metricCol);
bool moved = cursor.MoveNext();
env.Check(moved, "Expected an IDataView with a single row. Results dataset has no rows to extract.");
getter(ref outputValue);
env.Check(!cursor.MoveNext(), "Expected an IDataView with a single row. Results dataset has too many rows.");
}
return(outputValue);
}
private static void CheckBinaryLabel(bool user, IHostEnvironment env, IPredictorModel[] predictors)
{
int classCount = CheckLabelColumn(env, predictors, true);
if (classCount != 2)
{
var error = string.Format("Expected label to have exactly 2 classes, instead has {0}", classCount);
throw user?env.ExceptParam(nameof(predictors), error) : env.ExceptDecode(error);
}
}
public TypeName(IHostEnvironment env, float p, int foo)
{
Contracts.CheckValue(env, nameof(env));
env.CheckParam(0 <= p && p <= 1, nameof(p), "Should be in range [0,1]");
env.CheckParam(0 <= p && p <= 1, "p"); // Should fail.
env.CheckParam(0 <= p && p <= 1, nameof(p) + nameof(p)); // Should fail.
env.CheckValue(paramName: nameof(p), val: "p"); // Should succeed despite confusing order.
env.CheckValue(paramName: "p", val: nameof(p)); // Should fail despite confusing order.
env.CheckValue("p", nameof(p));
env.CheckUserArg(foo > 5, "foo", "Nice");
env.CheckUserArg(foo > 5, nameof(foo), "Nice");
env.Except(); // Not throwing or doing anything with the exception, so should fail.
Contracts.ExceptParam(nameof(env), "What a silly env"); // Should also fail.
if (false)
{
throw env.Except(); // Should not fail.
}
if (false)
{
throw env.ExceptParam(nameof(env), "What a silly env"); // Should not fail.
}
if (false)
{
throw env.ExceptParam("env", "What a silly env"); // Should fail due to name error.
}
var e = env.Except();
env.Check(true, $"Hello {foo} is cool");
env.Check(true, "Hello it is cool");
string coolMessage = "Hello it is cool";
env.Check(true, coolMessage);
env.Check(true, string.Format("Hello {0} is cool", foo));
env.Check(true, Messages.CoolMessage);
env.CheckDecode(true, "Not suspicious, no ModelLoadContext");
Contracts.Check(true, "Fine: " + nameof(env));
Contracts.Check(true, "Less fine: " + env.GetType().Name);
Contracts.CheckUserArg(0 <= p && p <= 1,
"p", "On a new line");
}
// Extracts the indices and types of the input columns to the whitening transform.
private static void GetColTypesAndIndex(IHostEnvironment env, IDataView inputData, ColumnInfo[] columns, out ColumnType[] srcTypes, out int[] cols)
{
cols = new int[columns.Length];
srcTypes = new ColumnType[columns.Length];
var inputSchema = inputData.Schema;
for (int i = 0; i < columns.Length; i++)
{
if (!inputSchema.TryGetColumnIndex(columns[i].Input, out cols[i]))
{
throw env.ExceptSchemaMismatch(nameof(inputSchema), "input", columns[i].Input);
}
srcTypes[i] = inputSchema.GetColumnType(cols[i]);
var reason = TestColumn(srcTypes[i]);
if (reason != null)
{
throw env.ExceptParam(nameof(inputData.Schema), reason);
}
}
}
// Extracts the indices and types of the input columns to the whitening transform.
private static void GetColTypesAndIndex(IHostEnvironment env, IDataView inputData, VectorWhiteningEstimator.ColumnOptions[] columns, out DataViewType[] srcTypes, out int[] cols)
{
cols = new int[columns.Length];
srcTypes = new DataViewType[columns.Length];
var inputSchema = inputData.Schema;
for (int i = 0; i < columns.Length; i++)
{
var col = inputSchema.GetColumnOrNull(columns[i].InputColumnName);
if (!col.HasValue)
{
throw env.ExceptSchemaMismatch(nameof(inputSchema), "input", columns[i].InputColumnName);
}
cols[i] = col.Value.Index;
srcTypes[i] = col.Value.Type;
var reason = TestColumn(srcTypes[i]);
if (reason != null)
{
throw env.ExceptParam(nameof(inputData.Schema), reason);
}
}
}
internal static DatabaseLoader CreateDatabaseLoader <TInput>(IHostEnvironment host)
{
var userType = typeof(TInput);
var fieldInfos = userType.GetFields(BindingFlags.Public | BindingFlags.Instance);
var propertyInfos =
userType
.GetProperties(BindingFlags.Public | BindingFlags.Instance)
.Where(x => x.CanRead && x.GetGetMethod() != null && x.GetIndexParameters().Length == 0);
var memberInfos = (fieldInfos as IEnumerable <MemberInfo>).Concat(propertyInfos).ToArray();
if (memberInfos.Length == 0)
{
throw host.ExceptParam(nameof(TInput), $"Should define at least one public, readable field or property in {nameof(TInput)}.");
}
var columns = new List <Column>();
for (int index = 0; index < memberInfos.Length; index++)
{
var memberInfo = memberInfos[index];
var mappingAttrName = memberInfo.GetCustomAttribute <ColumnNameAttribute>();
var column = new Column();
column.Name = mappingAttrName?.Name ?? memberInfo.Name;
var indexMappingAttr = memberInfo.GetCustomAttribute <LoadColumnAttribute>();
var nameMappingAttr = memberInfo.GetCustomAttribute <LoadColumnNameAttribute>();
if (indexMappingAttr is object)
{
if (nameMappingAttr is object)
{
throw Contracts.Except($"Cannot specify both {nameof(LoadColumnAttribute)} and {nameof(LoadColumnNameAttribute)}");
}
column.Source = indexMappingAttr.Sources.Select((source) => Range.FromTextLoaderRange(source)).ToArray();
}
else if (nameMappingAttr is object)
{
column.Source = nameMappingAttr.Sources.Select((source) => new Range(source)).ToArray();
}
InternalDataKind dk;
switch (memberInfo)
{
case FieldInfo field:
if (!InternalDataKindExtensions.TryGetDataKind(field.FieldType.IsArray ? field.FieldType.GetElementType() : field.FieldType, out dk))
{
throw Contracts.Except($"Field {memberInfo.Name} is of unsupported type.");
}
break;
case PropertyInfo property:
if (!InternalDataKindExtensions.TryGetDataKind(property.PropertyType.IsArray ? property.PropertyType.GetElementType() : property.PropertyType, out dk))
{
throw Contracts.Except($"Property {memberInfo.Name} is of unsupported type.");
}
break;
default:
Contracts.Assert(false);
throw Contracts.ExceptNotSupp("Expected a FieldInfo or a PropertyInfo");
}
column.Type = dk.ToDbType();
columns.Add(column);
}
var options = new Options
{
Columns = columns.ToArray()
};
return(new DatabaseLoader(host, options));
}
public static PipelineResultRow[] ExtractResults(IHostEnvironment env, IDataView data,
string graphColName, string metricColName, string idColName, string trainingMetricColName,
string firstInputColName, string predictorModelColName)
{
var results = new List <PipelineResultRow>();
var schema = data.Schema;
if (!schema.TryGetColumnIndex(graphColName, out var graphCol))
{
throw env.ExceptParam(nameof(graphColName), $"Column name {graphColName} not found");
}
if (!schema.TryGetColumnIndex(metricColName, out var metricCol))
{
throw env.ExceptParam(nameof(metricColName), $"Column name {metricColName} not found");
}
if (!schema.TryGetColumnIndex(trainingMetricColName, out var trainingMetricCol))
{
throw env.ExceptParam(nameof(trainingMetricColName), $"Column name {trainingMetricColName} not found");
}
if (!schema.TryGetColumnIndex(idColName, out var pipelineIdCol))
{
throw env.ExceptParam(nameof(idColName), $"Column name {idColName} not found");
}
if (!schema.TryGetColumnIndex(firstInputColName, out var firstInputCol))
{
throw env.ExceptParam(nameof(firstInputColName), $"Column name {firstInputColName} not found");
}
if (!schema.TryGetColumnIndex(predictorModelColName, out var predictorModelCol))
{
throw env.ExceptParam(nameof(predictorModelColName), $"Column name {predictorModelColName} not found");
}
using (var cursor = data.GetRowCursor(col => true))
{
var getter1 = cursor.GetGetter <double>(metricCol);
var getter2 = cursor.GetGetter <ReadOnlyMemory <char> >(graphCol);
var getter3 = cursor.GetGetter <ReadOnlyMemory <char> >(pipelineIdCol);
var getter4 = cursor.GetGetter <double>(trainingMetricCol);
var getter5 = cursor.GetGetter <ReadOnlyMemory <char> >(firstInputCol);
var getter6 = cursor.GetGetter <ReadOnlyMemory <char> >(predictorModelCol);
double metricValue = 0;
double trainingMetricValue = 0;
ReadOnlyMemory <char> graphJson = default;
ReadOnlyMemory <char> pipelineId = default;
ReadOnlyMemory <char> firstInput = default;
ReadOnlyMemory <char> predictorModel = default;
while (cursor.MoveNext())
{
getter1(ref metricValue);
getter2(ref graphJson);
getter3(ref pipelineId);
getter4(ref trainingMetricValue);
getter5(ref firstInput);
getter6(ref predictorModel);
results.Add(new PipelineResultRow(graphJson.ToString(),
metricValue, pipelineId.ToString(), trainingMetricValue,
firstInput.ToString(), predictorModel.ToString()));
}
}
return(results.ToArray());
}
// REVIEW: It would be nice to support propagation of select metadata.
public static IDataView Create <TSrc, TDst>(IHostEnvironment env, string name, IDataView input,
string src, string dst, ColumnType typeSrc, ColumnType typeDst, ValueMapper <TSrc, TDst> mapper,
ValueGetter <VBuffer <ReadOnlyMemory <char> > > keyValueGetter = null, ValueGetter <VBuffer <ReadOnlyMemory <char> > > slotNamesGetter = null)
{
Contracts.CheckValue(env, nameof(env));
env.CheckNonEmpty(name, nameof(name));
env.CheckValue(input, nameof(input));
env.CheckNonEmpty(src, nameof(src));
env.CheckNonEmpty(dst, nameof(dst));
env.CheckValue(typeSrc, nameof(typeSrc));
env.CheckValue(typeDst, nameof(typeDst));
env.CheckValue(mapper, nameof(mapper));
env.Check(keyValueGetter == null || typeDst.GetItemType() is KeyType);
env.Check(slotNamesGetter == null || typeDst.IsKnownSizeVector());
if (typeSrc.RawType != typeof(TSrc))
{
throw env.ExceptParam(nameof(mapper),
"The source column type '{0}' doesn't match the input type of the mapper", typeSrc);
}
if (typeDst.RawType != typeof(TDst))
{
throw env.ExceptParam(nameof(mapper),
"The destination column type '{0}' doesn't match the output type of the mapper", typeDst);
}
bool tmp = input.Schema.TryGetColumnIndex(src, out int colSrc);
if (!tmp)
{
throw env.ExceptParam(nameof(src), "The input data doesn't have a column named '{0}'", src);
}
var typeOrig = input.Schema[colSrc].Type;
// REVIEW: Ideally this should support vector-type conversion. It currently doesn't.
bool ident;
Delegate conv;
if (typeOrig.SameSizeAndItemType(typeSrc))
{
ident = true;
conv = null;
}
else if (!Conversions.Instance.TryGetStandardConversion(typeOrig, typeSrc, out conv, out ident))
{
throw env.ExceptParam(nameof(mapper),
"The type of column '{0}', '{1}', cannot be converted to the input type of the mapper '{2}'",
src, typeOrig, typeSrc);
}
var col = new Column(src, dst);
IDataView impl;
if (ident)
{
impl = new Impl <TSrc, TDst, TDst>(env, name, input, col, typeDst, mapper, keyValueGetter: keyValueGetter, slotNamesGetter: slotNamesGetter);
}
else
{
Func <IHostEnvironment, string, IDataView, Column, ColumnType, ValueMapper <int, int>,
ValueMapper <int, int>, ValueGetter <VBuffer <ReadOnlyMemory <char> > >, ValueGetter <VBuffer <ReadOnlyMemory <char> > >,
Impl <int, int, int> > del = CreateImpl <int, int, int>;
var meth = del.GetMethodInfo().GetGenericMethodDefinition()
.MakeGenericMethod(typeOrig.RawType, typeof(TSrc), typeof(TDst));
impl = (IDataView)meth.Invoke(null, new object[] { env, name, input, col, typeDst, conv, mapper, keyValueGetter, slotNamesGetter });
}
return(new OpaqueDataView(impl));
}