private static ValueGetter <StringBuilder> GetGetterAsStringBuilderCore <TSrc>(DataViewType typeSrc, DataViewRow row, int col)
{
Contracts.Assert(typeof(TSrc) == typeSrc.RawType);
var getter = row.GetGetter <TSrc>(row.Schema[col]);
var conv = Conversions.Instance.GetStringConversion <TSrc>(typeSrc);
var src = default(TSrc);
return
((ref StringBuilder dst) =>
{
getter(ref src);
conv(in src, ref dst);
});
}
/// <summary>
/// Given a type and annotation kind string, returns a key-value pair. This is useful when
/// implementing GetAnnotationTypes(col).
/// </summary>
public static KeyValuePair <string, DataViewType> GetPair(this DataViewType type, string kind)
{
Contracts.CheckValue(type, nameof(type));
return(new KeyValuePair <string, DataViewType>(kind, type));
}
private Delegate CreateGetterDelegateCore <TValue>(int col, DataViewType type)
{
return((Delegate)GetterDelegateCore <TValue>(col, type));
}
public Column(string columnName, DataViewType columnType, Delegate generator) :
this(columnName, columnType, null, generator, null)
{
}
//private Delegate CreateGetter(SchemaProxy schema, int index, Delegate peek)
private Delegate CreateGetter(DataViewType colType, InternalSchemaDefinition.Column column, Delegate peek)
{
var outputType = column.OutputType;
var genericType = outputType;
FuncInstanceMethodInfo1 <InputRowBase <TRow>, Delegate, Delegate> del;
if (outputType.IsArray)
{
VectorDataViewType vectorType = colType as VectorDataViewType;
Host.Assert(vectorType != null);
// String[] -> ReadOnlyMemory<char>
if (outputType.GetElementType() == typeof(string))
{
Host.Assert(vectorType.ItemType is TextDataViewType);
return(CreateConvertingArrayGetterDelegate <string, ReadOnlyMemory <char> >(peek, x => x != null ? x.AsMemory() : ReadOnlyMemory <char> .Empty));
}
// T[] -> VBuffer<T>
if (outputType.GetElementType().IsGenericType&& outputType.GetElementType().GetGenericTypeDefinition() == typeof(Nullable <>))
{
Host.Assert(Nullable.GetUnderlyingType(outputType.GetElementType()) == vectorType.ItemType.RawType);
}
else
{
Host.Assert(outputType.GetElementType() == vectorType.ItemType.RawType);
}
del = _createDirectArrayGetterDelegateMethodInfo;
genericType = outputType.GetElementType();
}
else if (colType is VectorDataViewType vectorType)
{
// VBuffer<T> -> VBuffer<T>
// REVIEW: Do we care about accommodating VBuffer<string> -> ReadOnlyMemory<char>?
Host.Assert(outputType.IsGenericType);
Host.Assert(outputType.GetGenericTypeDefinition() == typeof(VBuffer <>));
Host.Assert(outputType.GetGenericArguments()[0] == vectorType.ItemType.RawType);
del = _createDirectVBufferGetterDelegateMethodInfo;
genericType = vectorType.ItemType.RawType;
}
else if (colType is PrimitiveDataViewType)
{
if (outputType == typeof(string))
{
// String -> ReadOnlyMemory<char>
Host.Assert(colType is TextDataViewType);
return(CreateConvertingGetterDelegate <String, ReadOnlyMemory <char> >(peek, x => x != null ? x.AsMemory() : ReadOnlyMemory <char> .Empty));
}
// T -> T
if (outputType.IsGenericType && outputType.GetGenericTypeDefinition() == typeof(Nullable <>))
{
Host.Assert(colType.RawType == Nullable.GetUnderlyingType(outputType));
}
else
{
Host.Assert(colType.RawType == outputType);
}
if (!(colType is KeyDataViewType keyType))
{
del = _createDirectGetterDelegateMethodInfo;
}
else
{
var keyRawType = colType.RawType;
Func <Delegate, DataViewType, Delegate> delForKey = CreateKeyGetterDelegate <uint>;
return(Utils.MarshalInvoke(delForKey, keyRawType, peek, colType));
}
}
// REVIEW: It would be nice to support propagation of select metadata.
public static IDataView Create <TSrc, TDst>(IHostEnvironment env, string name, IDataView input,
string src, string dst, DataViewType typeSrc, DataViewType typeDst, ValueMapper <TSrc, TDst> mapper,
ValueGetter <VBuffer <ReadOnlyMemory <char> > > keyValueGetter = null, ValueGetter <VBuffer <ReadOnlyMemory <char> > > slotNamesGetter = null)
{
Contracts.CheckValue(env, nameof(env));
env.CheckNonEmpty(name, nameof(name));
env.CheckValue(input, nameof(input));
env.CheckNonEmpty(src, nameof(src));
env.CheckNonEmpty(dst, nameof(dst));
env.CheckValue(typeSrc, nameof(typeSrc));
env.CheckValue(typeDst, nameof(typeDst));
env.CheckValue(mapper, nameof(mapper));
env.Check(keyValueGetter == null || typeDst.GetItemType() is KeyDataViewType);
env.Check(slotNamesGetter == null || typeDst.IsKnownSizeVector());
if (typeSrc.RawType != typeof(TSrc))
{
throw env.ExceptParam(nameof(mapper),
"The source column type '{0}' doesn't match the input type of the mapper", typeSrc);
}
if (typeDst.RawType != typeof(TDst))
{
throw env.ExceptParam(nameof(mapper),
"The destination column type '{0}' doesn't match the output type of the mapper", typeDst);
}
bool tmp = input.Schema.TryGetColumnIndex(src, out int colSrc);
if (!tmp)
{
throw env.ExceptParam(nameof(src), "The input data doesn't have a column named '{0}'", src);
}
var typeOrig = input.Schema[colSrc].Type;
// REVIEW: Ideally this should support vector-type conversion. It currently doesn't.
bool ident;
Delegate conv;
if (typeOrig.SameSizeAndItemType(typeSrc))
{
ident = true;
conv = null;
}
else if (!Conversions.Instance.TryGetStandardConversion(typeOrig, typeSrc, out conv, out ident))
{
throw env.ExceptParam(nameof(mapper),
"The type of column '{0}', '{1}', cannot be converted to the input type of the mapper '{2}'",
src, typeOrig, typeSrc);
}
var col = new Column(src, dst);
IDataView impl;
if (ident)
{
impl = new Impl <TSrc, TDst, TDst>(env, name, input, col, typeDst, mapper, keyValueGetter: keyValueGetter, slotNamesGetter: slotNamesGetter);
}
else
{
Func <IHostEnvironment, string, IDataView, Column, DataViewType, ValueMapper <int, int>,
ValueMapper <int, int>, ValueGetter <VBuffer <ReadOnlyMemory <char> > >, ValueGetter <VBuffer <ReadOnlyMemory <char> > >,
Impl <int, int, int> > del = CreateImpl <int, int, int>;
var meth = del.GetMethodInfo().GetGenericMethodDefinition()
.MakeGenericMethod(typeOrig.RawType, typeof(TSrc), typeof(TDst));
impl = (IDataView)meth.Invoke(null, new object[] { env, name, input, col, typeDst, conv, mapper, keyValueGetter, slotNamesGetter });
}
return(new OpaqueDataView(impl));
}
public Column(string columnName, DataViewType columnType, MemberInfo memberInfo) :
this(columnName, columnType, memberInfo, null, null)
{
}
/// <summary> /// Zero return means it's not a key type. /// </summary> public static ulong GetKeyCount(this DataViewType columnType) => (columnType as KeyDataViewType)?.Count ?? 0;
/// <summary> /// For non-vector types, this returns the column type itself (i.e., return <paramref name="columnType"/>). /// For vector types, this returns the type of the items stored as values in vector. /// </summary> public static DataViewType GetItemType(this DataViewType columnType) => (columnType as VectorDataViewType)?.ItemType ?? columnType;
public static BindingsImpl Create(DataViewSchema input, ISchemaBoundRowMapper mapper, string suffix,
string scoreColKind, int scoreColIndex, DataViewType predColType, string predictedLabelColumnName = DefaultColumnNames.PredictedLabel)
{
Contracts.AssertValue(input);
Contracts.AssertValue(mapper);
Contracts.AssertValueOrNull(suffix);
Contracts.AssertNonEmpty(scoreColKind);
return(new BindingsImpl(input, mapper, suffix, scoreColKind, true,
scoreColIndex, predColType, predictedLabelColumnName));
}
/// <summary> /// Whether this type is a standard scalar type completely determined by its <see cref="DataViewType.RawType"/> /// (not a <see cref="KeyDataViewType"/> or <see cref="StructuredDataViewType"/>, etc). /// </summary> public static bool IsStandardScalar(this DataViewType columnType) => (columnType is NumberDataViewType) || (columnType is TextDataViewType) || (columnType is BooleanDataViewType) || (columnType is RowIdDataViewType) || (columnType is TimeSpanDataViewType) || (columnType is DateTimeDataViewType) || (columnType is DateTimeOffsetDataViewType);
private static bool OutputTypeMatches(DataViewType scoreType) => scoreType is VectorDataViewType vectorType && vectorType.IsKnownSize && vectorType.ItemType == NumberDataViewType.Single;
private static DataViewType GetPredColType(DataViewType scoreType, ISchemaBoundRowMapper mapper) => new KeyDataViewType(typeof(uint), scoreType.GetVectorSize());
internal static bool GetTypesForWrapping(ISchemaBoundMapper mapper, DataViewType labelNameType, string metaKind, out DataViewType scoreType)
{
Contracts.AssertValue(mapper);
Contracts.AssertValue(labelNameType);
scoreType = null;
ISchemaBoundRowMapper rowMapper = mapper as ISchemaBoundRowMapper;
if (rowMapper == null)
{
return(false); // We could cover this case, but it is of no practical worth as far as I see, so I decline to do so.
}
var outSchema = mapper.OutputSchema;
int scoreIdx;
var scoreCol = outSchema.GetColumnOrNull(AnnotationUtils.Const.ScoreValueKind.Score);
if (!outSchema.TryGetColumnIndex(AnnotationUtils.Const.ScoreValueKind.Score, out scoreIdx))
{
return(false); // The mapper doesn't even publish a score column to attach the metadata to.
}
if (outSchema[scoreIdx].Annotations.Schema.GetColumnOrNull(metaKind)?.Type != null)
{
return(false); // The mapper publishes a score column, and already produces its own metakind.
}
scoreType = outSchema[scoreIdx].Type;
return(true);
}
public static string TestGetLabelGetter(DataViewType type)
{
return(TestGetLabelGetter(type, true));
}
/// <summary> /// Zero return means either it's not a vector or the size is unknown. /// </summary> public static int GetVectorSize(this DataViewType columnType) => (columnType as VectorDataViewType)?.Size ?? 0;
private static ValueGetter <TDst> GetGetterAsCore <TSrc, TDst>(DataViewType typeSrc, DataViewType typeDst, DataViewRow row, int col)
{
Contracts.Assert(typeof(TSrc) == typeSrc.RawType);
Contracts.Assert(typeof(TDst) == typeDst.RawType);
var getter = row.GetGetter <TSrc>(row.Schema[col]);
bool identity;
var conv = Conversions.Instance.GetStandardConversion <TSrc, TDst>(typeSrc, typeDst, out identity);
if (identity)
{
Contracts.Assert(typeof(TSrc) == typeof(TDst));
return((ValueGetter <TDst>)(Delegate) getter);
}
var src = default(TSrc);
return
((ref TDst dst) =>
{
getter(ref src);
conv(in src, ref dst);
});
}
/// <summary> /// For non-vectors, this returns one. For unknown size vectors, it returns zero. /// For known sized vectors, it returns size. /// </summary> public static int GetValueCount(this DataViewType columnType) => (columnType as VectorDataViewType)?.Size ?? 1;
public static InternalSchemaDefinition Create(Type userType, SchemaDefinition userSchemaDefinition)
{
Contracts.AssertValue(userType);
Contracts.AssertValueOrNull(userSchemaDefinition);
if (userSchemaDefinition == null)
{
userSchemaDefinition = SchemaDefinition.Create(userType);
}
Column[] dstCols = new Column[userSchemaDefinition.Count];
for (int i = 0; i < userSchemaDefinition.Count; ++i)
{
var col = userSchemaDefinition[i];
if (col.MemberName == null)
{
throw Contracts.ExceptParam(nameof(userSchemaDefinition), "Null field name detected in schema definition");
}
bool isVector;
Type dataItemType;
MemberInfo memberInfo = null;
if (col.Generator == null)
{
memberInfo = userType.GetField(col.MemberName);
if (memberInfo == null)
{
memberInfo = userType.GetProperty(col.MemberName);
}
if (memberInfo == null)
{
throw Contracts.ExceptParam(nameof(userSchemaDefinition), "No field or property with name '{0}' found in type '{1}'",
col.MemberName,
userType.FullName);
}
//Clause to handle the field that may be used to expose the cursor channel.
//This field does not need a column.
if ((memberInfo is FieldInfo && (memberInfo as FieldInfo).FieldType == typeof(IChannel)) ||
(memberInfo is PropertyInfo && (memberInfo as PropertyInfo).PropertyType == typeof(IChannel)))
{
continue;
}
GetVectorAndItemType(memberInfo, out isVector, out dataItemType);
}
else
{
var parameterType = col.ReturnType;
if (parameterType == null)
{
throw Contracts.ExceptParam(nameof(userSchemaDefinition), "No return parameter found in computed column.");
}
GetVectorAndItemType(parameterType, "returnType", out isVector, out dataItemType);
}
// Infer the column name.
var colName = string.IsNullOrEmpty(col.ColumnName) ? col.MemberName : col.ColumnName;
// REVIEW: Because order is defined, we allow duplicate column names, since producing an IDataView
// with duplicate column names is completely legal. Possible objection is that we should make it less
// convenient to produce "hidden" columns, since this may not be of practical use to users.
DataViewType colType;
if (col.ColumnType == null)
{
// Infer a type as best we can.
PrimitiveDataViewType itemType = ColumnTypeExtensions.PrimitiveTypeFromType(dataItemType);
colType = isVector ? new VectorDataViewType(itemType) : (DataViewType)itemType;
}
else
{
// Make sure that the types are compatible with the declared type, including
// whether it is a vector type.
VectorDataViewType columnVectorType = col.ColumnType as VectorDataViewType;
if (isVector != (columnVectorType != null))
{
throw Contracts.ExceptParam(nameof(userSchemaDefinition), "Column '{0}' is supposed to be {1}, but type of associated field '{2}' is {3}",
colName, columnVectorType != null ? "vector" : "scalar", col.MemberName, isVector ? "vector" : "scalar");
}
DataViewType itemType = columnVectorType?.ItemType ?? col.ColumnType;
if (itemType.RawType != dataItemType)
{
throw Contracts.ExceptParam(nameof(userSchemaDefinition), "Column '{0}' is supposed to have item type {1}, but associated field has type {2}",
colName, itemType.RawType, dataItemType);
}
colType = col.ColumnType;
}
dstCols[i] = col.Generator != null ?
new Column(colName, colType, col.Generator, col.AnnotationInfos)
: new Column(colName, colType, memberInfo, col.AnnotationInfos);
}
return(new InternalSchemaDefinition(dstCols));
}
/// <summary> /// Whether this is a vector type with known size. Returns false for non-vector types. /// Equivalent to <c><see cref="GetVectorSize"/> > 0</c>. /// </summary> public static bool IsKnownSizeVector(this DataViewType columnType) => columnType.GetVectorSize() > 0;
public Column(string columnName, DataViewType columnType, MemberInfo memberInfo,
Dictionary <string, AnnotationInfo> metadataInfos) :
this(columnName, columnType, memberInfo, null, metadataInfos)
{
}
/// <summary>
/// Gets the equivalent <see cref="InternalDataKind"/> for the <paramref name="columnType"/>'s RawType.
/// This can return default(<see cref="InternalDataKind"/>) if the RawType doesn't have a corresponding
/// <see cref="InternalDataKind"/>.
/// </summary>
public static InternalDataKind GetRawKind(this DataViewType columnType)
{
columnType.RawType.TryGetDataKind(out InternalDataKind result);
return(result);
}
public Column(string columnName, DataViewType columnType, Delegate generator,
Dictionary <string, AnnotationInfo> metadataInfos) :
this(columnName, columnType, null, generator, metadataInfos)
{
}
public GetterInfoPrimitive(string kind, DataViewType type, TValue value)
: base(kind, type)
{
Contracts.Check(type.RawType == typeof(TValue), "Incompatible types");
Value = value;
}
private static bool OutputTypeMatches(DataViewType scoreType) => scoreType == NumberDataViewType.Single;
public Column(DataViewType type)
{
Type = type;
}