public GetterInfoDelegate(string kind, ColumnType type, MetadataUtils.MetadataGetter <TValue> getter)
: base(kind, type)
{
Contracts.Check(type.RawType == typeof(TValue), "Incompatible types");
Contracts.CheckValue(getter, nameof(getter));
Getter = getter;
}
public Impl(IHostEnvironment env, string name, IDataView input, OneToOneColumn col,
ColumnType typeDst, ValueMapper <T1, T2> map1, ValueMapper <T2, T3> map2 = null,
ValueGetter <VBuffer <DvText> > keyValueGetter = null, ValueGetter <VBuffer <DvText> > slotNamesGetter = null)
: base(env, name, new[] { col }, input, x => null)
{
Host.Assert(typeDst.RawType == typeof(T3));
Host.AssertValue(map1);
Host.Assert(map2 != null || typeof(T2) == typeof(T3));
_typeDst = typeDst;
_map1 = map1;
_map2 = map2;
if (keyValueGetter != null || slotNamesGetter != null)
{
using (var bldr = Metadata.BuildMetadata(0))
{
if (keyValueGetter != null)
{
Host.Assert(_typeDst.ItemType.KeyCount > 0);
MetadataUtils.MetadataGetter <VBuffer <DvText> > mdGetter =
(int c, ref VBuffer <DvText> dst) => keyValueGetter(ref dst);
bldr.AddGetter(MetadataUtils.Kinds.KeyValues, new VectorType(TextType.Instance, _typeDst.ItemType.KeyCount), mdGetter);
}
if (slotNamesGetter != null)
{
Host.Assert(_typeDst.VectorSize > 0);
MetadataUtils.MetadataGetter <VBuffer <DvText> > mdGetter =
(int c, ref VBuffer <DvText> dst) => slotNamesGetter(ref dst);
bldr.AddGetter(MetadataUtils.Kinds.SlotNames, new VectorType(TextType.Instance, _typeDst.VectorSize), mdGetter);
}
}
}
Metadata.Seal();
}
public Bindings(ModelLoadContext ctx, ISchema schemaInput)
: base(ctx, schemaInput, TestTypes)
{
// *** Binary format ***
// (base fields)
// if version >= VersionAddedAliases
// foreach column:
// foreach non-null alias
// int: index of the alias
// int: string id of the alias
// int: -1, marks the end of the list
_aliases = new string[Infos.Length][];
for (int i = 0; i < Infos.Length; i++)
{
var length = Infos[i].SrcIndices.Length;
_aliases[i] = new string[length];
if (ctx.Header.ModelVerReadable >= VersionAddedAliases)
{
for (; ;)
{
var j = ctx.Reader.ReadInt32();
if (j == -1)
{
break;
}
Contracts.CheckDecode(0 <= j && j < length);
Contracts.CheckDecode(_aliases[i][j] == null);
_aliases[i][j] = ctx.LoadNonEmptyString();
}
}
}
CacheTypes(out _types, out _typesSlotNames, out EchoSrc, out _isNormalized, out _typesCategoricals);
_getSlotNames = GetSlotNames;
}
public Bindings(Column[] columns, TaggedColumn[] taggedColumns, ISchema schemaInput)
: base(columns, schemaInput, TestTypes)
{
Contracts.Assert(taggedColumns == null || columns.Length == taggedColumns.Length);
_aliases = new string[columns.Length][];
for (int i = 0; i < columns.Length; i++)
{
_aliases[i] = new string[columns[i].Source.Length];
if (taggedColumns != null)
{
var column = taggedColumns[i];
Contracts.Assert(columns[i].Name == column.Name);
Contracts.AssertValue(columns[i].Source);
Contracts.AssertValue(column.Source);
Contracts.Assert(columns[i].Source.Length == column.Source.Length);
for (int j = 0; j < column.Source.Length; j++)
{
var kvp = column.Source[j];
Contracts.Assert(columns[i].Source[j] == kvp.Value);
if (!string.IsNullOrEmpty(kvp.Key))
{
_aliases[i][j] = kvp.Key;
}
}
}
}
CacheTypes(out _types, out _typesSlotNames, out EchoSrc, out _isNormalized, out _typesCategoricals);
_getSlotNames = GetSlotNames;
}
private BindingsImpl(DataViewSchema input, ISchemaBoundRowMapper mapper, string suffix, string scoreColumnKind,
bool user, int scoreColIndex, DataViewType predColType)
: base(input, mapper, suffix, user, DefaultColumnNames.PredictedLabel)
{
Contracts.AssertNonEmpty(scoreColumnKind);
Contracts.Assert(DerivedColumnCount == 1);
ScoreColumnIndex = scoreColIndex;
ScoreColumnKind = scoreColumnKind;
PredColType = predColType;
_getScoreColumnKind = GetScoreColumnKind;
_getScoreValueKind = GetScoreValueKind;
// REVIEW: This logic is very specific to multiclass, which is deeply
// regrettable, but the class structure as designed and the status of this schema
// bearing object makes pushing the logic into the multiclass scorer almost impossible.
if (predColType is KeyType predColKeyType && predColKeyType.Count > 0)
{
var scoreColMetadata = mapper.OutputSchema[scoreColIndex].Metadata;
var slotColumn = scoreColMetadata.Schema.GetColumnOrNull(MetadataUtils.Kinds.SlotNames);
if (slotColumn?.Type is VectorType slotColVecType && (ulong)slotColVecType.Size == predColKeyType.Count)
{
Contracts.Assert(slotColVecType.Size > 0);
_predColMetadata = Utils.MarshalInvoke(KeyValueMetadataFromMetadata <int>, slotColVecType.RawType,
scoreColMetadata, slotColumn.Value);
}
public FeatureNameCollectionSchema(FeatureNameCollection collection)
{
Contracts.CheckValue(collection, nameof(collection));
_collection = collection;
_colType = new VectorType(NumberType.R4, collection.Count);
_slotNamesType = new VectorType(TextType.Instance, collection.Count);
_getSlotNames = GetSlotNames;
}
public Bindings(IExceptionContext ectx, ISchema input, bool user, string labelCol, string scoreCol, string groupCol,
int truncationLevel)
: base(ectx, input, labelCol, scoreCol, groupCol, user, Ndcg, Dcg, MaxDcg)
{
_truncationLevel = truncationLevel;
_outputType = new VectorType(NumberType.R8, _truncationLevel);
_slotNamesType = new VectorType(TextType.Instance, _truncationLevel);
_slotNamesGetter = SlotNamesGetter;
}
public ScoreMapperSchemaBase(ColumnType scoreType, string scoreColumnKind)
{
Contracts.CheckValue(scoreType, nameof(scoreType));
Contracts.CheckNonEmpty(scoreColumnKind, nameof(scoreColumnKind));
ScoreType = scoreType;
ScoreColumnKind = scoreColumnKind;
ScoreValueKindGetter = GetScoreValueKind;
ScoreColumnKindGetter = GetScoreColumnKind;
}
protected override void GetMetadataCore <TValue>(string kind, int iinfo, ref TValue value)
{
if (kind == MetadataUtils.Kinds.SlotNames && _parent._slotNames != null && _parent._slotNames[iinfo].Length > 0)
{
MetadataUtils.MetadataGetter <VBuffer <ReadOnlyMemory <char> > > getTerms = _parent.GetTerms;
getTerms.Marshal(iinfo, ref value);
return;
}
base.GetMetadataCore(kind, iinfo, ref value);
}
public Schema(ColumnType scoreType, Double[] quantiles)
: base(scoreType, MetadataUtils.Const.ScoreColumnKind.QuantileRegression)
{
Contracts.Assert(Utils.Size(quantiles) > 0);
_slotNames = new string[quantiles.Length];
for (int i = 0; i < _slotNames.Length; i++)
{
_slotNames[i] = string.Format("Quantile-{0}", quantiles[i]);
}
_getSlotNames = GetSlotNames;
}
internal static ISchema GetModelSchema(IExceptionContext ectx, TFGraph graph, string opType = null)
{
var res = new List <KeyValuePair <string, ColumnType> >();
var opTypeGetters = new List <MetadataUtils.MetadataGetter <ReadOnlyMemory <char> > >();
var inputOpsGetters = new List <MetadataUtils.MetadataGetter <VBuffer <ReadOnlyMemory <char> > > >();
var inputOpsLengths = new List <int>();
foreach (var op in graph)
{
if (opType != null && opType != op.OpType)
{
continue;
}
var tfType = op[0].OutputType;
var mlType = Tf2MlNetTypeOrNull(tfType);
// If the type is not supported in ML.NET then we cannot represent it as a column in an ISchema.
// We also cannot output it with a TensorFlowTransform, so we skip it.
if (mlType == null)
{
continue;
}
var shape = graph.GetTensorShape(op[0]);
var shapeArray = shape.ToIntArray();
inputOpsLengths.Add(op.NumInputs);
MetadataUtils.MetadataGetter <VBuffer <ReadOnlyMemory <char> > > inputOpsGetter = null;
if (op.NumInputs > 0)
{
var inputOps = new ReadOnlyMemory <char> [op.NumInputs];
for (int i = 0; i < op.NumInputs; i++)
{
var input = op.GetInput(i);
inputOps[i] = new ReadOnlyMemory <char>(input.Operation.Name.ToArray());
}
inputOpsGetter = (int col, ref VBuffer <ReadOnlyMemory <char> > dst) =>
dst = new VBuffer <ReadOnlyMemory <char> >(op.NumInputs, inputOps);
}
inputOpsGetters.Add(inputOpsGetter);
MetadataUtils.MetadataGetter <ReadOnlyMemory <char> > opTypeGetter =
(int col, ref ReadOnlyMemory <char> dst) => dst = new ReadOnlyMemory <char>(op.OpType.ToArray());
opTypeGetters.Add(opTypeGetter);
var columnType = Utils.Size(shapeArray) == 1 && shapeArray[0] <= 0 ? new VectorType(mlType) :
Utils.Size(shapeArray) > 0 && shapeArray.Skip(1).All(x => x > 0) ?
new VectorType(mlType, shapeArray[0] > 0 ? shapeArray : shapeArray.Skip(1).ToArray())
: new VectorType(mlType);
res.Add(new KeyValuePair <string, ColumnType>(op.Name, columnType));
}
return(new TensorFlowSchema(ectx, res.ToArray(), opTypeGetters.ToArray(), inputOpsGetters.ToArray(), inputOpsLengths.ToArray()));
}
/// <summary>
/// Add metadata of the given kind. When requested, the metadata is fetched by calling the given delegate.
/// </summary>
public void AddGetter <TValue>(string kind, ColumnType type,
MetadataUtils.MetadataGetter <TValue> getter)
{
Contracts.Check(_md != null, "Builder disposed");
Contracts.CheckNonEmpty(kind, nameof(kind));
Contracts.CheckValue(type, nameof(type));
Contracts.CheckValue(getter, nameof(getter));
Contracts.CheckParam(type.RawType == typeof(TValue), nameof(type), "Given type doesn't match type parameter");
if (_getters != null && _getters.Any(g => g.Kind == kind))
{
throw Contracts.Except("Duplicate specification of metadata");
}
Utils.Add(ref _getters, new GetterInfoDelegate <TValue>(kind, type, getter));
}
/// <summary>
/// Constructs the schema.
/// </summary>
/// <param name="schema">The schema we will wrap.
/// Aside from presenting that additional piece of metadata, the constructed schema
/// will appear identical to this input schema.</param>
/// <param name="col">The column in <paramref name="schema"/> that has the metadata.</param>
/// <param name="names"></param>
/// <param name="length"></param>
public SchemaImpl(ISchema schema, int col, string[] names, int length)
{
Contracts.Assert(length > 0);
Contracts.Assert(Utils.Size(names) >= length);
Contracts.AssertValue(schema);
Contracts.Assert(0 <= col && col < schema.ColumnCount);
_parent = schema;
_col = col;
_names = names;
_namesLength = length;
_type = new VectorType(TextType.Instance, _namesLength);
Contracts.AssertValue(_type);
_getter = GetSlotNames;
}
// REVIEW: Future iterations may want slot names.
/// <summary>
/// Constructs an <see cref="ISchemaBoundMapper"/> with one column of a given type, called PredictedLabel.
/// If the input <paramref name="keyNames"/> has positive length, it is exposed as
/// <see cref="MetadataUtils.Kinds.KeyValues"/> metadata. Note that we do not copy
/// the input key names, but instead take a reference to it.
/// </summary>
public SequencePredictorSchema(ColumnType type, ref VBuffer <ReadOnlyMemory <char> > keyNames, string scoreColumnKind)
: base(type, scoreColumnKind)
{
if (keyNames.Length > 0)
{
Contracts.CheckParam(type.ItemType.IsKey,
nameof(keyNames), "keyNames valid only for key type");
Contracts.CheckParam(keyNames.Length == type.ItemType.KeyCount,
nameof(keyNames), "keyNames length must match type's key count");
// REVIEW: Assuming the caller takes some care, it seems
// like we can get away with
_keyNames = keyNames;
_keyNamesType = new VectorType(TextType.Instance, keyNames.Length);
_getKeyNames = GetKeyNames;
}
}
protected ScorerBindingsBase(Schema input, ISchemaBoundMapper mapper, string suffix, bool user, params string[] namesDerived)
: base(input, user, GetOutputNames(mapper, suffix, namesDerived))
{
Contracts.AssertValue(mapper);
Contracts.AssertValueOrNull(suffix);
Contracts.AssertValue(namesDerived);
Mapper = mapper;
DerivedColumnCount = namesDerived.Length;
Suffix = suffix ?? "";
int c;
var max = input.GetMaxMetadataKind(out c, MetadataUtils.Kinds.ScoreColumnSetId);
_crtScoreSet = checked(max + 1);
_getScoreColumnSetId = GetScoreColumnSetId;
}
public SchemaImpl(ISchema parent, int col, VectorType type, ValueGetter <VBuffer <T> > getter, string metadataKind)
{
Contracts.AssertValue(parent);
Contracts.Assert(0 <= col && col < parent.ColumnCount);
Contracts.AssertValue(type);
Contracts.AssertValue(getter);
Contracts.Assert(type.ItemType.RawType == typeof(T));
Contracts.AssertNonEmpty(metadataKind);
Contracts.Assert(parent.GetMetadataTypeOrNull(metadataKind, col) == null);
_parent = parent;
_scoreCol = col;
_labelNameType = type;
// We change to this metadata variant of the getter to enable the marshal call to work.
_labelNameGetter = (int c, ref VBuffer <T> val) => getter(ref val);
_metadataKind = metadataKind;
}
private BindingsImpl(ISchema input, ISchemaBoundRowMapper mapper, string suffix, string scoreColumnKind,
bool user, int scoreColIndex, ColumnType predColType)
: base(input, mapper, suffix, user, DefaultColumnNames.PredictedLabel)
{
Contracts.AssertNonEmpty(scoreColumnKind);
Contracts.Assert(DerivedColumnCount == 1);
ScoreColumnIndex = scoreColIndex;
ScoreColumnKind = scoreColumnKind;
PredColType = predColType;
_getScoreColumnKind = GetScoreColumnKind;
_getScoreValueKind = GetScoreValueKind;
// REVIEW: This logic is very specific to multiclass, which is deeply
// regrettable, but the class structure as designed and the status of this schema
// bearing object makes pushing the logic into the multiclass scorer almost impossible.
if (predColType.IsKey)
{
ColumnType scoreSlotsType = mapper.Schema.GetMetadataTypeOrNull(MetadataUtils.Kinds.SlotNames, scoreColIndex);
if (scoreSlotsType != null && scoreSlotsType.IsKnownSizeVector &&
scoreSlotsType.VectorSize == predColType.KeyCount)
{
Contracts.Assert(scoreSlotsType.VectorSize > 0);
IColumn col = Utils.MarshalInvoke(KeyValueMetadataFromMetadata <int>,
scoreSlotsType.RawType, mapper.Schema, scoreColIndex, MetadataUtils.Kinds.SlotNames);
_predColMetadata = RowColumnUtils.GetRow(null, col);
}
else
{
scoreSlotsType = mapper.Schema.GetMetadataTypeOrNull(MetadataUtils.Kinds.TrainingLabelValues, scoreColIndex);
if (scoreSlotsType != null && scoreSlotsType.IsKnownSizeVector &&
scoreSlotsType.VectorSize == predColType.KeyCount)
{
Contracts.Assert(scoreSlotsType.VectorSize > 0);
IColumn col = Utils.MarshalInvoke(KeyValueMetadataFromMetadata <int>,
scoreSlotsType.RawType, mapper.Schema, scoreColIndex, MetadataUtils.Kinds.TrainingLabelValues);
_predColMetadata = RowColumnUtils.GetRow(null, col);
}
}
}
}
