public DataView(IHostEnvironment env, ArrayDataViewBuilder builder, int rowCount)
{
Contracts.AssertValue(env, "env");
_host = env.Register("ArrayDataView");
_host.AssertValue(builder);
_host.Assert(rowCount >= 0);
_host.Assert(builder._names.Count == builder._columns.Count);
_columns = builder._columns.ToArray();
var schemaCols = new Schema.Column[_columns.Length];
for (int i = 0; i < schemaCols.Length; i++)
{
var meta = new Schema.Metadata.Builder();
if (builder._getSlotNames.TryGetValue(builder._names[i], out var slotNamesGetter))
{
meta.AddSlotNames(_columns[i].Type.VectorSize, slotNamesGetter);
}
if (builder._getKeyValues.TryGetValue(builder._names[i], out var keyValueGetter))
{
meta.AddKeyValues(_columns[i].Type.KeyCount, TextType.Instance, keyValueGetter);
}
schemaCols[i] = new Schema.Column(builder._names[i], _columns[i].Type, meta.GetMetadata());
}
_schema = new Schema(schemaCols);
_rowCount = rowCount;
}
public Schema.Column[] GetOutputColumns()
{
var meta = new Schema.Metadata.Builder();
meta.AddSlotNames(_parent._outputLength, GetSlotNames);
var info = new Schema.Column[1];
info[0] = new Schema.Column(_parent.OutputColumnName, new VectorType(NumberType.R8, _parent._outputLength), meta.GetMetadata());
return(info);
}
private void AddMetadata(int iinfo, Schema.Metadata.Builder builder)
{
builder.Add(InputSchema[_parent.ColumnPairs[iinfo].input].Metadata, name => name == MetadataUtils.Kinds.SlotNames);
ValueGetter <VBuffer <ReadOnlyMemory <char> > > getter =
(ref VBuffer <ReadOnlyMemory <char> > dst) =>
{
GetKeyValues(iinfo, ref dst);
};
builder.AddKeyValues(CharsCount, TextType.Instance, getter);
}
public override Schema.Column[] GetOutputColumns()
{
var result = new Schema.Column[_parent.ColumnPairs.Length];
for (int i = 0; i < _parent.ColumnPairs.Length; i++)
{
var meta = new Schema.Metadata.Builder();
meta.Add(InputSchema[ColMapNewToOld[i]].Metadata, name => name == MetadataUtils.Kinds.SlotNames);
result[i] = new Schema.Column(_parent.ColumnPairs[i].output, _types[i], meta.GetMetadata());
}
return(result);
}
public override Schema.Column[] GetOutputColumns()
{
var result = new Schema.Column[_parent.ColumnPairs.Length];
for (int i = 0; i < _parent.ColumnPairs.Length; i++)
{
var builder = new Schema.Metadata.Builder();
builder.Add(InputSchema[ColMapNewToOld[i]].Metadata, x => x == MetadataUtils.Kinds.KeyValues || x == MetadataUtils.Kinds.IsNormalized);
result[i] = new Schema.Column(_parent.ColumnPairs[i].output, _types[i], builder.GetMetadata());
}
return(result);
}
public override Schema.Column[] GetOutputColumns()
{
var result = new Schema.Column[_parent.ColumnPairs.Length];
for (int i = 0; i < _parent.ColumnPairs.Length; i++)
{
var builder = new Schema.Metadata.Builder();
AddMetadata(i, builder);
result[i] = new Schema.Column(_parent.ColumnPairs[i].output, _type, builder.GetMetadata());
}
return(result);
}
public override Schema.Column[] GetOutputColumns()
{
var infos = new Schema.Column[2];
var slotNamesType = new VectorType(TextType.Instance, _scoreSize);
var l1Metadata = new Schema.Metadata.Builder();
l1Metadata.AddSlotNames(_scoreSize, CreateSlotNamesGetter(L1));
var l2Metadata = new Schema.Metadata.Builder();
l2Metadata.AddSlotNames(_scoreSize, CreateSlotNamesGetter(L2));
infos[L1Col] = new Schema.Column(L1, _outputType, l1Metadata.GetMetadata());
infos[L2Col] = new Schema.Column(L2, _outputType, l2Metadata.GetMetadata());
return(infos);
}
internal static Schema.Column[] GetSchemaColumns(InternalSchemaDefinition schemaDefn)
{
Contracts.AssertValue(schemaDefn);
var columns = new Schema.Column[schemaDefn.Columns.Length];
for (int i = 0; i < columns.Length; i++)
{
var col = schemaDefn.Columns[i];
var meta = new Schema.Metadata.Builder();
foreach (var kvp in col.Metadata)
{
meta.Add(new Schema.Column(kvp.Value.Kind, kvp.Value.MetadataType, null), kvp.Value.GetGetterDelegate());
}
columns[i] = new Schema.Column(col.ColumnName, col.ColumnType, meta.GetMetadata());
}
return(columns);
}
public override Schema.Column[] GetOutputColumns()
{
var result = new Schema.Column[_parent.ColumnPairs.Length];
for (int iinfo = 0; iinfo < _infos.Length; iinfo++)
{
InputSchema.TryGetColumnIndex(_infos[iinfo].Input, out int colIndex);
Host.Assert(colIndex >= 0);
var builder = new Schema.Metadata.Builder();
builder.Add(InputSchema[colIndex].Metadata, x => x == MetadataUtils.Kinds.SlotNames);
ValueGetter <bool> getter = (ref bool dst) =>
{
dst = true;
};
builder.Add(new Schema.Column(MetadataUtils.Kinds.IsNormalized, BoolType.Instance, null), getter);
result[iinfo] = new Schema.Column(_infos[iinfo].Output, _infos[iinfo].OutputType, builder.GetMetadata());
}
return(result);
}
public override Schema.Column[] GetOutputColumns()
{
var infos = new Schema.Column[3];
infos[ClusterIdCol] = new Schema.Column(ClusterId, _types[ClusterIdCol], null);
var slotNamesType = new VectorType(TextType.Instance, _numClusters);
var sortedClusters = new Schema.Metadata.Builder();
sortedClusters.AddSlotNames(slotNamesType.VectorSize, CreateSlotNamesGetter(_numClusters, "Cluster"));
var sortedClusterScores = new Schema.Metadata.Builder();
sortedClusterScores.AddSlotNames(slotNamesType.VectorSize, CreateSlotNamesGetter(_numClusters, "Score"));
infos[SortedClusterCol] = new Schema.Column(SortedClusters, _types[SortedClusterCol], sortedClusters.GetMetadata());
infos[SortedClusterScoreCol] = new Schema.Column(SortedClusterScores, _types[SortedClusterScoreCol], sortedClusterScores.GetMetadata());
return(infos);
}
private void CheckInputColumnTypes(Schema schema, out ColumnType labelType, out ColumnType scoreType,
out Schema.Metadata labelMetadata, out Schema.Metadata scoreMetadata)
{
Host.AssertNonEmpty(ScoreCol);
Host.AssertNonEmpty(LabelCol);
var t = schema.GetColumnType(LabelIndex);
if (!t.IsKnownSizeVector || (t.ItemType != NumberType.R4 && t.ItemType != NumberType.R8))
{
throw Host.Except("Label column '{0}' has type '{1}' but must be a known-size vector of R4 or R8", LabelCol, t);
}
labelType = new VectorType(t.ItemType.AsPrimitive, t.VectorSize);
var slotNamesType = new VectorType(TextType.Instance, t.VectorSize);
var builder = new Schema.Metadata.Builder();
builder.AddSlotNames(t.VectorSize, CreateSlotNamesGetter(schema, LabelIndex, labelType.VectorSize, "True"));
labelMetadata = builder.GetMetadata();
t = schema.GetColumnType(ScoreIndex);
if (t.VectorSize == 0 || t.ItemType != NumberType.Float)
{
throw Host.Except("Score column '{0}' has type '{1}' but must be a known length vector of type R4", ScoreCol, t);
}
scoreType = new VectorType(t.ItemType.AsPrimitive, t.VectorSize);
builder = new Schema.Metadata.Builder();
builder.AddSlotNames(t.VectorSize, CreateSlotNamesGetter(schema, ScoreIndex, scoreType.VectorSize, "Predicted"));
ValueGetter <ReadOnlyMemory <char> > getter = GetScoreColumnKind;
builder.Add(new Schema.Column(MetadataUtils.Kinds.ScoreColumnKind, TextType.Instance, null), getter);
getter = GetScoreValueKind;
builder.Add(new Schema.Column(MetadataUtils.Kinds.ScoreValueKind, TextType.Instance, null), getter);
ValueGetter <uint> uintGetter = GetScoreColumnSetId(schema);
builder.Add(new Schema.Column(MetadataUtils.Kinds.ScoreColumnSetId, MetadataUtils.ScoreColumnSetIdType, null), uintGetter);
scoreMetadata = builder.GetMetadata();
}