/// <summary>
/// Returns a value getter delegate to fetch the value of column with the given columnIndex, from the row.
/// This throws if the column is not active in this row, or if the type
/// <typeparamref name="TValue"/> differs from this column's type.
/// </summary>
/// <typeparam name="TValue"> is the column's content type.</typeparam>
/// <param name="column"> is the output column whose getter should be returned.</param>
public override ValueGetter <TValue> GetGetter <TValue>(DataViewSchema.Column column)
{
Ch.Check(column.Index < Schema.Count);
Ch.Check(column.Index < _active.Length && _active[column.Index], "the requested column is not active");
var columnValue = _view._columns[column.Index] as Column <TValue>;
if (columnValue == null)
{
throw Ch.Except("Invalid TValue: '{0}'", typeof(TValue));
}
return
((ref TValue value) =>
{
Ch.Check(IsGood, RowCursorUtils.FetchValueStateError);
columnValue.CopyOut(MappedIndex(), ref value);
});
}
public override ValueGetter <T> GetGetter <T>(DataViewSchema.Column column)
{
bool isSrc;
int index = _bindings.MapColumnIndex(out isSrc, column.Index);
if (isSrc)
{
return(_input.GetGetter <T>(_input.Schema[index]));
}
Contracts.CheckParam(index < _getters.Length, nameof(column), "Invalid col value in GetGetter");
Contracts.Check(IsColumnActive(column));
var fn = _getters[index] as ValueGetter <T>;
if (fn == null)
{
throw Contracts.Except("Unexpected TValue in GetGetter");
}
return(fn);
}
public override ValueGetter <TValue> GetGetter <TValue>(DataViewSchema.Column col)
{
if (col.Index < _view.Source.Schema.Count)
{
return(_inputCursor.GetGetter <TValue>(col));
}
else if (col.Index == _view.Source.Schema.Count) // Cluster
{
return(GetGetterCluster() as ValueGetter <TValue>);
}
else if (col.Index == _view.Source.Schema.Count + 1) // Score
{
return(GetGetterScore() as ValueGetter <TValue>);
}
else
{
throw new IndexOutOfRangeException();
}
}
/// <summary>
/// Returns a value getter delegate to fetch the value of column with the given columnIndex, from the row.
/// This throws if the column is not active in this row, or if the type
/// <typeparamref name="TValue"/> differs from this column's type.
/// </summary>
/// <typeparam name="TValue"> is the column's content type.</typeparam>
/// <param name="column"> is the output column whose getter should be returned.</param>
public override ValueGetter <TValue> GetGetter <TValue>(DataViewSchema.Column column)
{
bool isSrc;
int index = _parent._bindings.MapColumnIndex(out isSrc, column.Index);
if (isSrc)
{
return(_input.GetGetter <TValue>(column));
}
Contracts.Assert(_getters[index] != null);
var fn = _getters[index] as ValueGetter <TValue>;
if (fn == null)
{
throw Contracts.Except("Invalid TValue in GetGetter: '{0}'", typeof(TValue));
}
return(fn);
}
/// <summary>
/// Returns a value getter delegate to fetch the value of column with the given columnIndex, from the row.
/// This throws if the column is not active in this row, or if the type
/// <typeparamref name="TValue"/> differs from this column's type.
/// </summary>
/// <typeparam name="TValue"> is the column's content type.</typeparam>
/// <param name="column"> is the output column whose getter should be returned.</param>
public override ValueGetter <TValue> GetGetter <TValue>(DataViewSchema.Column column)
{
Ch.Check(0 <= column.Index && column.Index < Schema.Count);
Ch.Check(IsColumnActive(column));
if (column.Index != Parent._index)
{
return(Input.GetGetter <TValue>(column));
}
var originFn = GetGetter();
var fn = originFn as ValueGetter <TValue>;
if (fn == null)
{
throw Ch.Except($"Invalid TValue in GetGetter: '{typeof(TValue)}', " +
$"expected type: '{originFn.GetType().GetGenericArguments().First()}'.");
}
return(fn);
}
public override ValueGetter <TValue> GetGetter <TValue>(DataViewSchema.Column col)
{
// If the column is part of the input view.
var schema = _inputCursor.Schema;
if (col.Index < schema.Count)
{
return(_inputCursor.GetGetter <TValue>(col));
}
// If it is the added column.
else if (col.Index == schema.Count)
{
return(PolynomialBuilder() as ValueGetter <TValue>);
}
// Otherwise, it is an error.
else
{
throw Contracts.Except("Unexpected columns {0} > {1}.", col, schema.Count);
}
}
void LoadCache <TClass>(Random rand, DataViewRowCursor cur, DataViewSchema.Column classColumn, TClass valueClass, IChannel ch)
{
_cacheReplica = new Dictionary <DataViewRowId, int>();
var hist = new Dictionary <TClass, long>();
var gid = cur.GetIdGetter();
var gcl = cur.GetGetter <TClass>(classColumn);
DataViewRowId did = default(DataViewRowId);
TClass cl = default(TClass);
long nbIn = 0;
long nbOut = 0;
int rep;
while (cur.MoveNext())
{
gcl(ref cl);
gid(ref did);
if (!hist.ContainsKey(cl))
{
hist[cl] = 1;
}
else
{
++hist[cl];
}
if (cl.Equals(valueClass))
{
rep = NextPoisson(_args.lambda, rand);
++nbIn;
}
else
{
rep = 1;
++nbOut;
}
_cacheReplica[did] = rep;
}
if (nbIn == 0)
{
ch.Warning(MessageSensitivity.UserData, "Resample on a condition never happened: nbIn={0} nbOut={1}", nbIn, nbOut);
}
}
/// <summary>
/// Returns the getter of an output column.
/// </summary>
/// <typeparam name="TValue"> is the output column's content type, for example, <see cref="VBuffer{T}"/>.</typeparam>
/// <param name="column"> is the output column whose getter should be returned.</param>
public override ValueGetter <TValue> GetGetter <TValue>(DataViewSchema.Column column)
{
// Although the input argument, col, is a output index, we check its range as if it's an input column index.
// It makes sense because the i-th output column is produced by either expanding or copying the i-th input column.
Ch.CheckParam(column.Index < _ungroupBinding.InputColumnCount, nameof(column));
if (!_ungroupBinding.IsPivot(column.Index))
{
return(Input.GetGetter <TValue>(column));
}
if (_cachedGetters[column.Index] == null)
{
_cachedGetters[column.Index] = MakeGetter <TValue>(column.Index, _ungroupBinding.GetPivotColumnOptionsByCol(column.Index).ItemType);
}
var result = _cachedGetters[column.Index] as ValueGetter <TValue>;
Ch.Check(result != null, "Unexpected getter type requested");
return(result);
}
/// <summary>
/// Returns a value getter delegate to fetch the value of column with the given columnIndex, from the row.
/// This throws if the column is not active in this row, or if the type
/// <typeparamref name="TValue"/> differs from this column's type.
/// </summary>
/// <typeparam name="TValue"> is the column's content type.</typeparam>
/// <param name="column"> is the output column whose getter should be returned.</param>
public override ValueGetter <TValue> GetGetter <TValue>(DataViewSchema.Column column)
{
Ch.Check(IsColumnActive(column));
bool isSrc;
int index = _bindings.MapColumnIndex(out isSrc, column.Index);
if (isSrc)
{
return(Input.GetGetter <TValue>(Input.Schema[index]));
}
Ch.Assert(_getters[index] != null);
var fn = _getters[index] as ValueGetter <TValue>;
if (fn == null)
{
throw Ch.Except("Invalid TValue in GetGetter: '{0}'", typeof(TValue));
}
return(fn);
}
ValueGetter <VBuffer <ReadOnlyMemory <char> > > GetGetterVector(DataViewSchema.Column col, ReadOnlyMemory <char> defval)
{
string alpha = "abcdefghijklmnopqrstuvwxyz";
var getter = _inputCursor.GetGetter <VBuffer <ReadOnlyMemory <char> > >(_view._columnMapping[col.Index]);
if (getter == null)
{
throw _view._host.Except($"Unable to create a getter for column {_view._columnMapping[col.Index]} from schema\n{SchemaHelper.ToString(_inputCursor.Schema)}.");
}
string cs;
return((ref VBuffer <ReadOnlyMemory <char> > value) =>
{
getter(ref value);
for (int i = 0; i < value.Length; ++i)
{
cs = value.Values[i].ToString();
cs = cs + alpha[_rand.Next() % alpha.Length];
value.Values[i] = new ReadOnlyMemory <char>(cs.ToCharArray());
}
});
}
/// <summary>
/// Returns a value getter delegate to fetch the value of column with the given columnIndex, from the row.
/// This throws if the column is not active in this row, or if the type
/// <typeparamref name="TValue"/> differs from this column's type.
/// </summary>
/// <typeparam name="TValue"> is the column's content type.</typeparam>
/// <param name="column"> is the output column whose getter should be returned.</param>
public override ValueGetter <TValue> GetGetter <TValue>(DataViewSchema.Column column)
{
Ch.Check(IsColumnActive(column));
bool isSrc;
int index = _bindings.MapColumnIndex(out isSrc, column.Index);
if (isSrc)
{
return(Input.GetGetter <TValue>(column));
}
Ch.AssertValue(_getters);
var getter = _getters[index];
Ch.AssertValue(getter);
if (getter is ValueGetter <TValue> fn)
{
return(fn);
}
throw Ch.Except("Invalid TValue in GetGetter: '{0}'", typeof(TValue));
}
/// <summary>
/// Returns a value getter delegate to fetch the value of column with the given columnIndex, from the row.
/// This throws if the column is not active in this row, or if the type
/// <typeparamref name="TValue"/> differs from this column's type.
/// </summary>
/// <typeparam name="TValue"> is the column's content type.</typeparam>
/// <param name="column"> is the output column whose getter should be returned.</param>
public override ValueGetter <TValue> GetGetter <TValue>(DataViewSchema.Column column)
{
bool isSrc;
int index = _parent.MapColumnIndex(out isSrc, column.Index);
if (isSrc)
{
return(Input.GetGetter <TValue>(Input.Schema[index]));
}
var originFn = _getters[index];
Contracts.Assert(originFn != null);
var fn = originFn as ValueGetter <TValue>;
if (fn == null)
{
throw Contracts.Except($"Invalid TValue in GetGetter: '{typeof(TValue)}', " +
$"expected type: '{originFn.GetType().GetGenericArguments().First()}'.");
}
return(fn);
}
public override ValueGetter <TValue> GetGetter <TValue>(DataViewSchema.Column col)
{
if (col.Index < _view.Source.Schema.Count)
{
return(_inputCursor.GetGetter <TValue>(col));
}
else if (col.Index == _view.Source.Schema.Count) // Ordering
{
return(GetGetterOrdering() as ValueGetter <TValue>);
}
else if (col.Index == _view.Source.Schema.Count + 1) // Reachability Distance
{
return(GetGetterReachabilityDistance() as ValueGetter <TValue>);
}
else if (col.Index == _view.Source.Schema.Count + 2) // Core Distance
{
return(GetGetterCoreDistance() as ValueGetter <TValue>);
}
else
{
throw new IndexOutOfRangeException();
}
}
/// <summary>
/// Returns a value getter delegate to fetch the value of column with the given columnIndex, from the row.
/// This throws if the column is not active in this row, or if the type
/// <typeparamref name="TValue"/> differs from this column's type.
/// </summary>
/// <typeparam name="TValue"> is the column's content type.</typeparam>
/// <param name="column"> is the output column whose getter should be returned.</param>
public override ValueGetter <TValue> GetGetter <TValue>(DataViewSchema.Column column)
{
Ch.CheckParam(column.Index < _bindings.ColumnCount, nameof(column));
Ch.CheckParam(IsColumnActive(column), nameof(column.Index));
bool isSrc;
int index = _bindings.MapColumnIndex(out isSrc, column.Index);
if (isSrc)
{
return(Input.GetGetter <TValue>(Input.Schema[index]));
}
Ch.Assert(index == 0);
Delegate idGetter = Input.GetIdGetter();
Ch.AssertValue(idGetter);
var fn = idGetter as ValueGetter <TValue>;
if (fn == null)
{
throw Ch.Except("Invalid TValue in GetGetter: '{0}'", typeof(TValue));
}
return(fn);
}
public override ValueGetter <TValue> GetGetter <TValue>(DataViewSchema.Column col)
{
if (col.Index == _view.ConstantCol)
{
return(GetGetterPrivate(col) as ValueGetter <TValue>);
}
else if (_otherValues != null)
{
return(_otherValues.GetGetter <TValue>(col));
}
else if (_ignoreOtherColumn)
{
return (ref TValue value) =>
{
value = default(TValue);
}
}
;
else
{
throw Contracts.Except("otherValues is null, unable to access other columns.");
}
}
/// <summary>
/// Returns a value getter delegate to fetch the value of column with the given columnIndex, from the row.
/// This throws if the column is not active in this row, or if the type
/// <typeparamref name="TValue"/> differs from this column's type.
/// </summary>
/// <typeparam name="TValue"> is the column's content type.</typeparam>
/// <param name="column"> is the output column whose getter should be returned.</param>
public override ValueGetter <TValue> GetGetter <TValue>(DataViewSchema.Column column)
{
Ch.Check(IsColumnActive(column));
bool isSrc;
int index = _bindings.MapColumnIndex(out isSrc, column.Index);
if (isSrc)
{
return(Input.GetGetter <TValue>(column));
}
Ch.AssertValue(_getters);
var getter = _getters[index];
Ch.AssertValue(getter);
if (getter is ValueGetter <TValue> fn)
{
return(fn);
}
throw Ch.Except($"Invalid TValue in GetGetter: '{typeof(TValue)}', " +
$"expected type: '{getter.GetType().GetGenericArguments().First()}'.");
}
public PolynomialState(IHostEnvironment host, IDataView input, Arguments args, Func <TInput, TInput, TInput> multiplication)
{
_host = host.Register("PolynomialState");
_host.CheckValue(input, "input");
_input = input;
// _lock = new object();
_args = args;
_multiplication = multiplication;
var column = _args.columns[0];
var schema = input.Schema;
using (var ch = _host.Start("PolynomialState"))
{
_inputCol = SchemaHelper.GetColumnIndexDC(schema, column.Source);
var type = schema[_inputCol.Index].Type;
if (!type.IsVector())
{
throw _host.Except("Input column type must be a vector.");
}
int dim = type.AsVector().DimCount();
if (dim > 1)
{
throw _host.Except("Input column type must be a vector of one dimension.");
}
int size = dim > 0 ? type.AsVector().GetDim(0) : 0;
if (size > 0)
{
size = TotalCumulated[_args.degree](size);
}
ch.Trace("PolynomialTransform {0}->{1}.", dim, size);
// We extend the input schema. The new type has the same type as the input.
_schema = ExtendedSchema.Create(new ExtendedSchema(input.Schema,
new[] { column.Name },
new[] { new VectorDataViewType(type.AsVector().ItemType(), size) }));
}
}
public override ValueGetter <TValue> GetGetter <TValue>(DataViewSchema.Column column)
{
Contracts.CheckParam(IsColumnActive(column), nameof(column), "requested column is not active");
var col = _parent.SchemaBindings.MapColumnIndex(out bool isSrc, column.Index);
if (isSrc)
{
Contracts.AssertValue(_input);
return(_input.GetGetter <TValue>(_input.Schema[col]));
}
Ch.AssertValue(_getters);
var getter = _getters[col];
Ch.Assert(getter != null);
var fn = getter as ValueGetter <TValue>;
if (fn == null)
{
throw Ch.Except("Invalid TValue in GetGetter: '{0}'", typeof(TValue));
}
return(fn);
}
private static void AddSlotNames(OnnxContextImpl ctx, DataViewSchema.Column column)
{
VBuffer <ReadOnlyMemory <char> > slotNames = default;
column.GetSlotNames(ref slotNames);
IEnumerable <string> slotNamesAsStrings = slotNames.DenseValues().Select(name => name.ToString());
string opType = "LabelEncoder";
string labelEncoderInputName = $"mlnet.{column.Name}.unusedInput";
string labelEncoderOutputName = $"mlnet.{column.Name}.unusedOutput";
string labelEncoderNodeName = $"mlnet.{column.Name}.SlotNames";
string[] oneVals = new string[] { "one" };
long[] dims = new long[] { 1, 1 };
var one = ctx.AddInitializer(oneVals, dims, labelEncoderNodeName);
var labelEncoderOutput = ctx.AddIntermediateVariable(NumberDataViewType.Int64, labelEncoderOutputName, true);
var node = ctx.CreateNode(opType, one, labelEncoderOutput, labelEncoderNodeName);
node.AddAttribute("keys_strings", slotNamesAsStrings);
node.AddAttribute("values_int64s", Enumerable.Range(0, slotNames.Length).Select(x => (long)x));
ctx.AddOutputVariable(NumberDataViewType.Int64, labelEncoderOutput);
}
/// <summary>
/// Returns a value getter delegate to fetch the value of column with the given columnIndex, from the row.
/// This throws if the column is not active in this row, or if the type
/// <typeparamref name="TValue"/> differs from this column's type.
/// </summary>
/// <typeparam name="TValue"> is the column's content type.</typeparam>
/// <param name="column"> is the output column whose getter should be returned.</param>
public override ValueGetter <TValue> GetGetter <TValue>(DataViewSchema.Column column)
{
Ch.Check(IsColumnActive(column), nameof(column));
return(Input.GetGetter <TValue>(column));
}
/// <summary>
/// Returns whether the given column is active in this row.
/// </summary>
public override bool IsColumnActive(DataViewSchema.Column column)
{
Ch.Check(column.Index < Schema.Count, nameof(column));
return(Input.IsColumnActive(column));
}
/// <summary>
/// Checks whether a column kind in a <see cref="RoleMappedData"/> is unique, and its type
/// is a <see cref="InternalDataKind.U4"/> key of known cardinality.
/// </summary>
/// <param name="data">The training examples</param>
/// <param name="role">The column role to try to extract</param>
/// <param name="col">The extracted schema column</param>
/// <param name="isDecode">Whether a non-user error should be thrown as a decode</param>
/// <returns>The type cast to a key-type</returns>
private static KeyDataViewType CheckRowColumnType(RoleMappedData data, RoleMappedSchema.ColumnRole role, out DataViewSchema.Column col, bool isDecode)
{
Contracts.AssertValue(data);
Contracts.AssertValue(role.Value);
const string format2 = "There should be exactly one column with role {0}, but {1} were found instead";
if (!data.Schema.HasUnique(role))
{
int kindCount = Utils.Size(data.Schema.GetColumns(role));
if (isDecode)
{
throw Contracts.ExceptDecode(format2, role.Value, kindCount);
}
throw Contracts.Except(format2, role.Value, kindCount);
}
col = data.Schema.GetColumns(role)[0];
// REVIEW tfinley: Should we be a bit less restrictive? This doesn't seem like
// too terrible of a restriction.
const string format = "Column '{0}' with role {1} should be a known cardinality U4 key, but is instead '{2}'";
KeyDataViewType keyType;
if (!TryMarshalGoodRowColumnType(col.Type, out keyType))
{
if (isDecode)
{
throw Contracts.ExceptDecode(format, col.Name, role.Value, col.Type);
}
throw Contracts.Except(format, col.Name, role.Value, col.Type);
}
return(keyType);
}
public override bool IsColumnActive(DataViewSchema.Column column) => _getters[column.Index] != null;
/// <summary> /// Returns a value getter delegate to fetch the value of column with the given columnIndex, from the row. /// This throws if the column is not active in this row, or if the type /// <typeparamref name="TValue"/> differs from this column's type. /// </summary> /// <typeparam name="TValue"> is the column's content type.</typeparam> /// <param name="column"> is the output column whose getter should be returned.</param> public override ValueGetter <TValue> GetGetter <TValue>(DataViewSchema.Column column) => Input.GetGetter <TValue>(column);
/// <summary> /// Returns a value getter delegate to fetch the value of column with the given columnIndex, from the row. /// This throws if the column is not active in this row, or if the type /// <typeparamref name="TValue"/> differs from this column's type. /// </summary> /// <typeparam name="TValue"> is the column's content type.</typeparam> /// <param name="column"> is the output column whose getter should be returned.</param> public override ValueGetter <TValue> GetGetter <TValue>(DataViewSchema.Column column) => _annotations.GetGetter <TValue>(column);
/// <summary>
/// Returns whether the given column is active in this row.
/// </summary>
public override bool IsColumnActive(DataViewSchema.Column column)
{
_zipBinding.CheckColumnInRange(column.Index);
return(_isColumnActive[column.Index]);
}
/// <summary> /// Returns whether the given column is active in this row. /// </summary> public override bool IsColumnActive(DataViewSchema.Column column) => true;
/// <summary>
/// Check if the considered data, <see cref="RoleMappedData"/>, contains column roles specified by <see cref="MatrixColumnIndexKind"/> and <see cref="MatrixRowIndexKind"/>.
/// If the column roles, <see cref="MatrixColumnIndexKind"/> and <see cref="MatrixRowIndexKind"/>, uniquely exist in data, their <see cref="DataViewSchema.Column"/> would be assigned
/// to the two out parameters below.
/// </summary>
/// <param name="data">The considered data being checked</param>
/// <param name="matrixColumnIndexColumn">The schema column as the row in the input data</param>
/// <param name="matrixRowIndexColumn">The schema column as the column in the input data</param>
/// <param name="isDecode">Whether a non-user error should be thrown as a decode</param>
public static void CheckAndGetMatrixIndexColumns(RoleMappedData data, out DataViewSchema.Column matrixColumnIndexColumn, out DataViewSchema.Column matrixRowIndexColumn, bool isDecode)
{
Contracts.AssertValue(data);
CheckRowColumnType(data, MatrixColumnIndexKind, out matrixColumnIndexColumn, isDecode);
CheckRowColumnType(data, MatrixRowIndexKind, out matrixRowIndexColumn, isDecode);
}
/// <summary>
/// Returns whether the given column is active in this row.
/// </summary>
public override bool IsColumnActive(DataViewSchema.Column column)
{
Ch.Check(column.Index < _bindings.ColumnCount);
return(_active == null || _active[column.Index]);
}
/// <summary> /// Returns whether the given column is active in this row. /// </summary> public override bool IsColumnActive(DataViewSchema.Column column) => Input.IsColumnActive(column);
