protected static ValueGetter <T> GetGetterFromRow <T>(Row output, int col)
{
Contracts.AssertValue(output);
Contracts.Assert(0 <= col && col < output.Schema.Count);
Contracts.Assert(output.IsColumnActive(col));
return(output.GetGetter <T>(col));
}
// REVIEW: The converting getter invokes a type conversion delegate on every call, so it's inherently slower
// than the 'direct' getter. We don't have good indication of this to the user, and the selection
// of affected types is pretty arbitrary (signed integers and bools, but not uints and floats).
private Action <TRow> CreateConvertingVBufferSetter <TSrc, TDst>(Row input, int col, Delegate poke, Delegate peek, Func <TSrc, TDst> convert)
{
var getter = input.GetGetter <VBuffer <TSrc> >(col);
var typedPoke = poke as Poke <TRow, TDst[]>;
var typedPeek = peek as Peek <TRow, TDst[]>;
Contracts.AssertValue(typedPoke);
Contracts.AssertValue(typedPeek);
VBuffer <TSrc> value = default;
TDst[] buf = null;
return(row =>
{
getter(ref value);
typedPeek(row, Position, ref buf);
if (Utils.Size(buf) != value.Length)
{
buf = new TDst[value.Length];
}
foreach (var pair in value.Items(true))
{
buf[pair.Key] = convert(pair.Value);
}
typedPoke(row, buf);
});
}
public State(IExceptionContext ectx, Row input, TreeEnsembleModelParameters ensemble, int numLeaves, int featureIndex)
{
Contracts.AssertValue(ectx);
_ectx = ectx;
_ectx.AssertValue(input);
_ectx.AssertValue(ensemble);
_ectx.Assert(ensemble.TrainedEnsemble.NumTrees > 0);
_input = input;
_ensemble = ensemble;
_numTrees = _ensemble.TrainedEnsemble.NumTrees;
_numLeaves = numLeaves;
_src = default(VBuffer <float>);
_featureGetter = input.GetGetter <VBuffer <float> >(featureIndex);
_cachedPosition = -1;
_leafIds = new int[_numTrees];
_pathIds = new List <int> [_numTrees];
for (int i = 0; i < _numTrees; i++)
{
_pathIds[i] = new List <int>();
}
_cachedLeafBuilderPosition = -1;
_cachedPathBuilderPosition = -1;
}
protected override Delegate GetPredictionGetter(Row input, int colSrc)
{
Contracts.AssertValue(input);
Contracts.Assert(0 <= colSrc && colSrc < input.Schema.Count);
var typeSrc = input.Schema[colSrc].Type as VectorType;
Contracts.Assert(typeSrc != null && typeSrc.ItemType == NumberType.Float);
Contracts.Assert(ValueMapper == null ||
typeSrc.Size == ValueMapper.InputType.GetVectorSize() || ValueMapper.InputType.GetVectorSize() == 0);
Contracts.Assert(Utils.Size(_quantiles) > 0);
var featureGetter = input.GetGetter <VBuffer <Float> >(colSrc);
var featureCount = ValueMapper != null?ValueMapper.InputType.GetVectorSize() : 0;
var quantiles = new Float[_quantiles.Length];
for (int i = 0; i < quantiles.Length; i++)
{
quantiles[i] = (Float)_quantiles[i];
}
var map = _qpred.GetMapper(quantiles);
var features = default(VBuffer <Float>);
ValueGetter <VBuffer <Float> > del =
(ref VBuffer <Float> value) =>
{
featureGetter(ref features);
Contracts.Check(features.Length == featureCount || featureCount == 0);
map(in features, ref value);
};
private static Func <bool> GetIsNewGroupDelegateCore <T>(Row cursor, int col)
{
var getter = cursor.GetGetter <T>(col);
bool first = true;
T old = default(T);
T val = default(T);
var compare = EqualityComparer <T> .Default;
return(() =>
{
getter(ref val);
if (first)
{
first = false;
old = val;
return true;
}
if (compare.Equals(val, old))
{
return false;
}
old = val;
return true;
});
}
protected ValueGetter<T> GetSrcGetter<T>(Row input, int iinfo)
{
Host.AssertValue(input);
Host.Assert(0 <= iinfo && iinfo < Infos.Length);
int src = Infos[iinfo].Source;
Host.Assert(input.IsColumnActive(src));
return input.GetGetter<T>(src);
}
internal override void InitializeNextPass(Row row, RoleMappedSchema schema)
{
Contracts.Assert(PassNum < 1);
Contracts.Assert(schema.Label.HasValue);
var score = schema.GetUniqueColumn(MetadataUtils.Const.ScoreValueKind.Score);
_labelGetter = RowCursorUtils.GetVecGetterAs <Float>(NumberType.Float, row, schema.Label.Value.Index);
_scoreGetter = row.GetGetter <VBuffer <Float> >(score.Index);
Contracts.AssertValue(_labelGetter);
Contracts.AssertValue(_scoreGetter);
if (schema.Weight.HasValue)
{
_weightGetter = row.GetGetter <Float>(schema.Weight.Value.Index);
}
}
protected private static T Fetch <T>(IExceptionContext ectx, Row row, string name)
{
if (!row.Schema.TryGetColumnIndex(name, out int col))
{
throw ectx.Except($"Could not find column '{name}'");
}
T val = default;
row.GetGetter <T>(col)(ref val);
return(val);
}
public static ValueGetter <Single> GetLabelGetter(Row cursor, int labelIndex)
{
var type = cursor.Schema[labelIndex].Type;
if (type == NumberType.R4)
{
return(cursor.GetGetter <Single>(labelIndex));
}
if (type == NumberType.R8)
{
var getSingleSrc = cursor.GetGetter <Double>(labelIndex);
return
((ref Single dst) =>
{
Double src = Double.NaN;
getSingleSrc(ref src);
dst = Convert.ToSingle(src);
});
}
return(GetLabelGetterNotFloat(cursor, labelIndex));
}
private static Action <TRow> CreateConvertingActionSetter <TSrc, TDst>(Row input, int col, Delegate poke, Func <TSrc, TDst> convert)
{
var getter = input.GetGetter <TSrc>(col);
var typedPoke = poke as Poke <TRow, TDst>;
Contracts.AssertValue(typedPoke);
TSrc value = default;
return(row =>
{
getter(ref value);
var toPoke = convert(value);
typedPoke(row, toPoke);
});
}
private static ValueGetter <Single> GetLabelGetterNotFloat(Row cursor, int labelIndex)
{
var type = cursor.Schema[labelIndex].Type;
Contracts.Assert(type != NumberType.R4 && type != NumberType.R8);
// boolean type label mapping: True -> 1, False -> 0.
if (type is BoolType)
{
var getBoolSrc = cursor.GetGetter <bool>(labelIndex);
return
((ref Single dst) =>
{
bool src = default;
getBoolSrc(ref src);
dst = Convert.ToSingle(src);
});
}
if (!(type is KeyType keyType))
{
throw Contracts.Except("Only floating point number, boolean, and key type values can be used as label.");
}
Contracts.Assert(TestGetLabelGetter(type) == null);
ulong keyMax = (ulong)keyType.Count;
if (keyMax == 0)
{
keyMax = ulong.MaxValue;
}
var getSrc = RowCursorUtils.GetGetterAs <ulong>(NumberType.U8, cursor, labelIndex);
return
((ref Single dst) =>
{
ulong src = 0;
getSrc(ref src);
if (0 < src && src <= keyMax)
{
dst = src - 1;
}
else
{
dst = Single.NaN;
}
});
}
private static ValueGetter <StringBuilder> GetGetterAsStringBuilderCore <TSrc>(ColumnType typeSrc, Row row, int col)
{
Contracts.Assert(typeof(TSrc) == typeSrc.RawType);
var getter = row.GetGetter <TSrc>(col);
var conv = Conversions.Instance.GetStringConversion <TSrc>(typeSrc);
var src = default(TSrc);
return
((ref StringBuilder dst) =>
{
getter(ref src);
conv(in src, ref dst);
});
}
private ValueGetter <TDst> GetValueGetter <TSrc, TDst>(Row input, int colSrc)
{
Contracts.AssertValue(input);
Contracts.Assert(ValueMapper != null);
var featureGetter = input.GetGetter <TSrc>(colSrc);
var map = ValueMapper.GetMapper <TSrc, TDst>();
var features = default(TSrc);
return
((ref TDst dst) =>
{
featureGetter(ref features);
map(in features, ref dst);
});
}
private static Action <TRow> CreateDirectSetter <TDst>(Row input, int col, Delegate poke, Delegate peek)
{
// Awkward to have a parameter that's always null, but slightly more convenient for generalizing the setter.
Contracts.Assert(peek == null);
var getter = input.GetGetter <TDst>(col);
var typedPoke = poke as Poke <TRow, TDst>;
Contracts.AssertValue(typedPoke);
TDst value = default(TDst);
return(row =>
{
getter(ref value);
typedPoke(row, value);
});
}
private Action <TRow> CreateVBufferToVBufferSetter <TDst>(Row input, int col, Delegate poke, Delegate peek)
{
var getter = input.GetGetter <VBuffer <TDst> >(col);
var typedPoke = poke as Poke <TRow, VBuffer <TDst> >;
var typedPeek = peek as Peek <TRow, VBuffer <TDst> >;
Contracts.AssertValue(typedPoke);
Contracts.AssertValue(typedPeek);
VBuffer <TDst> value = default(VBuffer <TDst>);
return(row =>
{
typedPeek(row, Position, ref value);
getter(ref value);
typedPoke(row, value);
});
}
private Action <RowInfo, List <object> > MakeSetter <T>(Row row, int col)
{
var getter = row.GetGetter <T>(col);
string name = row.Schema[col].Name;
Action <RowInfo, List <object> > result = (rowInfo, list) =>
{
T value = default;
getter(ref value);
rowInfo.Values[col] = new KeyValuePair <string, object>(name, value);
// Call getter again on another buffer, since we store it in two places.
value = default;
getter(ref value);
list.Add(value);
};
return(result);
}
private ValueGetter <VBuffer <float> > GetValueGetter <TSrc>(Row input, int colSrc)
{
Contracts.AssertValue(input);
Contracts.AssertValue(Predictor);
var featureGetter = input.GetGetter <TSrc>(colSrc);
// REVIEW: Scorer can call Sparsification\Norm routine.
var map = Predictor.GetFeatureContributionMapper <TSrc, VBuffer <float> >(_topContributionsCount, _bottomContributionsCount, _normalize);
var features = default(TSrc);
return
((ref VBuffer <float> dst) =>
{
featureGetter(ref features);
map(in features, ref dst);
});
}
private Action <TRow> CreateDirectVBufferSetter <TDst>(Row input, int col, Delegate poke, Delegate peek)
{
var getter = input.GetGetter <VBuffer <TDst> >(col);
var typedPoke = poke as Poke <TRow, TDst[]>;
var typedPeek = peek as Peek <TRow, TDst[]>;
Contracts.AssertValue(typedPoke);
Contracts.AssertValue(typedPeek);
VBuffer <TDst> value = default(VBuffer <TDst>);
TDst[] buf = null;
return(row =>
{
typedPeek(row, Position, ref buf);
getter(ref value);
if (value.Length == Utils.Size(buf) && value.IsDense)
{
// In this case, buf (which came from the input object) is the
// right size to represent the vector.
// Otherwise, we are either sparse (and need densifying), or value.GetValues()
// is a different length than buf.
value.CopyTo(buf);
}
else
{
buf = new TDst[value.Length];
if (value.IsDense)
{
value.GetValues().CopyTo(buf);
}
else
{
foreach (var pair in value.Items(true))
{
buf[pair.Key] = pair.Value;
}
}
}
typedPoke(row, buf);
});
}
private ValueGetter <ReadOnlyMemory <char> > GetTextValueGetter <TSrc>(Row input, int colSrc, VBuffer <ReadOnlyMemory <char> > slotNames)
{
Contracts.AssertValue(input);
Contracts.AssertValue(Predictor);
var featureGetter = input.GetGetter <TSrc>(colSrc);
var map = Predictor.GetFeatureContributionMapper <TSrc, VBuffer <float> >(_topContributionsCount, _bottomContributionsCount, _normalize);
var features = default(TSrc);
var contributions = default(VBuffer <float>);
return
((ref ReadOnlyMemory <char> dst) =>
{
featureGetter(ref features);
map(in features, ref contributions);
var editor = VBufferEditor.CreateFromBuffer(ref contributions);
var indices = contributions.IsDense ? Utils.GetIdentityPermutation(contributions.Length) : editor.Indices;
var values = editor.Values;
var count = values.Length;
var sb = new StringBuilder();
GenericSpanSortHelper <float> .Sort(values, indices, 0, count);
for (var i = 0; i < count; i++)
{
var val = values[i];
var ind = indices[i];
var name = GetSlotName(ind, slotNames);
sb.AppendFormat("{0}: {1}, ", name, val);
}
if (sb.Length > 0)
{
_env.Assert(sb.Length >= 2);
sb.Remove(sb.Length - 2, 2);
}
dst = new ReadOnlyMemory <char>(sb.ToString().ToCharArray());
});
}
protected override Delegate GetPredictedLabelGetter(Row output, out Delegate scoreGetter)
{
Contracts.AssertValue(output);
Contracts.Assert(output.Schema == Bindings.RowMapper.OutputSchema);
Contracts.Assert(output.IsColumnActive(Bindings.ScoreColumnIndex));
ValueGetter <VBuffer <Float> > mapperScoreGetter = output.GetGetter <VBuffer <Float> >(Bindings.ScoreColumnIndex);
long cachedPosition = -1;
VBuffer <Float> score = default(VBuffer <Float>);
int keyCount = Bindings.PredColType is KeyType key?key.GetCountAsInt32(Host) : 0;
int scoreLength = keyCount;
ValueGetter <uint> predFn =
(ref uint dst) =>
{
EnsureCachedPosition(ref cachedPosition, ref score, output, mapperScoreGetter);
Contracts.Check(score.Length == scoreLength);
int index = VectorUtils.ArgMin(in score);
if (index < 0)
{
dst = 0;
}
else
{
dst = (uint)index + 1;
}
};
ValueGetter <VBuffer <Float> > scoreFn =
(ref VBuffer <Float> dst) =>
{
EnsureCachedPosition(ref cachedPosition, ref score, output, mapperScoreGetter);
Contracts.Check(score.Length == scoreLength);
score.CopyTo(ref dst);
};
scoreGetter = scoreFn;
return(predFn);
}
protected override Delegate GetPredictedLabelGetter(Row output, out Delegate scoreGetter)
{
Host.AssertValue(output);
Host.Assert(output.Schema == Bindings.RowMapper.OutputSchema);
Host.Assert(output.IsColumnActive(Bindings.ScoreColumnIndex));
ValueGetter <Float> mapperScoreGetter = output.GetGetter <Float>(Bindings.ScoreColumnIndex);
long cachedPosition = -1;
Float score = 0;
ValueGetter <Float> scoreFn =
(ref Float dst) =>
{
EnsureCachedPosition(ref cachedPosition, ref score, output, mapperScoreGetter);
dst = score;
};
scoreGetter = scoreFn;
if (Bindings.PredColType is KeyType)
{
ValueGetter <uint> predFnAsKey =
(ref uint dst) =>
{
EnsureCachedPosition(ref cachedPosition, ref score, output, mapperScoreGetter);
GetPredictedLabelCoreAsKey(score, ref dst);
};
return(predFnAsKey);
}
ValueGetter <bool> predFn =
(ref bool dst) =>
{
EnsureCachedPosition(ref cachedPosition, ref score, output, mapperScoreGetter);
GetPredictedLabelCore(score, ref dst);
};
return(predFn);
}
private static ValueGetter <TDst> GetGetterAsCore <TSrc, TDst>(ColumnType typeSrc, ColumnType typeDst, Row row, int col)
{
Contracts.Assert(typeof(TSrc) == typeSrc.RawType);
Contracts.Assert(typeof(TDst) == typeDst.RawType);
var getter = row.GetGetter <TSrc>(col);
bool identity;
var conv = Conversions.Instance.GetStandardConversion <TSrc, TDst>(typeSrc, typeDst, out identity);
if (identity)
{
Contracts.Assert(typeof(TSrc) == typeof(TDst));
return((ValueGetter <TDst>)(Delegate) getter);
}
var src = default(TSrc);
return
((ref TDst dst) =>
{
getter(ref src);
conv(in src, ref dst);
});
}
public override ValueGetter <TValue> GetGetter <TValue>()
{
return(_row.GetGetter <TValue>(_col));
}
protected override ValueGetter <Single> GetScoreGetter(Row row)
{
return(row.GetGetter <Single>(_bindings.ScoreIndex));
}
private Delegate[] CreateGetters(Row input, bool[] active)
{
Contracts.Assert(Utils.Size(active) == 2);
Contracts.Assert(_parent._distMapper != null);
var getters = new Delegate[2];
if (active[0] || active[1])
{
// Put all captured locals at this scope.
var featureGetter = InputRoleMappedSchema.Feature?.Index is int idx?input.GetGetter <VBuffer <Float> >(idx) : null;
Float prob = 0;
Float score = 0;
long cachedPosition = -1;
var features = default(VBuffer <Float>);
ValueMapper <VBuffer <Float>, Float, Float> mapper;
mapper = _parent._distMapper.GetMapper <VBuffer <Float>, Float, Float>();
if (active[0])
{
ValueGetter <Float> getScore =
(ref Float dst) =>
{
EnsureCachedResultValueMapper(mapper, ref cachedPosition, featureGetter, ref features, ref score, ref prob, input);
dst = score;
};
getters[0] = getScore;
}
if (active[1])
{
ValueGetter <Float> getProb =
(ref Float dst) =>
{
EnsureCachedResultValueMapper(mapper, ref cachedPosition, featureGetter, ref features, ref score, ref prob, input);
dst = prob;
};
getters[1] = getProb;
}
}
return(getters);
}
public override ValueGetter <TValue> GetGetter <TValue>(int col) => _row.GetGetter <TValue>(col);
private static Delegate GetGetterAsDelegateCore <TValue>(Row row, int col)
{
return(row.GetGetter <TValue>(col));
}
private protected override Delegate[] CreateGettersCore(Row input, Func <int, bool> activeCols, out Action disposer)
{
Host.Assert(LabelIndex >= 0);
Host.Assert(ScoreIndex >= 0);
disposer = null;
long cachedPosition = -1;
var label = default(VBuffer <Float>);
var score = default(VBuffer <Float>);
ValueGetter <VBuffer <Float> > nullGetter = (ref VBuffer <Float> vec) => vec = default(VBuffer <Float>);
var labelGetter = activeCols(LabelOutput) || activeCols(L1Output) || activeCols(L2Output) || activeCols(DistCol)
? RowCursorUtils.GetVecGetterAs <Float>(NumberType.Float, input, LabelIndex)
: nullGetter;
var scoreGetter = activeCols(ScoreOutput) || activeCols(L1Output) || activeCols(L2Output) || activeCols(DistCol)
? input.GetGetter <VBuffer <Float> >(ScoreIndex)
: nullGetter;
Action updateCacheIfNeeded =
() =>
{
if (cachedPosition != input.Position)
{
labelGetter(ref label);
scoreGetter(ref score);
cachedPosition = input.Position;
}
};
var getters = new Delegate[5];
if (activeCols(LabelOutput))
{
ValueGetter <VBuffer <Float> > labelFn =
(ref VBuffer <Float> dst) =>
{
updateCacheIfNeeded();
label.CopyTo(ref dst);
};
getters[LabelOutput] = labelFn;
}
if (activeCols(ScoreOutput))
{
ValueGetter <VBuffer <Float> > scoreFn =
(ref VBuffer <Float> dst) =>
{
updateCacheIfNeeded();
score.CopyTo(ref dst);
};
getters[ScoreOutput] = scoreFn;
}
if (activeCols(L1Output))
{
ValueGetter <double> l1Fn =
(ref double dst) =>
{
updateCacheIfNeeded();
dst = VectorUtils.L1Distance(in label, in score);
};
getters[L1Output] = l1Fn;
}