public DebugParameterInfo(MemberInfo member, string name, Type parameterType, int position, ValueGetter getter)
{
this.member = member;
this.name = name;
this.parameterType = parameterType;
this.position = position;
this.getter = getter;
}
public DebugLocalVariableInfo(string name, int localIndex, int startOffset, int endOffset, DebugType localType, ValueGetter getter)
{
this.Name = name;
this.localIndex = localIndex;
this.StartOffset = startOffset;
this.EndOffset = endOffset;
this.localType = localType;
this.getter = getter;
}
//private ILabel _displayText;
//private event EventHandler _valueChanged;
///<summary>
/// The Constructor for the <see cref="DateTimePickerManager"/>
///</summary>
///<param name="controlFactory"></param>
///<param name="dateTimePicker"></param>
///<param name="valueGetter"></param>
///<param name="valueSetter"></param>
///<exception cref="ArgumentNullException"></exception>
public DateTimePickerManager(IControlFactory controlFactory, IDateTimePicker dateTimePicker,
ValueGetter<DateTime> valueGetter, ValueSetter<DateTime> valueSetter)
{
if (valueGetter == null) throw new ArgumentNullException("valueGetter");
if (valueSetter == null) throw new ArgumentNullException("valueSetter");
_controlFactory = controlFactory;
_dateTimePicker = dateTimePicker;
_valueGetter = valueGetter;
_valueSetter = valueSetter;
SetupNullDisplayBox();
ApplyBlankFormat();
}
public NameOnnxValueGetter(DataViewRow input, int colIndex)
{
_colName = input.Schema[colIndex].Name;
_srcGetter = input.GetGetter <T>(input.Schema[colIndex]);
}
private FieldAwareFactorizationMachineModelParameters TrainCore(IChannel ch, IProgressChannel pch, RoleMappedData data,
RoleMappedData validData = null, FieldAwareFactorizationMachineModelParameters predictor = null)
{
_host.AssertValue(ch);
_host.AssertValue(pch);
data.CheckBinaryLabel();
var featureColumns = data.Schema.GetColumns(RoleMappedSchema.ColumnRole.Feature);
int fieldCount = featureColumns.Count;
int totalFeatureCount = 0;
int[] fieldColumnIndexes = new int[fieldCount];
for (int f = 0; f < fieldCount; f++)
{
var col = featureColumns[f];
_host.Assert(!col.IsHidden);
if (!(col.Type is VectorDataViewType vectorType) ||
!vectorType.IsKnownSize ||
vectorType.ItemType != NumberDataViewType.Single)
{
throw ch.ExceptParam(nameof(data), "Training feature column '{0}' must be a known-size vector of Single, but has type: {1}.", col.Name, col.Type);
}
_host.Assert(vectorType.Size > 0);
fieldColumnIndexes[f] = col.Index;
totalFeatureCount += vectorType.Size;
}
ch.Check(checked (totalFeatureCount * fieldCount * _latentDimAligned) <= Utils.ArrayMaxSize, "Latent dimension or the number of fields too large");
if (predictor != null)
{
ch.Check(predictor.FeatureCount == totalFeatureCount, "Input model's feature count mismatches training feature count");
ch.Check(predictor.LatentDimension == _latentDim, "Input model's latent dimension mismatches trainer's");
}
if (validData != null)
{
validData.CheckBinaryLabel();
var validFeatureColumns = data.Schema.GetColumns(RoleMappedSchema.ColumnRole.Feature);
_host.Assert(fieldCount == validFeatureColumns.Count);
for (int f = 0; f < fieldCount; f++)
{
var featCol = featureColumns[f];
var validFeatCol = validFeatureColumns[f];
_host.Assert(featCol.Name == validFeatCol.Name);
_host.Assert(featCol.Type == validFeatCol.Type);
}
}
bool shuffle = _shuffle;
if (shuffle && !data.Data.CanShuffle)
{
ch.Warning("Training data does not support shuffling, so ignoring request to shuffle");
shuffle = false;
}
var rng = shuffle ? _host.Rand : null;
var featureGetters = new ValueGetter <VBuffer <float> > [fieldCount];
var featureBuffer = new VBuffer <float>();
var featureValueBuffer = new float[totalFeatureCount];
var featureIndexBuffer = new int[totalFeatureCount];
var featureFieldBuffer = new int[totalFeatureCount];
var latentSum = new AlignedArray(fieldCount * fieldCount * _latentDimAligned, 16);
var metricNames = new List <string>()
{
"Training-loss"
};
if (validData != null)
{
metricNames.Add("Validation-loss");
}
int iter = 0;
long exampleCount = 0;
long badExampleCount = 0;
long validBadExampleCount = 0;
double loss = 0;
double validLoss = 0;
pch.SetHeader(new ProgressHeader(metricNames.ToArray(), new string[] { "iterations", "examples" }), entry =>
{
entry.SetProgress(0, iter, _numIterations);
entry.SetProgress(1, exampleCount);
});
var columns = data.Schema.Schema.Where(x => fieldColumnIndexes.Contains(x.Index)).ToList();
columns.Add(data.Schema.Label.Value);
if (data.Schema.Weight != null)
{
columns.Add(data.Schema.Weight.Value);
}
InitializeTrainingState(fieldCount, totalFeatureCount, predictor, out float[] linearWeights,
out AlignedArray latentWeightsAligned, out float[] linearAccSqGrads, out AlignedArray latentAccSqGradsAligned);
// refer to Algorithm 3 in https://github.com/wschin/fast-ffm/blob/master/fast-ffm.pdf
while (iter++ < _numIterations)
{
using (var cursor = data.Data.GetRowCursor(columns, rng))
{
var labelGetter = RowCursorUtils.GetLabelGetter(cursor, data.Schema.Label.Value.Index);
var weightGetter = data.Schema.Weight?.Index is int weightIdx?RowCursorUtils.GetGetterAs <float>(NumberDataViewType.Single, cursor, weightIdx) : null;
for (int i = 0; i < fieldCount; i++)
{
featureGetters[i] = cursor.GetGetter <VBuffer <float> >(cursor.Schema[fieldColumnIndexes[i]]);
}
loss = 0;
exampleCount = 0;
badExampleCount = 0;
while (cursor.MoveNext())
{
float label = 0;
float weight = 1;
int count = 0;
float modelResponse = 0;
labelGetter(ref label);
weightGetter?.Invoke(ref weight);
float annihilation = label - label + weight - weight;
if (!FloatUtils.IsFinite(annihilation))
{
badExampleCount++;
continue;
}
if (!FieldAwareFactorizationMachineUtils.LoadOneExampleIntoBuffer(featureGetters, featureBuffer, _norm, ref count,
featureFieldBuffer, featureIndexBuffer, featureValueBuffer))
{
badExampleCount++;
continue;
}
// refer to Algorithm 1 in [3] https://github.com/wschin/fast-ffm/blob/master/fast-ffm.pdf
FieldAwareFactorizationMachineInterface.CalculateIntermediateVariables(fieldCount, _latentDimAligned, count,
featureFieldBuffer, featureIndexBuffer, featureValueBuffer, linearWeights, latentWeightsAligned, latentSum, ref modelResponse);
var slope = CalculateLossSlope(label, modelResponse);
// refer to Algorithm 2 in [3] https://github.com/wschin/fast-ffm/blob/master/fast-ffm.pdf
FieldAwareFactorizationMachineInterface.CalculateGradientAndUpdate(_lambdaLinear, _lambdaLatent, _learningRate, fieldCount, _latentDimAligned, weight, count,
featureFieldBuffer, featureIndexBuffer, featureValueBuffer, latentSum, slope, linearWeights, latentWeightsAligned, linearAccSqGrads, latentAccSqGradsAligned);
loss += weight * CalculateLoss(label, modelResponse);
exampleCount++;
}
loss /= exampleCount;
}
if (_verbose)
{
if (validData == null)
{
pch.Checkpoint(loss, iter, exampleCount);
}
else
{
validLoss = CalculateAvgLoss(ch, validData, _norm, linearWeights, latentWeightsAligned, _latentDimAligned, latentSum,
featureFieldBuffer, featureIndexBuffer, featureValueBuffer, featureBuffer, ref validBadExampleCount);
pch.Checkpoint(loss, validLoss, iter, exampleCount);
}
}
}
if (badExampleCount != 0)
{
ch.Warning($"Skipped {badExampleCount} examples with bad label/weight/features in training set");
}
if (validBadExampleCount != 0)
{
ch.Warning($"Skipped {validBadExampleCount} examples with bad label/weight/features in validation set");
}
return(new FieldAwareFactorizationMachineModelParameters(_host, _norm, fieldCount, totalFeatureCount, _latentDim, linearWeights, latentWeightsAligned));
}
private static T GetValue<T>(ValueGetter<T> valueGetter)
{
T value = default;
valueGetter(ref value);
return value;
}
public Impl(IHostEnvironment env, string name, IDataView input, OneToOneColumn col,
DataViewType typeDst, ValueMapper <T1, T2> map1, ValueMapper <T2, T3> map2 = null,
ValueGetter <VBuffer <ReadOnlyMemory <char> > > keyValueGetter = null, ValueGetter <VBuffer <ReadOnlyMemory <char> > > slotNamesGetter = null)
: base(env, name, new[] { col }, input, x => null)
{
Host.Assert(typeDst.RawType == typeof(T3));
Host.AssertValue(map1);
Host.Assert(map2 != null || typeof(T2) == typeof(T3));
_typeDst = typeDst;
_map1 = map1;
_map2 = map2;
if (keyValueGetter != null || slotNamesGetter != null)
{
using (var bldr = Metadata.BuildMetadata(0))
{
if (keyValueGetter != null)
{
AnnotationUtils.AnnotationGetter <VBuffer <ReadOnlyMemory <char> > > mdGetter =
(int c, ref VBuffer <ReadOnlyMemory <char> > dst) => keyValueGetter(ref dst);
bldr.AddGetter(AnnotationUtils.Kinds.KeyValues, new VectorDataViewType(TextDataViewType.Instance, _typeDst.GetItemType().GetKeyCountAsInt32(Host)), mdGetter);
}
if (slotNamesGetter != null)
{
int vectorSize = _typeDst.GetVectorSize();
Host.Assert(vectorSize > 0);
AnnotationUtils.AnnotationGetter <VBuffer <ReadOnlyMemory <char> > > mdGetter =
(int c, ref VBuffer <ReadOnlyMemory <char> > dst) => slotNamesGetter(ref dst);
bldr.AddGetter(AnnotationUtils.Kinds.SlotNames, new VectorDataViewType(TextDataViewType.Instance, vectorSize), mdGetter);
}
}
}
Metadata.Seal();
}
public ValueWriter(RowCursor cursor, PrimitiveType type, int source, char sep)
: base(type, source, sep)
{
_getSrc = cursor.GetGetter <T>(source);
_columnName = cursor.Schema[source].Name;
}
protected bool CheckSameValues(IRowCursor curs1, IRowCursor curs2, bool exactTypes, bool exactDoubles, bool checkId, bool checkIdCollisions = true)
{
Contracts.Assert(curs1.Schema.ColumnCount == curs2.Schema.ColumnCount);
// Get the comparison delegates for each column.
int colLim = curs1.Schema.ColumnCount;
Func <bool>[] comps = new Func <bool> [colLim];
for (int col = 0; col < colLim; col++)
{
var f1 = curs1.IsColumnActive(col);
var f2 = curs2.IsColumnActive(col);
if (f1 && f2)
{
var type1 = curs1.Schema.GetColumnType(col);
var type2 = curs2.Schema.GetColumnType(col);
if (!EqualTypes(type1, type2, exactTypes))
{
Fail("Different types");
return(Failed());
}
comps[col] = GetColumnComparer(curs1, curs2, col, type1, exactDoubles);
}
}
ValueGetter <UInt128> idGetter = null;
Func <bool> idComp = checkId ? GetIdComparer(curs1, curs2, out idGetter) : null;
HashSet <UInt128> idsSeen = null;
if (checkIdCollisions && idGetter == null)
{
idGetter = curs1.GetIdGetter();
}
long idCollisions = 0;
UInt128 id = default(UInt128);
for (; ;)
{
bool f1 = curs1.MoveNext();
bool f2 = curs2.MoveNext();
if (f1 != f2)
{
if (f1)
{
Fail("Left has more rows at position: {0}", curs1.Position);
}
else
{
Fail("Right has more rows at position: {0}", curs2.Position);
}
return(Failed());
}
if (!f1)
{
if (idCollisions > 0)
{
Fail("{0} id collisions among {1} items", idCollisions, Utils.Size(idsSeen) + idCollisions);
}
return(idCollisions == 0);
}
else if (checkIdCollisions)
{
idGetter(ref id);
if (!Utils.Add(ref idsSeen, id))
{
if (idCollisions == 0)
{
idCollisions++;
}
}
}
Contracts.Assert(curs1.Position == curs2.Position);
for (int col = 0; col < colLim; col++)
{
var comp = comps[col];
if (comp != null && !comp())
{
Fail("Different values in column {0} of row {1}", col, curs1.Position);
return(Failed());
}
if (idComp != null && !idComp())
{
Fail("Different values in ID of row {0}", curs1.Position);
return(Failed());
}
}
}
}
public static void AddValue <TOwner, TValue>(this IList <InspectorPropertyInfo> infos, string name, ValueGetter <TOwner, TValue> getter, ValueSetter <TOwner, TValue> setter, int order = 0, SerializationBackend backend = SerializationBackend.None)
{
AddValue(infos, name, getter, setter, 0, SerializationBackend.None, null);
}
static void AddCapturedLocalVariables(List<DebugLocalVariableInfo> vars, int scopeStartOffset, int scopeEndOffset, ValueGetter getCaptureClass, DebugType captureClassType)
{
if (captureClassType.IsDisplayClass || captureClassType.IsYieldEnumerator) {
foreach(DebugFieldInfo fieldInfo in captureClassType.GetFields()) {
DebugFieldInfo fieldInfoCopy = fieldInfo;
if (fieldInfo.Name.StartsWith("CS$")) continue; // Ignore
DebugLocalVariableInfo locVar = new DebugLocalVariableInfo(
fieldInfo.Name,
-1,
scopeStartOffset,
scopeEndOffset,
(DebugType)fieldInfo.FieldType,
delegate(StackFrame context) {
return getCaptureClass(context).GetFieldValue(fieldInfoCopy);
}
);
locVar.IsCaptured = true;
if (locVar.Name.StartsWith("<>")) {
bool hasThis = false;
foreach(DebugLocalVariableInfo l in vars) {
if (l.IsThis) {
hasThis = true;
break;
}
}
if (!hasThis && locVar.Name.EndsWith("__this")) {
locVar.Name = "this";
locVar.IsThis = true;
} else {
continue; // Ignore
}
}
if (locVar.Name.StartsWith("<")) {
int endIndex = locVar.Name.IndexOf('>');
if (endIndex == -1) continue; // Ignore
locVar.Name = fieldInfo.Name.Substring(1, endIndex - 1);
}
vars.Add(locVar);
}
}
}
/// <summary>
/// This is the constructor called for the initial wrapping.
/// </summary>
public Bound(IHostEnvironment env, ISchemaBoundRowMapper mapper, VectorType type, ValueGetter <VBuffer <T> > getter,
string metadataKind, Func <ISchemaBoundMapper, ColumnType, bool> canWrap)
{
Contracts.CheckValue(env, nameof(env));
_host = env.Register(LoaderSignature);
_host.CheckValue(mapper, nameof(mapper));
_host.CheckValue(type, nameof(type));
_host.CheckValue(getter, nameof(getter));
_host.CheckNonEmpty(metadataKind, nameof(metadataKind));
_host.CheckValueOrNull(canWrap);
_mapper = mapper;
int scoreIdx;
bool result = mapper.Schema.TryGetColumnIndex(MetadataUtils.Const.ScoreValueKind.Score, out scoreIdx);
if (!result)
{
throw env.ExceptParam(nameof(mapper), "Mapper did not have a '{0}' column", MetadataUtils.Const.ScoreValueKind.Score);
}
_labelNameType = type;
_labelNameGetter = getter;
_metadataKind = metadataKind;
_outSchema = new SchemaImpl(mapper.Schema, scoreIdx, _labelNameType, _labelNameGetter, _metadataKind);
_canWrap = canWrap;
}
public static void AddValue <TOwner, TValue>(this IList <InspectorPropertyInfo> infos, string name, ValueGetter <TOwner, TValue> getter, ValueSetter <TOwner, TValue> setter, params Attribute[] attributes)
{
AddValue(infos, name, getter, setter, 0, SerializationBackend.None, attributes);
}
static void AddCapturedLocalVariables(List <LocalVariable> vars, IMethod method, ILRange[] ilranges, ValueGetter getCaptureClass, IType captureClassType)
{
if (captureClassType.IsDisplayClass())
{
foreach (IField fieldInfo in captureClassType.GetFields())
{
IField fieldInfoCopy = fieldInfo;
if (fieldInfo.Name.StartsWith("CS$"))
{
continue; // Ignore
}
LocalVariable locVar = new LocalVariable(
method,
-1,
fieldInfo.Type,
fieldInfo.Name,
ilranges,
// TODO: Use eval thread
context => getCaptureClass(context).GetFieldValue(context.Thread, fieldInfoCopy)
);
locVar.IsCaptured = true;
if (locVar.Name.StartsWith("<>"))
{
if (locVar.Name.EndsWith("__this"))
{
locVar.Name = "this";
locVar.IsThis = true;
}
else
{
continue; // Ignore
}
}
if (locVar.Name.StartsWith("<"))
{
int endIndex = locVar.Name.IndexOf('>');
if (endIndex == -1)
{
continue; // Ignore
}
locVar.Name = fieldInfo.Name.Substring(1, endIndex - 1);
}
vars.Add(locVar);
}
}
}
public CustomProperty( object parent, FieldInfo field, object[] displayAttributes )
{
this.parent = parent;
this.parentType = parent.GetType();
this.identifier = 0;
m_field = field;
valueType = field.FieldType;
getter = id => { return m_field.GetValue( parent ); };
setter = ( id, v ) => { m_field.SetValue( parent, v ); };
ApplyAttributes( displayAttributes );
}
public CustomProperty( string sCategory, string sName, string sDescription, int identifier, ValueGetter g, ValueSetter s, bool bReadOnly, bool bVisible )
{
this.sCategory = sCategory;
this.sDescription = sDescription;
this.sName = sName;
this.bReadOnly = bReadOnly;
this.bVisible = bVisible;
this.identifier = identifier;
this.getter = g;
this.setter = s;
valueType = g( identifier ).GetType();
readOnlyTester = delegate() { return bReadOnly; };
visibleTester = delegate() { return bVisible; };
}
private protected override Delegate[] CreateGettersCore(Row input, Func <int, bool> activeCols, out Action disposer)
{
Host.Assert(LabelIndex >= 0);
Host.Assert(ScoreIndex >= 0);
disposer = null;
long cachedPosition = -1;
var label = default(VBuffer <Float>);
var score = default(VBuffer <Float>);
ValueGetter <VBuffer <Float> > nullGetter = (ref VBuffer <Float> vec) => vec = default(VBuffer <Float>);
var labelGetter = activeCols(LabelOutput) || activeCols(L1Output) || activeCols(L2Output) || activeCols(DistCol)
? RowCursorUtils.GetVecGetterAs <Float>(NumberType.Float, input, LabelIndex)
: nullGetter;
var scoreGetter = activeCols(ScoreOutput) || activeCols(L1Output) || activeCols(L2Output) || activeCols(DistCol)
? input.GetGetter <VBuffer <Float> >(ScoreIndex)
: nullGetter;
Action updateCacheIfNeeded =
() =>
{
if (cachedPosition != input.Position)
{
labelGetter(ref label);
scoreGetter(ref score);
cachedPosition = input.Position;
}
};
var getters = new Delegate[5];
if (activeCols(LabelOutput))
{
ValueGetter <VBuffer <Float> > labelFn =
(ref VBuffer <Float> dst) =>
{
updateCacheIfNeeded();
label.CopyTo(ref dst);
};
getters[LabelOutput] = labelFn;
}
if (activeCols(ScoreOutput))
{
ValueGetter <VBuffer <Float> > scoreFn =
(ref VBuffer <Float> dst) =>
{
updateCacheIfNeeded();
score.CopyTo(ref dst);
};
getters[ScoreOutput] = scoreFn;
}
if (activeCols(L1Output))
{
ValueGetter <double> l1Fn =
(ref double dst) =>
{
updateCacheIfNeeded();
dst = VectorUtils.L1Distance(in label, in score);
};
getters[L1Output] = l1Fn;
}
public static void AddValue <TOwner, TValue>(this IList <InspectorPropertyInfo> infos, string name, ValueGetter <TOwner, TValue> getter, ValueSetter <TOwner, TValue> setter, int order = 0, SerializationBackend backend = SerializationBackend.None, params Attribute[] attributes)
{
infos.Add(InspectorPropertyInfo.CreateValue(name, order, backend, new GetterSetter <TOwner, TValue>(getter, setter), attributes));
}
static void AddCapturedLocalVariables(List<LocalVariable> vars, IMethod method, ILRange[] ilranges, ValueGetter getCaptureClass, IType captureClassType)
{
if (captureClassType.IsDisplayClass()) {
foreach(IField fieldInfo in captureClassType.GetFields()) {
IField fieldInfoCopy = fieldInfo;
if (fieldInfo.Name.StartsWith("CS$")) continue; // Ignore
LocalVariable locVar = new LocalVariable(
method,
-1,
fieldInfo.Type,
fieldInfo.Name,
ilranges,
// TODO: Use eval thread
context => getCaptureClass(context).GetFieldValue(context.Thread, fieldInfoCopy)
);
locVar.IsCaptured = true;
if (locVar.Name.StartsWith("<>")) {
if (locVar.Name.EndsWith("__this")) {
locVar.Name = "this";
locVar.IsThis = true;
} else {
continue; // Ignore
}
}
if (locVar.Name.StartsWith("<")) {
int endIndex = locVar.Name.IndexOf('>');
if (endIndex == -1) continue; // Ignore
locVar.Name = fieldInfo.Name.Substring(1, endIndex - 1);
}
vars.Add(locVar);
}
}
}
public DebugLocalVariableInfo(string name, int localIndex, int startOffset, int endOffset, DebugType localType, ValueGetter getter)
{
this.Name = name;
this.localIndex = localIndex;
this.StartOffset = startOffset;
this.EndOffset = endOffset;
this.localType = localType;
this.getter = getter;
}
public void NoMatch(ValueGetter getter)
{
throw new NotSupportedException("Compiling types is not supported for " + getter.GetType().Name);;
}
// The multi-output regression evaluator prints only the per-label metrics for each fold.
protected override void PrintFoldResultsCore(IChannel ch, Dictionary <string, IDataView> metrics)
{
IDataView fold;
if (!metrics.TryGetValue(MetricKinds.OverallMetrics, out fold))
{
throw ch.Except("No overall metrics found");
}
int isWeightedCol;
bool needWeighted = fold.Schema.TryGetColumnIndex(MetricKinds.ColumnNames.IsWeighted, out isWeightedCol);
int stratCol;
bool hasStrats = fold.Schema.TryGetColumnIndex(MetricKinds.ColumnNames.StratCol, out stratCol);
int stratVal;
bool hasStratVals = fold.Schema.TryGetColumnIndex(MetricKinds.ColumnNames.StratVal, out stratVal);
ch.Assert(hasStrats == hasStratVals);
var colCount = fold.Schema.ColumnCount;
var vBufferGetters = new ValueGetter <VBuffer <double> > [colCount];
using (var cursor = fold.GetRowCursor(col => true))
{
bool isWeighted = false;
ValueGetter <bool> isWeightedGetter;
if (needWeighted)
{
isWeightedGetter = cursor.GetGetter <bool>(isWeightedCol);
}
else
{
isWeightedGetter = (ref bool dst) => dst = false;
}
ValueGetter <uint> stratGetter;
if (hasStrats)
{
var type = cursor.Schema.GetColumnType(stratCol);
stratGetter = RowCursorUtils.GetGetterAs <uint>(type, cursor, stratCol);
}
else
{
stratGetter = (ref uint dst) => dst = 0;
}
int labelCount = 0;
for (int i = 0; i < fold.Schema.ColumnCount; i++)
{
if (fold.Schema.IsHidden(i) || (needWeighted && i == isWeightedCol) ||
(hasStrats && (i == stratCol || i == stratVal)))
{
continue;
}
var type = fold.Schema.GetColumnType(i);
if (type.IsKnownSizeVector && type.ItemType == NumberType.R8)
{
vBufferGetters[i] = cursor.GetGetter <VBuffer <double> >(i);
if (labelCount == 0)
{
labelCount = type.VectorSize;
}
else
{
ch.Check(labelCount == type.VectorSize, "All vector metrics should contain the same number of slots");
}
}
}
var labelNames = new ReadOnlyMemory <char> [labelCount];
for (int j = 0; j < labelCount; j++)
{
labelNames[j] = string.Format("Label_{0}", j).AsMemory();
}
var sb = new StringBuilder();
sb.AppendLine("Per-label metrics:");
sb.AppendFormat("{0,12} ", " ");
for (int i = 0; i < labelCount; i++)
{
sb.AppendFormat(" {0,20}", labelNames[i]);
}
sb.AppendLine();
VBuffer <Double> metricVals = default(VBuffer <Double>);
bool foundWeighted = !needWeighted;
bool foundUnweighted = false;
uint strat = 0;
while (cursor.MoveNext())
{
isWeightedGetter(ref isWeighted);
if (foundWeighted && isWeighted || foundUnweighted && !isWeighted)
{
throw ch.Except("Multiple {0} rows found in overall metrics data view",
isWeighted ? "weighted" : "unweighted");
}
if (isWeighted)
{
foundWeighted = true;
}
else
{
foundUnweighted = true;
}
stratGetter(ref strat);
if (strat > 0)
{
continue;
}
for (int i = 0; i < colCount; i++)
{
if (vBufferGetters[i] != null)
{
vBufferGetters[i](ref metricVals);
ch.Assert(metricVals.Length == labelCount);
sb.AppendFormat("{0}{1,12}:", isWeighted ? "Weighted " : "", fold.Schema.GetColumnName(i));
foreach (var metric in metricVals.Items(all: true))
{
sb.AppendFormat(" {0,20:G20}", metric.Value);
}
sb.AppendLine();
}
}
if (foundUnweighted && foundWeighted)
{
break;
}
}
ch.Assert(foundUnweighted && foundWeighted);
ch.Info(sb.ToString());
}
}
public static Type InferPredictorCategoryType(IDataView data, PurposeInference.Column[] columns)
{
List <PurposeInference.Column> labels = columns.Where(col => col.Purpose == ColumnPurpose.Label).ToList();
if (labels.Count == 0)
{
return(typeof(SignatureClusteringTrainer));
}
if (labels.Count > 1)
{
return(typeof(SignatureMultiOutputRegressorTrainer));
}
PurposeInference.Column label = labels.First();
HashSet <string> uniqueLabelValues = new HashSet <string>();
data = data.Take(1000);
using (var cursor = data.GetRowCursor(index => index == label.ColumnIndex))
{
ValueGetter <ReadOnlyMemory <char> > getter = DataViewUtils.PopulateGetterArray(cursor, new List <int> {
label.ColumnIndex
})[0];
while (cursor.MoveNext())
{
var currentLabel = default(ReadOnlyMemory <char>);
getter(ref currentLabel);
string currentLabelString = currentLabel.ToString();
if (!String.IsNullOrEmpty(currentLabelString) && !uniqueLabelValues.Contains(currentLabelString))
{
//Missing values in float and doubles are converted to "NaN" in text and they should not
//be treated as label values.
if ((label.ItemKind == DataKind.R4 || label.ItemKind == DataKind.R8) && currentLabelString == "?")
{
continue;
}
uniqueLabelValues.Add(currentLabelString);
}
}
}
if (uniqueLabelValues.Count == 1)
{
return(typeof(SignatureAnomalyDetectorTrainer));
}
if (uniqueLabelValues.Count == 2)
{
return(typeof(SignatureBinaryClassifierTrainer));
}
if (uniqueLabelValues.Count > 2)
{
if ((label.ItemKind == DataKind.R4) &&
uniqueLabelValues.Any(val =>
{
float fVal;
return(float.TryParse(val, out fVal) && (fVal > 50 || fVal < 0 || val.Contains('.')));
}))
{
return(typeof(SignatureRegressorTrainer));
}
if (label.ItemKind == DataKind.R4 ||
label.ItemKind == DataKind.TX ||
data.Schema.GetColumnType(label.ColumnIndex).IsKey)
{
if (columns.Any(col => col.Purpose == ColumnPurpose.Group))
{
return(typeof(SignatureRankerTrainer));
}
else
{
return(typeof(SignatureMultiClassClassifierTrainer));
}
}
}
return(null);
}
private void InitMapMurmurHashV2 <T>(T val, DataViewType type, int numberOfBits = 20, ValueGetter <T> getter = null)
{
if (getter == null)
{
getter = (ref T dst) => dst = val;
}
_inRow = RowImpl.Create(type, getter);
// One million features is a nice, typical number.
var info = new HashingEstimator.ColumnOptionsInternal("Bar", "Foo", numberOfBits: numberOfBits);
var xf = new HashingTransformer(_env, new[] { info });
var mapper = ((ITransformer)xf).GetRowToRowMapper(_inRow.Schema);
var column = mapper.OutputSchema["Bar"];
var outRow = mapper.GetRow(_inRow, column);
if (type is VectorDataViewType)
{
_vecGetter = outRow.GetGetter <VBuffer <uint> >(column);
}
else
{
_getter = outRow.GetGetter <uint>(column);
}
}
// Loads all relevant data for whitening training into memory.
private static float[][] LoadDataAsDense(IHostEnvironment env, IChannel ch, IDataView inputData, out int[] actualRowCounts,
ColumnType[] srcTypes, int[] cols, params ColumnInfo[] columns)
{
long crowData = GetRowCount(inputData, columns);
var columnData = new float[columns.Length][];
actualRowCounts = new int[columns.Length];
int maxActualRowCount = 0;
for (int i = 0; i < columns.Length; i++)
{
ch.Assert(srcTypes[i].IsVector && srcTypes[i].IsKnownSizeVector);
// Use not more than MaxRow number of rows.
var ex = columns[i];
if (crowData <= ex.MaxRow)
{
actualRowCounts[i] = (int)crowData;
}
else
{
ch.Info(MessageSensitivity.Schema, "Only {0:N0} rows of column '{1}' will be used for whitening transform.", ex.MaxRow, columns[i].Output);
actualRowCounts[i] = ex.MaxRow;
}
int cslot = srcTypes[i].ValueCount;
// Check that total number of values in matrix does not exceed int.MaxValue and adjust row count if necessary.
if ((long)cslot * actualRowCounts[i] > int.MaxValue)
{
actualRowCounts[i] = int.MaxValue / cslot;
ch.Info(MessageSensitivity.Schema, "Only {0:N0} rows of column '{1}' will be used for whitening transform.", actualRowCounts[i], columns[i].Output);
}
columnData[i] = new float[cslot * actualRowCounts[i]];
if (actualRowCounts[i] > maxActualRowCount)
{
maxActualRowCount = actualRowCounts[i];
}
}
var idxDst = new int[columns.Length];
var colsSet = new HashSet <int>(cols);
using (var cursor = inputData.GetRowCursor(colsSet.Contains))
{
var getters = new ValueGetter <VBuffer <float> > [columns.Length];
for (int i = 0; i < columns.Length; i++)
{
getters[i] = cursor.GetGetter <VBuffer <float> >(cols[i]);
}
var val = default(VBuffer <float>);
int irow = 0;
while (irow < maxActualRowCount && cursor.MoveNext())
{
for (int i = 0; i < columns.Length; i++)
{
if (irow >= actualRowCounts[i] || columnData[i].Length == 0)
{
continue;
}
getters[i](ref val);
val.CopyTo(columnData[i], idxDst[i]);
idxDst[i] += srcTypes[i].ValueCount;
}
irow++;
}
#if DEBUG
for (int i = 0; i < columns.Length; i++)
{
ch.Assert(idxDst[i] == columnData[i].Length);
}
#endif
}
return(columnData);
}
public MethodDescriptor(string name, ValueGetter<T> getter, Type returnType) : base(name, null) {
this.getter = getter;
this.returnType = returnType;
}
private ValueGetter <float> GetGetter(ValueGetter <uint> matrixColumnIndexGetter, ValueGetter <uint> matrixRowIndexGetter)
{
_host.AssertValue(matrixColumnIndexGetter);
_host.AssertValue(matrixRowIndexGetter);
uint matrixColumnIndex = 0;
uint matrixRowIndex = 0;
var mapper = GetMapper <uint, uint, float>();
ValueGetter <float> del =
(ref float value) =>
{
matrixColumnIndexGetter(ref matrixColumnIndex);
matrixRowIndexGetter(ref matrixRowIndex);
mapper(in matrixColumnIndex, ref matrixRowIndex, ref value);
};
public ValueOne(Cursor cursor, ValueGetter <T> getSrc, InPredicate <T> hasBad)
: base(cursor, getSrc, hasBad)
{
_getter = GetValue;
}
// REVIEW: It would be nice to support propagation of select metadata.
public static IDataView Create <TSrc, TDst>(IHostEnvironment env, string name, IDataView input,
string src, string dst, DataViewType typeSrc, DataViewType typeDst, ValueMapper <TSrc, TDst> mapper,
ValueGetter <VBuffer <ReadOnlyMemory <char> > > keyValueGetter = null, ValueGetter <VBuffer <ReadOnlyMemory <char> > > slotNamesGetter = null)
{
Contracts.CheckValue(env, nameof(env));
env.CheckNonEmpty(name, nameof(name));
env.CheckValue(input, nameof(input));
env.CheckNonEmpty(src, nameof(src));
env.CheckNonEmpty(dst, nameof(dst));
env.CheckValue(typeSrc, nameof(typeSrc));
env.CheckValue(typeDst, nameof(typeDst));
env.CheckValue(mapper, nameof(mapper));
env.Check(keyValueGetter == null || typeDst.GetItemType() is KeyDataViewType);
env.Check(slotNamesGetter == null || typeDst.IsKnownSizeVector());
if (typeSrc.RawType != typeof(TSrc))
{
throw env.ExceptParam(nameof(mapper),
"The source column type '{0}' doesn't match the input type of the mapper", typeSrc);
}
if (typeDst.RawType != typeof(TDst))
{
throw env.ExceptParam(nameof(mapper),
"The destination column type '{0}' doesn't match the output type of the mapper", typeDst);
}
bool tmp = input.Schema.TryGetColumnIndex(src, out int colSrc);
if (!tmp)
{
throw env.ExceptParam(nameof(src), "The input data doesn't have a column named '{0}'", src);
}
var typeOrig = input.Schema[colSrc].Type;
// REVIEW: Ideally this should support vector-type conversion. It currently doesn't.
bool ident;
Delegate conv;
if (typeOrig.SameSizeAndItemType(typeSrc))
{
ident = true;
conv = null;
}
else if (!Conversions.Instance.TryGetStandardConversion(typeOrig, typeSrc, out conv, out ident))
{
throw env.ExceptParam(nameof(mapper),
"The type of column '{0}', '{1}', cannot be converted to the input type of the mapper '{2}'",
src, typeOrig, typeSrc);
}
var col = new Column(src, dst);
IDataView impl;
if (ident)
{
impl = new Impl <TSrc, TDst, TDst>(env, name, input, col, typeDst, mapper, keyValueGetter: keyValueGetter, slotNamesGetter: slotNamesGetter);
}
else
{
Func <IHostEnvironment, string, IDataView, Column, DataViewType, ValueMapper <int, int>,
ValueMapper <int, int>, ValueGetter <VBuffer <ReadOnlyMemory <char> > >, ValueGetter <VBuffer <ReadOnlyMemory <char> > >,
Impl <int, int, int> > del = CreateImpl <int, int, int>;
var meth = del.GetMethodInfo().GetGenericMethodDefinition()
.MakeGenericMethod(typeOrig.RawType, typeof(TSrc), typeof(TDst));
impl = (IDataView)meth.Invoke(null, new object[] { env, name, input, col, typeDst, conv, mapper, keyValueGetter, slotNamesGetter });
}
return(new OpaqueDataView(impl));
}
public ValueVec(Cursor cursor, ValueGetter <VBuffer <T> > getSrc, InPredicate <VBuffer <T> > hasBad)
: base(cursor, getSrc, hasBad)
{
_getter = GetValue;
}
private MatrixFactorizationModelParameters TrainCore(IChannel ch, RoleMappedData data, RoleMappedData validData = null)
{
_host.AssertValue(ch);
ch.AssertValue(data);
ch.AssertValueOrNull(validData);
ch.CheckParam(data.Schema.Label.HasValue, nameof(data), "Input data did not have a unique label");
RecommenderUtils.CheckAndGetMatrixIndexColumns(data, out var matrixColumnIndexColInfo, out var matrixRowIndexColInfo, isDecode: false);
var labelCol = data.Schema.Label.Value;
if (labelCol.Type != NumberDataViewType.Single && labelCol.Type != NumberDataViewType.Double)
{
throw ch.Except("Column '{0}' for label should be floating point, but is instead {1}", labelCol.Name, labelCol.Type);
}
MatrixFactorizationModelParameters predictor;
if (validData != null)
{
ch.CheckValue(validData, nameof(validData));
ch.CheckParam(validData.Schema.Label.HasValue, nameof(validData), "Input validation data did not have a unique label");
RecommenderUtils.CheckAndGetMatrixIndexColumns(validData, out var validMatrixColumnIndexColInfo, out var validMatrixRowIndexColInfo, isDecode: false);
var validLabelCol = validData.Schema.Label.Value;
if (validLabelCol.Type != NumberDataViewType.Single && validLabelCol.Type != NumberDataViewType.Double)
{
throw ch.Except("Column '{0}' for validation label should be floating point, but is instead {1}", validLabelCol.Name, validLabelCol.Type);
}
if (!matrixColumnIndexColInfo.Type.Equals(validMatrixColumnIndexColInfo.Type))
{
throw ch.ExceptParam(nameof(validData), "Train and validation sets' matrix-column types differed, {0} vs. {1}",
matrixColumnIndexColInfo.Type, validMatrixColumnIndexColInfo.Type);
}
if (!matrixRowIndexColInfo.Type.Equals(validMatrixRowIndexColInfo.Type))
{
throw ch.ExceptParam(nameof(validData), "Train and validation sets' matrix-row types differed, {0} vs. {1}",
matrixRowIndexColInfo.Type, validMatrixRowIndexColInfo.Type);
}
}
int colCount = matrixColumnIndexColInfo.Type.GetKeyCountAsInt32(_host);
int rowCount = matrixRowIndexColInfo.Type.GetKeyCountAsInt32(_host);
ch.Assert(rowCount > 0);
ch.Assert(colCount > 0);
// Checks for equality on the validation set ensure it is correct here.
using (var cursor = data.Data.GetRowCursor(matrixColumnIndexColInfo, matrixRowIndexColInfo, data.Schema.Label.Value))
{
// LibMF works only over single precision floats, but we want to be able to consume either.
var labGetter = RowCursorUtils.GetGetterAs <float>(NumberDataViewType.Single, cursor, data.Schema.Label.Value.Index);
var matrixColumnIndexGetter = RowCursorUtils.GetGetterAs <uint>(NumberDataViewType.UInt32, cursor, matrixColumnIndexColInfo.Index);
var matrixRowIndexGetter = RowCursorUtils.GetGetterAs <uint>(NumberDataViewType.UInt32, cursor, matrixRowIndexColInfo.Index);
if (validData == null)
{
// Have the trainer do its work.
using (var buffer = PrepareBuffer())
{
buffer.Train(ch, rowCount, colCount, cursor, labGetter, matrixRowIndexGetter, matrixColumnIndexGetter);
predictor = new MatrixFactorizationModelParameters(_host, buffer, (KeyType)matrixColumnIndexColInfo.Type, (KeyType)matrixRowIndexColInfo.Type);
}
}
else
{
RecommenderUtils.CheckAndGetMatrixIndexColumns(validData, out var validMatrixColumnIndexColInfo, out var validMatrixRowIndexColInfo, isDecode: false);
using (var validCursor = validData.Data.GetRowCursor(matrixColumnIndexColInfo, matrixRowIndexColInfo, data.Schema.Label.Value))
{
ValueGetter <float> validLabelGetter = RowCursorUtils.GetGetterAs <float>(NumberDataViewType.Single, validCursor, validData.Schema.Label.Value.Index);
var validMatrixColumnIndexGetter = RowCursorUtils.GetGetterAs <uint>(NumberDataViewType.UInt32, validCursor, validMatrixColumnIndexColInfo.Index);
var validMatrixRowIndexGetter = RowCursorUtils.GetGetterAs <uint>(NumberDataViewType.UInt32, validCursor, validMatrixRowIndexColInfo.Index);
// Have the trainer do its work.
using (var buffer = PrepareBuffer())
{
buffer.TrainWithValidation(ch, rowCount, colCount,
cursor, labGetter, matrixRowIndexGetter, matrixColumnIndexGetter,
validCursor, validLabelGetter, validMatrixRowIndexGetter, validMatrixColumnIndexGetter);
predictor = new MatrixFactorizationModelParameters(_host, buffer, (KeyType)matrixColumnIndexColInfo.Type, (KeyType)matrixRowIndexColInfo.Type);
}
}
}
}
return(predictor);
}
private void Train(IChannel ch, IDataView trainingData, LdaState[] states)
{
Host.AssertValue(ch);
ch.AssertValue(trainingData);
ch.AssertValue(states);
ch.Assert(states.Length == Infos.Length);
bool[] activeColumns = new bool[trainingData.Schema.ColumnCount];
int[] numVocabs = new int[Infos.Length];
for (int i = 0; i < Infos.Length; i++)
{
activeColumns[Infos[i].Source] = true;
numVocabs[i] = 0;
}
//the current lda needs the memory allocation before feedin data, so needs two sweeping of the data,
//one for the pre-calc memory, one for feedin data really
//another solution can be prepare these two value externally and put them in the beginning of the input file.
long[] corpusSize = new long[Infos.Length];
int[] numDocArray = new int[Infos.Length];
using (var cursor = trainingData.GetRowCursor(col => activeColumns[col]))
{
var getters = new ValueGetter <VBuffer <Double> > [Utils.Size(Infos)];
for (int i = 0; i < Infos.Length; i++)
{
corpusSize[i] = 0;
numDocArray[i] = 0;
getters[i] = RowCursorUtils.GetVecGetterAs <Double>(NumberType.R8, cursor, Infos[i].Source);
}
VBuffer <Double> src = default(VBuffer <Double>);
long rowCount = 0;
while (cursor.MoveNext())
{
++rowCount;
for (int i = 0; i < Infos.Length; i++)
{
int docSize = 0;
getters[i](ref src);
// compute term, doc instance#.
for (int termID = 0; termID < src.Count; termID++)
{
int termFreq = GetFrequency(src.Values[termID]);
if (termFreq < 0)
{
// Ignore this row.
docSize = 0;
break;
}
if (docSize >= _exes[i].NumMaxDocToken - termFreq)
{
break; //control the document length
}
//if legal then add the term
docSize += termFreq;
}
// Ignore empty doc
if (docSize == 0)
{
continue;
}
numDocArray[i]++;
corpusSize[i] += docSize * 2 + 1; // in the beggining of each doc, there is a cursor variable
// increase numVocab if needed.
if (numVocabs[i] < src.Length)
{
numVocabs[i] = src.Length;
}
}
}
for (int i = 0; i < Infos.Length; ++i)
{
if (numDocArray[i] != rowCount)
{
ch.Assert(numDocArray[i] < rowCount);
ch.Warning($"Column '{Infos[i].Name}' has skipped {rowCount - numDocArray[i]} of {rowCount} rows either empty or with negative, non-finite, or fractional values.");
}
}
}
// Initialize all LDA states
for (int i = 0; i < Infos.Length; i++)
{
var state = new LdaState(Host, _exes[i], numVocabs[i]);
if (numDocArray[i] == 0 || corpusSize[i] == 0)
{
throw ch.Except("The specified documents are all empty in column '{0}'.", Infos[i].Name);
}
state.AllocateDataMemory(numDocArray[i], corpusSize[i]);
states[i] = state;
}
using (var cursor = trainingData.GetRowCursor(col => activeColumns[col]))
{
int[] docSizeCheck = new int[Infos.Length];
// This could be optimized so that if multiple trainers consume the same column, it is
// fed into the train method once.
var getters = new ValueGetter <VBuffer <Double> > [Utils.Size(Infos)];
for (int i = 0; i < Infos.Length; i++)
{
docSizeCheck[i] = 0;
getters[i] = RowCursorUtils.GetVecGetterAs <Double>(NumberType.R8, cursor, Infos[i].Source);
}
VBuffer <Double> src = default(VBuffer <Double>);
while (cursor.MoveNext())
{
for (int i = 0; i < Infos.Length; i++)
{
getters[i](ref src);
docSizeCheck[i] += states[i].FeedTrain(Host, ref src);
}
}
for (int i = 0; i < Infos.Length; i++)
{
Host.Assert(corpusSize[i] == docSizeCheck[i]);
states[i].CompleteTrain();
}
}
}
private ValueGetter <VBuffer <ushort> > MakeGetterVec(Row input, int iinfo)
{
Host.AssertValue(input);
int cv = input.Schema[ColMapNewToOld[iinfo]].Type.GetVectorSize();
Contracts.Assert(cv >= 0);
var getSrc = input.GetGetter <VBuffer <ReadOnlyMemory <char> > >(ColMapNewToOld[iinfo]);
var src = default(VBuffer <ReadOnlyMemory <char> >);
ValueGetter <VBuffer <ushort> > getterWithStartEndSep = (ref VBuffer <ushort> dst) =>
{
getSrc(ref src);
int len = 0;
var srcValues = src.GetValues();
for (int i = 0; i < srcValues.Length; i++)
{
if (!srcValues[i].IsEmpty)
{
len += srcValues[i].Length;
if (_parent._useMarkerChars)
{
len += TextMarkersCount;
}
}
}
var editor = VBufferEditor.Create(ref dst, len);
if (len > 0)
{
int index = 0;
for (int i = 0; i < srcValues.Length; i++)
{
if (srcValues[i].IsEmpty)
{
continue;
}
if (_parent._useMarkerChars)
{
editor.Values[index++] = TextStartMarker;
}
var span = srcValues[i].Span;
for (int ich = 0; ich < srcValues[i].Length; ich++)
{
editor.Values[index++] = span[ich];
}
if (_parent._useMarkerChars)
{
editor.Values[index++] = TextEndMarker;
}
}
Contracts.Assert(index == len);
}
dst = editor.Commit();
};
ValueGetter <VBuffer <ushort> > getterWithUnitSep = (ref VBuffer <ushort> dst) =>
{
getSrc(ref src);
int len = 0;
var srcValues = src.GetValues();
for (int i = 0; i < srcValues.Length; i++)
{
if (!srcValues[i].IsEmpty)
{
len += srcValues[i].Length;
if (i > 0)
{
len += 1; // add UnitSeparator character to len that will be added
}
}
}
if (_parent._useMarkerChars)
{
len += TextMarkersCount;
}
var editor = VBufferEditor.Create(ref dst, len);
if (len > 0)
{
int index = 0;
// ReadOnlyMemory can be a result of either concatenating text columns together
// or application of word tokenizer before char tokenizer in TextFeaturizingEstimator.
//
// Considering VBuffer<ReadOnlyMemory> as a single text stream.
// Therefore, prepend and append start and end markers only once i.e. at the start and at end of vector.
// Insert UnitSeparator after every piece of text in the vector.
if (_parent._useMarkerChars)
{
editor.Values[index++] = TextStartMarker;
}
for (int i = 0; i < srcValues.Length; i++)
{
if (srcValues[i].IsEmpty)
{
continue;
}
if (i > 0)
{
editor.Values[index++] = UnitSeparator;
}
var span = srcValues[i].Span;
for (int ich = 0; ich < srcValues[i].Length; ich++)
{
editor.Values[index++] = span[ich];
}
}
if (_parent._useMarkerChars)
{
editor.Values[index++] = TextEndMarker;
}
Contracts.Assert(index == len);
}
dst = editor.Commit();
};
return(_parent._isSeparatorStartEnd ? getterWithStartEndSep : getterWithUnitSep);
}
public ListAggregator(IRow row, int col)
{
Contracts.AssertValue(row);
_srcGetter = row.GetGetter <TValue>(col);
_getter = (ValueGetter <VBuffer <TValue> >)Getter;
}
private static ValueGetter AsUntypedGetter(this ValueGetter <object, object> typedGetter)
{
return(t => typedGetter(t));
}
public void Execute(MadLevelIcon icon, ValueGetter getter, ValueSetter setter) {
var animations = MadAnim.FindAnimations(icon.gameObject, animationName);
for (int i = 0; i < animations.Count; ++i) {
var animation = animations[i];
float baseValue = getter(animation);
switch (modifierFunction) {
case ModifierFunc.Custom:
setter(animation, customModifierFunction(icon));
break;
case ModifierFunc.Predefined:
float firstParameter = GetFirstParameterValue(icon);
float rightSideValue = Compute(firstParameter, secondParameter, valueOperator);
float leftSideValue = Compute(baseValue, rightSideValue, baseOperator);
setter(animation, leftSideValue);
break;
default:
Debug.LogError("Uknown modifier function:" + modifierFunction);
setter(animation, baseValue);
break;
}
}
}
public NameOnnxValueGetter(Row input, string colName, int colIndex)
{
_colName = colName;
_srcgetter = input.GetGetter <T>(colIndex);
}
public override Delegate[] CreateGetters(IRow input, Func <int, bool> activeOutput, out Action disposer)
{
disposer = null;
var getters = new Delegate[3];
if (!activeOutput(ClusterIdCol) && !activeOutput(SortedClusterCol) && !activeOutput(SortedClusterScoreCol))
{
return(getters);
}
long cachedPosition = -1;
VBuffer <Single> scores = default(VBuffer <Single>);
var scoresArr = new Single[_numClusters];
int[] sortedIndices = new int[_numClusters];
var scoreGetter = input.GetGetter <VBuffer <Single> >(ScoreIndex);
Action updateCacheIfNeeded =
() =>
{
if (cachedPosition != input.Position)
{
scoreGetter(ref scores);
scores.CopyTo(scoresArr);
int j = 0;
foreach (var index in Enumerable.Range(0, scoresArr.Length).OrderBy(i => scoresArr[i]))
{
sortedIndices[j++] = index;
}
cachedPosition = input.Position;
}
};
if (activeOutput(ClusterIdCol))
{
ValueGetter <uint> assignedFn =
(ref uint dst) =>
{
updateCacheIfNeeded();
dst = (uint)sortedIndices[0] + 1;
};
getters[ClusterIdCol] = assignedFn;
}
if (activeOutput(SortedClusterScoreCol))
{
ValueGetter <VBuffer <Single> > topKScoresFn =
(ref VBuffer <Single> dst) =>
{
updateCacheIfNeeded();
var values = dst.Values;
if (Utils.Size(values) < _numClusters)
{
values = new Single[_numClusters];
}
for (int i = 0; i < _numClusters; i++)
{
values[i] = scores.GetItemOrDefault(sortedIndices[i]);
}
dst = new VBuffer <Single>(_numClusters, values);
};
getters[SortedClusterScoreCol] = topKScoresFn;
}
if (activeOutput(SortedClusterCol))
{
ValueGetter <VBuffer <uint> > topKClassesFn =
(ref VBuffer <uint> dst) =>
{
updateCacheIfNeeded();
var values = dst.Values;
if (Utils.Size(values) < _numClusters)
{
values = new uint[_numClusters];
}
for (int i = 0; i < _numClusters; i++)
{
values[i] = (uint)sortedIndices[i] + 1;
}
dst = new VBuffer <uint>(_numClusters, values);
};
getters[SortedClusterCol] = topKClassesFn;
}
return(getters);
}
private static void EnsureCachedResultValueMapper(ValueMapper <VBuffer <Float>, Float, Float> mapper,
ref long cachedPosition, ValueGetter <VBuffer <Float> > featureGetter, ref VBuffer <Float> features,
ref Float score, ref Float prob, Row input)
{
Contracts.AssertValue(mapper);
if (cachedPosition != input.Position)
{
if (featureGetter != null)
{
featureGetter(ref features);
}
mapper(in features, ref score, ref prob);
cachedPosition = input.Position;
}
}
/// <summary>
/// Determines whether the present value matches the value on
/// the initialSetValue (which can be a single value or a set)
/// </summary>
/// <param name="value">Value from the datasource</param>
/// <param name="initialSetValue">Value from the initial selection set</param>
/// <param name="propertyOnInitialSet">Optional. Property to obtain the value from</param>
/// <param name="isMultiple"><c>true</c> if the initial selection is a set</param>
/// <returns><c>true</c> if it's selected</returns>
protected internal static bool IsPresent(object value, object initialSetValue,
ValueGetter propertyOnInitialSet, bool isMultiple)
{
if (!isMultiple)
{
object valueToCompare = initialSetValue;
if (propertyOnInitialSet != null)
{
// propertyOnInitialSet.GetValue(initialSetValue, null);
valueToCompare = propertyOnInitialSet.GetValue(initialSetValue);
}
return AreEqual(value, valueToCompare);
}
else
{
foreach(object item in (IEnumerable) initialSetValue)
{
object valueToCompare = item;
if (propertyOnInitialSet != null)
{
// valueToCompare = propertyOnInitialSet.GetValue(item, null);
valueToCompare = propertyOnInitialSet.GetValue(item);
}
if (AreEqual(value, valueToCompare))
{
return true;
}
}
}
return false;
}