private FieldAwareFactorizationMachineModelParameters(IHostEnvironment env, ModelLoadContext ctx) : base(env, LoaderSignature)
{
Host.AssertValue(ctx);
// *** Binary format ***
// bool: whether to normalize feature vectors
// int: number of fields
// int: number of features
// int: latent dimension
// float[]: linear coefficients
// float[]: latent representation of features
var norm = ctx.Reader.ReadBoolean();
var fieldCount = ctx.Reader.ReadInt32();
Host.CheckDecode(fieldCount > 0);
var featureCount = ctx.Reader.ReadInt32();
Host.CheckDecode(featureCount > 0);
var latentDim = ctx.Reader.ReadInt32();
Host.CheckDecode(latentDim > 0);
LatentDimAligned = FieldAwareFactorizationMachineUtils.GetAlignedVectorLength(latentDim);
Host.Check(checked (featureCount * fieldCount * LatentDimAligned) <= Utils.ArrayMaxSize, "Latent dimension too large");
var linearWeights = ctx.Reader.ReadFloatArray();
Host.CheckDecode(Utils.Size(linearWeights) == featureCount);
var latentWeights = ctx.Reader.ReadFloatArray();
Host.CheckDecode(Utils.Size(latentWeights) == featureCount * fieldCount * latentDim);
_norm = norm;
FieldCount = fieldCount;
FeatureCount = featureCount;
LatentDim = latentDim;
_linearWeights = linearWeights;
_latentWeightsAligned = new AlignedArray(FeatureCount * FieldCount * LatentDimAligned, 16);
for (int j = 0; j < FeatureCount; j++)
{
for (int f = 0; f < FieldCount; f++)
{
int vBias = j * FieldCount * LatentDim + f * LatentDim;
int vBiasAligned = j * FieldCount * LatentDimAligned + f * LatentDimAligned;
for (int k = 0; k < LatentDimAligned; k++)
{
if (k < LatentDim)
{
_latentWeightsAligned[vBiasAligned + k] = latentWeights[vBias + k];
}
else
{
_latentWeightsAligned[vBiasAligned + k] = 0;
}
}
}
}
}
private static double CalculateAvgLoss(IChannel ch, RoleMappedData data, bool norm, float[] linearWeights, AlignedArray latentWeightsAligned,
int latentDimAligned, AlignedArray latentSum, int[] featureFieldBuffer, int[] featureIndexBuffer, float[] featureValueBuffer, VBuffer <float> buffer, ref long badExampleCount)
{
var featureColumns = data.Schema.GetColumns(RoleMappedSchema.ColumnRole.Feature);
var getters = new ValueGetter <VBuffer <float> > [featureColumns.Count];
float label = 0;
float weight = 1;
double loss = 0;
float modelResponse = 0;
long exampleCount = 0;
badExampleCount = 0;
int count = 0;
var columns = new List <DataViewSchema.Column>(featureColumns);
columns.Add(data.Schema.Label.Value);
if (data.Schema.Weight != null)
{
columns.Add(data.Schema.Weight.Value);
}
using (var cursor = data.Data.GetRowCursor(columns))
{
var labelGetter = RowCursorUtils.GetLabelGetter(cursor, data.Schema.Label.Value.Index);
var weightGetter = data.Schema.Weight?.Index is int weightIdx?cursor.GetGetter <float>(weightIdx) : null;
for (int f = 0; f < featureColumns.Count; f++)
{
getters[f] = cursor.GetGetter <VBuffer <float> >(featureColumns[f].Index);
}
while (cursor.MoveNext())
{
labelGetter(ref label);
weightGetter?.Invoke(ref weight);
float annihilation = label - label + weight - weight;
if (!FloatUtils.IsFinite(annihilation))
{
badExampleCount++;
continue;
}
if (!FieldAwareFactorizationMachineUtils.LoadOneExampleIntoBuffer(getters, buffer, norm, ref count,
featureFieldBuffer, featureIndexBuffer, featureValueBuffer))
{
badExampleCount++;
continue;
}
FieldAwareFactorizationMachineInterface.CalculateIntermediateVariables(featureColumns.Count, latentDimAligned, count,
featureFieldBuffer, featureIndexBuffer, featureValueBuffer, linearWeights, latentWeightsAligned, latentSum, ref modelResponse);
loss += weight * CalculateLoss(label, modelResponse);
exampleCount++;
}
}
return(loss / exampleCount);
}
internal float CalculateResponse(ValueGetter <VBuffer <float> >[] getters, VBuffer <float> featureBuffer,
int[] featureFieldBuffer, int[] featureIndexBuffer, float[] featureValueBuffer, AlignedArray latentSum)
{
int count = 0;
float modelResponse = 0;
FieldAwareFactorizationMachineUtils.LoadOneExampleIntoBuffer(getters, featureBuffer, _norm, ref count,
featureFieldBuffer, featureIndexBuffer, featureValueBuffer);
FieldAwareFactorizationMachineInterface.CalculateIntermediateVariables(FieldCount, LatentDimAligned, count,
featureFieldBuffer, featureIndexBuffer, featureValueBuffer, _linearWeights, _latentWeightsAligned, latentSum, ref modelResponse);
return(modelResponse);
}
internal FieldAwareFactorizationMachineModelParameters(IHostEnvironment env, bool norm, int fieldCount, int featureCount, int latentDim,
float[] linearWeights, AlignedArray latentWeightsAligned) : base(env, LoaderSignature)
{
Host.Assert(fieldCount > 0);
Host.Assert(featureCount > 0);
Host.Assert(latentDim > 0);
Host.Assert(Utils.Size(linearWeights) == featureCount);
LatentDimAligned = FieldAwareFactorizationMachineUtils.GetAlignedVectorLength(latentDim);
Host.Assert(latentWeightsAligned.Size == checked (featureCount * fieldCount * LatentDimAligned));
_norm = norm;
FieldCount = fieldCount;
FeatureCount = featureCount;
LatentDim = latentDim;
_linearWeights = linearWeights;
_latentWeightsAligned = latentWeightsAligned;
}
/// <summary>
/// Initializes the instance. Shared between the two constructors.
/// REVIEW: Once the legacy constructor goes away, this can move to the only constructor and most of the fields can be back to readonly.
/// </summary>
/// <param name="env"></param>
/// <param name="options"></param>
private void Initialize(IHostEnvironment env, Options options)
{
Host.CheckUserArg(options.LatentDim > 0, nameof(options.LatentDim), "Must be positive");
Host.CheckUserArg(options.LambdaLinear >= 0, nameof(options.LambdaLinear), "Must be non-negative");
Host.CheckUserArg(options.LambdaLatent >= 0, nameof(options.LambdaLatent), "Must be non-negative");
Host.CheckUserArg(options.LearningRate > 0, nameof(options.LearningRate), "Must be positive");
Host.CheckUserArg(options.Iters >= 0, nameof(options.Iters), "Must be non-negative");
_latentDim = options.LatentDim;
_latentDimAligned = FieldAwareFactorizationMachineUtils.GetAlignedVectorLength(_latentDim);
_lambdaLinear = options.LambdaLinear;
_lambdaLatent = options.LambdaLatent;
_learningRate = options.LearningRate;
_numIterations = options.Iters;
_norm = options.Norm;
_shuffle = options.Shuffle;
_verbose = options.Verbose;
_radius = options.Radius;
}
/// <summary>
/// Initialize model parameters with a trained model.
/// </summary>
/// <param name="env">The host environment</param>
/// <param name="norm">True if user wants to normalize feature vector to unit length.</param>
/// <param name="fieldCount">The number of fileds, which is the symbol `m` in the doc: https://github.com/wschin/fast-ffm/blob/master/fast-ffm.pdf </param>
/// <param name="featureCount">The number of features, which is the symbol `n` in the doc: https://github.com/wschin/fast-ffm/blob/master/fast-ffm.pdf </param>
/// <param name="latentDim">The latent dimensions, which is the length of `v_{j, f}` in the doc: https://github.com/wschin/fast-ffm/blob/master/fast-ffm.pdf </param>
/// <param name="linearWeights">The linear coefficients of the features, which is the symbol `w` in the doc: https://github.com/wschin/fast-ffm/blob/master/fast-ffm.pdf </param>
/// <param name="latentWeights">Latent representation of each feature. Note that one feature may have <see cref="FieldCount"/> latent vectors
/// and each latent vector contains <see cref="LatentDim"/> values. In the f-th field, the j-th feature's latent vector, `v_{j, f}` in the doc
/// https://github.com/wschin/fast-ffm/blob/master/fast-ffm.pdf, starts at latentWeights[j * fieldCount * latentDim + f * latentDim].
/// The k-th element in v_{j, f} is latentWeights[j * fieldCount * latentDim + f * latentDim + k]. The size of the array must be featureCount x fieldCount x latentDim.</param>
public FieldAwareFactorizationMachineModelParameters(IHostEnvironment env, bool norm, int fieldCount, int featureCount, int latentDim,
float[] linearWeights, float[] latentWeights) : base(env, LoaderSignature)
{
Host.Assert(fieldCount > 0);
Host.Assert(featureCount > 0);
Host.Assert(latentDim > 0);
Host.Assert(Utils.Size(linearWeights) == featureCount);
LatentDimAligned = FieldAwareFactorizationMachineUtils.GetAlignedVectorLength(latentDim);
Host.Assert(Utils.Size(latentWeights) == checked (featureCount * fieldCount * LatentDimAligned));
_norm = norm;
FieldCount = fieldCount;
FeatureCount = featureCount;
LatentDim = latentDim;
_linearWeights = linearWeights;
_latentWeightsAligned = new AlignedArray(FeatureCount * FieldCount * LatentDimAligned, 16);
for (int j = 0; j < FeatureCount; j++)
{
for (int f = 0; f < FieldCount; f++)
{
int index = j * FieldCount * LatentDim + f * LatentDim;
int indexAligned = j * FieldCount * LatentDimAligned + f * LatentDimAligned;
for (int k = 0; k < LatentDimAligned; k++)
{
if (k < LatentDim)
{
_latentWeightsAligned[indexAligned + k] = latentWeights[index + k];
}
else
{
_latentWeightsAligned[indexAligned + k] = 0;
}
}
}
}
}
private FieldAwareFactorizationMachineModelParameters TrainCore(IChannel ch, IProgressChannel pch, RoleMappedData data,
RoleMappedData validData = null, FieldAwareFactorizationMachineModelParameters predictor = null)
{
Host.AssertValue(ch);
Host.AssertValue(pch);
data.CheckBinaryLabel();
var featureColumns = data.Schema.GetColumns(RoleMappedSchema.ColumnRole.Feature);
int fieldCount = featureColumns.Count;
int totalFeatureCount = 0;
int[] fieldColumnIndexes = new int[fieldCount];
for (int f = 0; f < fieldCount; f++)
{
var col = featureColumns[f];
Host.Assert(!col.IsHidden);
if (!(col.Type is VectorType vectorType) ||
!vectorType.IsKnownSize ||
vectorType.ItemType != NumberType.Float)
{
throw ch.ExceptParam(nameof(data), "Training feature column '{0}' must be a known-size vector of R4, but has type: {1}.", col.Name, col.Type);
}
Host.Assert(vectorType.Size > 0);
fieldColumnIndexes[f] = col.Index;
totalFeatureCount += vectorType.Size;
}
ch.Check(checked (totalFeatureCount * fieldCount * _latentDimAligned) <= Utils.ArrayMaxSize, "Latent dimension or the number of fields too large");
if (predictor != null)
{
ch.Check(predictor.FeatureCount == totalFeatureCount, "Input model's feature count mismatches training feature count");
ch.Check(predictor.LatentDim == _latentDim, "Input model's latent dimension mismatches trainer's");
}
if (validData != null)
{
validData.CheckBinaryLabel();
var validFeatureColumns = data.Schema.GetColumns(RoleMappedSchema.ColumnRole.Feature);
Host.Assert(fieldCount == validFeatureColumns.Count);
for (int f = 0; f < fieldCount; f++)
{
var featCol = featureColumns[f];
var validFeatCol = validFeatureColumns[f];
Host.Assert(featCol.Name == validFeatCol.Name);
Host.Assert(featCol.Type == validFeatCol.Type);
}
}
bool shuffle = _shuffle;
if (shuffle && !data.Data.CanShuffle)
{
ch.Warning("Training data does not support shuffling, so ignoring request to shuffle");
shuffle = false;
}
var rng = shuffle ? Host.Rand : null;
var featureGetters = new ValueGetter <VBuffer <float> > [fieldCount];
var featureBuffer = new VBuffer <float>();
var featureValueBuffer = new float[totalFeatureCount];
var featureIndexBuffer = new int[totalFeatureCount];
var featureFieldBuffer = new int[totalFeatureCount];
var latentSum = new AlignedArray(fieldCount * fieldCount * _latentDimAligned, 16);
var metricNames = new List <string>()
{
"Training-loss"
};
if (validData != null)
{
metricNames.Add("Validation-loss");
}
int iter = 0;
long exampleCount = 0;
long badExampleCount = 0;
long validBadExampleCount = 0;
double loss = 0;
double validLoss = 0;
pch.SetHeader(new ProgressHeader(metricNames.ToArray(), new string[] { "iterations", "examples" }), entry =>
{
entry.SetProgress(0, iter, _numIterations);
entry.SetProgress(1, exampleCount);
});
var columns = data.Schema.Schema.Where(x => fieldColumnIndexes.Contains(x.Index)).ToList();
columns.Add(data.Schema.Label.Value);
if (data.Schema.Weight != null)
{
columns.Add(data.Schema.Weight.Value);
}
InitializeTrainingState(fieldCount, totalFeatureCount, predictor, out float[] linearWeights,
out AlignedArray latentWeightsAligned, out float[] linearAccSqGrads, out AlignedArray latentAccSqGradsAligned);
// refer to Algorithm 3 in https://github.com/wschin/fast-ffm/blob/master/fast-ffm.pdf
while (iter++ < _numIterations)
{
using (var cursor = data.Data.GetRowCursor(columns, rng))
{
var labelGetter = RowCursorUtils.GetLabelGetter(cursor, data.Schema.Label.Value.Index);
var weightGetter = data.Schema.Weight?.Index is int weightIdx?RowCursorUtils.GetGetterAs <float>(NumberType.R4, cursor, weightIdx) : null;
for (int i = 0; i < fieldCount; i++)
{
featureGetters[i] = cursor.GetGetter <VBuffer <float> >(fieldColumnIndexes[i]);
}
loss = 0;
exampleCount = 0;
badExampleCount = 0;
while (cursor.MoveNext())
{
float label = 0;
float weight = 1;
int count = 0;
float modelResponse = 0;
labelGetter(ref label);
weightGetter?.Invoke(ref weight);
float annihilation = label - label + weight - weight;
if (!FloatUtils.IsFinite(annihilation))
{
badExampleCount++;
continue;
}
if (!FieldAwareFactorizationMachineUtils.LoadOneExampleIntoBuffer(featureGetters, featureBuffer, _norm, ref count,
featureFieldBuffer, featureIndexBuffer, featureValueBuffer))
{
badExampleCount++;
continue;
}
// refer to Algorithm 1 in [3] https://github.com/wschin/fast-ffm/blob/master/fast-ffm.pdf
FieldAwareFactorizationMachineInterface.CalculateIntermediateVariables(fieldCount, _latentDimAligned, count,
featureFieldBuffer, featureIndexBuffer, featureValueBuffer, linearWeights, latentWeightsAligned, latentSum, ref modelResponse);
var slope = CalculateLossSlope(label, modelResponse);
// refer to Algorithm 2 in [3] https://github.com/wschin/fast-ffm/blob/master/fast-ffm.pdf
FieldAwareFactorizationMachineInterface.CalculateGradientAndUpdate(_lambdaLinear, _lambdaLatent, _learningRate, fieldCount, _latentDimAligned, weight, count,
featureFieldBuffer, featureIndexBuffer, featureValueBuffer, latentSum, slope, linearWeights, latentWeightsAligned, linearAccSqGrads, latentAccSqGradsAligned);
loss += weight * CalculateLoss(label, modelResponse);
exampleCount++;
}
loss /= exampleCount;
}
if (_verbose)
{
if (validData == null)
{
pch.Checkpoint(loss, iter, exampleCount);
}
else
{
validLoss = CalculateAvgLoss(ch, validData, _norm, linearWeights, latentWeightsAligned, _latentDimAligned, latentSum,
featureFieldBuffer, featureIndexBuffer, featureValueBuffer, featureBuffer, ref validBadExampleCount);
pch.Checkpoint(loss, validLoss, iter, exampleCount);
}
}
}
if (badExampleCount != 0)
{
ch.Warning($"Skipped {badExampleCount} examples with bad label/weight/features in training set");
}
if (validBadExampleCount != 0)
{
ch.Warning($"Skipped {validBadExampleCount} examples with bad label/weight/features in validation set");
}
return(new FieldAwareFactorizationMachineModelParameters(Host, _norm, fieldCount, totalFeatureCount, _latentDim, linearWeights, latentWeightsAligned));
}