private FieldAwareFactorizationMachinePredictor(IHostEnvironment env, ModelLoadContext ctx) : base(env, LoaderSignature) { Host.AssertValue(ctx); // *** Binary format *** // bool: whether to normalize feature vectors // int: number of fields // int: number of features // int: latent dimension // float[]: linear coefficients // float[]: latent representation of features var norm = ctx.Reader.ReadBoolean(); var fieldCount = ctx.Reader.ReadInt32(); Host.CheckDecode(fieldCount > 0); var featureCount = ctx.Reader.ReadInt32(); Host.CheckDecode(featureCount > 0); var latentDim = ctx.Reader.ReadInt32(); Host.CheckDecode(latentDim > 0); LatentDimAligned = FieldAwareFactorizationMachineUtils.GetAlignedVectorLength(latentDim); Host.Check(checked (featureCount * fieldCount * LatentDimAligned) <= Utils.ArrayMaxSize, "Latent dimension too large"); var linearWeights = ctx.Reader.ReadFloatArray(); Host.CheckDecode(Utils.Size(linearWeights) == featureCount); var latentWeights = ctx.Reader.ReadFloatArray(); Host.CheckDecode(Utils.Size(latentWeights) == featureCount * fieldCount * latentDim); _norm = norm; FieldCount = fieldCount; FeatureCount = featureCount; LatentDim = latentDim; _linearWeights = linearWeights; _latentWeightsAligned = new AlignedArray(FeatureCount * FieldCount * LatentDimAligned, 16); for (int j = 0; j < FeatureCount; j++) { for (int f = 0; f < FieldCount; f++) { int vBias = j * FieldCount * LatentDim + f * LatentDim; int vBiasAligned = j * FieldCount * LatentDimAligned + f * LatentDimAligned; for (int k = 0; k < LatentDimAligned; k++) { if (k < LatentDim) { _latentWeightsAligned[vBiasAligned + k] = latentWeights[vBias + k]; } else { _latentWeightsAligned[vBiasAligned + k] = 0; } } } } }
internal float CalculateResponse(ValueGetter <VBuffer <float> >[] getters, VBuffer <float> featureBuffer, int[] featureFieldBuffer, int[] featureIndexBuffer, float[] featureValueBuffer, AlignedArray latentSum) { int count = 0; float modelResponse = 0; FieldAwareFactorizationMachineUtils.LoadOneExampleIntoBuffer(getters, featureBuffer, _norm, ref count, featureFieldBuffer, featureIndexBuffer, featureValueBuffer); FieldAwareFactorizationMachineInterface.CalculateIntermediateVariables(FieldCount, LatentDimAligned, count, featureFieldBuffer, featureIndexBuffer, featureValueBuffer, _linearWeights, _latentWeightsAligned, latentSum, ref modelResponse); return(modelResponse); }
private static double CalculateAvgLoss(IChannel ch, RoleMappedData data, bool norm, float[] linearWeights, AlignedArray latentWeightsAligned, int latentDimAligned, AlignedArray latentSum, int[] featureFieldBuffer, int[] featureIndexBuffer, float[] featureValueBuffer, VBuffer <float> buffer, ref long badExampleCount) { var featureColumns = data.Schema.GetColumns(RoleMappedSchema.ColumnRole.Feature); Func <int, bool> pred = c => featureColumns.Select(ci => ci.Index).Contains(c) || c == data.Schema.Label.Value.Index || c == data.Schema.Weight?.Index; var getters = new ValueGetter <VBuffer <float> > [featureColumns.Count]; float label = 0; float weight = 1; double loss = 0; float modelResponse = 0; long exampleCount = 0; badExampleCount = 0; int count = 0; using (var cursor = data.Data.GetRowCursor(pred)) { var labelGetter = RowCursorUtils.GetLabelGetter(cursor, data.Schema.Label.Value.Index); var weightGetter = data.Schema.Weight?.Index is int weightIdx?cursor.GetGetter <float>(weightIdx) : null; for (int f = 0; f < featureColumns.Count; f++) { getters[f] = cursor.GetGetter <VBuffer <float> >(featureColumns[f].Index); } while (cursor.MoveNext()) { labelGetter(ref label); weightGetter?.Invoke(ref weight); float annihilation = label - label + weight - weight; if (!FloatUtils.IsFinite(annihilation)) { badExampleCount++; continue; } if (!FieldAwareFactorizationMachineUtils.LoadOneExampleIntoBuffer(getters, buffer, norm, ref count, featureFieldBuffer, featureIndexBuffer, featureValueBuffer)) { badExampleCount++; continue; } FieldAwareFactorizationMachineInterface.CalculateIntermediateVariables(featureColumns.Count, latentDimAligned, count, featureFieldBuffer, featureIndexBuffer, featureValueBuffer, linearWeights, latentWeightsAligned, latentSum, ref modelResponse); loss += weight * CalculateLoss(label, modelResponse); exampleCount++; } } return(loss / exampleCount); }
internal FieldAwareFactorizationMachinePredictor(IHostEnvironment env, bool norm, int fieldCount, int featureCount, int latentDim, float[] linearWeights, AlignedArray latentWeightsAligned) : base(env, LoaderSignature) { Host.Assert(fieldCount > 0); Host.Assert(featureCount > 0); Host.Assert(latentDim > 0); Host.Assert(Utils.Size(linearWeights) == featureCount); LatentDimAligned = FieldAwareFactorizationMachineUtils.GetAlignedVectorLength(latentDim); Host.Assert(latentWeightsAligned.Size == checked (featureCount * fieldCount * LatentDimAligned)); _norm = norm; FieldCount = fieldCount; FeatureCount = featureCount; LatentDim = latentDim; _linearWeights = linearWeights; _latentWeightsAligned = latentWeightsAligned; }
public FieldAwareFactorizationMachineTrainer(IHostEnvironment env, Arguments args) : base(env, LoadName) { Host.CheckUserArg(args.LatentDim > 0, nameof(args.LatentDim), "Must be positive"); Host.CheckUserArg(args.LambdaLinear >= 0, nameof(args.LambdaLinear), "Must be non-negative"); Host.CheckUserArg(args.LambdaLatent >= 0, nameof(args.LambdaLatent), "Must be non-negative"); Host.CheckUserArg(args.LearningRate > 0, nameof(args.LearningRate), "Must be positive"); Host.CheckUserArg(args.Iters >= 0, nameof(args.Iters), "Must be non-negative"); _latentDim = args.LatentDim; _latentDimAligned = FieldAwareFactorizationMachineUtils.GetAlignedVectorLength(_latentDim); _lambdaLinear = args.LambdaLinear; _lambdaLatent = args.LambdaLatent; _learningRate = args.LearningRate; _numIterations = args.Iters; _norm = args.Norm; _shuffle = args.Shuffle; _verbose = args.Verbose; _radius = args.Radius; }
/// <summary> /// Initialize model parameters with a trained model. /// </summary> /// <param name="env">The host environment</param> /// <param name="norm">True if user wants to normalize feature vector to unit length.</param> /// <param name="fieldCount">The number of fileds, which is the symbol `m` in the doc: https://github.com/wschin/fast-ffm/blob/master/fast-ffm.pdf </param> /// <param name="featureCount">The number of features, which is the symbol `n` in the doc: https://github.com/wschin/fast-ffm/blob/master/fast-ffm.pdf </param> /// <param name="latentDim">The latent dimensions, which is the length of `v_{j, f}` in the doc: https://github.com/wschin/fast-ffm/blob/master/fast-ffm.pdf </param> /// <param name="linearWeights">The linear coefficients of the features, which is the symbol `w` in the doc: https://github.com/wschin/fast-ffm/blob/master/fast-ffm.pdf </param> /// <param name="latentWeights">Latent representation of each feature. Note that one feature may have <see cref="FieldCount"/> latent vectors /// and each latent vector contains <see cref="LatentDim"/> values. In the f-th field, the j-th feature's latent vector, `v_{j, f}` in the doc /// https://github.com/wschin/fast-ffm/blob/master/fast-ffm.pdf, starts at latentWeights[j * fieldCount * latentDim + f * latentDim]. /// The k-th element in v_{j, f} is latentWeights[j * fieldCount * latentDim + f * latentDim + k]. The size of the array must be featureCount x fieldCount x latentDim.</param> public FieldAwareFactorizationMachineModelParameters(IHostEnvironment env, bool norm, int fieldCount, int featureCount, int latentDim, float[] linearWeights, float[] latentWeights) : base(env, LoaderSignature) { Host.Assert(fieldCount > 0); Host.Assert(featureCount > 0); Host.Assert(latentDim > 0); Host.Assert(Utils.Size(linearWeights) == featureCount); LatentDimAligned = FieldAwareFactorizationMachineUtils.GetAlignedVectorLength(latentDim); Host.Assert(Utils.Size(latentWeights) == checked (featureCount * fieldCount * LatentDimAligned)); _norm = norm; FieldCount = fieldCount; FeatureCount = featureCount; LatentDim = latentDim; _linearWeights = linearWeights; _latentWeightsAligned = new AlignedArray(FeatureCount * FieldCount * LatentDimAligned, 16); for (int j = 0; j < FeatureCount; j++) { for (int f = 0; f < FieldCount; f++) { int index = j * FieldCount * LatentDim + f * LatentDim; int indexAligned = j * FieldCount * LatentDimAligned + f * LatentDimAligned; for (int k = 0; k < LatentDimAligned; k++) { if (k < LatentDim) { _latentWeightsAligned[indexAligned + k] = latentWeights[index + k]; } else { _latentWeightsAligned[indexAligned + k] = 0; } } } } }
private void TrainCore(IChannel ch, IProgressChannel pch, RoleMappedData data, RoleMappedData validData, FieldAwareFactorizationMachinePredictor predictor) { Host.AssertValue(ch); Host.AssertValue(pch); data.CheckBinaryLabel(); var featureColumns = data.Schema.GetColumns(RoleMappedSchema.ColumnRole.Feature); int fieldCount = featureColumns.Count; int totalFeatureCount = 0; int[] fieldColumnIndexes = new int[fieldCount]; for (int f = 0; f < fieldCount; f++) { var col = featureColumns[f]; Host.Assert(col.Type.AsVector.VectorSize > 0); if (col == null) throw ch.ExceptParam(nameof(data), "Empty feature column not allowed"); Host.Assert(!data.Schema.Schema.IsHidden(col.Index)); if (!col.Type.IsKnownSizeVector || col.Type.ItemType != NumberType.Float) throw ch.ExceptParam(nameof(data), "Training feature column '{0}' must be a known-size vector of R4, but has type: {1}.", col.Name, col.Type); fieldColumnIndexes[f] = col.Index; totalFeatureCount += col.Type.AsVector.VectorSize; } ch.Check(checked(totalFeatureCount * fieldCount * _latentDimAligned) <= Utils.ArrayMaxSize, "Latent dimension or the number of fields too large"); if (predictor != null) { ch.Check(predictor.FeatureCount == totalFeatureCount, "Input model's feature count mismatches training feature count"); ch.Check(predictor.LatentDim == _latentDim, "Input model's latent dimension mismatches trainer's"); } if (validData != null) { validData.CheckBinaryLabel(); var validFeatureColumns = data.Schema.GetColumns(RoleMappedSchema.ColumnRole.Feature); Host.Assert(fieldCount == validFeatureColumns.Count); for (int f = 0; f < fieldCount; f++) Host.Assert(featureColumns[f] == validFeatureColumns[f]); } bool shuffle = _shuffle; if (shuffle && !data.Data.CanShuffle) { ch.Warning("Training data does not support shuffling, so ignoring request to shuffle"); shuffle = false; } var rng = shuffle ? Host.Rand : null; var featureGetters = new ValueGetter<VBuffer<float>>[fieldCount]; var featureBuffer = new VBuffer<float>(); var featureValueBuffer = new float[totalFeatureCount]; var featureIndexBuffer = new int[totalFeatureCount]; var featureFieldBuffer = new int[totalFeatureCount]; var latentSum = new AlignedArray(fieldCount * fieldCount * _latentDimAligned, 16); var metricNames = new List<string>() { "Training-loss" }; if (validData != null) metricNames.Add("Validation-loss"); int iter = 0; long exampleCount = 0; long badExampleCount = 0; long validBadExampleCount = 0; double loss = 0; double validLoss = 0; pch.SetHeader(new ProgressHeader(metricNames.ToArray(), new string[] { "iterations", "examples" }), entry => { entry.SetProgress(0, iter, _numIterations); entry.SetProgress(1, exampleCount); }); Func<int, bool> pred = c => fieldColumnIndexes.Contains(c) || c == data.Schema.Label.Index || (data.Schema.Weight != null && c == data.Schema.Weight.Index); InitializeTrainingState(fieldCount, totalFeatureCount, predictor, out float[] linearWeights, out AlignedArray latentWeightsAligned, out float[] linearAccSqGrads, out AlignedArray latentAccSqGradsAligned); // refer to Algorithm 3 in https://github.com/wschin/fast-ffm/blob/master/fast-ffm.pdf while (iter++ < _numIterations) { using (var cursor = data.Data.GetRowCursor(pred, rng)) { var labelGetter = RowCursorUtils.GetLabelGetter(cursor, data.Schema.Label.Index); var weightGetter = data.Schema.Weight == null ? null : RowCursorUtils.GetGetterAs<float>(NumberType.R4, cursor, data.Schema.Weight.Index); for (int i = 0; i < fieldCount; i++) featureGetters[i] = cursor.GetGetter<VBuffer<float>>(fieldColumnIndexes[i]); loss = 0; exampleCount = 0; badExampleCount = 0; while (cursor.MoveNext()) { float label = 0; float weight = 1; int count = 0; float modelResponse = 0; labelGetter(ref label); weightGetter?.Invoke(ref weight); float annihilation = label - label + weight - weight; if (!FloatUtils.IsFinite(annihilation)) { badExampleCount++; continue; } if (!FieldAwareFactorizationMachineUtils.LoadOneExampleIntoBuffer(featureGetters, featureBuffer, _norm, ref count, featureFieldBuffer, featureIndexBuffer, featureValueBuffer)) { badExampleCount++; continue; } // refer to Algorithm 1 in [3] https://github.com/wschin/fast-ffm/blob/master/fast-ffm.pdf FieldAwareFactorizationMachineInterface.CalculateIntermediateVariables(fieldCount, _latentDimAligned, count, featureFieldBuffer, featureIndexBuffer, featureValueBuffer, linearWeights, latentWeightsAligned, latentSum, ref modelResponse); var slope = CalculateLossSlope(label, modelResponse); // refer to Algorithm 2 in [3] https://github.com/wschin/fast-ffm/blob/master/fast-ffm.pdf FieldAwareFactorizationMachineInterface.CalculateGradientAndUpdate(_lambdaLinear, _lambdaLatent, _learningRate, fieldCount, _latentDimAligned, weight, count, featureFieldBuffer, featureIndexBuffer, featureValueBuffer, latentSum, slope, linearWeights, latentWeightsAligned, linearAccSqGrads, latentAccSqGradsAligned); loss += weight * CalculateLoss(label, modelResponse); exampleCount++; } loss /= exampleCount; } if (_verbose) { if (validData == null) pch.Checkpoint(loss, iter, exampleCount); else { validLoss = CalculateAvgLoss(ch, validData, _norm, linearWeights, latentWeightsAligned, _latentDimAligned, latentSum, featureFieldBuffer, featureIndexBuffer, featureValueBuffer, featureBuffer, ref validBadExampleCount); pch.Checkpoint(loss, validLoss, iter, exampleCount); } } } if (badExampleCount != 0) ch.Warning($"Skipped {badExampleCount} examples with bad label/weight/features in training set"); if (validBadExampleCount != 0) ch.Warning($"Skipped {validBadExampleCount} examples with bad label/weight/features in validation set"); _pred = new FieldAwareFactorizationMachinePredictor(Host, _norm, fieldCount, totalFeatureCount, _latentDim, linearWeights, latentWeightsAligned); }