/// <summary>
/// The gradient being used by the optimizer
/// </summary>
protected virtual float DifferentiableFunction(ref VBuffer <float> x, ref VBuffer <float> gradient,
IProgressChannelProvider progress)
{
Contracts.Assert((_numChunks == 0) != (_data == null));
Contracts.Assert((_cursorFactory == null) == (_data == null));
Contracts.Assert(x.Length == BiasCount + WeightCount);
Contracts.Assert(gradient.Length == BiasCount + WeightCount);
// REVIEW: if/when LBFGS test code is removed, the progress provider needs to become required.
Contracts.AssertValueOrNull(progress);
float scaleFactor = 1 / (float)WeightSum;
VBuffer <float> xDense = default(VBuffer <float>);
if (x.IsDense)
{
xDense = x;
}
else
{
x.CopyToDense(ref xDense);
}
IProgressChannel pch = progress != null?progress.StartProgressChannel("Gradient") : null;
float loss;
using (pch)
{
loss = _data == null
? DifferentiableFunctionMultithreaded(ref xDense, ref gradient, pch)
: DifferentiableFunctionStream(_cursorFactory, ref xDense, ref gradient, pch);
}
float regLoss = 0;
if (L2Weight > 0)
{
Contracts.Assert(xDense.IsDense);
var values = xDense.Values;
Double r = 0;
for (int i = BiasCount; i < values.Length; i++)
{
var xx = values[i];
r += xx * xx;
}
regLoss = (float)(r * L2Weight * 0.5);
// Here we probably want to use sparse x
VBufferUtils.ApplyWithEitherDefined(ref x, ref gradient,
(int ind, float v1, ref float v2) => { if (ind >= BiasCount)
{
v2 += L2Weight * v1;
}
});
}
VectorUtils.ScaleBy(ref gradient, scaleFactor);
// REVIEW: The regularization component of the loss is being scaled as well,
// but it's unclear that it should be scaled.
return((loss + regLoss) * scaleFactor);
}
protected override void PrintIterationMessage(IChannel ch, IProgressChannel pch)
{
// REVIEW: Shift this to use progress channels.
#if OLD_TRACING
ch.Info("Finished iteration {0}", Ensemble.NumTrees);
//This needs to be executed every iteration
if (PruningTest != null)
{
if (PruningTest is TestWindowWithTolerance)
{
if (PruningTest.BestIteration != -1)
{
ch.Info("Iteration {0} \t(Best tolerated validation moving average iter {1}:{2}~{3})",
Ensemble.NumTrees,
PruningTest.BestIteration,
(PruningTest as TestWindowWithTolerance).BestAverageValue,
(PruningTest as TestWindowWithTolerance).CurrentAverageValue);
}
else
{
ch.Info("Iteration {0}", Ensemble.NumTrees);
}
}
else
{
ch.Info("Iteration {0} \t(best validation iter {1}:{2}>{3})",
Ensemble.NumTrees,
PruningTest.BestIteration,
PruningTest.BestResult.FinalValue,
PruningTest.ComputeTests().First().FinalValue);
}
}
else
{
base.PrintIterationMessage(ch, pch);
}
#else
base.PrintIterationMessage(ch, pch);
#endif
}
private void TrainCore(IChannel ch, IProgressChannel pch, Dataset dtrain, CategoricalMetaData catMetaData, Dataset dvalid = null)
{
Host.AssertValue(ch);
Host.AssertValue(pch);
Host.AssertValue(dtrain);
Host.AssertValueOrNull(dvalid);
// For multi class, the number of labels is required.
ch.Assert(PredictionKind != PredictionKind.MultiClassClassification || Options.ContainsKey("num_class"),
"LightGBM requires the number of classes to be specified in the parameters.");
// Only enable one trainer to run at one time.
lock (LightGbmShared.LockForMultiThreadingInside)
{
ch.Info("LightGBM objective={0}", Options["objective"]);
using (Booster bst = WrappedLightGbmTraining.Train(ch, pch, Options, dtrain,
dvalid: dvalid, numIteration: Args.NumBoostRound,
verboseEval: Args.VerboseEval, earlyStoppingRound: Args.EarlyStoppingRound))
{
TrainedEnsemble = bst.GetModel(catMetaData.CategoricalBoudaries);
}
}
}
public override void PrintIterationMessage(IChannel ch, IProgressChannel pch)
{
// REVIEW: Shift this to use progress channels.
#if OLD_TRACING
if (PruningTest != null)
{
if (PruningTest is TestWindowWithTolerance)
{
if (PruningTest.BestIteration != -1)
{
ch.Info("Iteration {0} \t(Best tolerated validation moving average WinLossSurplus {1}:{2:00.00}~{3:00.00})",
Ensemble.NumTrees,
PruningTest.BestIteration,
(PruningTest as TestWindowWithTolerance).BestAverageValue,
(PruningTest as TestWindowWithTolerance).CurrentAverageValue);
}
else
{
ch.Info("Iteration {0}", Ensemble.NumTrees);
}
}
else
{
ch.Info("Iteration {0} \t(best validation WinLoss {1}:{2:00.00}>{3:00.00})",
Ensemble.NumTrees,
PruningTest.BestIteration,
PruningTest.BestResult.FinalValue,
PruningTest.ComputeTests().First().FinalValue);
}
}
else
{
base.PrintIterationMessage(ch, pch);
}
#else
base.PrintIterationMessage(ch, pch);
#endif
}
private void FetchWorker(BlockingCollection <Block> toCompress, IDataView data,
ColumnCodec[] activeColumns, int rowsPerBlock, Stopwatch sw, IChannel ch, IProgressChannel pch, ExceptionMarshaller exMarshaller)
{
Contracts.AssertValue(ch);
Contracts.AssertValueOrNull(pch);
ch.AssertValue(exMarshaller);
try
{
ch.AssertValue(toCompress);
ch.AssertValue(data);
ch.AssertValue(activeColumns);
ch.AssertValue(sw);
ch.Assert(rowsPerBlock > 0);
// The main thread handles fetching from the cursor, and storing it into blocks passed to toCompress.
HashSet <int> activeSet = new HashSet <int>(activeColumns.Select(col => col.SourceIndex));
long blockIndex = 0;
int remainingInBlock = rowsPerBlock;
using (RowCursor cursor = data.GetRowCursor(activeSet.Contains))
{
WritePipe[] pipes = new WritePipe[activeColumns.Length];
for (int c = 0; c < activeColumns.Length; ++c)
{
pipes[c] = WritePipe.Create(this, cursor, activeColumns[c]);
}
for (int c = 0; c < pipes.Length; ++c)
{
pipes[c].BeginBlock();
}
long rows = 0;
if (pch != null)
{
pch.SetHeader(new ProgressHeader(new[] { "rows" }), e => e.SetProgress(0, rows));
}
while (cursor.MoveNext())
{
for (int c = 0; c < pipes.Length; ++c)
{
pipes[c].FetchAndWrite();
}
if (--remainingInBlock == 0)
{
for (int c = 0; c < pipes.Length; ++c)
{
// REVIEW: It may be better if EndBlock got moved to a different worker thread.
toCompress.Add(new Block(pipes[c].EndBlock(), c, blockIndex), exMarshaller.Token);
pipes[c].BeginBlock();
}
remainingInBlock = rowsPerBlock;
blockIndex++;
}
rows++;
}
if (remainingInBlock < rowsPerBlock)
{
for (int c = 0; c < pipes.Length; ++c)
{
toCompress.Add(new Block(pipes[c].EndBlock(), c, blockIndex), exMarshaller.Token);
}
}
Contracts.Assert(rows == (blockIndex + 1) * rowsPerBlock - remainingInBlock);
_rowCount = rows;
if (pch != null)
{
pch.Checkpoint(rows);
}
}
toCompress.CompleteAdding();
}
catch (Exception ex)
{
exMarshaller.Set("cursoring", ex);
}
}
protected override void PrintIterationMessage(IChannel ch, IProgressChannel pch)
{
base.PrintIterationMessage(ch, pch);
}
private FieldAwareFactorizationMachineModelParameters TrainCore(IChannel ch, IProgressChannel pch, RoleMappedData data,
RoleMappedData validData = null, FieldAwareFactorizationMachineModelParameters predictor = null)
{
_host.AssertValue(ch);
_host.AssertValue(pch);
data.CheckBinaryLabel();
var featureColumns = data.Schema.GetColumns(RoleMappedSchema.ColumnRole.Feature);
int fieldCount = featureColumns.Count;
int totalFeatureCount = 0;
int[] fieldColumnIndexes = new int[fieldCount];
for (int f = 0; f < fieldCount; f++)
{
var col = featureColumns[f];
_host.Assert(!col.IsHidden);
if (!(col.Type is VectorDataViewType vectorType) ||
!vectorType.IsKnownSize ||
vectorType.ItemType != NumberDataViewType.Single)
{
throw ch.ExceptParam(nameof(data), "Training feature column '{0}' must be a known-size vector of Single, but has type: {1}.", col.Name, col.Type);
}
_host.Assert(vectorType.Size > 0);
fieldColumnIndexes[f] = col.Index;
totalFeatureCount += vectorType.Size;
}
ch.Check(checked (totalFeatureCount * fieldCount * _latentDimAligned) <= Utils.ArrayMaxSize, "Latent dimension or the number of fields too large");
if (predictor != null)
{
ch.Check(predictor.FeatureCount == totalFeatureCount, "Input model's feature count mismatches training feature count");
ch.Check(predictor.LatentDimension == _latentDim, "Input model's latent dimension mismatches trainer's");
}
if (validData != null)
{
validData.CheckBinaryLabel();
var validFeatureColumns = data.Schema.GetColumns(RoleMappedSchema.ColumnRole.Feature);
_host.Assert(fieldCount == validFeatureColumns.Count);
for (int f = 0; f < fieldCount; f++)
{
var featCol = featureColumns[f];
var validFeatCol = validFeatureColumns[f];
_host.Assert(featCol.Name == validFeatCol.Name);
_host.Assert(featCol.Type == validFeatCol.Type);
}
}
bool shuffle = _shuffle;
if (shuffle && !data.Data.CanShuffle)
{
ch.Warning("Training data does not support shuffling, so ignoring request to shuffle");
shuffle = false;
}
var rng = shuffle ? _host.Rand : null;
var featureGetters = new ValueGetter <VBuffer <float> > [fieldCount];
var featureBuffer = new VBuffer <float>();
var featureValueBuffer = new float[totalFeatureCount];
var featureIndexBuffer = new int[totalFeatureCount];
var featureFieldBuffer = new int[totalFeatureCount];
var latentSum = new AlignedArray(fieldCount * fieldCount * _latentDimAligned, 16);
var metricNames = new List <string>()
{
"Training-loss"
};
if (validData != null)
{
metricNames.Add("Validation-loss");
}
int iter = 0;
long exampleCount = 0;
long badExampleCount = 0;
long validBadExampleCount = 0;
double loss = 0;
double validLoss = 0;
pch.SetHeader(new ProgressHeader(metricNames.ToArray(), new string[] { "iterations", "examples" }), entry =>
{
entry.SetProgress(0, iter, _numIterations);
entry.SetProgress(1, exampleCount);
});
var columns = data.Schema.Schema.Where(x => fieldColumnIndexes.Contains(x.Index)).ToList();
columns.Add(data.Schema.Label.Value);
if (data.Schema.Weight != null)
{
columns.Add(data.Schema.Weight.Value);
}
InitializeTrainingState(fieldCount, totalFeatureCount, predictor, out float[] linearWeights,
out AlignedArray latentWeightsAligned, out float[] linearAccSqGrads, out AlignedArray latentAccSqGradsAligned);
// refer to Algorithm 3 in https://github.com/wschin/fast-ffm/blob/master/fast-ffm.pdf
while (iter++ < _numIterations)
{
using (var cursor = data.Data.GetRowCursor(columns, rng))
{
var labelGetter = RowCursorUtils.GetLabelGetter(cursor, data.Schema.Label.Value.Index);
var weightGetter = data.Schema.Weight?.Index is int weightIdx?RowCursorUtils.GetGetterAs <float>(NumberDataViewType.Single, cursor, weightIdx) : null;
for (int i = 0; i < fieldCount; i++)
{
featureGetters[i] = cursor.GetGetter <VBuffer <float> >(cursor.Schema[fieldColumnIndexes[i]]);
}
loss = 0;
exampleCount = 0;
badExampleCount = 0;
while (cursor.MoveNext())
{
float label = 0;
float weight = 1;
int count = 0;
float modelResponse = 0;
labelGetter(ref label);
weightGetter?.Invoke(ref weight);
float annihilation = label - label + weight - weight;
if (!FloatUtils.IsFinite(annihilation))
{
badExampleCount++;
continue;
}
if (!FieldAwareFactorizationMachineUtils.LoadOneExampleIntoBuffer(featureGetters, featureBuffer, _norm, ref count,
featureFieldBuffer, featureIndexBuffer, featureValueBuffer))
{
badExampleCount++;
continue;
}
// refer to Algorithm 1 in [3] https://github.com/wschin/fast-ffm/blob/master/fast-ffm.pdf
FieldAwareFactorizationMachineInterface.CalculateIntermediateVariables(fieldCount, _latentDimAligned, count,
featureFieldBuffer, featureIndexBuffer, featureValueBuffer, linearWeights, latentWeightsAligned, latentSum, ref modelResponse);
var slope = CalculateLossSlope(label, modelResponse);
// refer to Algorithm 2 in [3] https://github.com/wschin/fast-ffm/blob/master/fast-ffm.pdf
FieldAwareFactorizationMachineInterface.CalculateGradientAndUpdate(_lambdaLinear, _lambdaLatent, _learningRate, fieldCount, _latentDimAligned, weight, count,
featureFieldBuffer, featureIndexBuffer, featureValueBuffer, latentSum, slope, linearWeights, latentWeightsAligned, linearAccSqGrads, latentAccSqGradsAligned);
loss += weight * CalculateLoss(label, modelResponse);
exampleCount++;
}
loss /= exampleCount;
}
if (_verbose)
{
if (validData == null)
{
pch.Checkpoint(loss, iter, exampleCount);
}
else
{
validLoss = CalculateAvgLoss(ch, validData, _norm, linearWeights, latentWeightsAligned, _latentDimAligned, latentSum,
featureFieldBuffer, featureIndexBuffer, featureValueBuffer, featureBuffer, ref validBadExampleCount);
pch.Checkpoint(loss, validLoss, iter, exampleCount);
}
}
}
if (badExampleCount != 0)
{
ch.Warning($"Skipped {badExampleCount} examples with bad label/weight/features in training set");
}
if (validBadExampleCount != 0)
{
ch.Warning($"Skipped {validBadExampleCount} examples with bad label/weight/features in validation set");
}
return(new FieldAwareFactorizationMachineModelParameters(_host, _norm, fieldCount, totalFeatureCount, _latentDim, linearWeights, latentWeightsAligned));
}
/// <inheritdoc/>
private protected override bool CheckConvergence(
IProgressChannel pch,
int iter,
FloatLabelCursor.Factory cursorFactory,
DualsTableBase duals,
IdToIdxLookup idToIdx,
VBuffer <float>[] weights,
VBuffer <float>[] bestWeights,
float[] biasUnreg,
float[] bestBiasUnreg,
float[] biasReg,
float[] bestBiasReg,
long count,
Double[] metrics,
ref Double bestPrimalLoss,
ref int bestIter)
{
Contracts.AssertValue(weights);
Contracts.AssertValue(duals);
int numClasses = weights.Length;
Contracts.Assert(duals.Length >= numClasses * count);
Contracts.AssertValueOrNull(idToIdx);
Contracts.Assert(Utils.Size(weights) == numClasses);
Contracts.Assert(Utils.Size(biasReg) == numClasses);
Contracts.Assert(Utils.Size(biasUnreg) == numClasses);
Contracts.Assert(Utils.Size(metrics) == 6);
var reportedValues = new Double?[metrics.Length + 1];
reportedValues[metrics.Length] = iter;
var lossSum = new CompensatedSum();
var dualLossSum = new CompensatedSum();
int numFeatures = weights[0].Length;
using (var cursor = cursorFactory.Create())
{
long row = 0;
Func <DataViewRowId, long, long> getIndexFromIdAndRow = GetIndexFromIdAndRowGetter(idToIdx, biasReg.Length);
// Iterates through data to compute loss function.
while (cursor.MoveNext())
{
var instanceWeight = GetInstanceWeight(cursor);
var features = cursor.Features;
var label = (int)cursor.Label;
var labelOutput = WDot(in features, in weights[label], biasReg[label] + biasUnreg[label]);
Double subLoss = 0;
Double subDualLoss = 0;
long idx = getIndexFromIdAndRow(cursor.Id, row);
long dualIndex = idx * numClasses;
for (int iClass = 0; iClass < numClasses; iClass++)
{
if (iClass == label)
{
dualIndex++;
continue;
}
var currentClassOutput = WDot(in features, in weights[iClass], biasReg[iClass] + biasUnreg[iClass]);
subLoss += _loss.Loss(labelOutput - currentClassOutput, 1);
Contracts.Assert(dualIndex == iClass + idx * numClasses);
var dual = duals[dualIndex++];
subDualLoss += _loss.DualLoss(1, dual);
}
lossSum.Add(subLoss * instanceWeight);
dualLossSum.Add(subDualLoss * instanceWeight);
row++;
}
Host.Assert(idToIdx == null || row * numClasses == duals.Length);
}
Contracts.Assert(SdcaTrainerOptions.L2Regularization.HasValue);
Contracts.Assert(SdcaTrainerOptions.L1Threshold.HasValue);
Double l2Const = SdcaTrainerOptions.L2Regularization.Value;
Double l1Threshold = SdcaTrainerOptions.L1Threshold.Value;
Double weightsL1Norm = 0;
Double weightsL2NormSquared = 0;
Double biasRegularizationAdjustment = 0;
for (int iClass = 0; iClass < numClasses; iClass++)
{
weightsL1Norm += VectorUtils.L1Norm(in weights[iClass]) + Math.Abs(biasReg[iClass]);
weightsL2NormSquared += VectorUtils.NormSquared(weights[iClass]) + biasReg[iClass] * biasReg[iClass];
biasRegularizationAdjustment += biasReg[iClass] * biasUnreg[iClass];
}
Double l1Regularizer = SdcaTrainerOptions.L1Threshold.Value * l2Const * weightsL1Norm;
var l2Regularizer = l2Const * weightsL2NormSquared * 0.5;
var newLoss = lossSum.Sum / count + l2Regularizer + l1Regularizer;
var newDualLoss = dualLossSum.Sum / count - l2Regularizer - l2Const * biasRegularizationAdjustment;
var dualityGap = newLoss - newDualLoss;
metrics[(int)MetricKind.Loss] = newLoss;
metrics[(int)MetricKind.DualLoss] = newDualLoss;
metrics[(int)MetricKind.DualityGap] = dualityGap;
metrics[(int)MetricKind.BiasUnreg] = biasUnreg[0];
metrics[(int)MetricKind.BiasReg] = biasReg[0];
metrics[(int)MetricKind.L1Sparsity] = SdcaTrainerOptions.L1Threshold == 0 ? 1 : weights.Sum(
weight => weight.GetValues().Count(w => w != 0)) / (numClasses * numFeatures);
bool converged = dualityGap / newLoss < SdcaTrainerOptions.ConvergenceTolerance;
if (metrics[(int)MetricKind.Loss] < bestPrimalLoss)
{
for (int iClass = 0; iClass < numClasses; iClass++)
{
// Maintain a copy of weights and bias with best primal loss thus far.
// This is some extra work and uses extra memory, but it seems worth doing it.
// REVIEW: Sparsify bestWeights?
weights[iClass].CopyTo(ref bestWeights[iClass]);
bestBiasReg[iClass] = biasReg[iClass];
bestBiasUnreg[iClass] = biasUnreg[iClass];
}
bestPrimalLoss = metrics[(int)MetricKind.Loss];
bestIter = iter;
}
for (int i = 0; i < metrics.Length; i++)
{
reportedValues[i] = metrics[i];
}
if (pch != null)
{
pch.Checkpoint(reportedValues);
}
return(converged);
}
private void TrainCore(IChannel ch, IProgressChannel pch, RoleMappedData data, TPredictor predictor)
{
// Verifications.
_host.AssertValue(ch);
ch.CheckValue(data, nameof(data));
ValidateTrainInput(ch, data);
var featureColumns = data.Schema.GetColumns(RoleMappedSchema.ColumnRole.Feature);
ch.Check(featureColumns.Count == 1, "Only one vector of features is allowed.");
// Data dimension.
int fi = data.Schema.Feature.Index;
var colType = data.Schema.Schema.GetColumnType(fi);
ch.Assert(colType.IsVector, "Feature must be a vector.");
ch.Assert(colType.VectorSize > 0, "Feature dimension must be known.");
int nbDim = colType.VectorSize;
IDataView view = data.Data;
long nbRows = DataViewUtils.ComputeRowCount(view);
Float[] labels;
uint[] groupCount;
DMatrix dtrain;
// REVIEW xadupre: this can be avoided by using method XGDMatrixCreateFromDataIter from the XGBoost API.
// XGBoost removes NaN values from a dense matrix and stores it in sparse format anyway.
bool isDense = DetectDensity(data);
var dt = DateTime.Now;
if (isDense)
{
dtrain = FillDenseMatrix(ch, nbDim, nbRows, data, out labels, out groupCount);
ch.Info("Dense matrix created with nbFeatures={0} and nbRows={1} in {2}.", nbDim, nbRows, DateTime.Now - dt);
}
else
{
dtrain = FillSparseMatrix(ch, nbDim, nbRows, data, out labels, out groupCount);
ch.Info("Sparse matrix created with nbFeatures={0} and nbRows={1} in {2}.", nbDim, nbRows, DateTime.Now - dt);
}
// Some options are filled based on the data.
var options = _args.ToDict(_host);
UpdateXGBoostOptions(ch, options, labels, groupCount);
// For multi class, the number of labels is required.
ch.Assert(PredictionKind != PredictionKind.MultiClassClassification || options.ContainsKey("num_class"),
"XGBoost requires the number of classes to be specified in the parameters.");
ch.Info("XGBoost objective={0}", options["objective"]);
int numTrees;
Booster res = WrappedXGBoostTraining.Train(ch, pch, out numTrees, options, dtrain,
numBoostRound: _args.numBoostRound,
obj: null, verboseEval: _args.verboseEval,
xgbModel: predictor == null ? null : predictor.GetBooster(),
saveBinaryDMatrix: _args.saveXGBoostDMatrixAsBinary);
int nbTrees = res.GetNumTrees();
ch.Info("Training is complete. Number of added trees={0}, total={1}.", numTrees, nbTrees);
_model = res.SaveRaw();
_nbFeaturesXGboost = (int)dtrain.GetNumCols();
_nbFeaturesML = nbDim;
}
protected float DifferentiableFunctionStream(FloatLabelCursor.Factory cursorFactory, ref VBuffer <float> xDense, ref VBuffer <float> grad, IProgressChannel pch)
{
Contracts.AssertValue(cursorFactory);
VBufferUtils.Clear(ref grad);
VBufferUtils.Densify(ref grad);
float[] scratch = null;
double loss = 0;
long count = 0;
if (pch != null)
{
pch.SetHeader(new ProgressHeader(null, new[] { "examples" }), e => e.SetProgress(0, count));
}
using (var cursor = cursorFactory.Create())
{
while (cursor.MoveNext())
{
loss += AccumulateOneGradient(ref cursor.Features, cursor.Label, cursor.Weight,
ref xDense, ref grad, ref scratch);
count++;
}
}
// we need use double type to accumulate loss to avoid roundoff error
// please see http://mathworld.wolfram.com/RoundoffError.html for roundoff error definition
// finally we need to convert double type to float for function definition
return((float)loss);
}
protected float DifferentiableFunctionComputeChunk(int ichk, ref VBuffer <float> xDense, ref VBuffer <float> grad, IProgressChannel pch)
{
Contracts.Assert(0 <= ichk && ichk < _numChunks);
Contracts.AssertValueOrNull(pch);
VBufferUtils.Clear(ref grad);
VBufferUtils.Densify(ref grad);
float[] scratch = null;
double loss = 0;
int ivMin = _ranges[ichk];
int ivLim = _ranges[ichk + 1];
int iv = ivMin;
if (pch != null)
{
pch.SetHeader(new ProgressHeader(null, new[] { "examples" }), e => e.SetProgress(0, iv - ivMin, ivLim - ivMin));
}
for (iv = ivMin; iv < ivLim; iv++)
{
float weight = _weights != null ? _weights[iv] : 1;
loss += AccumulateOneGradient(ref _features[iv], _labels[iv], weight, ref xDense, ref grad, ref scratch);
}
// we need use double type to accumulate loss to avoid roundoff error
// please see http://mathworld.wolfram.com/RoundoffError.html for roundoff error definition
// finally we need to convert double type to float for function definition
return((float)loss);
}
/// <summary>
/// Batch-parallel optimizer
/// </summary>
/// <remarks>
/// REVIEW: consider getting rid of multithread-targeted members
/// Using TPL, the distinction between Multithreaded and Sequential implementations is unnecessary
/// </remarks>
protected virtual float DifferentiableFunctionMultithreaded(ref VBuffer <float> xDense, ref VBuffer <float> gradient, IProgressChannel pch)
{
Contracts.Assert(_data == null);
Contracts.Assert(_cursorFactory == null);
Contracts.Assert(_numChunks > 0);
Contracts.Assert(Utils.Size(_ranges) == _numChunks + 1);
Contracts.Assert(Utils.Size(_localLosses) == _numChunks);
Contracts.Assert(Utils.Size(_localGradients) + 1 == _numChunks);
Contracts.AssertValueOrNull(pch);
// Declare a local variable, since the lambda cannot capture the xDense. The gradient
// calculation will modify the local gradients, but not this xx value.
var xx = xDense;
var gg = gradient;
Parallel.For(0, _numChunks,
ichk =>
{
if (ichk == 0)
{
_localLosses[ichk] = DifferentiableFunctionComputeChunk(ichk, ref xx, ref gg, pch);
}
else
{
_localLosses[ichk] = DifferentiableFunctionComputeChunk(ichk, ref xx, ref _localGradients[ichk - 1], null);
}
});
gradient = gg;
float loss = _localLosses[0];
for (int i = 1; i < _numChunks; i++)
{
VectorUtils.Add(ref _localGradients[i - 1], ref gradient);
loss += _localLosses[i];
}
return(loss);
}
/// <summary>
/// Train and return a booster.
/// </summary>
public static Booster Train(IChannel ch, IProgressChannel pch,
Dictionary <string, object> parameters, Dataset dtrain, Dataset dvalid = null, int numIteration = 100,
bool verboseEval = true, int earlyStoppingRound = 0)
{
// create Booster.
Booster bst = new Booster(parameters, dtrain, dvalid);
// Disable early stopping if we don't have validation data.
if (dvalid == null && earlyStoppingRound > 0)
{
earlyStoppingRound = 0;
ch.Warning("Validation dataset not present, early stopping will be disabled.");
}
int bestIter = 0;
double bestScore = double.MaxValue;
double factorToSmallerBetter = 1.0;
var metric = (string)parameters["metric"];
if (earlyStoppingRound > 0 && (metric == "auc" || metric == "ndcg" || metric == "map"))
{
factorToSmallerBetter = -1.0;
}
const int evalFreq = 50;
var metrics = new List <string>()
{
"Iteration"
};
var units = new List <string>()
{
"iterations"
};
if (verboseEval)
{
ch.Assert(parameters.ContainsKey("metric"));
metrics.Add("Training-" + parameters["metric"]);
if (dvalid != null)
{
metrics.Add("Validation-" + parameters["metric"]);
}
}
var header = new ProgressHeader(metrics.ToArray(), units.ToArray());
int iter = 0;
double trainError = double.NaN;
double validError = double.NaN;
pch.SetHeader(header, e =>
{
e.SetProgress(0, iter, numIteration);
if (verboseEval)
{
e.SetProgress(1, trainError);
if (dvalid != null)
{
e.SetProgress(2, validError);
}
}
});
for (iter = 0; iter < numIteration; ++iter)
{
if (bst.Update())
{
break;
}
if (earlyStoppingRound > 0)
{
validError = bst.EvalValid();
if (validError * factorToSmallerBetter < bestScore)
{
bestScore = validError * factorToSmallerBetter;
bestIter = iter;
}
if (iter - bestIter >= earlyStoppingRound)
{
ch.Info($"Met early stopping, best iteration: {bestIter + 1}, best score: {bestScore / factorToSmallerBetter}");
break;
}
}
if ((iter + 1) % evalFreq == 0)
{
if (verboseEval)
{
trainError = bst.EvalTrain();
if (dvalid == null)
{
pch.Checkpoint(new double?[] { iter + 1, trainError });
}
else
{
if (earlyStoppingRound == 0)
{
validError = bst.EvalValid();
}
pch.Checkpoint(new double?[] { iter + 1,
trainError, validError });
}
}
else
{
pch.Checkpoint(new double?[] { iter + 1 });
}
}
}
// Set the BestIteration.
if (iter != numIteration && earlyStoppingRound > 0)
{
bst.BestIteration = bestIter + 1;
}
return(bst);
}
/// <summary>
/// Train and returns a booster.
/// </summary>
/// <param name="ch">IChannel</param>
/// <param name="pch">IProgressChannel</param>
/// <param name="numberOfTrees">Number of trained trees</param>
/// <param name="parameters">Parameters see <see cref="XGBoostArguments"/></param>
/// <param name="dtrain">Training set</param>
/// <param name="numBoostRound">Number of trees to train</param>
/// <param name="obj">Custom objective</param>
/// <param name="maximize">Whether to maximize feval.</param>
/// <param name="verboseEval">Requires at least one item in evals.
/// If "verbose_eval" is True then the evaluation metric on the validation set is
/// printed at each boosting stage.</param>
/// <param name="xgbModel">For continuous training.</param>
/// <param name="saveBinaryDMatrix">Save DMatrix in binary format (for debugging purpose).</param>
public static Booster Train(IChannel ch, IProgressChannel pch, out int numberOfTrees,
Dictionary <string, string> parameters, DMatrix dtrain, int numBoostRound = 10,
Booster.FObjType obj = null, bool maximize = false,
bool verboseEval = true, Booster xgbModel = null,
string saveBinaryDMatrix = null)
{
#if (!XGBOOST_RABIT)
if (WrappedXGBoostInterface.RabitIsDistributed() == 1)
{
var pname = WrappedXGBoostInterface.RabitGetProcessorName();
ch.Info("[WrappedXGBoostTraining.Train] start {0}:{1}", pname, WrappedXGBoostInterface.RabitGetRank());
}
#endif
if (!string.IsNullOrEmpty(saveBinaryDMatrix))
{
dtrain.SaveBinary(saveBinaryDMatrix);
}
Booster bst = new Booster(parameters, dtrain, xgbModel);
int numParallelTree = 1;
int nboost = 0;
if (parameters != null && parameters.ContainsKey("num_parallel_tree"))
{
numParallelTree = Convert.ToInt32(parameters["num_parallel_tree"]);
nboost /= numParallelTree;
}
if (parameters.ContainsKey("num_class"))
{
int numClass = Convert.ToInt32(parameters["num_class"]);
nboost /= numClass;
}
var prediction = new VBuffer <Float>();
var grad = new VBuffer <Float>();
var hess = new VBuffer <Float>();
var start = DateTime.Now;
#if (!XGBOOST_RABIT)
int version = bst.LoadRabitCheckpoint();
ch.Check(WrappedXGBoostInterface.RabitGetWorldSize() != 1 || version == 0);
#else
int version = 0;
#endif
int startIteration = version / 2;
nboost += startIteration;
int logten = 0;
int temp = numBoostRound * 5;
while (temp > 0)
{
logten += 1;
temp /= 10;
}
temp = Math.Max(logten - 2, 0);
logten = 1;
while (temp-- > 0)
{
logten *= 10;
}
var metrics = new List <string>()
{
"Iteration", "Training Time"
};
var units = new List <string>()
{
"iterations", "seconds"
};
if (verboseEval)
{
metrics.Add("Training Error");
metrics.Add(parameters["objective"]);
}
var header = new ProgressHeader(metrics.ToArray(), units.ToArray());
int iter = 0;
double trainTime = 0;
double trainError = double.NaN;
pch.SetHeader(header, e =>
{
e.SetProgress(0, iter, numBoostRound - startIteration);
e.SetProgress(1, trainTime);
if (verboseEval)
{
e.SetProgress(2, trainError);
}
});
for (iter = startIteration; iter < numBoostRound; ++iter)
{
if (version % 2 == 0)
{
bst.Update(dtrain, iter, ref grad, ref hess, ref prediction, obj);
#if (!XGBOOST_RABIT)
bst.SaveRabitCheckpoint();
#endif
version += 1;
}
#if (!XGBOOST_RABIT)
ch.Check(WrappedXGBoostInterface.RabitGetWorldSize() == 1 ||
version == WrappedXGBoostInterface.RabitVersionNumber());
#endif
nboost += 1;
trainTime = (DateTime.Now - start).TotalMilliseconds;
if (verboseEval)
{
pch.Checkpoint(new double?[] { iter, trainTime, trainError });
if (iter == startIteration || iter == numBoostRound - 1 || iter % logten == 0 ||
(DateTime.Now - start) > TimeSpan.FromMinutes(2))
{
string strainError = bst.EvalSet(new[] { dtrain }, new[] { "Train" }, iter);
// Example: "[0]\tTrain-error:0.028612"
if (!string.IsNullOrEmpty(strainError) && strainError.Contains(":"))
{
double val;
if (double.TryParse(strainError.Split(':').Last(), out val))
{
trainError = val;
}
}
}
}
else
{
pch.Checkpoint(new double?[] { iter, trainTime });
}
version += 1;
}
numberOfTrees = numBoostRound * numParallelTree;
if (WrappedXGBoostInterface.RabitIsDistributed() == 1)
{
var pname = WrappedXGBoostInterface.RabitGetProcessorName();
ch.Info("[WrappedXGBoostTraining.Train] end {0}:{1}", pname, WrappedXGBoostInterface.RabitGetRank());
}
return(bst);
}
