public override RegressionTree TrainingIteration(IChannel ch, bool[] activeFeatures)
{
Contracts.CheckValue(ch, nameof(ch));
AgdScoreTracker trainingScores = TrainingScores as AgdScoreTracker;
//First Let's make XK=YK as we want to fit YK and LineSearch YK
// and call base class that uses fits XK (in our case will fir YK thanks to the swap)
var xk = trainingScores.XK;
trainingScores.XK = trainingScores.YK;
trainingScores.YK = null;
//Invoke standard gradient descent on YK rather than XK(Scores)
RegressionTree tree = base.TrainingIteration(ch, activeFeatures);
//Reverse the XK/YK swap
trainingScores.YK = trainingScores.XK;
trainingScores.XK = xk;
if (tree == null)
{
return(null); // No tree was actually learnt. Give up.
}
// ... and update the training scores that we omitted from update
// in AcceleratedGradientDescent.UpdateScores
// Here we could use faster way of comuting train scores taking advantage of scores precompited by LineSearch
// But that would make the code here even more difficult/complex
trainingScores.AddScores(tree, TreeLearner.Partitioning, 1.0);
//Now rescale all previous trees based on ratio of new_desired_tree_scale/previous_tree_scale
for (int t = 0; t < Ensemble.NumTrees - 1; t++)
{
Ensemble.GetTreeAt(t).ScaleOutputsBy(AgdScoreTracker.TreeMultiplier(t, Ensemble.NumTrees) / AgdScoreTracker.TreeMultiplier(t, Ensemble.NumTrees - 1));
}
return(tree);
}
public OptimizationAlgorithm(Ensemble ensemble, Dataset trainData, double[] initTrainScores)
{
Ensemble = ensemble;
TrainingScores = ConstructScoreTracker("train", trainData, initTrainScores);
TrackedScores = new List <ScoreTracker>();
TrackedScores.Add(TrainingScores);
DropoutRng = new Random();
UseFastTrainingScoresUpdate = true;
}
internal override InternalRegressionTree TrainingIteration(IChannel ch, bool[] activeFeatures)
{
Contracts.CheckValue(ch, nameof(ch));
// Fit a regression tree to the gradient using least squares.
InternalRegressionTree tree = TreeLearner.FitTargets(ch, activeFeatures, AdjustTargetsAndSetWeights(ch));
if (tree == null)
{
return(null); // Could not learn a tree. Exit.
}
// Adjust output values of tree by performing a Newton step.
// REVIEW: This should be part of OptimizingAlgorithm.
using (Timer.Time(TimerEvent.TreeLearnerAdjustTreeOutputs))
{
double[] backupScores = null;
// when doing dropouts we need to replace the TrainingScores with the scores without the dropped trees
if (DropoutRate > 0)
{
backupScores = TrainingScores.Scores;
TrainingScores.Scores = _scores;
}
if (AdjustTreeOutputsOverride != null)
{
AdjustTreeOutputsOverride.AdjustTreeOutputs(ch, tree, TreeLearner.Partitioning, TrainingScores);
}
else if (ObjectiveFunction is IStepSearch)
{
(ObjectiveFunction as IStepSearch).AdjustTreeOutputs(ch, tree, TreeLearner.Partitioning, TrainingScores);
}
else
{
throw ch.Except("No AdjustTreeOutputs defined. Objective function should define IStepSearch or AdjustTreeOutputsOverride should be set");
}
if (DropoutRate > 0)
{
// Returning the original scores.
TrainingScores.Scores = backupScores;
}
}
if (Smoothing != 0.0)
{
SmoothTree(tree, Smoothing);
UseFastTrainingScoresUpdate = false;
}
if (DropoutRate > 0)
{
// Don't do shrinkage if you do dropouts.
double scaling = (1.0 / (1.0 + _numberOfDroppedTrees));
tree.ScaleOutputsBy(scaling);
_treeScores.Add(tree.GetOutputs(TrainingScores.Dataset));
}
UpdateAllScores(ch, tree);
Ensemble.AddTree(tree);
return(tree);
}
// Divides output values of leaves to bag count.
// This brings back the final scores generated by model on a same
// range as when we didn't use bagging
public void ScaleEnsembleLeaves(int numTrees, int bagSize, Ensemble ensemble)
{
int bagCount = GetBagCount(numTrees, bagSize);
for (int t = 0; t < ensemble.NumTrees; t++)
{
RegressionTree tree = ensemble.GetTreeAt(t);
tree.ScaleOutputsBy(1.0 / bagCount);
}
}
public override void FinalizeLearning(int bestIteration)
{
if (bestIteration != Ensemble.NumTrees)
{
// Restore multiplier for each tree as it was set during bestIteration
for (int t = 0; t < bestIteration; t++)
{
Ensemble.GetTreeAt(t).ScaleOutputsBy(AgdScoreTracker.TreeMultiplier(t, bestIteration) / AgdScoreTracker.TreeMultiplier(t, Ensemble.NumTrees));
}
}
base.FinalizeLearning(bestIteration);
}
private protected virtual double[] GetGradient(IChannel ch)
{
Contracts.AssertValue(ch);
if (DropoutRate > 0)
{
if (_droppedScores == null)
{
_droppedScores = new double[TrainingScores.Scores.Length];
}
else
{
Array.Clear(_droppedScores, 0, _droppedScores.Length);
}
if (_scores == null)
{
_scores = new double[TrainingScores.Scores.Length];
}
int numberOfTrees = Ensemble.NumTrees;
int[] droppedTrees =
Enumerable.Range(0, numberOfTrees).Where(t => (DropoutRng.NextDouble() < DropoutRate)).ToArray();
_numberOfDroppedTrees = droppedTrees.Length;
if ((_numberOfDroppedTrees == 0) && (numberOfTrees > 0))
{
droppedTrees = new int[] { DropoutRng.Next(numberOfTrees) };
// force at least a single tree to be dropped
_numberOfDroppedTrees = droppedTrees.Length;
}
ch.Trace("dropout: Dropping {0} trees of {1} for rate {2}",
_numberOfDroppedTrees, numberOfTrees, DropoutRate);
foreach (int i in droppedTrees)
{
double[] s = _treeScores[i];
for (int j = 0; j < _droppedScores.Length; j++)
{
_droppedScores[j] += s[j]; // summing up the weights of the dropped tree
s[j] *= _numberOfDroppedTrees / (1.0 + _numberOfDroppedTrees); // rescaling the dropped tree
}
Ensemble.GetTreeAt(i).ScaleOutputsBy(_numberOfDroppedTrees / (1.0 + _numberOfDroppedTrees));
}
for (int j = 0; j < _scores.Length; j++)
{
_scores[j] = TrainingScores.Scores[j] - _droppedScores[j];
TrainingScores.Scores[j] -= _droppedScores[j] / (1.0 + _numberOfDroppedTrees);
}
return(ObjectiveFunction.GetGradient(ch, _scores));
}
else
{
return(ObjectiveFunction.GetGradient(ch, TrainingScores.Scores));
}
}
public override RegressionTree TrainingIteration(IChannel ch, bool[] activeFeatures)
{
Contracts.CheckValue(ch, nameof(ch));
double[] sampleWeights = null;
double[] targets = GetGradient(ch);
double[] weightedTargets = _gradientWrapper.AdjustTargetAndSetWeights(targets, ObjectiveFunction, out sampleWeights);
RegressionTree tree = ((RandomForestLeastSquaresTreeLearner)TreeLearner).FitTargets(ch, activeFeatures, weightedTargets,
targets, sampleWeights);
if (tree != null)
{
Ensemble.AddTree(tree);
}
return(tree);
}
public bool Compress(IChannel ch, Ensemble ensemble, double[] trainScores, int bestIteration, int maxTreesAfterCompression)
{
LoadTargets(trainScores, bestIteration);
LassoFit fit = GetLassoFit(ch, maxTreesAfterCompression);
int numberOfSolutions = fit.NumberOfLambdas;
int bestSolutionIdx = 0;
ch.Info("Compression R2 values:");
for (int i = 0; i < numberOfSolutions; i++)
{
ch.Info("Solution {0}:\t{1}\t{2}", i + 1, fit.NonZeroWeights[i], fit.Rsquared[i]);
}
bestSolutionIdx = numberOfSolutions - 1;
_compressedEnsemble = GetEnsembleFromSolution(fit, bestSolutionIdx, ensemble);
return(true);
}
private Ensemble GetEnsembleFromSolution(LassoFit fit, int solutionIdx, Ensemble originalEnsemble)
{
Ensemble ensemble = new Ensemble();
int weightsCount = fit.NumberOfWeights[solutionIdx];
for (int i = 0; i < weightsCount; i++)
{
double weight = fit.CompressedWeights[solutionIdx][i];
if (weight != 0)
{
RegressionTree tree = originalEnsemble.GetTreeAt(fit.Indices[i]);
tree.Weight = weight;
ensemble.AddTree(tree);
}
}
ensemble.Bias = fit.Intercepts[solutionIdx];
return(ensemble);
}
// REVIEW: When the FastTree appliation is decoupled with tree learner and boosting logic, this class should be removed.
public RandomForestOptimizer(Ensemble ensemble, Dataset trainData, double[] initTrainScores, IGradientAdjuster gradientWrapper)
: base(ensemble, trainData, initTrainScores, gradientWrapper)
{
_gradientWrapper = gradientWrapper;
}
public AcceleratedGradientDescent(Ensemble ensemble, Dataset trainData, double[] initTrainScores, IGradientAdjuster gradientWrapper)
: base(ensemble, trainData, initTrainScores, gradientWrapper)
{
UseFastTrainingScoresUpdate = false;
}
public GradientDescent(Ensemble ensemble, Dataset trainData, double[] initTrainScores, IGradientAdjuster gradientWrapper)
: base(ensemble, trainData, initTrainScores)
{
_gradientWrapper = gradientWrapper;
_treeScores = new List <double[]>();
}
public ConjugateGradientDescent(Ensemble ensemble, Dataset trainData, double[] initTrainScores, IGradientAdjuster gradientWrapper)
: base(ensemble, trainData, initTrainScores, gradientWrapper)
{
_currentDk = new double[trainData.NumDocs];
}
public IPredictor CombineModels(IEnumerable<IPredictor> models)
{
_host.CheckValue(models, nameof(models));
var ensemble = new Ensemble();
int modelCount = 0;
int featureCount = -1;
bool binaryClassifier = false;
foreach (var model in models)
{
modelCount++;
var predictor = model;
_host.CheckValue(predictor, nameof(models), "One of the models is null");
var calibrated = predictor as CalibratedPredictorBase;
double paramA = 1;
if (calibrated != null)
{
_host.Check(calibrated.Calibrator is PlattCalibrator,
"Combining FastTree models can only be done when the models are calibrated with Platt calibrator");
predictor = calibrated.SubPredictor;
paramA = -(calibrated.Calibrator as PlattCalibrator).ParamA;
}
var tree = predictor as FastTreePredictionWrapper;
if (tree == null)
throw _host.Except("Model is not a tree ensemble");
foreach (var t in tree.TrainedEnsemble.Trees)
{
var bytes = new byte[t.SizeInBytes()];
int position = -1;
t.ToByteArray(bytes, ref position);
position = -1;
var tNew = new RegressionTree(bytes, ref position);
if (paramA != 1)
{
for (int i = 0; i < tNew.NumLeaves; i++)
tNew.SetOutput(i, tNew.LeafValues[i] * paramA);
}
ensemble.AddTree(tNew);
}
if (modelCount == 1)
{
binaryClassifier = calibrated != null;
featureCount = tree.InputType.ValueCount;
}
else
{
_host.Check((calibrated != null) == binaryClassifier, "Ensemble contains both calibrated and uncalibrated models");
_host.Check(featureCount == tree.InputType.ValueCount, "Found models with different number of features");
}
}
var scale = 1 / (double)modelCount;
foreach (var t in ensemble.Trees)
{
for (int i = 0; i < t.NumLeaves; i++)
t.SetOutput(i, t.LeafValues[i] * scale);
}
switch (_kind)
{
case PredictionKind.BinaryClassification:
if (!binaryClassifier)
return new FastTreeBinaryPredictor(_host, ensemble, featureCount, null);
var cali = new PlattCalibrator(_host, -1, 0);
return new FeatureWeightsCalibratedPredictor(_host, new FastTreeBinaryPredictor(_host, ensemble, featureCount, null), cali);
case PredictionKind.Regression:
return new FastTreeRegressionPredictor(_host, ensemble, featureCount, null);
case PredictionKind.Ranking:
return new FastTreeRankingPredictor(_host, ensemble, featureCount, null);
default:
_host.Assert(false);
throw _host.ExceptNotSupp();
}
}