internal override InternalRegressionTree TrainingIteration(IChannel ch, bool[] activeFeatures) { Contracts.CheckValue(ch, nameof(ch)); // Fit a regression tree to the gradient using least squares. InternalRegressionTree tree = TreeLearner.FitTargets(ch, activeFeatures, AdjustTargetsAndSetWeights(ch)); if (tree == null) { return(null); // Could not learn a tree. Exit. } // Adjust output values of tree by performing a Newton step. // REVIEW: This should be part of OptimizingAlgorithm. using (Timer.Time(TimerEvent.TreeLearnerAdjustTreeOutputs)) { double[] backupScores = null; // when doing dropouts we need to replace the TrainingScores with the scores without the dropped trees if (DropoutRate > 0) { backupScores = TrainingScores.Scores; TrainingScores.Scores = _scores; } if (AdjustTreeOutputsOverride != null) { AdjustTreeOutputsOverride.AdjustTreeOutputs(ch, tree, TreeLearner.Partitioning, TrainingScores); } else if (ObjectiveFunction is IStepSearch) { (ObjectiveFunction as IStepSearch).AdjustTreeOutputs(ch, tree, TreeLearner.Partitioning, TrainingScores); } else { throw ch.Except("No AdjustTreeOutputs defined. Objective function should define IStepSearch or AdjustTreeOutputsOverride should be set"); } if (DropoutRate > 0) { // Returning the original scores. TrainingScores.Scores = backupScores; } } if (Smoothing != 0.0) { SmoothTree(tree, Smoothing); UseFastTrainingScoresUpdate = false; } if (DropoutRate > 0) { // Don't do shrinkage if you do dropouts. double scaling = (1.0 / (1.0 + _numberOfDroppedTrees)); tree.ScaleOutputsBy(scaling); _treeScores.Add(tree.GetOutputs(TrainingScores.Dataset)); } UpdateAllScores(ch, tree); Ensemble.AddTree(tree); return(tree); }
// Divides output values of leaves to bag count. // This brings back the final scores generated by model on a same // range as when we didn't use bagging internal void ScaleEnsembleLeaves(int numTrees, int bagSize, InternalTreeEnsemble ensemble) { int bagCount = GetBagCount(numTrees, bagSize); for (int t = 0; t < ensemble.NumTrees; t++) { InternalRegressionTree tree = ensemble.GetTreeAt(t); tree.ScaleOutputsBy(1.0 / bagCount); } }
void IStepSearch.AdjustTreeOutputs(IChannel ch, InternalRegressionTree tree, DocumentPartitioning partitioning, ScoreTracker previousScores) { _lo.Initialize(tree, partitioning, previousScores); _hi.Initialize(tree, partitioning, previousScores); _left.Initialize(tree, partitioning, previousScores); _right.Initialize(tree, partitioning, previousScores); _lo.Step = _historicStepSize / _phi; _left.Step = _historicStepSize; if (_lo.Loss.CompareTo(_left.Loss) == 1) // backtrack { do { Rotate(ref _hi, ref _left, ref _lo); if (_hi.Step <= _minStepSize) { goto FINISHED; } _lo.Step = _left.Step / _phi; } while (_lo.Loss.CompareTo(_left.Loss) == 1); } else // extend (or stay) { _hi.Step = _historicStepSize * _phi; while (_hi.Loss.CompareTo(_left.Loss) == 1) { Rotate(ref _lo, ref _left, ref _hi); _hi.Step = _left.Step * _phi; } } if (_numPostbracketSteps > 0) { _right.Step = _lo.Step + (_hi.Step - _lo.Step) / _phi; for (int step = 0; step < _numPostbracketSteps; ++step) { int cmp = _right.Loss.CompareTo(_left.Loss); if (cmp == 0) { break; } if (cmp == 1) // move right { Rotate(ref _lo, ref _left, ref _right); _right.Step = _lo.Step + (_hi.Step - _lo.Step) / _phi; } else // move left { Rotate(ref _hi, ref _right, ref _left); if (_hi.Step <= _minStepSize) { goto FINISHED; } _left.Step = _hi.Step - (_hi.Step - _lo.Step) / _phi; } } // prepare to return _left if (_right.Loss.CompareTo(_left.Loss) == 1) { Swap(ref _left, ref _right); } } FINISHED: if (_hi.Step < _minStepSize) { _left.Step = _minStepSize; } else if (_hi.Step == _minStepSize) { Swap(ref _hi, ref _left); } double bestStep = _left.Step; ch.Info("multiplier: {0}", bestStep); _historicStepSize = bestStep; tree.ScaleOutputsBy(bestStep); }