/// <summary>
/// Regression learner for XGBoost
/// </summary>
/// <param name="maximumTreeDepth">Maximum tree depth for base learners. (default is 3)</param>
/// <param name="learningRate">Boosting learning rate (xgb's "eta"). 0 indicates no limit. (default is 0.1)</param>
/// <param name="estimators">Number of estimators to fit. (default is 100)</param>
/// <param name="silent">Whether to print messages while running boosting. (default is false)</param>
/// <param name="objective">Specify the learning task and the corresponding learning objective. (default is LinearRegression)</param>
/// <param name="boosterType"> which booster to use, can be gbtree, gblinear or dart.
/// gbtree and dart use tree based model while gblinear uses linear function (default is gbtree)</param>
/// <param name="treeMethod">The tree construction algorithm used in XGBoost. See reference paper: https://arxiv.org/abs/1603.02754. (default is auto)</param>
/// <param name="samplerType">Type of sampling algorithm for DART. (default is uniform)</param>
/// <param name="normalizeType">Type of normalization algorithm for DART. (default is tree)</param>
/// <param name="dropoutRate">Dropout rate for DART (a fraction of previous trees to drop during the dropout). (default is 0.0)</param>
/// <param name="oneDrop">When this is true, at least one tree is always dropped during the dropout.
/// Allows Binomial-plus-one or epsilon-dropout from the original DART paper. (default is false)</param>
/// <param name="skipDrop">Probability of skipping the dropout procedure during a boosting iteration. (default is 0.0)
/// If a dropout is skipped, new trees are added in the same manner as gbtree.
/// Note that non-zero skip_drop has higher priority than rate_drop or one_drop.</param>
/// <param name="numberOfThreads">Number of parallel threads used to run xgboost. -1 means use all thread avialable. (default is -1)</param>
/// <param name="gamma">Minimum loss reduction required to make a further partition on a leaf node of the tree. (default is 0) </param>
/// <param name="minChildWeight">Minimum sum of instance weight(hessian) needed in a child. (default is 1)</param>
/// <param name="maxDeltaStep">Maximum delta step we allow each tree's weight estimation to be. (default is 0)</param>
/// <param name="subSample">Subsample ratio of the training instance. (default is 1)</param>
/// <param name="colSampleByTree">Subsample ratio of columns when constructing each tree. (defualt is 1)</param>
/// <param name="colSampleByLevel">Subsample ratio of columns for each split, in each level. (defualt is 1)</param>
/// <param name="l1Regularization">L1 regularization term on weights. Also known as RegAlpha. (default is 0)</param>
/// <param name="l2Reguralization">L2 regularization term on weights. Also known as regLambda. (default is 1)</param>
/// <param name="scalePosWeight">Balancing of positive and negative weights. (default is 1)</param>
/// <param name="baseScore">The initial prediction score of all instances, global bias. (default is 0.5)</param>
/// <param name="seed">Random number seed. (defaukt is 0)</param>
/// <param name="missing">Value in the data which needs to be present as a missing value. (default is NaN)</param>
public RegressionXGBoostLearner(int maximumTreeDepth = 3, double learningRate = 0.1, int estimators = 100,
bool silent = true,
RegressionObjective objective = RegressionObjective.LinearRegression,
BoosterType boosterType = BoosterType.GBTree,
TreeMethod treeMethod = TreeMethod.Auto,
SamplerType samplerType = SamplerType.Uniform,
NormalizeType normalizeType = NormalizeType.Tree,
double dropoutRate = 0.0,
bool oneDrop = false,
double skipDrop = 0.0,
int numberOfThreads = -1, double gamma = 0, int minChildWeight = 1,
int maxDeltaStep = 0, double subSample = 1, double colSampleByTree = 1,
double colSampleByLevel = 1, double l1Regularization = 0, double l2Reguralization = 1,
double scalePosWeight = 1, double baseScore = 0.5, int seed = 0,
double missing = double.NaN)
{
ArgumentChecks.ThrowOnArgumentLessThan(nameof(maximumTreeDepth), maximumTreeDepth, 0);
ArgumentChecks.ThrowOnArgumentLessThanOrHigherThan(nameof(learningRate), learningRate, 0, 1.0);
ArgumentChecks.ThrowOnArgumentLessThan(nameof(estimators), estimators, 1);
ArgumentChecks.ThrowOnArgumentLessThan(nameof(numberOfThreads), numberOfThreads, -1);
ArgumentChecks.ThrowOnArgumentLessThan(nameof(gamma), gamma, 0);
ArgumentChecks.ThrowOnArgumentLessThan(nameof(minChildWeight), minChildWeight, 0);
ArgumentChecks.ThrowOnArgumentLessThan(nameof(maxDeltaStep), maxDeltaStep, 0);
ArgumentChecks.ThrowOnArgumentLessThanOrHigherThan(nameof(subSample), subSample, 0, 1.0);
ArgumentChecks.ThrowOnArgumentLessThanOrHigherThan(nameof(colSampleByTree), colSampleByTree, 0, 1.0);
ArgumentChecks.ThrowOnArgumentLessThanOrHigherThan(nameof(colSampleByLevel), colSampleByLevel, 0, 1.0);
ArgumentChecks.ThrowOnArgumentLessThan(nameof(l1Regularization), l1Regularization, 0);
ArgumentChecks.ThrowOnArgumentLessThan(nameof(l2Reguralization), l2Reguralization, 0);
ArgumentChecks.ThrowOnArgumentLessThan(nameof(scalePosWeight), scalePosWeight, 0);
m_parameters[ParameterNames.MaxDepth] = maximumTreeDepth;
m_parameters[ParameterNames.LearningRate] = (float)learningRate;
m_parameters[ParameterNames.Estimators] = estimators;
m_parameters[ParameterNames.Silent] = silent;
m_parameters[ParameterNames.objective] = objective.ToXGBoostString();
m_parameters[ParameterNames.Threads] = numberOfThreads;
m_parameters[ParameterNames.Gamma] = (float)gamma;
m_parameters[ParameterNames.MinChildWeight] = minChildWeight;
m_parameters[ParameterNames.MaxDeltaStep] = maxDeltaStep;
m_parameters[ParameterNames.SubSample] = (float)subSample;
m_parameters[ParameterNames.ColSampleByTree] = (float)colSampleByTree;
m_parameters[ParameterNames.ColSampleByLevel] = (float)colSampleByLevel;
m_parameters[ParameterNames.RegAlpha] = (float)l1Regularization;
m_parameters[ParameterNames.RegLambda] = (float)l2Reguralization;
m_parameters[ParameterNames.ScalePosWeight] = (float)scalePosWeight;
m_parameters[ParameterNames.BaseScore] = (float)baseScore;
m_parameters[ParameterNames.Seed] = seed;
m_parameters[ParameterNames.Missing] = (float)missing;
m_parameters[ParameterNames.ExistingBooster] = null;
m_parameters[ParameterNames.Booster] = boosterType.ToXGBoostString();
m_parameters[ParameterNames.TreeMethod] = treeMethod.ToXGBoostString();
m_parameters[ParameterNames.SampleType] = samplerType.ToXGBoostString();
m_parameters[ParameterNames.NormalizeType] = normalizeType.ToXGBoostString();
m_parameters[ParameterNames.RateDrop] = (float)dropoutRate;
m_parameters[ParameterNames.OneDrop] = oneDrop ? 1 : 0;
m_parameters[ParameterNames.SkipDrop] = (float)skipDrop;
}
/// <summary>
/// Convert regression objective to the xgboost parameter string.
/// </summary>
/// <param name="type"></param>
/// <returns></returns>
public static string ToXGBoostString(this TreeMethod type)
{
switch (type)
{
case TreeMethod.Auto:
return("auto");
case TreeMethod.Exact:
return("exact");
case TreeMethod.Approx:
return("approx");
case TreeMethod.Hist:
return("hist");
case TreeMethod.GPUExact:
return("gpu_exact");
case TreeMethod.GPUHist:
return("gpu_hist");
default:
throw new ArgumentException("Unknown TreeMethod type: " + type);
}
}
/// <summary>
/// Classification learner for XGBoost. For classification problems,
/// XGBoost requires that target values are sequntial and start at 0.
/// </summary>
/// <param name="maximumTreeDepth">Maximum tree depth for base learners. (default is 3)</param>
/// <param name="learningRate">Boosting learning rate (xgb's "eta"). 0 indicates no limit. (default is 0.1)</param>
/// <param name="estimators">Number of estimators to fit. (default is 100)</param>
/// <param name="silent">Whether to print messages while running boosting. (default is false)</param>
/// <param name="objective">Specify the learning task and the corresponding learning objective. (default is softmax)</param>
/// <param name="boosterType"> which booster to use, can be gbtree, gblinear or dart.
/// gbtree and dart use tree based model while gblinear uses linear function (default is gbtree)</param>
/// <param name="treeMethod">The tree construction algorithm used in XGBoost. See reference paper: https://arxiv.org/abs/1603.02754. (default is auto)</param>
/// <param name="samplerType">Type of sampling algorithm for DART. (default is uniform)</param>
/// <param name="normalizeType">Type of normalization algorithm for DART. (default is tree)</param>
/// <param name="dropoutRate">Dropout rate for DART (a fraction of previous trees to drop during the dropout). (default is 0.0)</param>
/// <param name="oneDrop">When this is true, at least one tree is always dropped during the dropout.
/// Allows Binomial-plus-one or epsilon-dropout from the original DART paper. (default is false)</param>
/// <param name="skipDrop">Probability of skipping the dropout procedure during a boosting iteration. (default is 0.0)
/// If a dropout is skipped, new trees are added in the same manner as gbtree.
/// Note that non-zero skip_drop has higher priority than rate_drop or one_drop.</param>
/// <param name="numberOfThreads">Number of parallel threads used to run xgboost. -1 means use all thread avialable. (default is -1)</param>
/// <param name="gamma">Minimum loss reduction required to make a further partition on a leaf node of the tree. (default is 0) </param>
/// <param name="minChildWeight">Minimum sum of instance weight(hessian) needed in a child. (default is 1)</param>
/// <param name="maxDeltaStep">Maximum delta step we allow each tree's weight estimation to be. (default is 0)</param>
/// <param name="subSample">Subsample ratio of the training instance. (default is 1)</param>
/// <param name="colSampleByTree">Subsample ratio of columns when constructing each tree. (defualt is 1)</param>
/// <param name="colSampleByLevel">Subsample ratio of columns for each split, in each level. (defualt is 1)</param>
/// <param name="l1Regularization">L1 regularization term on weights. Also known as RegAlpha. (default is 0)</param>
/// <param name="l2Reguralization">L2 regularization term on weights. Also known as regLambda. (default is 1)</param>
/// <param name="scalePosWeight">Balancing of positive and negative weights. (default is 1)</param>
/// <param name="baseScore">The initial prediction score of all instances, global bias. (default is 0.5)</param>
/// <param name="seed">Random number seed. (defaukt is 0)</param>
/// <param name="missing">Value in the data which needs to be present as a missing value. (default is NaN)</param>
public ClassificationXGBoostLearner(int maximumTreeDepth = 3, double learningRate = 0.1, int estimators = 100,
bool silent = true,
ClassificationObjective objective = ClassificationObjective.Softmax,
BoosterType boosterType = BoosterType.GBTree,
TreeMethod treeMethod = TreeMethod.Auto,
SamplerType samplerType = SamplerType.Uniform,
NormalizeType normalizeType = NormalizeType.Tree,
double dropoutRate = 0.0,
bool oneDrop = false,
double skipDrop = 0.0,
int numberOfThreads = -1, double gamma = 0, int minChildWeight = 1,
int maxDeltaStep = 0, double subSample = 1, double colSampleByTree = 1,
double colSampleByLevel = 1, double l1Regularization = 0, double l2Reguralization = 1,
double scalePosWeight = 1, double baseScore = 0.5, int seed = 0,
double missing = double.NaN)
{
ArgumentChecks.ThrowOnArgumentLessThan(nameof(maximumTreeDepth), maximumTreeDepth, 0);
ArgumentChecks.ThrowOnArgumentLessThanOrHigherThan(nameof(learningRate), learningRate, 0, 1.0);
ArgumentChecks.ThrowOnArgumentLessThan(nameof(estimators), estimators, 1);
ArgumentChecks.ThrowOnArgumentLessThan(nameof(numberOfThreads), numberOfThreads, -1);
ArgumentChecks.ThrowOnArgumentLessThan(nameof(gamma), gamma, 0);
ArgumentChecks.ThrowOnArgumentLessThan(nameof(minChildWeight), minChildWeight, 0);
ArgumentChecks.ThrowOnArgumentLessThan(nameof(maxDeltaStep), maxDeltaStep, 0);
ArgumentChecks.ThrowOnArgumentLessThanOrHigherThan(nameof(subSample), subSample, 0, 1.0);
ArgumentChecks.ThrowOnArgumentLessThanOrHigherThan(nameof(colSampleByTree), colSampleByTree, 0, 1.0);
ArgumentChecks.ThrowOnArgumentLessThanOrHigherThan(nameof(colSampleByLevel), colSampleByLevel, 0, 1.0);
ArgumentChecks.ThrowOnArgumentLessThan(nameof(l1Regularization), l1Regularization, 0);
ArgumentChecks.ThrowOnArgumentLessThan(nameof(l2Reguralization), l2Reguralization, 0);
ArgumentChecks.ThrowOnArgumentLessThan(nameof(scalePosWeight), scalePosWeight, 0);
m_parameters[ParameterNames.MaxDepth] = maximumTreeDepth;
m_parameters[ParameterNames.LearningRate] = (float)learningRate;
m_parameters[ParameterNames.Estimators] = estimators;
m_parameters[ParameterNames.Silent] = silent;
if (objective == ClassificationObjective.Softmax)
{
// SoftMax and SoftProp are the same objective,
// but softprop returns probabilities.
// So in order to always support PredictProbability,
// always use softprop for multi-class.
// Conversions to class labels is handled in the
// ClassificationXGBoostModel.
objective = ClassificationObjective.SoftProb;
}
m_parameters[ParameterNames.objective] = objective.ToXGBoostString();
m_parameters[ParameterNames.Threads] = numberOfThreads;
m_parameters[ParameterNames.Gamma] = (float)gamma;
m_parameters[ParameterNames.MinChildWeight] = minChildWeight;
m_parameters[ParameterNames.MaxDeltaStep] = maxDeltaStep;
m_parameters[ParameterNames.SubSample] = (float)subSample;
m_parameters[ParameterNames.ColSampleByTree] = (float)colSampleByTree;
m_parameters[ParameterNames.ColSampleByLevel] = (float)colSampleByLevel;
m_parameters[ParameterNames.RegAlpha] = (float)l1Regularization;
m_parameters[ParameterNames.RegLambda] = (float)l2Reguralization;
m_parameters[ParameterNames.ScalePosWeight] = (float)scalePosWeight;
m_parameters[ParameterNames.BaseScore] = (float)baseScore;
m_parameters[ParameterNames.Seed] = seed;
m_parameters[ParameterNames.Missing] = (float)missing;
m_parameters[ParameterNames.ExistingBooster] = null;
m_parameters[ParameterNames.Booster] = boosterType.ToXGBoostString();
m_parameters[ParameterNames.TreeMethod] = treeMethod.ToXGBoostString();
m_parameters[ParameterNames.SampleType] = samplerType.ToXGBoostString();
m_parameters[ParameterNames.NormalizeType] = normalizeType.ToXGBoostString();
m_parameters[ParameterNames.RateDrop] = (float)dropoutRate;
m_parameters[ParameterNames.OneDrop] = oneDrop ? 1 : 0;
m_parameters[ParameterNames.SkipDrop] = (float)skipDrop;
}
