public void TrainRanking() { var rand = new Random(Seed); for (int test = 0; test < 5; ++test) { int numColumns = rand.Next(1, 10); var pms = GenerateParameters(rand, ObjectiveType.LambdaRank, numColumns); pms.Objective.EvalAt = new int[] { 5 }; // TODO: need at most one or get 'Expected at most one metric' error var numRanks = rand.Next(2, 4); Dictionary <int, int> categorical = null; var trainData = CreateRandomDenseClassifyData(rand, numRanks, ref categorical, pms.Dataset.UseMissing, numColumns); trainData.Groups = GenGroups(rand, trainData.NumRows); var validData = (pms.Learning.EarlyStoppingRound > 0 || rand.Next(2) == 0) ? CreateRandomDenseClassifyData(rand, numRanks, ref categorical, pms.Dataset.UseMissing, numColumns) : null; if (validData != null) { validData.Groups = GenGroups(rand, validData.NumRows); } pms.Dataset.CategoricalFeature = categorical.Keys.ToArray(); var learningRateSchedule = (rand.Next(2) == 0) ? (Func <int, double>)null : (iter => pms.Learning.LearningRate * Math.Pow(0.99, iter)); try { using (var datasets = (rand.Next(2) == 0) ? new Datasets(pms.Common, pms.Dataset, trainData, validData) : new Datasets(pms.Common, pms.Dataset, Dense2Sparse(trainData), Dense2Sparse(validData))) using (var trainer = new RankingTrainer(pms.Learning, pms.Objective)) { var model = trainer.Train(datasets, learningRateSchedule); model.Managed.MaxThreads = rand.Next(1, Environment.ProcessorCount); // possibly use subset of trees var numIterations = -1; if (rand.Next(2) == 0) { numIterations = rand.Next(1, model.Managed.MaxNumTrees); model.Managed.MaxNumTrees = numIterations; model.Native.MaxNumTrees = numIterations; } RankingPredictor model2 = null; using (var ms = new System.IO.MemoryStream()) using (var writer = new System.IO.BinaryWriter(ms)) using (var reader = new System.IO.BinaryReader(ms)) { PredictorPersist.Save(model.Managed, writer); ms.Position = 0; model2 = PredictorPersist.Load <double>(reader) as RankingPredictor; Assert.Equal(ms.Position, ms.Length); } RankingNativePredictor model2native = null; using (var ms = new System.IO.MemoryStream()) using (var writer = new System.IO.BinaryWriter(ms)) using (var reader = new System.IO.BinaryReader(ms)) { NativePredictorPersist.Save(model.Native, writer); ms.Position = 0; model2native = NativePredictorPersist.Load <double>(reader) as RankingNativePredictor; Assert.Equal(ms.Position, ms.Length); } var output3s = trainer.Evaluate(Booster.PredictType.Normal, trainData.Features, numIterations); Assert.Equal(trainData.Features.Length, output3s.GetLength(0)); Assert.Equal(1, output3s.GetLength(1)); var output3natives = model.Native.GetOutputs(trainData.Features); Assert.Equal(trainData.Features.Length, output3s.Length); for (int i = 0; i < trainData.Features.Length; i++) { var row = trainData.Features[i]; double output = 0; var input = new VBuffer <float>(row.Length, row); model.Managed.GetOutput(ref input, ref output); // TODO: NFI what output represents... double output2 = 0; model2.GetOutput(ref input, ref output2); Compare(output, output2); var output3 = trainer.Evaluate(Booster.PredictType.Normal, row, numIterations); Assert.Single(output3); Assert.Equal(output3[0], output3s[i, 0]); Assert.Equal(output3[0], output3natives[i]); Compare(output, output3[0]); //Console.WriteLine(trainer.GetModelString()); //throw new Exception($"Output mismatch {output} vs {output3[0]} (error: {Math.Abs(output - output3[0])}) input: {String.Join(", ", row)}"); } var normalise = rand.Next(2) == 0; var getSplits = rand.Next(2) == 0; var gains = model.Managed.GetFeatureWeights(normalise, getSplits); var gainsNative = model.Native.GetFeatureWeights(normalise, getSplits); Assert.Equal(gains.Count, gainsNative.Count); foreach (var kv in gains) { Assert.True(0 <= kv.Key && kv.Key < trainData.NumColumns); Assert.True(0.0 <= kv.Value); Compare(kv.Value, gainsNative[kv.Key]); } if (!getSplits && !normalise) { var totGain1 = gains.Values.Sum(); var totGain2 = Enumerable.Range(0, trainData.NumColumns).SelectMany(i => model.Managed.GetFeatureGains(i)).Sum(); Compare(totGain1, totGain2); } } } catch (Exception e) { throw new Exception($"Failed: {Seed} #{test} {pms}", e); } } }
public void TrainMultiClass() { var rand = new Random(Seed); for (int test = 0; test < 5; ++test) { int numColumns = rand.Next(1, 10); var objective = (rand.Next(2) == 0) ? ObjectiveType.MultiClass : ObjectiveType.MultiClassOva; var pms = GenerateParameters(rand, objective, numColumns); Dictionary <int, int> categorical = null; var trainData = CreateRandomDenseClassifyData(rand, pms.Objective.NumClass, ref categorical, pms.Dataset.UseMissing, numColumns); var validData = (pms.Learning.EarlyStoppingRound > 0 || rand.Next(2) == 0) ? CreateRandomDenseClassifyData(rand, pms.Objective.NumClass, ref categorical, pms.Dataset.UseMissing, numColumns) : null; pms.Dataset.CategoricalFeature = categorical.Keys.ToArray(); var learningRateSchedule = (rand.Next(2) == 0) ? (Func <int, double>)null : (iter => pms.Learning.LearningRate * Math.Pow(0.99, iter)); //if (test != 1) continue; try { using (var datasets = (rand.Next(2) == 0) ? new Datasets(pms.Common, pms.Dataset, trainData, validData) : new Datasets(pms.Common, pms.Dataset, Dense2Sparse(trainData), Dense2Sparse(validData))) using (var trainer = new MulticlassTrainer(pms.Learning, pms.Objective)) { //trainer.ToCommandLineFiles(datasets); var model = trainer.Train(datasets, learningRateSchedule); model.Managed.MaxThreads = rand.Next(1, Environment.ProcessorCount); // possibly use subset of trees var numIterations = -1; if (rand.Next(2) == 0) { numIterations = rand.Next(1, model.Managed.MaxNumTrees); model.Managed.MaxNumTrees = numIterations; model.Native.MaxNumTrees = numIterations; } OvaPredictor model2 = null; using (var ms = new System.IO.MemoryStream()) using (var writer = new System.IO.BinaryWriter(ms)) using (var reader = new System.IO.BinaryReader(ms)) { PredictorPersist.Save(model.Managed, writer); ms.Position = 0; model2 = PredictorPersist.Load <double[]>(reader) as OvaPredictor; Assert.Equal(ms.Position, ms.Length); } MulticlassNativePredictor model2native = null; using (var ms = new System.IO.MemoryStream()) using (var writer = new System.IO.BinaryWriter(ms)) using (var reader = new System.IO.BinaryReader(ms)) { NativePredictorPersist.Save(model.Native, writer); ms.Position = 0; model2native = NativePredictorPersist.Load <double[]>(reader) as MulticlassNativePredictor; Assert.Equal(ms.Position, ms.Length); } var rawscore3s = trainer.Evaluate(Booster.PredictType.RawScore, trainData.Features, numIterations); Assert.Equal(trainData.Features.Length, rawscore3s.GetLength(0)); Assert.Equal(pms.Objective.NumClass, rawscore3s.GetLength(1)); var output3s = trainer.Evaluate(Booster.PredictType.Normal, trainData.Features, numIterations); Assert.Equal(trainData.Features.Length, output3s.GetLength(0)); Assert.Equal(pms.Objective.NumClass, output3s.GetLength(1)); var output3natives = model.Native.GetOutputs(trainData.Features); Assert.Equal(trainData.Features.Length, output3natives.Length); for (var irow = 0; irow < trainData.Features.Length; irow++) { var row = trainData.Features[irow]; // check evaluation of managed model double[] output = null; var input = new VBuffer <float>(row.Length, row); model.Managed.GetOutput(ref input, ref output); foreach (var p in output) { Assert.True(p >= 0); Assert.True(p <= 1); } Assert.Equal(1, output.Sum(), 5); Assert.Equal(output.Length, pms.Objective.NumClass); // compare with output of serialised model double[] output2 = null; model2.GetOutput(ref input, ref output2); Assert.Equal(output.Length, output2.Length); for (var i = 0; i < output.Length; i++) { Compare(output[i], output2[i]); } // check raw scores against native booster object var isRf = (pms.Learning.Boosting == BoostingType.RandomForest); var rawscores = (model.Managed as OvaPredictor).Predictors.Select(p => { double outputi = 0; if (p is CalibratedPredictor) { (p as CalibratedPredictor).SubPredictor.GetOutput(ref input, ref outputi); } else { p.GetOutput(ref input, ref outputi); } return(outputi, p.MaxNumTrees); }).ToArray(); var rawscores3 = trainer.Evaluate(Booster.PredictType.RawScore, row, numIterations); Assert.Equal(pms.Objective.NumClass, rawscores.Length); Assert.Equal(pms.Objective.NumClass, rawscores3.Length); for (var i = 0; i < rawscores.Length; i++) { (var rawscore, var numTrees) = rawscores[i]; Compare(isRf ? rawscore * numTrees : rawscore, rawscores3[i]); Assert.Equal(rawscores3[i], rawscore3s[irow, i]); } //Console.WriteLine(trainer.GetModelString()); //throw new Exception($"Raw score mismatch at row {irow}: {rawscores[i]} vs {rawscores3[i]} (error: {Math.Abs(rawscores[i] - rawscores3[i])}) input: {String.Join(", ", row)}"); double [] outputNative = null; model.Native.GetOutput(ref input, ref outputNative); double[] outputNative2 = null; model2native.GetOutput(ref input, ref outputNative2); // check probabilities against native booster object var output3 = trainer.Evaluate(Booster.PredictType.Normal, row, numIterations); for (var i = 0; i < output3.Length; i++) { Assert.Equal(output3s[irow, i], output3[i]); Assert.Equal(output3natives[irow][i], output3[i]); Assert.Equal(outputNative[i], output3[i]); Assert.Equal(outputNative2[i], output3[i]); } if (objective == ObjectiveType.MultiClassOva) { // booster object doesn't return normalised probabilities for OVA var sum = output3.Sum(); for (var i = 0; i < output3.Length; i++) { output3[i] /= sum; } } Assert.Equal(pms.Objective.NumClass, output3.Length); for (var i = 0; i < output3.Length; i++) { Assert.Equal(output[i], output3[i], 3); } } var normalise = rand.Next(2) == 0; var getSplits = rand.Next(2) == 0; var gains = model.Managed.GetFeatureWeights(normalise, getSplits); var gainsNative = model.Native.GetFeatureWeights(normalise, getSplits); Assert.Equal(gains.Count, gainsNative.Count); foreach (var kv in gains) { Assert.True(0 <= kv.Key && kv.Key < trainData.NumColumns); Assert.True(0.0 <= kv.Value); Compare(kv.Value, gainsNative[kv.Key]); } if (!getSplits && !normalise) { var totGain1 = gains.Values.Sum(); var totGain2 = Enumerable.Range(0, trainData.NumColumns).SelectMany(i => model.Managed.GetFeatureGains(i)).Sum(); Compare(totGain1, totGain2); } } } catch (Exception e) { throw new Exception($"Failed: {Seed} #{test} {pms}", e); } } }
public void TrainRegression() { var objectiveTypes = new ObjectiveType[] { ObjectiveType.Regression, ObjectiveType.RegressionL1, ObjectiveType.Huber, ObjectiveType.Fair, ObjectiveType.Poisson, ObjectiveType.Quantile, ObjectiveType.Mape, ObjectiveType.Gamma, ObjectiveType.Tweedie }; var rand = new Random(Seed); for (int test = 0; test < 5; ++test) { int numColumns = rand.Next(1, 10); var objective = objectiveTypes[rand.Next(objectiveTypes.Length)]; var pms = GenerateParameters(rand, objective, numColumns); if (rand.Next(2) == 0) { pms.Objective.RegSqrt = true; } var learningRateSchedule = (rand.Next(2) == 0) ? (Func <int, double>)null : (iter => pms.Learning.LearningRate * Math.Pow(0.99, iter)); try { Dictionary <int, int> categorical = null; var trainData = CreateRandomDenseRegressionData(rand, ref categorical, pms.Dataset.UseMissing, numColumns); var validData = (pms.Learning.EarlyStoppingRound > 0 || rand.Next(2) == 0) ? CreateRandomDenseRegressionData(rand, ref categorical, pms.Dataset.UseMissing, numColumns) : null; pms.Dataset.CategoricalFeature = categorical.Keys.ToArray(); // make labels positive for certain objective types if (objective == ObjectiveType.Poisson || objective == ObjectiveType.Gamma || objective == ObjectiveType.Tweedie) { for (var i = 0; i < trainData.Labels.Length; i++) { trainData.Labels[i] = Math.Abs(trainData.Labels[i]); } if (validData != null) { for (var i = 0; i < validData.Labels.Length; i++) { validData.Labels[i] = Math.Abs(validData.Labels[i]); } } } // uncomment to select particular iteration //if (test != 3) // continue; using (var datasets = (rand.Next(2) == 0) ? new Datasets(pms.Common, pms.Dataset, trainData, validData) : new Datasets(pms.Common, pms.Dataset, Dense2Sparse(trainData), Dense2Sparse(validData))) using (var trainer = new RegressionTrainer(pms.Learning, pms.Objective)) { //if (true) // trainer.ToCommandLineFiles(datasets); var model = trainer.Train(datasets, learningRateSchedule); model.Managed.MaxThreads = rand.Next(1, Environment.ProcessorCount); // possibly use subset of trees var numIterations = -1; if (rand.Next(2) == 0) { numIterations = rand.Next(1, model.Managed.MaxNumTrees); model.Managed.MaxNumTrees = numIterations; model.Native.MaxNumTrees = numIterations; } IPredictorWithFeatureWeights <double> model2 = null; using (var ms = new System.IO.MemoryStream()) using (var writer = new System.IO.BinaryWriter(ms)) using (var reader = new System.IO.BinaryReader(ms)) { PredictorPersist.Save(model.Managed, writer); ms.Position = 0; model2 = PredictorPersist.Load <double>(reader); Assert.Equal(ms.Position, ms.Length); } IPredictorWithFeatureWeights <double> model2native = null; using (var ms = new System.IO.MemoryStream()) using (var writer = new System.IO.BinaryWriter(ms)) using (var reader = new System.IO.BinaryReader(ms)) { NativePredictorPersist.Save(model.Native, writer); ms.Position = 0; model2native = NativePredictorPersist.Load <double>(reader); Assert.Equal(ms.Position, ms.Length); } var output3s = trainer.Evaluate(Booster.PredictType.Normal, trainData.Features, numIterations); Assert.Equal(trainData.Features.Length, output3s.GetLength(0)); Assert.Equal(1, output3s.GetLength(1)); var output3natives = model.Native.GetOutputs(trainData.Features); Assert.Equal(trainData.Features.Length, output3s.Length); for (int i = 0; i < trainData.Features.Length; i++) { var row = trainData.Features[i]; double output = 0; var input = new VBuffer <float>(row.Length, row); model.Managed.GetOutput(ref input, ref output); Assert.False(double.IsNaN(output)); double output2 = 0; model2.GetOutput(ref input, ref output2); Compare(output, output2); var output3 = trainer.Evaluate(Booster.PredictType.Normal, row, numIterations); Assert.Single(output3); Assert.Equal(output3[0], output3s[i, 0]); Assert.Equal(output3[0], output3natives[i]); Compare(output, output3[0]); //Console.WriteLine(trainer.GetModelString()); //throw new Exception($"Output mismatch {output} vs {output3[0]} (error: {Math.Abs(output - output3[0])}) input: {String.Join(", ", row)}"); double outputNative = 0; model.Native.GetOutput(ref input, ref outputNative); Assert.Equal(outputNative, output3[0]); model2native.GetOutput(ref input, ref outputNative); Assert.Equal(outputNative, output3[0]); } var normalise = rand.Next(2) == 0; var getSplits = rand.Next(2) == 0; var gains = model.Managed.GetFeatureWeights(normalise, getSplits); var gainsNative = model.Native.GetFeatureWeights(normalise, getSplits); Assert.Equal(gains.Count, gainsNative.Count); foreach (var kv in gains) { Assert.True(0 <= kv.Key && kv.Key < trainData.NumColumns); Assert.True(0.0 <= kv.Value); Compare(kv.Value, gainsNative[kv.Key]); } if (!getSplits && !normalise) { var totGain1 = gains.Values.Sum(); var totGain2 = Enumerable.Range(0, trainData.NumColumns).SelectMany(i => model.Managed.GetFeatureGains(i)).Sum(); Compare(totGain1, totGain2); } } } catch (Exception e) { throw new Exception($"Failed: {Seed} #{test} {pms}", e); } } }
public void TrainBinary() { var rand = new Random(Seed); for (int test = 0; test < 5; ++test) { int numColumns = rand.Next(1, 10); var pms = GenerateParameters(rand, ObjectiveType.Binary, numColumns); Dictionary <int, int> categorical = null; var trainData = CreateRandomDenseClassifyData(rand, 2, ref categorical, pms.Dataset.UseMissing, numColumns); var validData = (pms.Learning.EarlyStoppingRound > 0 || rand.Next(2) == 0) ? CreateRandomDenseClassifyData(rand, 2, ref categorical, pms.Dataset.UseMissing, numColumns) : null; pms.Dataset.CategoricalFeature = categorical.Keys.ToArray(); var learningRateSchedule = (rand.Next(2) == 0) ? (Func <int, double>)null : (iter => pms.Learning.LearningRate * Math.Pow(0.99, iter)); try { using (var datasets = (rand.Next(2) == 0) ? new Datasets(pms.Common, pms.Dataset, trainData, validData) : new Datasets(pms.Common, pms.Dataset, Dense2Sparse(trainData), Dense2Sparse(validData))) using (var trainer = new BinaryTrainer(pms.Learning, pms.Objective)) { //trainer.ToCommandLineFiles(datasets); var datasets2 = (rand.Next(2) == 0) ? null : datasets.Training.GetSubset(Enumerable.Range(0, datasets.Training.NumRows / 2).ToArray()); var model = trainer.Train(datasets, learningRateSchedule); { if (datasets2 != null) { model.Dispose(); model = trainer.ContinueTraining(datasets2, learningRateSchedule); } model.Managed.MaxThreads = rand.Next(1, Environment.ProcessorCount); // possibly use subset of trees var numIterations = -1; if (rand.Next(2) == 0) { numIterations = rand.Next(1, model.Managed.MaxNumTrees); model.Managed.MaxNumTrees = numIterations; model.Native.MaxNumTrees = numIterations; } CalibratedPredictor model2 = null; using (var ms = new System.IO.MemoryStream()) using (var writer = new System.IO.BinaryWriter(ms)) using (var reader = new System.IO.BinaryReader(ms)) { PredictorPersist.Save(model.Managed, writer); ms.Position = 0; model2 = PredictorPersist.Load <double>(reader) as CalibratedPredictor; Assert.Equal(ms.Position, ms.Length); } BinaryNativePredictor model2native = null; using (var ms = new System.IO.MemoryStream()) using (var writer = new System.IO.BinaryWriter(ms)) using (var reader = new System.IO.BinaryReader(ms)) { NativePredictorPersist.Save(model.Native, writer); ms.Position = 0; model2native = NativePredictorPersist.Load <double>(reader) as BinaryNativePredictor; Assert.Equal(ms.Position, ms.Length); } var rawscore2s = trainer.Evaluate(Booster.PredictType.RawScore, trainData.Features, numIterations); Assert.Equal(trainData.Features.Length, rawscore2s.GetLength(0)); Assert.Equal(1, rawscore2s.GetLength(1)); var output3s = trainer.Evaluate(Booster.PredictType.Normal, trainData.Features, numIterations); Assert.Equal(trainData.Features.Length, output3s.GetLength(0)); Assert.Equal(1, output3s.GetLength(1)); var output3natives = model.Native.GetOutputs(trainData.Features); Assert.Equal(trainData.Features.Length, output3s.Length); for (int i = 0; i < trainData.Features.Length; i++) { var row = trainData.Features[i]; double output = 0; var input = new VBuffer <float>(row.Length, row); model.Managed.GetOutput(ref input, ref output); Assert.True(output >= 0); Assert.True(output <= 1); double output2 = 0; model2.GetOutput(ref input, ref output2); Compare(output, output2); // check raw score against native booster object var rawscore = 0.0; (model.Managed as CalibratedPredictor).SubPredictor.GetOutput(ref input, ref rawscore); var rawscore2 = trainer.Evaluate(Booster.PredictType.RawScore, row, numIterations); Assert.Single(rawscore2); Assert.Equal(rawscore2[0], rawscore2s[i, 0]); var isRf = (pms.Learning.Boosting == BoostingType.RandomForest); Compare(isRf ? rawscore * model.Managed.MaxNumTrees : rawscore, rawscore2[0]); var output3 = trainer.Evaluate(Booster.PredictType.Normal, row, numIterations); Assert.Single(output3); Assert.Equal(output3[0], output3s[i, 0]); Assert.Equal(output3[0], output3natives[i]); Compare(output, output3[0]); double outputNative = 0; model.Native.GetOutput(ref input, ref outputNative); Assert.Equal(outputNative, output3[0]); model2native.GetOutput(ref input, ref outputNative); Assert.Equal(outputNative, output3[0]); //Console.WriteLine(trainer.GetModelString()); //throw new Exception($"Output mismatch {output} vs {output3[0]} (error: {Math.Abs(output - output3[0])}) input: {String.Join(", ", row)}"); } var normalise = rand.Next(2) == 0; var getSplits = rand.Next(2) == 0; var gains = model.Managed.GetFeatureWeights(normalise, getSplits); var gainsNative = model.Native.GetFeatureWeights(normalise, getSplits); Assert.Equal(gains.Count, gainsNative.Count); foreach (var kv in gains) { Assert.True(0 <= kv.Key && kv.Key < trainData.NumColumns); Assert.True(0.0 <= kv.Value); Compare(kv.Value, gainsNative[kv.Key]); } if (!getSplits && !normalise) { var totGain1 = gains.Values.Sum(); var totGain2 = Enumerable.Range(0, trainData.NumColumns).SelectMany(i => model.Managed.GetFeatureGains(i)).Sum(); Compare(totGain1, totGain2); } } if (datasets2 != null) { datasets2.Dispose(); } if (model != null) { model.Dispose(); } } } catch (Exception e) { throw new Exception($"Failed: {Seed} #{test} {pms}", e); } } }