public void TestCrossValidationMacroWithStratification() { var dataPath = GetDataPath(@"breast-cancer.txt"); using (var env = new TlcEnvironment(42)) { var subGraph = env.CreateExperiment(); var nop = new ML.Transforms.NoOperation(); var nopOutput = subGraph.Add(nop); var learnerInput = new ML.Trainers.StochasticDualCoordinateAscentBinaryClassifier { TrainingData = nopOutput.OutputData, NumThreads = 1 }; var learnerOutput = subGraph.Add(learnerInput); var modelCombine = new ML.Transforms.ManyHeterogeneousModelCombiner { TransformModels = new ArrayVar <ITransformModel>(nopOutput.Model), PredictorModel = learnerOutput.PredictorModel }; var modelCombineOutput = subGraph.Add(modelCombine); var experiment = env.CreateExperiment(); var importInput = new ML.Data.TextLoader(dataPath); importInput.Arguments.Column = new ML.Data.TextLoaderColumn[] { new ML.Data.TextLoaderColumn { Name = "Label", Source = new[] { new ML.Data.TextLoaderRange(0) } }, new ML.Data.TextLoaderColumn { Name = "Strat", Source = new[] { new ML.Data.TextLoaderRange(1) } }, new ML.Data.TextLoaderColumn { Name = "Features", Source = new[] { new ML.Data.TextLoaderRange(2, 9) } } }; var importOutput = experiment.Add(importInput); var crossValidate = new ML.Models.CrossValidator { Data = importOutput.Data, Nodes = subGraph, TransformModel = null, StratificationColumn = "Strat" }; crossValidate.Inputs.Data = nop.Data; crossValidate.Outputs.PredictorModel = modelCombineOutput.PredictorModel; var crossValidateOutput = experiment.Add(crossValidate); experiment.Compile(); experiment.SetInput(importInput.InputFile, new SimpleFileHandle(env, dataPath, false, false)); experiment.Run(); var data = experiment.GetOutput(crossValidateOutput.OverallMetrics); var schema = data.Schema; var b = schema.TryGetColumnIndex("AUC", out int metricCol); Assert.True(b); b = schema.TryGetColumnIndex("Fold Index", out int foldCol); Assert.True(b); using (var cursor = data.GetRowCursor(col => col == metricCol || col == foldCol)) { var getter = cursor.GetGetter <double>(metricCol); var foldGetter = cursor.GetGetter <DvText>(foldCol); DvText fold = default; // Get the verage. b = cursor.MoveNext(); Assert.True(b); double avg = 0; getter(ref avg); foldGetter(ref fold); Assert.True(fold.EqualsStr("Average")); // Get the standard deviation. b = cursor.MoveNext(); Assert.True(b); double stdev = 0; getter(ref stdev); foldGetter(ref fold); Assert.True(fold.EqualsStr("Standard Deviation")); Assert.Equal(0.00485, stdev, 5); double sum = 0; double val = 0; for (int f = 0; f < 2; f++) { b = cursor.MoveNext(); Assert.True(b); getter(ref val); foldGetter(ref fold); sum += val; Assert.True(fold.EqualsStr("Fold " + f)); } Assert.Equal(avg, sum / 2); b = cursor.MoveNext(); Assert.False(b); } } }
public void TestCrossValidationMacroWithMultiClass() { var dataPath = GetDataPath(@"Train-Tiny-28x28.txt"); using (var env = new TlcEnvironment(42)) { var subGraph = env.CreateExperiment(); var nop = new ML.Transforms.NoOperation(); var nopOutput = subGraph.Add(nop); var learnerInput = new ML.Trainers.StochasticDualCoordinateAscentClassifier { TrainingData = nopOutput.OutputData, NumThreads = 1 }; var learnerOutput = subGraph.Add(learnerInput); var modelCombine = new ML.Transforms.ManyHeterogeneousModelCombiner { TransformModels = new ArrayVar <ITransformModel>(nopOutput.Model), PredictorModel = learnerOutput.PredictorModel }; var modelCombineOutput = subGraph.Add(modelCombine); var experiment = env.CreateExperiment(); var importInput = new ML.Data.TextLoader(dataPath); var importOutput = experiment.Add(importInput); var crossValidate = new ML.Models.CrossValidator { Data = importOutput.Data, Nodes = subGraph, Kind = ML.Models.MacroUtilsTrainerKinds.SignatureMultiClassClassifierTrainer, TransformModel = null }; crossValidate.Inputs.Data = nop.Data; crossValidate.Outputs.PredictorModel = modelCombineOutput.PredictorModel; var crossValidateOutput = experiment.Add(crossValidate); experiment.Compile(); importInput.SetInput(env, experiment); experiment.Run(); var data = experiment.GetOutput(crossValidateOutput.OverallMetrics); var schema = data.Schema; var b = schema.TryGetColumnIndex("Accuracy(micro-avg)", out int metricCol); Assert.True(b); b = schema.TryGetColumnIndex("Fold Index", out int foldCol); Assert.True(b); using (var cursor = data.GetRowCursor(col => col == metricCol || col == foldCol)) { var getter = cursor.GetGetter <double>(metricCol); var foldGetter = cursor.GetGetter <DvText>(foldCol); DvText fold = default; // Get the verage. b = cursor.MoveNext(); Assert.True(b); double avg = 0; getter(ref avg); foldGetter(ref fold); Assert.True(fold.EqualsStr("Average")); // Get the standard deviation. b = cursor.MoveNext(); Assert.True(b); double stdev = 0; getter(ref stdev); foldGetter(ref fold); Assert.True(fold.EqualsStr("Standard Deviation")); Assert.Equal(0.025, stdev, 3); double sum = 0; double val = 0; for (int f = 0; f < 2; f++) { b = cursor.MoveNext(); Assert.True(b); getter(ref val); foldGetter(ref fold); sum += val; Assert.True(fold.EqualsStr("Fold " + f)); } Assert.Equal(avg, sum / 2); b = cursor.MoveNext(); Assert.False(b); } var confusion = experiment.GetOutput(crossValidateOutput.ConfusionMatrix); schema = confusion.Schema; b = schema.TryGetColumnIndex("Count", out int countCol); Assert.True(b); b = schema.TryGetColumnIndex("Fold Index", out foldCol); Assert.True(b); var type = schema.GetMetadataTypeOrNull(MetadataUtils.Kinds.SlotNames, countCol); Assert.True(type != null && type.ItemType.IsText && type.VectorSize == 10); var slotNames = default(VBuffer <DvText>); schema.GetMetadata(MetadataUtils.Kinds.SlotNames, countCol, ref slotNames); Assert.True(slotNames.Values.Select((s, i) => s.EqualsStr(i.ToString())).All(x => x)); using (var curs = confusion.GetRowCursor(col => true)) { var countGetter = curs.GetGetter <VBuffer <double> >(countCol); var foldGetter = curs.GetGetter <DvText>(foldCol); var confCount = default(VBuffer <double>); var foldIndex = default(DvText); int rowCount = 0; var foldCur = "Fold 0"; while (curs.MoveNext()) { countGetter(ref confCount); foldGetter(ref foldIndex); rowCount++; Assert.True(foldIndex.EqualsStr(foldCur)); if (rowCount == 10) { rowCount = 0; foldCur = "Fold 1"; } } Assert.Equal(0, rowCount); } var warnings = experiment.GetOutput(crossValidateOutput.Warnings); using (var cursor = warnings.GetRowCursor(col => true)) Assert.False(cursor.MoveNext()); } }
public void TestCrossValidationMacroMultiClassWithWarnings() { var dataPath = GetDataPath(@"Train-Tiny-28x28.txt"); using (var env = new TlcEnvironment(42)) { var subGraph = env.CreateExperiment(); var nop = new ML.Transforms.NoOperation(); var nopOutput = subGraph.Add(nop); var learnerInput = new ML.Trainers.LogisticRegressionClassifier { TrainingData = nopOutput.OutputData, NumThreads = 1 }; var learnerOutput = subGraph.Add(learnerInput); var experiment = env.CreateExperiment(); var importInput = new ML.Data.TextLoader(dataPath); var importOutput = experiment.Add(importInput); var filter = new ML.Transforms.RowRangeFilter(); filter.Data = importOutput.Data; filter.Column = "Label"; filter.Min = 0; filter.Max = 5; var filterOutput = experiment.Add(filter); var term = new ML.Transforms.TextToKeyConverter(); term.Column = new[] { new ML.Transforms.TermTransformColumn() { Source = "Label", Name = "Strat", Sort = ML.Transforms.TermTransformSortOrder.Value } }; term.Data = filterOutput.OutputData; var termOutput = experiment.Add(term); var crossValidate = new ML.Models.CrossValidator { Data = termOutput.OutputData, Nodes = subGraph, Kind = ML.Models.MacroUtilsTrainerKinds.SignatureMultiClassClassifierTrainer, TransformModel = null, StratificationColumn = "Strat" }; crossValidate.Inputs.Data = nop.Data; crossValidate.Outputs.PredictorModel = learnerOutput.PredictorModel; var crossValidateOutput = experiment.Add(crossValidate); experiment.Compile(); importInput.SetInput(env, experiment); experiment.Run(); var warnings = experiment.GetOutput(crossValidateOutput.Warnings); var schema = warnings.Schema; var b = schema.TryGetColumnIndex("WarningText", out int warningCol); Assert.True(b); using (var cursor = warnings.GetRowCursor(col => col == warningCol)) { var getter = cursor.GetGetter <DvText>(warningCol); b = cursor.MoveNext(); Assert.True(b); var warning = default(DvText); getter(ref warning); Assert.Contains("test instances with class values not seen in the training set.", warning.ToString()); b = cursor.MoveNext(); Assert.True(b); getter(ref warning); Assert.Contains("Detected columns of variable length: SortedScores, SortedClasses", warning.ToString()); b = cursor.MoveNext(); Assert.False(b); } } }
public void TestCrossValidationMacro() { var dataPath = GetDataPath(TestDatasets.winequality.trainFilename); using (var env = new TlcEnvironment(42)) { var subGraph = env.CreateExperiment(); var nop = new ML.Transforms.NoOperation(); var nopOutput = subGraph.Add(nop); var generate = new ML.Transforms.RandomNumberGenerator(); generate.Column = new[] { new ML.Transforms.GenerateNumberTransformColumn() { Name = "Weight1" } }; generate.Data = nopOutput.OutputData; var generateOutput = subGraph.Add(generate); var learnerInput = new ML.Trainers.PoissonRegressor { TrainingData = generateOutput.OutputData, NumThreads = 1, WeightColumn = "Weight1" }; var learnerOutput = subGraph.Add(learnerInput); var modelCombine = new ML.Transforms.ManyHeterogeneousModelCombiner { TransformModels = new ArrayVar <ITransformModel>(nopOutput.Model, generateOutput.Model), PredictorModel = learnerOutput.PredictorModel }; var modelCombineOutput = subGraph.Add(modelCombine); var experiment = env.CreateExperiment(); var importInput = new ML.Data.TextLoader(dataPath) { Arguments = new TextLoaderArguments { Separator = new[] { ';' }, HasHeader = true, Column = new[] { new TextLoaderColumn() { Name = "Label", Source = new [] { new TextLoaderRange(11) }, Type = ML.Data.DataKind.Num }, new TextLoaderColumn() { Name = "Features", Source = new [] { new TextLoaderRange(0, 10) }, Type = ML.Data.DataKind.Num } } } }; var importOutput = experiment.Add(importInput); var crossValidate = new ML.Models.CrossValidator { Data = importOutput.Data, Nodes = subGraph, Kind = ML.Models.MacroUtilsTrainerKinds.SignatureRegressorTrainer, TransformModel = null, WeightColumn = "Weight1" }; crossValidate.Inputs.Data = nop.Data; crossValidate.Outputs.PredictorModel = modelCombineOutput.PredictorModel; var crossValidateOutput = experiment.Add(crossValidate); experiment.Compile(); importInput.SetInput(env, experiment); experiment.Run(); var data = experiment.GetOutput(crossValidateOutput.OverallMetrics); var schema = data.Schema; var b = schema.TryGetColumnIndex("L1(avg)", out int metricCol); Assert.True(b); b = schema.TryGetColumnIndex("Fold Index", out int foldCol); Assert.True(b); b = schema.TryGetColumnIndex("IsWeighted", out int isWeightedCol); using (var cursor = data.GetRowCursor(col => col == metricCol || col == foldCol || col == isWeightedCol)) { var getter = cursor.GetGetter <double>(metricCol); var foldGetter = cursor.GetGetter <DvText>(foldCol); var isWeightedGetter = cursor.GetGetter <DvBool>(isWeightedCol); DvText fold = default; DvBool isWeighted = default; double avg = 0; double weightedAvg = 0; for (int w = 0; w < 2; w++) { // Get the average. b = cursor.MoveNext(); Assert.True(b); if (w == 1) { getter(ref weightedAvg); } else { getter(ref avg); } foldGetter(ref fold); Assert.True(fold.EqualsStr("Average")); isWeightedGetter(ref isWeighted); Assert.True(isWeighted.IsTrue == (w == 1)); // Get the standard deviation. b = cursor.MoveNext(); Assert.True(b); double stdev = 0; getter(ref stdev); foldGetter(ref fold); Assert.True(fold.EqualsStr("Standard Deviation")); if (w == 1) { Assert.Equal(0.002827, stdev, 6); } else { Assert.Equal(0.002376, stdev, 6); } isWeightedGetter(ref isWeighted); Assert.True(isWeighted.IsTrue == (w == 1)); } double sum = 0; double weightedSum = 0; for (int f = 0; f < 2; f++) { for (int w = 0; w < 2; w++) { b = cursor.MoveNext(); Assert.True(b); double val = 0; getter(ref val); foldGetter(ref fold); if (w == 1) { weightedSum += val; } else { sum += val; } Assert.True(fold.EqualsStr("Fold " + f)); isWeightedGetter(ref isWeighted); Assert.True(isWeighted.IsTrue == (w == 1)); } } Assert.Equal(weightedAvg, weightedSum / 2); Assert.Equal(avg, sum / 2); b = cursor.MoveNext(); Assert.False(b); } } }