public void TestCrossValidationMacroWithNonDefaultNames()
{
string dataPath = GetDataPath(@"adult.tiny.with-schema.txt");
var env = new MLContext(42);
var subGraph = env.CreateExperiment();
var textToKey = new Legacy.Transforms.TextToKeyConverter();
textToKey.Column = new[] { new Legacy.Transforms.ValueToKeyMappingTransformerColumn()
{
Name = "Label1", Source = "Label"
} };
var textToKeyOutput = subGraph.Add(textToKey);
var hash = new Legacy.Transforms.HashConverter();
hash.Column = new[] { new Legacy.Transforms.HashJoiningTransformColumn()
{
Name = "GroupId1", Source = "Workclass"
} };
hash.Data = textToKeyOutput.OutputData;
var hashOutput = subGraph.Add(hash);
var learnerInput = new Legacy.Trainers.FastTreeRanker
{
TrainingData = hashOutput.OutputData,
NumThreads = 1,
LabelColumn = "Label1",
GroupIdColumn = "GroupId1"
};
var learnerOutput = subGraph.Add(learnerInput);
var modelCombine = new Legacy.Transforms.ManyHeterogeneousModelCombiner
{
TransformModels = new ArrayVar <TransformModel>(textToKeyOutput.Model, hashOutput.Model),
PredictorModel = learnerOutput.PredictorModel
};
var modelCombineOutput = subGraph.Add(modelCombine);
var experiment = env.CreateExperiment();
var importInput = new Legacy.Data.TextLoader(dataPath);
importInput.Arguments.HasHeader = true;
importInput.Arguments.Column = new TextLoaderColumn[]
{
new TextLoaderColumn {
Name = "Label", Source = new[] { new TextLoaderRange(0) }
},
new TextLoaderColumn {
Name = "Workclass", Source = new[] { new TextLoaderRange(1) }, Type = Legacy.Data.DataKind.Text
},
new TextLoaderColumn {
Name = "Features", Source = new[] { new TextLoaderRange(9, 14) }
}
};
var importOutput = experiment.Add(importInput);
var crossValidate = new Legacy.Models.CrossValidator
{
Data = importOutput.Data,
Nodes = subGraph,
TransformModel = null,
LabelColumn = "Label1",
GroupColumn = "GroupId1",
NameColumn = "Workclass",
Kind = Legacy.Models.MacroUtilsTrainerKinds.SignatureRankerTrainer
};
crossValidate.Inputs.Data = textToKey.Data;
crossValidate.Outputs.PredictorModel = modelCombineOutput.PredictorModel;
var crossValidateOutput = experiment.Add(crossValidate);
experiment.Compile();
experiment.SetInput(importInput.InputFile, new SimpleFileHandle(env, dataPath, false, false));
experiment.Run();
var data = experiment.GetOutput(crossValidateOutput.OverallMetrics);
var schema = data.Schema;
var b = schema.TryGetColumnIndex("NDCG", out int metricCol);
Assert.True(b);
b = schema.TryGetColumnIndex("Fold Index", out int foldCol);
Assert.True(b);
using (var cursor = data.GetRowCursor(col => col == metricCol || col == foldCol))
{
var getter = cursor.GetGetter <VBuffer <double> >(metricCol);
var foldGetter = cursor.GetGetter <ReadOnlyMemory <char> >(foldCol);
ReadOnlyMemory <char> fold = default;
// Get the verage.
b = cursor.MoveNext();
Assert.True(b);
var avg = default(VBuffer <double>);
getter(ref avg);
foldGetter(ref fold);
Assert.True(ReadOnlyMemoryUtils.EqualsStr("Average", fold));
// Get the standard deviation.
b = cursor.MoveNext();
Assert.True(b);
var stdev = default(VBuffer <double>);
getter(ref stdev);
foldGetter(ref fold);
Assert.True(ReadOnlyMemoryUtils.EqualsStr("Standard Deviation", fold));
var stdevValues = stdev.GetValues();
Assert.Equal(2.462, stdevValues[0], 3);
Assert.Equal(2.763, stdevValues[1], 3);
Assert.Equal(3.273, stdevValues[2], 3);
var sumBldr = new BufferBuilder <double>(R8Adder.Instance);
sumBldr.Reset(avg.Length, true);
var val = default(VBuffer <double>);
for (int f = 0; f < 2; f++)
{
b = cursor.MoveNext();
Assert.True(b);
getter(ref val);
foldGetter(ref fold);
sumBldr.AddFeatures(0, in val);
Assert.True(ReadOnlyMemoryUtils.EqualsStr("Fold " + f, fold));
}
var sum = default(VBuffer <double>);
sumBldr.GetResult(ref sum);
var avgValues = avg.GetValues();
var sumValues = sum.GetValues();
for (int i = 0; i < avgValues.Length; i++)
{
Assert.Equal(avgValues[i], sumValues[i] / 2);
}
b = cursor.MoveNext();
Assert.False(b);
}
data = experiment.GetOutput(crossValidateOutput.PerInstanceMetrics);
Assert.True(data.Schema.TryGetColumnIndex("Instance", out int nameCol));
using (var cursor = data.GetRowCursor(col => col == nameCol))
{
var getter = cursor.GetGetter <ReadOnlyMemory <char> >(nameCol);
while (cursor.MoveNext())
{
ReadOnlyMemory <char> name = default;
getter(ref name);
Assert.Subset(new HashSet <string>()
{
"Private", "?", "Federal-gov"
}, new HashSet <string>()
{
name.ToString()
});
if (cursor.Position > 4)
{
break;
}
}
}
}
public void TestCrossValidationMacroMultiClassWithWarnings()
{
var dataPath = GetDataPath(@"Train-Tiny-28x28.txt");
var env = new MLContext(42);
var subGraph = env.CreateExperiment();
var nop = new Legacy.Transforms.NoOperation();
var nopOutput = subGraph.Add(nop);
var learnerInput = new Legacy.Trainers.LogisticRegressionClassifier
{
TrainingData = nopOutput.OutputData,
NumThreads = 1
};
var learnerOutput = subGraph.Add(learnerInput);
var experiment = env.CreateExperiment();
var importInput = new Legacy.Data.TextLoader(dataPath);
var importOutput = experiment.Add(importInput);
var filter = new Legacy.Transforms.RowRangeFilter();
filter.Data = importOutput.Data;
filter.Column = "Label";
filter.Min = 0;
filter.Max = 5;
var filterOutput = experiment.Add(filter);
var term = new Legacy.Transforms.TextToKeyConverter();
term.Column = new[]
{
new Legacy.Transforms.ValueToKeyMappingTransformerColumn()
{
Source = "Label", Name = "Strat", Sort = Legacy.Transforms.ValueToKeyMappingTransformerSortOrder.Value
}
};
term.Data = filterOutput.OutputData;
var termOutput = experiment.Add(term);
var crossValidate = new Legacy.Models.CrossValidator
{
Data = termOutput.OutputData,
Nodes = subGraph,
Kind = Legacy.Models.MacroUtilsTrainerKinds.SignatureMultiClassClassifierTrainer,
TransformModel = null,
StratificationColumn = "Strat"
};
crossValidate.Inputs.Data = nop.Data;
crossValidate.Outputs.PredictorModel = learnerOutput.PredictorModel;
var crossValidateOutput = experiment.Add(crossValidate);
experiment.Compile();
importInput.SetInput(env, experiment);
experiment.Run();
var warnings = experiment.GetOutput(crossValidateOutput.Warnings);
var schema = warnings.Schema;
var b = schema.TryGetColumnIndex("WarningText", out int warningCol);
Assert.True(b);
using (var cursor = warnings.GetRowCursor(col => col == warningCol))
{
var getter = cursor.GetGetter <ReadOnlyMemory <char> >(warningCol);
b = cursor.MoveNext();
Assert.True(b);
var warning = default(ReadOnlyMemory <char>);
getter(ref warning);
Assert.Contains("test instances with class values not seen in the training set.", warning.ToString());
b = cursor.MoveNext();
Assert.True(b);
getter(ref warning);
Assert.Contains("Detected columns of variable length: SortedScores, SortedClasses", warning.ToString());
b = cursor.MoveNext();
Assert.False(b);
}
}