public void AllowedTrainersWhitelistTest()
{
var whitelist = new[] { TrainerName.AveragedPerceptronBinary, TrainerName.FastForestBinary };
var trainers = RecipeInference.AllowedTrainers(new MLContext(1), TaskKind.BinaryClassification, new ColumnInformation(), whitelist);
Assert.Equal(whitelist.Count(), trainers.Count());
}
public void TestPipelineNodeCloning()
{
using (var env = new TlcEnvironment())
{
var lr1 = RecipeInference
.AllowedLearners(env, MacroUtils.TrainerKinds.SignatureBinaryClassifierTrainer)
.First(learner => learner.PipelineNode != null && learner.LearnerName.Contains("LogisticRegression"));
var sdca1 = RecipeInference
.AllowedLearners(env, MacroUtils.TrainerKinds.SignatureBinaryClassifierTrainer)
.First(learner => learner.PipelineNode != null && learner.LearnerName.Contains("StochasticDualCoordinateAscent"));
// Clone and change hyperparam values
var lr2 = lr1.Clone();
lr1.PipelineNode.SweepParams[0].RawValue = 1.2f;
lr2.PipelineNode.SweepParams[0].RawValue = 3.5f;
var sdca2 = sdca1.Clone();
sdca1.PipelineNode.SweepParams[0].RawValue = 3;
sdca2.PipelineNode.SweepParams[0].RawValue = 0;
// Make sure the changes are propagated to entry point objects
env.Check(lr1.PipelineNode.UpdateProperties());
env.Check(lr2.PipelineNode.UpdateProperties());
env.Check(sdca1.PipelineNode.UpdateProperties());
env.Check(sdca2.PipelineNode.UpdateProperties());
env.Check(lr1.PipelineNode.CheckEntryPointStateMatchesParamValues());
env.Check(lr2.PipelineNode.CheckEntryPointStateMatchesParamValues());
env.Check(sdca1.PipelineNode.CheckEntryPointStateMatchesParamValues());
env.Check(sdca2.PipelineNode.CheckEntryPointStateMatchesParamValues());
// Make sure second object's set of changes didn't overwrite first object's
env.Check(!lr1.PipelineNode.SweepParams[0].RawValue.Equals(lr2.PipelineNode.SweepParams[0].RawValue));
env.Check(!sdca2.PipelineNode.SweepParams[0].RawValue.Equals(sdca1.PipelineNode.SweepParams[0].RawValue));
}
}
public static void Run()
{
// load data
var context = new MLContext();
var columnInference = context.Data.InferColumns(TrainDataPath, Label, true);
var textLoader = context.Data.CreateTextReader(columnInference);
var data = textLoader.Read(TrainDataPath);
// get trainers & transforms
var transforms = TransformInferenceApi.InferTransforms(context, data, Label);
var availableTrainers = RecipeInference.AllowedTrainers(context, TaskKind.BinaryClassification, 4);
// get next pipeline loop
var history = new List <PipelineRunResult>();
for (var i = 0; i < 100; i++)
{
// get next pipeline
var pipeline = PipelineSuggester.GetNextPipeline(history, transforms, availableTrainers);
if (pipeline == null)
{
break;
}
Console.WriteLine($"{i}\t{pipeline}");
// mock pipeline run
var pipelineScore = AutoMlUtils.Random.NextDouble();
var result = new PipelineRunResult(null, null, pipeline, pipelineScore, null);
history.Add(result);
}
Console.ReadLine();
}
public void GetNextPipelineMock()
{
var context = new MLContext(1);
var uciAdult = DatasetUtil.GetUciAdultDataView();
var columns = DatasetColumnInfoUtil.GetDatasetColumnInfo(context, uciAdult, new ColumnInformation()
{
LabelColumnName = DatasetUtil.UciAdultLabel
});
// Get next pipeline loop
var history = new List <PipelineScore>();
var task = TaskKind.BinaryClassification;
var maxIterations = 60;
for (var i = 0; i < maxIterations; i++)
{
// Get next pipeline
var pipeline = PipelineSuggester.GetNextPipeline(context, history, columns, task, ((IChannelProvider)context).Start("AutoMLTest"));
if (pipeline == null)
{
break;
}
var result = new PipelineScore(pipeline, AutoMlUtils.Random.Value.NextDouble(), true);
history.Add(result);
}
Assert.Equal(maxIterations, history.Count);
// Get all 'Stage 1' and 'Stage 2' runs from Pipeline Suggester
var allAvailableTrainers = RecipeInference.AllowedTrainers(context, task, new ColumnInformation(), null);
var stage1Runs = history.Take(allAvailableTrainers.Count());
var stage2Runs = history.Skip(allAvailableTrainers.Count());
// Get the trainer names from top 3 Stage 1 runs
var topStage1Runs = stage1Runs.OrderByDescending(r => r.Score).Take(3);
var topStage1TrainerNames = topStage1Runs.Select(r => r.Pipeline.Nodes.Last().Name);
// Get unique trainer names from Stage 2 runs
var stage2TrainerNames = stage2Runs.Select(r => r.Pipeline.Nodes.Last().Name).Distinct();
// Assert that are only 3 unique trainers used in stage 2
Assert.Equal(3, stage2TrainerNames.Count());
// Assert that all trainers in stage 2 were the top trainers from stage 1
Assert.False(topStage1TrainerNames.Except(stage2TrainerNames).Any());
}
private void RunCore(IChannel ch)
{
_host.AssertValue(ch);
Type predictorType;
string settingsString;
TransformInference.InferenceResult inferenceResult;
RecipeInference.SuggestedRecipe[] recipes = RecipeInference.InferRecipesFromData(_host, _dataFile, _schemaDefinitionFile, out predictorType, out settingsString, out inferenceResult);
if (!string.IsNullOrEmpty(_rspOutFile))
{
using (var sw = new StreamWriter(_rspOutFile))
PrintRecipe(ch, recipes, settingsString, sw);
}
else
{
PrintRecipe(ch, recipes, settingsString);
}
}
public void AllowedTrainersWhitelistNullTest()
{
var trainers = RecipeInference.AllowedTrainers(new MLContext(1), TaskKind.BinaryClassification, new ColumnInformation(), null);
Assert.True(trainers.Any());
}