public void GetNextPipeline()
{
var context = new MLContext(1);
var uciAdult = DatasetUtil.GetUciAdultDataView();
var columns = DatasetColumnInfoUtil.GetDatasetColumnInfo(context, uciAdult, new ColumnInformation()
{
LabelColumnName = DatasetUtil.UciAdultLabel
});
// get next pipeline
var pipeline = PipelineSuggester.GetNextPipeline(context, new List <PipelineScore>(), columns,
TaskKind.BinaryClassification, ((IChannelProvider)context).Start("AutoMLTest"));
// serialize & deserialize pipeline
var serialized = JsonConvert.SerializeObject(pipeline);
Console.WriteLine(serialized);
var deserialized = JsonConvert.DeserializeObject <Pipeline>(serialized);
// run pipeline
var estimator = deserialized.ToEstimator(context);
var scoredData = estimator.Fit(uciAdult).Transform(uciAdult);
var score = context.BinaryClassification.EvaluateNonCalibrated(scoredData).Accuracy;
var result = new PipelineScore(deserialized, score, true);
Assert.NotNull(result);
}
public static void Run()
{
// load data
var context = new MLContext();
var columnInference = context.Data.InferColumns(TrainDataPath, Label, true);
var textLoader = context.Data.CreateTextReader(columnInference);
var data = textLoader.Read(TrainDataPath);
// get trainers & transforms
var transforms = TransformInferenceApi.InferTransforms(context, data, Label);
var availableTrainers = RecipeInference.AllowedTrainers(context, TaskKind.BinaryClassification, 4);
// get next pipeline loop
var history = new List <PipelineRunResult>();
for (var i = 0; i < 100; i++)
{
// get next pipeline
var pipeline = PipelineSuggester.GetNextPipeline(history, transforms, availableTrainers);
if (pipeline == null)
{
break;
}
Console.WriteLine($"{i}\t{pipeline}");
// mock pipeline run
var pipelineScore = AutoMlUtils.Random.NextDouble();
var result = new PipelineRunResult(null, null, pipeline, pipelineScore, null);
history.Add(result);
}
Console.ReadLine();
}
public void GetNextPipelineMock()
{
var context = new MLContext(1);
var uciAdult = DatasetUtil.GetUciAdultDataView();
var columns = DatasetColumnInfoUtil.GetDatasetColumnInfo(context, uciAdult, new ColumnInformation()
{
LabelColumnName = DatasetUtil.UciAdultLabel
});
// Get next pipeline loop
var history = new List <PipelineScore>();
var task = TaskKind.BinaryClassification;
var maxIterations = 60;
for (var i = 0; i < maxIterations; i++)
{
// Get next pipeline
var pipeline = PipelineSuggester.GetNextPipeline(context, history, columns, task, ((IChannelProvider)context).Start("AutoMLTest"));
if (pipeline == null)
{
break;
}
var result = new PipelineScore(pipeline, AutoMlUtils.Random.Value.NextDouble(), true);
history.Add(result);
}
Assert.Equal(maxIterations, history.Count);
// Get all 'Stage 1' and 'Stage 2' runs from Pipeline Suggester
var allAvailableTrainers = RecipeInference.AllowedTrainers(context, task, new ColumnInformation(), null);
var stage1Runs = history.Take(allAvailableTrainers.Count());
var stage2Runs = history.Skip(allAvailableTrainers.Count());
// Get the trainer names from top 3 Stage 1 runs
var topStage1Runs = stage1Runs.OrderByDescending(r => r.Score).Take(3);
var topStage1TrainerNames = topStage1Runs.Select(r => r.Pipeline.Nodes.Last().Name);
// Get unique trainer names from Stage 2 runs
var stage2TrainerNames = stage2Runs.Select(r => r.Pipeline.Nodes.Last().Name).Distinct();
// Assert that are only 3 unique trainers used in stage 2
Assert.Equal(3, stage2TrainerNames.Count());
// Assert that all trainers in stage 2 were the top trainers from stage 1
Assert.False(topStage1TrainerNames.Except(stage2TrainerNames).Any());
}
