public void AllowedTrainersWhitelistTest() { var whitelist = new[] { TrainerName.AveragedPerceptronBinary, TrainerName.FastForestBinary }; var trainers = RecipeInference.AllowedTrainers(new MLContext(1), TaskKind.BinaryClassification, new ColumnInformation(), whitelist); Assert.Equal(whitelist.Count(), trainers.Count()); }
public static void Run() { // load data var context = new MLContext(); var columnInference = context.Data.InferColumns(TrainDataPath, Label, true); var textLoader = context.Data.CreateTextReader(columnInference); var data = textLoader.Read(TrainDataPath); // get trainers & transforms var transforms = TransformInferenceApi.InferTransforms(context, data, Label); var availableTrainers = RecipeInference.AllowedTrainers(context, TaskKind.BinaryClassification, 4); // get next pipeline loop var history = new List <PipelineRunResult>(); for (var i = 0; i < 100; i++) { // get next pipeline var pipeline = PipelineSuggester.GetNextPipeline(history, transforms, availableTrainers); if (pipeline == null) { break; } Console.WriteLine($"{i}\t{pipeline}"); // mock pipeline run var pipelineScore = AutoMlUtils.Random.NextDouble(); var result = new PipelineRunResult(null, null, pipeline, pipelineScore, null); history.Add(result); } Console.ReadLine(); }
public void GetNextPipelineMock() { var context = new MLContext(1); var uciAdult = DatasetUtil.GetUciAdultDataView(); var columns = DatasetColumnInfoUtil.GetDatasetColumnInfo(context, uciAdult, new ColumnInformation() { LabelColumnName = DatasetUtil.UciAdultLabel }); // Get next pipeline loop var history = new List <PipelineScore>(); var task = TaskKind.BinaryClassification; var maxIterations = 60; for (var i = 0; i < maxIterations; i++) { // Get next pipeline var pipeline = PipelineSuggester.GetNextPipeline(context, history, columns, task, ((IChannelProvider)context).Start("AutoMLTest")); if (pipeline == null) { break; } var result = new PipelineScore(pipeline, AutoMlUtils.Random.Value.NextDouble(), true); history.Add(result); } Assert.Equal(maxIterations, history.Count); // Get all 'Stage 1' and 'Stage 2' runs from Pipeline Suggester var allAvailableTrainers = RecipeInference.AllowedTrainers(context, task, new ColumnInformation(), null); var stage1Runs = history.Take(allAvailableTrainers.Count()); var stage2Runs = history.Skip(allAvailableTrainers.Count()); // Get the trainer names from top 3 Stage 1 runs var topStage1Runs = stage1Runs.OrderByDescending(r => r.Score).Take(3); var topStage1TrainerNames = topStage1Runs.Select(r => r.Pipeline.Nodes.Last().Name); // Get unique trainer names from Stage 2 runs var stage2TrainerNames = stage2Runs.Select(r => r.Pipeline.Nodes.Last().Name).Distinct(); // Assert that are only 3 unique trainers used in stage 2 Assert.Equal(3, stage2TrainerNames.Count()); // Assert that all trainers in stage 2 were the top trainers from stage 1 Assert.False(topStage1TrainerNames.Except(stage2TrainerNames).Any()); }
public void AllowedTrainersWhitelistNullTest() { var trainers = RecipeInference.AllowedTrainers(new MLContext(1), TaskKind.BinaryClassification, new ColumnInformation(), null); Assert.True(trainers.Any()); }