public void GetNextPipeline()
        {
            var context  = new MLContext(1);
            var uciAdult = DatasetUtil.GetUciAdultDataView();
            var columns  = DatasetColumnInfoUtil.GetDatasetColumnInfo(context, uciAdult, new ColumnInformation()
            {
                LabelColumnName = DatasetUtil.UciAdultLabel
            });

            // get next pipeline
            var pipeline = PipelineSuggester.GetNextPipeline(context, new List <PipelineScore>(), columns,
                                                             TaskKind.BinaryClassification, ((IChannelProvider)context).Start("AutoMLTest"));

            // serialize & deserialize pipeline
            var serialized = JsonConvert.SerializeObject(pipeline);

            Console.WriteLine(serialized);
            var deserialized = JsonConvert.DeserializeObject <Pipeline>(serialized);

            // run pipeline
            var estimator  = deserialized.ToEstimator(context);
            var scoredData = estimator.Fit(uciAdult).Transform(uciAdult);
            var score      = context.BinaryClassification.EvaluateNonCalibrated(scoredData).Accuracy;
            var result     = new PipelineScore(deserialized, score, true);

            Assert.NotNull(result);
        }
示例#2
0
        public static void Run()
        {
            // load data
            var context         = new MLContext();
            var columnInference = context.Data.InferColumns(TrainDataPath, Label, true);
            var textLoader      = context.Data.CreateTextReader(columnInference);
            var data            = textLoader.Read(TrainDataPath);

            // get trainers & transforms
            var transforms        = TransformInferenceApi.InferTransforms(context, data, Label);
            var availableTrainers = RecipeInference.AllowedTrainers(context, TaskKind.BinaryClassification, 4);

            // get next pipeline loop
            var history = new List <PipelineRunResult>();

            for (var i = 0; i < 100; i++)
            {
                // get next pipeline
                var pipeline = PipelineSuggester.GetNextPipeline(history, transforms, availableTrainers);
                if (pipeline == null)
                {
                    break;
                }
                Console.WriteLine($"{i}\t{pipeline}");

                // mock pipeline run
                var pipelineScore = AutoMlUtils.Random.NextDouble();
                var result        = new PipelineRunResult(null, null, pipeline, pipelineScore, null);

                history.Add(result);
            }

            Console.ReadLine();
        }
        public void GetNextPipelineMock()
        {
            var context  = new MLContext(1);
            var uciAdult = DatasetUtil.GetUciAdultDataView();
            var columns  = DatasetColumnInfoUtil.GetDatasetColumnInfo(context, uciAdult, new ColumnInformation()
            {
                LabelColumnName = DatasetUtil.UciAdultLabel
            });

            // Get next pipeline loop
            var history       = new List <PipelineScore>();
            var task          = TaskKind.BinaryClassification;
            var maxIterations = 60;

            for (var i = 0; i < maxIterations; i++)
            {
                // Get next pipeline
                var pipeline = PipelineSuggester.GetNextPipeline(context, history, columns, task, ((IChannelProvider)context).Start("AutoMLTest"));
                if (pipeline == null)
                {
                    break;
                }

                var result = new PipelineScore(pipeline, AutoMlUtils.Random.Value.NextDouble(), true);
                history.Add(result);
            }

            Assert.Equal(maxIterations, history.Count);

            // Get all 'Stage 1' and 'Stage 2' runs from Pipeline Suggester
            var allAvailableTrainers = RecipeInference.AllowedTrainers(context, task, new ColumnInformation(), null);
            var stage1Runs           = history.Take(allAvailableTrainers.Count());
            var stage2Runs           = history.Skip(allAvailableTrainers.Count());

            // Get the trainer names from top 3 Stage 1 runs
            var topStage1Runs         = stage1Runs.OrderByDescending(r => r.Score).Take(3);
            var topStage1TrainerNames = topStage1Runs.Select(r => r.Pipeline.Nodes.Last().Name);

            // Get unique trainer names from Stage 2 runs
            var stage2TrainerNames = stage2Runs.Select(r => r.Pipeline.Nodes.Last().Name).Distinct();

            // Assert that are only 3 unique trainers used in stage 2
            Assert.Equal(3, stage2TrainerNames.Count());
            // Assert that all trainers in stage 2 were the top trainers from stage 1
            Assert.False(topStage1TrainerNames.Except(stage2TrainerNames).Any());
        }