public static InferredPipeline GetNextPipeline(IEnumerable <PipelineRunResult> history, IEnumerable <SuggestedTransform> transforms, IEnumerable <SuggestedTrainer> availableTrainers, bool isMaximizingMetric = true) { // if we haven't run all pipelines once if (history.Count() < availableTrainers.Count()) { return(GetNextFirstStagePipeline(history, availableTrainers, transforms)); } // get next trainer var topTrainers = GetTopTrainers(history, availableTrainers, isMaximizingMetric); var nextTrainerIndex = (history.Count() - availableTrainers.Count()) % topTrainers.Count(); var trainer = topTrainers.ElementAt(nextTrainerIndex).Clone(); // make sure we have not seen pipeline before. // repeat until passes or runs out of chances. var visitedPipelines = new HashSet <InferredPipeline>(history.Select(h => h.Pipeline)); const int maxNumberAttempts = 10; var count = 0; do { SampleHyperparameters(trainer, history, isMaximizingMetric); var pipeline = new InferredPipeline(transforms, trainer); if (!visitedPipelines.Contains(pipeline)) { return(pipeline); } } while (++count <= maxNumberAttempts); return(null); }
// local public static Pipeline GetPipeline(TaskKind task, IDataView data, string label) { var mlContext = new MLContext(); var availableTransforms = TransformInferenceApi.InferTransforms(mlContext, data, label); var availableTrainers = RecipeInference.AllowedTrainers(mlContext, task, 1); var pipeline = new InferredPipeline(availableTransforms, availableTrainers.First(), mlContext); return(pipeline.ToPipeline()); }
public PipelineRunResult(object evaluatedMetrics, ITransformer model, InferredPipeline pipeline, double score, IDataView scoredValidationData, bool runSucceeded = true) { EvaluatedMetrics = evaluatedMetrics; Model = model; Pipeline = pipeline; Score = score; ScoredValidationData = scoredValidationData; RunSucceded = runSucceeded; }
private void ProcessPipeline(InferredPipeline pipeline) { // run pipeline var stopwatch = Stopwatch.StartNew(); PipelineRunResult runResult; try { var pipelineModel = pipeline.TrainTransformer(_trainData); var scoredValidationData = pipelineModel.Transform(_validationData); var evaluatedMetrics = GetEvaluatedMetrics(scoredValidationData); var score = GetPipelineScore(evaluatedMetrics); runResult = new PipelineRunResult(evaluatedMetrics, pipelineModel, pipeline, score, scoredValidationData); } catch (Exception ex) { WriteDebugLog(DebugStream.Exception, $"{pipeline.Trainer} Crashed {ex}"); runResult = new PipelineRunResult(pipeline, false); } // save pipeline run _history.Add(runResult); // debug log pipeline result if (runResult.RunSucceded) { var transformsSb = new StringBuilder(); foreach (var transform in pipeline.Transforms) { transformsSb.Append("xf="); transformsSb.Append(transform); transformsSb.Append(" "); } var commandLineStr = $"{transformsSb.ToString()} tr={pipeline.Trainer}"; WriteDebugLog(DebugStream.RunResult, $"{_history.Count}\t{runResult.Score}\t{stopwatch.Elapsed}\t{commandLineStr}"); } }
public PipelineRunResult(InferredPipeline pipeline, bool runSucceeded) { Pipeline = pipeline; RunSucceded = runSucceeded; }