public static InferredPipeline GetNextPipeline(IEnumerable <PipelineRunResult> history,
IEnumerable <SuggestedTransform> transforms,
IEnumerable <SuggestedTrainer> availableTrainers,
bool isMaximizingMetric = true)
{
// if we haven't run all pipelines once
if (history.Count() < availableTrainers.Count())
{
return(GetNextFirstStagePipeline(history, availableTrainers, transforms));
}
// get next trainer
var topTrainers = GetTopTrainers(history, availableTrainers, isMaximizingMetric);
var nextTrainerIndex = (history.Count() - availableTrainers.Count()) % topTrainers.Count();
var trainer = topTrainers.ElementAt(nextTrainerIndex).Clone();
// make sure we have not seen pipeline before.
// repeat until passes or runs out of chances.
var visitedPipelines = new HashSet <InferredPipeline>(history.Select(h => h.Pipeline));
const int maxNumberAttempts = 10;
var count = 0;
do
{
SampleHyperparameters(trainer, history, isMaximizingMetric);
var pipeline = new InferredPipeline(transforms, trainer);
if (!visitedPipelines.Contains(pipeline))
{
return(pipeline);
}
} while (++count <= maxNumberAttempts);
return(null);
}
// local
public static Pipeline GetPipeline(TaskKind task, IDataView data, string label)
{
var mlContext = new MLContext();
var availableTransforms = TransformInferenceApi.InferTransforms(mlContext, data, label);
var availableTrainers = RecipeInference.AllowedTrainers(mlContext, task, 1);
var pipeline = new InferredPipeline(availableTransforms, availableTrainers.First(), mlContext);
return(pipeline.ToPipeline());
}
public PipelineRunResult(object evaluatedMetrics, ITransformer model, InferredPipeline pipeline, double score, IDataView scoredValidationData,
bool runSucceeded = true)
{
EvaluatedMetrics = evaluatedMetrics;
Model = model;
Pipeline = pipeline;
Score = score;
ScoredValidationData = scoredValidationData;
RunSucceded = runSucceeded;
}
private void ProcessPipeline(InferredPipeline pipeline)
{
// run pipeline
var stopwatch = Stopwatch.StartNew();
PipelineRunResult runResult;
try
{
var pipelineModel = pipeline.TrainTransformer(_trainData);
var scoredValidationData = pipelineModel.Transform(_validationData);
var evaluatedMetrics = GetEvaluatedMetrics(scoredValidationData);
var score = GetPipelineScore(evaluatedMetrics);
runResult = new PipelineRunResult(evaluatedMetrics, pipelineModel, pipeline, score, scoredValidationData);
}
catch (Exception ex)
{
WriteDebugLog(DebugStream.Exception, $"{pipeline.Trainer} Crashed {ex}");
runResult = new PipelineRunResult(pipeline, false);
}
// save pipeline run
_history.Add(runResult);
// debug log pipeline result
if (runResult.RunSucceded)
{
var transformsSb = new StringBuilder();
foreach (var transform in pipeline.Transforms)
{
transformsSb.Append("xf=");
transformsSb.Append(transform);
transformsSb.Append(" ");
}
var commandLineStr = $"{transformsSb.ToString()} tr={pipeline.Trainer}";
WriteDebugLog(DebugStream.RunResult, $"{_history.Count}\t{runResult.Score}\t{stopwatch.Elapsed}\t{commandLineStr}");
}
}
public PipelineRunResult(InferredPipeline pipeline, bool runSucceeded)
{
Pipeline = pipeline;
RunSucceded = runSucceeded;
}
