private CrossValidationExperimentResult <TMetrics> ExecuteCrossVal(IDataView[] trainDatasets,
ColumnInformation columnInfo,
IDataView[] validationDatasets,
IEstimator <ITransformer> preFeaturizer,
IProgress <CrossValidationRunDetail <TMetrics> > progressHandler)
{
columnInfo = columnInfo ?? new ColumnInformation();
UserInputValidationUtil.ValidateExperimentExecuteArgs(trainDatasets[0], columnInfo, validationDatasets[0], _task);
// Apply pre-featurizer
ITransformer[] preprocessorTransforms = null;
(trainDatasets, validationDatasets, preprocessorTransforms) = ApplyPreFeaturizerCrossVal(trainDatasets, validationDatasets, preFeaturizer);
var runner = new CrossValRunner <TMetrics>(Context, trainDatasets, validationDatasets, MetricsAgent, preFeaturizer,
preprocessorTransforms, columnInfo.GroupIdColumnName, columnInfo.LabelColumnName, _logger);
var columns = DatasetColumnInfoUtil.GetDatasetColumnInfo(Context, trainDatasets[0], columnInfo);
// Execute experiment & get all pipelines run
var experiment = new Experiment <CrossValidationRunDetail <TMetrics>, TMetrics>(Context, _task, OptimizingMetricInfo, progressHandler,
Settings, MetricsAgent, _trainerAllowList, columns, runner, _logger);
var runDetails = experiment.Execute();
var bestRun = GetBestCrossValRun(runDetails);
var experimentResult = new CrossValidationExperimentResult <TMetrics>(runDetails, bestRun);
return(experimentResult);
}
public override CrossValidationExperimentResult <BinaryClassificationMetrics> Execute(IDataView trainData, uint numberOfCVFolds, ColumnInformation columnInformation = null, IEstimator <ITransformer> preFeaturizer = null, IProgress <CrossValidationRunDetail <BinaryClassificationMetrics> > progressHandler = null)
{
var label = columnInformation.LabelColumnName;
_experiment.SetEvaluateMetric(Settings.OptimizingMetric, label);
_experiment.SetTrainingTimeInSeconds(Settings.MaxExperimentTimeInSeconds);
_experiment.SetDataset(trainData, (int)numberOfCVFolds);
MultiModelPipeline pipeline = new MultiModelPipeline();
if (preFeaturizer != null)
{
pipeline = pipeline.Append(preFeaturizer);
}
pipeline = pipeline.Append(Context.Auto().Featurizer(trainData, columnInformation, "__Features__"))
.Append(Context.Auto().BinaryClassification(label, featureColumnName: Features));
_experiment.SetPipeline(pipeline);
var monitor = new BinaryClassificationTrialResultMonitor();
monitor.OnTrialCompleted += (o, e) =>
{
var runDetails = ToCrossValidationRunDetail(e);
progressHandler?.Report(runDetails);
};
_experiment.SetMonitor(monitor);
_experiment.Run();
var runDetails = monitor.RunDetails.Select(e => ToCrossValidationRunDetail(e));
var bestResult = ToCrossValidationRunDetail(monitor.BestRun);
var result = new CrossValidationExperimentResult <BinaryClassificationMetrics>(runDetails, bestResult);
return(result);
}