private CrossValidationExperimentResult <TMetrics> ExecuteCrossVal(IDataView[] trainDatasets,
ColumnInformation columnInfo,
IDataView[] validationDatasets,
IEstimator <ITransformer> preFeaturizer,
IProgress <CrossValidationRunDetail <TMetrics> > progressHandler)
{
columnInfo = columnInfo ?? new ColumnInformation();
UserInputValidationUtil.ValidateExperimentExecuteArgs(trainDatasets[0], columnInfo, validationDatasets[0], _task);
// Apply pre-featurizer
ITransformer[] preprocessorTransforms = null;
(trainDatasets, validationDatasets, preprocessorTransforms) = ApplyPreFeaturizerCrossVal(trainDatasets, validationDatasets, preFeaturizer);
var runner = new CrossValRunner <TMetrics>(Context, trainDatasets, validationDatasets, MetricsAgent, preFeaturizer,
preprocessorTransforms, columnInfo.GroupIdColumnName, columnInfo.LabelColumnName, _logger);
var columns = DatasetColumnInfoUtil.GetDatasetColumnInfo(Context, trainDatasets[0], columnInfo);
// Execute experiment & get all pipelines run
var experiment = new Experiment <CrossValidationRunDetail <TMetrics>, TMetrics>(Context, _task, OptimizingMetricInfo, progressHandler,
Settings, MetricsAgent, _trainerAllowList, columns, runner, _logger);
var runDetails = experiment.Execute();
var bestRun = GetBestCrossValRun(runDetails);
var experimentResult = new CrossValidationExperimentResult <TMetrics>(runDetails, bestRun);
return(experimentResult);
}
private ExperimentResult <TMetrics> ExecuteTrainValidate(IDataView trainData,
ColumnInformation columnInfo,
IDataView validationData,
IEstimator <ITransformer> preFeaturizer,
IProgress <RunDetail <TMetrics> > progressHandler)
{
columnInfo = columnInfo ?? new ColumnInformation();
UserInputValidationUtil.ValidateExperimentExecuteArgs(trainData, columnInfo, validationData, _task);
// Apply pre-featurizer
ITransformer preprocessorTransform = null;
if (preFeaturizer != null)
{
preprocessorTransform = preFeaturizer.Fit(trainData);
trainData = preprocessorTransform.Transform(trainData);
validationData = preprocessorTransform.Transform(validationData);
}
var runner = new TrainValidateRunner <TMetrics>(Context, trainData, validationData, columnInfo.GroupIdColumnName, columnInfo.LabelColumnName, MetricsAgent,
preFeaturizer, preprocessorTransform, _logger);
var columns = DatasetColumnInfoUtil.GetDatasetColumnInfo(Context, trainData, columnInfo);
return(Execute(columnInfo, columns, preFeaturizer, progressHandler, runner));
}
private ExperimentResult <TMetrics> ExecuteCrossValSummary(IDataView[] trainDatasets,
ColumnInformation columnInfo,
IDataView[] validationDatasets,
IEstimator <ITransformer> preFeaturizer,
IProgress <RunDetail <TMetrics> > progressHandler)
{
columnInfo = columnInfo ?? new ColumnInformation();
UserInputValidationUtil.ValidateExperimentExecuteArgs(trainDatasets[0], columnInfo, validationDatasets[0], _task);
// Apply pre-featurizer
ITransformer[] preprocessorTransforms = null;
(trainDatasets, validationDatasets, preprocessorTransforms) = ApplyPreFeaturizerCrossVal(trainDatasets, validationDatasets, preFeaturizer);
var runner = new CrossValSummaryRunner <TMetrics>(Context, trainDatasets, validationDatasets, MetricsAgent, preFeaturizer,
preprocessorTransforms, columnInfo.GroupIdColumnName, columnInfo.LabelColumnName, OptimizingMetricInfo, _logger);
var columns = DatasetColumnInfoUtil.GetDatasetColumnInfo(Context, trainDatasets[0], columnInfo);
return(Execute(columnInfo, columns, preFeaturizer, progressHandler, runner));
}
