private ExperimentResult <TMetrics> ExecuteTrainValidate(IDataView trainData,
ColumnInformation columnInfo,
IDataView validationData,
IEstimator <ITransformer> preFeaturizer,
IProgress <RunDetail <TMetrics> > progressHandler)
{
columnInfo = columnInfo ?? new ColumnInformation();
UserInputValidationUtil.ValidateExperimentExecuteArgs(trainData, columnInfo, validationData, _task);
// Apply pre-featurizer
ITransformer preprocessorTransform = null;
if (preFeaturizer != null)
{
preprocessorTransform = preFeaturizer.Fit(trainData);
trainData = preprocessorTransform.Transform(trainData);
validationData = preprocessorTransform.Transform(validationData);
}
var runner = new TrainValidateRunner <TMetrics>(Context, trainData, validationData, columnInfo.GroupIdColumnName, columnInfo.LabelColumnName, MetricsAgent,
preFeaturizer, preprocessorTransform, _logger);
var columns = DatasetColumnInfoUtil.GetDatasetColumnInfo(Context, trainData, columnInfo);
return(Execute(columnInfo, columns, preFeaturizer, progressHandler, runner));
}
private CrossValidationExperimentResult <TMetrics> ExecuteCrossVal(IDataView[] trainDatasets,
ColumnInformation columnInfo,
IDataView[] validationDatasets,
IEstimator <ITransformer> preFeaturizer,
IProgress <CrossValidationRunDetail <TMetrics> > progressHandler)
{
columnInfo = columnInfo ?? new ColumnInformation();
UserInputValidationUtil.ValidateExperimentExecuteArgs(trainDatasets[0], columnInfo, validationDatasets[0], _task);
// Apply pre-featurizer
ITransformer[] preprocessorTransforms = null;
(trainDatasets, validationDatasets, preprocessorTransforms) = ApplyPreFeaturizerCrossVal(trainDatasets, validationDatasets, preFeaturizer);
var runner = new CrossValRunner <TMetrics>(Context, trainDatasets, validationDatasets, MetricsAgent, preFeaturizer,
preprocessorTransforms, columnInfo.GroupIdColumnName, columnInfo.LabelColumnName, _logger);
var columns = DatasetColumnInfoUtil.GetDatasetColumnInfo(Context, trainDatasets[0], columnInfo);
// Execute experiment & get all pipelines run
var experiment = new Experiment <CrossValidationRunDetail <TMetrics>, TMetrics>(Context, _task, OptimizingMetricInfo, progressHandler,
Settings, MetricsAgent, _trainerAllowList, columns, runner, _logger);
var runDetails = experiment.Execute();
var bestRun = GetBestCrossValRun(runDetails);
var experimentResult = new CrossValidationExperimentResult <TMetrics>(runDetails, bestRun);
return(experimentResult);
}
/// <summary>
/// Executes an AutoML experiment.
/// </summary>
/// <param name="trainData">The training data to be used by the AutoML experiment.</param>
/// <param name="numberOfCVFolds">The number of cross validation folds into which the training data should be divided when fitting a model.</param>
/// <param name="columnInformation">Column information for the dataset.</param>
/// <param name="preFeaturizer">Pre-featurizer that AutoML will apply to the data during an
/// experiment. (The pre-featurizer will be fit only on the training data split to produce a
/// trained transform. Then, the trained transform will be applied to both the training
/// data split and corresponding validation data split.)</param>
/// <param name="progressHandler">A user-defined object that implements
/// the <see cref="IProgress{T}"/> interface. AutoML will invoke the method
/// <see cref="IProgress{T}.Report(T)"/> after each model it produces during the
/// course of the experiment.
/// </param>
/// <returns>The cross validation experiment result.</returns>
/// <remarks>
/// Depending on the size of your data, the AutoML experiment could take a long time to execute.
/// </remarks>
public CrossValidationExperimentResult <TMetrics> Execute(IDataView trainData, uint numberOfCVFolds, ColumnInformation columnInformation = null, IEstimator <ITransformer> preFeaturizer = null, IProgress <CrossValidationRunDetail <TMetrics> > progressHandler = null)
{
UserInputValidationUtil.ValidateNumberOfCVFoldsArg(numberOfCVFolds);
var splitResult = SplitUtil.CrossValSplit(Context, trainData, numberOfCVFolds, columnInformation?.SamplingKeyColumnName);
return(ExecuteCrossVal(splitResult.trainDatasets, columnInformation, splitResult.validationDatasets, preFeaturizer, progressHandler));
}
/// <summary>
/// Infers information about the columns of a dataset in a file located at <paramref name="path"/>.
/// </summary>
/// <param name="path">Path to a dataset file.</param>
/// <param name="columnInformation">Column information for the dataset.</param>
/// <param name="separatorChar">The character used as separator between data elements in a row. If <see langword="null"/>, AutoML will try to infer this value.</param>
/// <param name="allowQuoting">Whether the file can contain columns defined by a quoted string. If <see langword="null"/>, AutoML will try to infer this value.</param>
/// <param name="allowSparse">Whether the file can contain numerical vectors in sparse format. If <see langword="null"/>, AutoML will try to infer this value.</param>
/// <param name="trimWhitespace">Whether trailing whitespace should be removed from dataset file lines.</param>
/// <param name="groupColumns">Whether to group together (when possible) original columns in the dataset file into vector columns in the resulting data structures. See <see cref="TextLoader.Range"/> for more information.</param>
/// <returns>Information inferred about the columns in the provided dataset.</returns>
/// <remarks>
/// Infers information about the name, data type, and purpose of each column.
/// The returned <see cref="ColumnInferenceResults.TextLoaderOptions" /> can be used to
/// instantiate a <see cref="TextLoader" />. The <see cref="TextLoader" /> can be used to
/// obtain an <see cref="IDataView"/> that can be fed into an AutoML experiment,
/// or used elsewhere in the ML.NET ecosystem (ie in <see cref="IEstimator{TTransformer}.Fit(IDataView)"/>.
/// The <see cref="ColumnInformation"/> contains the inferred purpose of each column in the dataset.
/// (For instance, is the column categorical, numeric, or text data? Should the column be ignored? Etc.)
/// The <see cref="ColumnInformation"/> can be inspected and modified (or kept as is) and used by an AutoML experiment.
/// </remarks>
public ColumnInferenceResults InferColumns(string path, ColumnInformation columnInformation, char?separatorChar = null, bool?allowQuoting = null,
bool?allowSparse = null, bool trimWhitespace = false, bool groupColumns = true)
{
columnInformation = columnInformation ?? new ColumnInformation();
UserInputValidationUtil.ValidateInferColumnsArgs(path, columnInformation);
return(ColumnInferenceApi.InferColumns(_context, path, columnInformation, separatorChar, allowQuoting, allowSparse, trimWhitespace, groupColumns));
}
private string GetSamplingKey(string groupIdColumnName, string samplingKeyColumnName)
{
UserInputValidationUtil.ValidateSamplingKey(samplingKeyColumnName, groupIdColumnName, _task);
if (_task == TaskKind.Ranking)
{
return(groupIdColumnName ?? DefaultColumnNames.GroupId);
}
return(samplingKeyColumnName);
}
private ExperimentResult <TMetrics> ExecuteCrossValSummary(IDataView[] trainDatasets,
ColumnInformation columnInfo,
IDataView[] validationDatasets,
IEstimator <ITransformer> preFeaturizer,
IProgress <RunDetail <TMetrics> > progressHandler)
{
columnInfo = columnInfo ?? new ColumnInformation();
UserInputValidationUtil.ValidateExperimentExecuteArgs(trainDatasets[0], columnInfo, validationDatasets[0], _task);
// Apply pre-featurizer
ITransformer[] preprocessorTransforms = null;
(trainDatasets, validationDatasets, preprocessorTransforms) = ApplyPreFeaturizerCrossVal(trainDatasets, validationDatasets, preFeaturizer);
var runner = new CrossValSummaryRunner <TMetrics>(Context, trainDatasets, validationDatasets, MetricsAgent, preFeaturizer,
preprocessorTransforms, columnInfo.GroupIdColumnName, columnInfo.LabelColumnName, OptimizingMetricInfo, _logger);
var columns = DatasetColumnInfoUtil.GetDatasetColumnInfo(Context, trainDatasets[0], columnInfo);
return(Execute(columnInfo, columns, preFeaturizer, progressHandler, runner));
}
/// <summary>
/// Infers information about the columns of a dataset in a file located at <paramref name="path"/>.
/// </summary>
/// <param name="path">Path to a dataset file.</param>
/// <param name="labelColumnIndex">Column index of the label column in the dataset.</param>
/// <param name="hasHeader">Whether or not the dataset file has a header row.</param>
/// <param name="separatorChar">The character used as separator between data elements in a row. If <see langword="null"/>, AutoML will try to infer this value.</param>
/// <param name="allowQuoting">Whether the file can contain columns defined by a quoted string. If <see langword="null"/>, AutoML will try to infer this value.</param>
/// <param name="allowSparse">Whether the file can contain numerical vectors in sparse format. If <see langword="null"/>, AutoML will try to infer this value.</param>
/// <param name="trimWhitespace">Whether trailing whitespace should be removed from dataset file lines.</param>
/// <param name="groupColumns">Whether to group together (when possible) original columns in the dataset file into vector columns in the resulting data structures. See <see cref="TextLoader.Range"/> for more information.</param>
/// <returns>Information inferred about the columns in the provided dataset.</returns>
/// <remarks>
/// Infers information about the name, data type, and purpose of each column.
/// The returned <see cref="ColumnInferenceResults.TextLoaderOptions" /> can be used to
/// instantiate a <see cref="TextLoader" />. The <see cref="TextLoader" /> can be used to
/// obtain an <see cref="IDataView"/> that can be fed into an AutoML experiment,
/// or used elsewhere in the ML.NET ecosystem (ie in <see cref="IEstimator{TTransformer}.Fit(IDataView)"/>.
/// The <see cref="ColumnInformation"/> contains the inferred purpose of each column in the dataset.
/// (For instance, is the column categorical, numeric, or text data? Should the column be ignored? Etc.)
/// The <see cref="ColumnInformation"/> can be inspected and modified (or kept as is) and used by an AutoML experiment.
/// </remarks>
public ColumnInferenceResults InferColumns(string path, uint labelColumnIndex, bool hasHeader = false, char?separatorChar = null,
bool?allowQuoting = null, bool?allowSparse = null, bool trimWhitespace = false, bool groupColumns = true)
{
UserInputValidationUtil.ValidateInferColumnsArgs(path);
return(ColumnInferenceApi.InferColumns(_context, path, labelColumnIndex, hasHeader, separatorChar, allowQuoting, allowSparse, trimWhitespace, groupColumns));
}
/// <summary>
/// Infers information about the columns of a dataset in a file located at <paramref name="path"/>.
/// </summary>
/// <param name="path">Path to a dataset file.</param>
/// <param name="labelColumnName">The name of the label column.</param>
/// <param name="separatorChar">The character used as separator between data elements in a row. If <see langword="null"/>, AutoML will try to infer this value.</param>
/// <param name="allowQuoting">Whether the file can contain columns defined by a quoted string. If <see langword="null"/>, AutoML will try to infer this value.</param>
/// <param name="allowSparse">Whether the file can contain numerical vectors in sparse format. If <see langword="null"/>, AutoML will try to infer this value.</param>
/// <param name="trimWhitespace">Whether trailing whitespace should be removed from dataset file lines.</param>
/// <param name="groupColumns">Whether to group together (when possible) original columns in the dataset file into vector columns in the resulting data structures. See <see cref="TextLoader.Range"/> for more information.</param>
/// <returns>Information inferred about the columns in the provided dataset.</returns>
/// <remarks>
/// Infers information about the name, data type, and purpose of each column.
/// The returned <see cref="ColumnInferenceResults.TextLoaderOptions" /> can be used to
/// instantiate a <see cref="TextLoader" />. The <see cref="TextLoader" /> can be used to
/// obtain an <see cref="IDataView"/> that can be fed into an AutoML experiment,
/// or used elsewhere in the ML.NET ecosystem (ie in <see cref="IEstimator{TTransformer}.Fit(IDataView)"/>.
/// The <see cref="ColumnInformation"/> contains the inferred purpose of each column in the dataset.
/// (For instance, is the column categorical, numeric, or text data? Should the column be ignored? Etc.)
/// The <see cref="ColumnInformation"/> can be inspected and modified (or kept as is) and used by an AutoML experiment.
/// </remarks>
public ColumnInferenceResults InferColumns(string path, string labelColumnName = DefaultColumnNames.Label, char?separatorChar = null, bool?allowQuoting = null,
bool?allowSparse = null, bool trimWhitespace = false, bool groupColumns = true)
{
UserInputValidationUtil.ValidateInferColumnsArgs(path, labelColumnName);
return(ColumnInferenceApi.InferColumns(_context, path, labelColumnName, separatorChar, allowQuoting, allowSparse, trimWhitespace, groupColumns));
}
