Esempio n. 1
0
        private ExperimentResult <TMetrics> ExecuteTrainValidate(IDataView trainData,
                                                                 ColumnInformation columnInfo,
                                                                 IDataView validationData,
                                                                 IEstimator <ITransformer> preFeaturizer,
                                                                 IProgress <RunDetail <TMetrics> > progressHandler)
        {
            columnInfo = columnInfo ?? new ColumnInformation();
            UserInputValidationUtil.ValidateExperimentExecuteArgs(trainData, columnInfo, validationData, _task);

            // Apply pre-featurizer
            ITransformer preprocessorTransform = null;

            if (preFeaturizer != null)
            {
                preprocessorTransform = preFeaturizer.Fit(trainData);
                trainData             = preprocessorTransform.Transform(trainData);
                validationData        = preprocessorTransform.Transform(validationData);
            }

            var runner = new TrainValidateRunner <TMetrics>(Context, trainData, validationData, columnInfo.GroupIdColumnName, columnInfo.LabelColumnName, MetricsAgent,
                                                            preFeaturizer, preprocessorTransform, _logger);
            var columns = DatasetColumnInfoUtil.GetDatasetColumnInfo(Context, trainData, columnInfo);

            return(Execute(columnInfo, columns, preFeaturizer, progressHandler, runner));
        }
Esempio n. 2
0
        private CrossValidationExperimentResult <TMetrics> ExecuteCrossVal(IDataView[] trainDatasets,
                                                                           ColumnInformation columnInfo,
                                                                           IDataView[] validationDatasets,
                                                                           IEstimator <ITransformer> preFeaturizer,
                                                                           IProgress <CrossValidationRunDetail <TMetrics> > progressHandler)
        {
            columnInfo = columnInfo ?? new ColumnInformation();
            UserInputValidationUtil.ValidateExperimentExecuteArgs(trainDatasets[0], columnInfo, validationDatasets[0], _task);

            // Apply pre-featurizer
            ITransformer[] preprocessorTransforms = null;
            (trainDatasets, validationDatasets, preprocessorTransforms) = ApplyPreFeaturizerCrossVal(trainDatasets, validationDatasets, preFeaturizer);

            var runner = new CrossValRunner <TMetrics>(Context, trainDatasets, validationDatasets, MetricsAgent, preFeaturizer,
                                                       preprocessorTransforms, columnInfo.GroupIdColumnName, columnInfo.LabelColumnName, _logger);
            var columns = DatasetColumnInfoUtil.GetDatasetColumnInfo(Context, trainDatasets[0], columnInfo);

            // Execute experiment & get all pipelines run
            var experiment = new Experiment <CrossValidationRunDetail <TMetrics>, TMetrics>(Context, _task, OptimizingMetricInfo, progressHandler,
                                                                                            Settings, MetricsAgent, _trainerAllowList, columns, runner, _logger);
            var runDetails = experiment.Execute();

            var bestRun          = GetBestCrossValRun(runDetails);
            var experimentResult = new CrossValidationExperimentResult <TMetrics>(runDetails, bestRun);

            return(experimentResult);
        }
Esempio n. 3
0
        /// <summary>
        /// Executes an AutoML experiment.
        /// </summary>
        /// <param name="trainData">The training data to be used by the AutoML experiment.</param>
        /// <param name="numberOfCVFolds">The number of cross validation folds into which the training data should be divided when fitting a model.</param>
        /// <param name="columnInformation">Column information for the dataset.</param>
        /// <param name="preFeaturizer">Pre-featurizer that AutoML will apply to the data during an
        /// experiment. (The pre-featurizer will be fit only on the training data split to produce a
        /// trained transform. Then, the trained transform will be applied to both the training
        /// data split and corresponding validation data split.)</param>
        /// <param name="progressHandler">A user-defined object that implements
        /// the <see cref="IProgress{T}"/> interface. AutoML will invoke the method
        /// <see cref="IProgress{T}.Report(T)"/> after each model it produces during the
        /// course of the experiment.
        /// </param>
        /// <returns>The cross validation experiment result.</returns>
        /// <remarks>
        /// Depending on the size of your data, the AutoML experiment could take a long time to execute.
        /// </remarks>
        public CrossValidationExperimentResult <TMetrics> Execute(IDataView trainData, uint numberOfCVFolds, ColumnInformation columnInformation = null, IEstimator <ITransformer> preFeaturizer = null, IProgress <CrossValidationRunDetail <TMetrics> > progressHandler = null)
        {
            UserInputValidationUtil.ValidateNumberOfCVFoldsArg(numberOfCVFolds);
            var splitResult = SplitUtil.CrossValSplit(Context, trainData, numberOfCVFolds, columnInformation?.SamplingKeyColumnName);

            return(ExecuteCrossVal(splitResult.trainDatasets, columnInformation, splitResult.validationDatasets, preFeaturizer, progressHandler));
        }
Esempio n. 4
0
 /// <summary>
 /// Infers information about the columns of a dataset in a file located at <paramref name="path"/>.
 /// </summary>
 /// <param name="path">Path to a dataset file.</param>
 /// <param name="columnInformation">Column information for the dataset.</param>
 /// <param name="separatorChar">The character used as separator between data elements in a row. If <see langword="null"/>, AutoML will try to infer this value.</param>
 /// <param name="allowQuoting">Whether the file can contain columns defined by a quoted string. If <see langword="null"/>, AutoML will try to infer this value.</param>
 /// <param name="allowSparse">Whether the file can contain numerical vectors in sparse format. If <see langword="null"/>, AutoML will try to infer this value.</param>
 /// <param name="trimWhitespace">Whether trailing whitespace should be removed from dataset file lines.</param>
 /// <param name="groupColumns">Whether to group together (when possible) original columns in the dataset file into vector columns in the resulting data structures. See <see cref="TextLoader.Range"/> for more information.</param>
 /// <returns>Information inferred about the columns in the provided dataset.</returns>
 /// <remarks>
 /// Infers information about the name, data type, and purpose of each column.
 /// The returned <see cref="ColumnInferenceResults.TextLoaderOptions" /> can be used to
 /// instantiate a <see cref="TextLoader" />. The <see cref="TextLoader" /> can be used to
 /// obtain an <see cref="IDataView"/> that can be fed into an AutoML experiment,
 /// or used elsewhere in the ML.NET ecosystem (ie in <see cref="IEstimator{TTransformer}.Fit(IDataView)"/>.
 /// The <see cref="ColumnInformation"/> contains the inferred purpose of each column in the dataset.
 /// (For instance, is the column categorical, numeric, or text data? Should the column be ignored? Etc.)
 /// The <see cref="ColumnInformation"/> can be inspected and modified (or kept as is) and used by an AutoML experiment.
 /// </remarks>
 public ColumnInferenceResults InferColumns(string path, ColumnInformation columnInformation, char?separatorChar = null, bool?allowQuoting = null,
                                            bool?allowSparse = null, bool trimWhitespace = false, bool groupColumns = true)
 {
     columnInformation = columnInformation ?? new ColumnInformation();
     UserInputValidationUtil.ValidateInferColumnsArgs(path, columnInformation);
     return(ColumnInferenceApi.InferColumns(_context, path, columnInformation, separatorChar, allowQuoting, allowSparse, trimWhitespace, groupColumns));
 }
Esempio n. 5
0
 private string GetSamplingKey(string groupIdColumnName, string samplingKeyColumnName)
 {
     UserInputValidationUtil.ValidateSamplingKey(samplingKeyColumnName, groupIdColumnName, _task);
     if (_task == TaskKind.Ranking)
     {
         return(groupIdColumnName ?? DefaultColumnNames.GroupId);
     }
     return(samplingKeyColumnName);
 }
Esempio n. 6
0
        private ExperimentResult <TMetrics> ExecuteCrossValSummary(IDataView[] trainDatasets,
                                                                   ColumnInformation columnInfo,
                                                                   IDataView[] validationDatasets,
                                                                   IEstimator <ITransformer> preFeaturizer,
                                                                   IProgress <RunDetail <TMetrics> > progressHandler)
        {
            columnInfo = columnInfo ?? new ColumnInformation();
            UserInputValidationUtil.ValidateExperimentExecuteArgs(trainDatasets[0], columnInfo, validationDatasets[0], _task);

            // Apply pre-featurizer
            ITransformer[] preprocessorTransforms = null;
            (trainDatasets, validationDatasets, preprocessorTransforms) = ApplyPreFeaturizerCrossVal(trainDatasets, validationDatasets, preFeaturizer);

            var runner = new CrossValSummaryRunner <TMetrics>(Context, trainDatasets, validationDatasets, MetricsAgent, preFeaturizer,
                                                              preprocessorTransforms, columnInfo.GroupIdColumnName, columnInfo.LabelColumnName, OptimizingMetricInfo, _logger);
            var columns = DatasetColumnInfoUtil.GetDatasetColumnInfo(Context, trainDatasets[0], columnInfo);

            return(Execute(columnInfo, columns, preFeaturizer, progressHandler, runner));
        }
Esempio n. 7
0
 /// <summary>
 /// Infers information about the columns of a dataset in a file located at <paramref name="path"/>.
 /// </summary>
 /// <param name="path">Path to a dataset file.</param>
 /// <param name="labelColumnIndex">Column index of the label column in the dataset.</param>
 /// <param name="hasHeader">Whether or not the dataset file has a header row.</param>
 /// <param name="separatorChar">The character used as separator between data elements in a row. If <see langword="null"/>, AutoML will try to infer this value.</param>
 /// <param name="allowQuoting">Whether the file can contain columns defined by a quoted string. If <see langword="null"/>, AutoML will try to infer this value.</param>
 /// <param name="allowSparse">Whether the file can contain numerical vectors in sparse format. If <see langword="null"/>, AutoML will try to infer this value.</param>
 /// <param name="trimWhitespace">Whether trailing whitespace should be removed from dataset file lines.</param>
 /// <param name="groupColumns">Whether to group together (when possible) original columns in the dataset file into vector columns in the resulting data structures. See <see cref="TextLoader.Range"/> for more information.</param>
 /// <returns>Information inferred about the columns in the provided dataset.</returns>
 /// <remarks>
 /// Infers information about the name, data type, and purpose of each column.
 /// The returned <see cref="ColumnInferenceResults.TextLoaderOptions" /> can be used to
 /// instantiate a <see cref="TextLoader" />. The <see cref="TextLoader" /> can be used to
 /// obtain an <see cref="IDataView"/> that can be fed into an AutoML experiment,
 /// or used elsewhere in the ML.NET ecosystem (ie in <see cref="IEstimator{TTransformer}.Fit(IDataView)"/>.
 /// The <see cref="ColumnInformation"/> contains the inferred purpose of each column in the dataset.
 /// (For instance, is the column categorical, numeric, or text data? Should the column be ignored? Etc.)
 /// The <see cref="ColumnInformation"/> can be inspected and modified (or kept as is) and used by an AutoML experiment.
 /// </remarks>
 public ColumnInferenceResults InferColumns(string path, uint labelColumnIndex, bool hasHeader = false, char?separatorChar = null,
                                            bool?allowQuoting = null, bool?allowSparse = null, bool trimWhitespace = false, bool groupColumns = true)
 {
     UserInputValidationUtil.ValidateInferColumnsArgs(path);
     return(ColumnInferenceApi.InferColumns(_context, path, labelColumnIndex, hasHeader, separatorChar, allowQuoting, allowSparse, trimWhitespace, groupColumns));
 }
Esempio n. 8
0
 /// <summary>
 /// Infers information about the columns of a dataset in a file located at <paramref name="path"/>.
 /// </summary>
 /// <param name="path">Path to a dataset file.</param>
 /// <param name="labelColumnName">The name of the label column.</param>
 /// <param name="separatorChar">The character used as separator between data elements in a row. If <see langword="null"/>, AutoML will try to infer this value.</param>
 /// <param name="allowQuoting">Whether the file can contain columns defined by a quoted string. If <see langword="null"/>, AutoML will try to infer this value.</param>
 /// <param name="allowSparse">Whether the file can contain numerical vectors in sparse format. If <see langword="null"/>, AutoML will try to infer this value.</param>
 /// <param name="trimWhitespace">Whether trailing whitespace should be removed from dataset file lines.</param>
 /// <param name="groupColumns">Whether to group together (when possible) original columns in the dataset file into vector columns in the resulting data structures. See <see cref="TextLoader.Range"/> for more information.</param>
 /// <returns>Information inferred about the columns in the provided dataset.</returns>
 /// <remarks>
 /// Infers information about the name, data type, and purpose of each column.
 /// The returned <see cref="ColumnInferenceResults.TextLoaderOptions" /> can be used to
 /// instantiate a <see cref="TextLoader" />. The <see cref="TextLoader" /> can be used to
 /// obtain an <see cref="IDataView"/> that can be fed into an AutoML experiment,
 /// or used elsewhere in the ML.NET ecosystem (ie in <see cref="IEstimator{TTransformer}.Fit(IDataView)"/>.
 /// The <see cref="ColumnInformation"/> contains the inferred purpose of each column in the dataset.
 /// (For instance, is the column categorical, numeric, or text data? Should the column be ignored? Etc.)
 /// The <see cref="ColumnInformation"/> can be inspected and modified (or kept as is) and used by an AutoML experiment.
 /// </remarks>
 public ColumnInferenceResults InferColumns(string path, string labelColumnName = DefaultColumnNames.Label, char?separatorChar = null, bool?allowQuoting = null,
                                            bool?allowSparse = null, bool trimWhitespace = false, bool groupColumns = true)
 {
     UserInputValidationUtil.ValidateInferColumnsArgs(path, labelColumnName);
     return(ColumnInferenceApi.InferColumns(_context, path, labelColumnName, separatorChar, allowQuoting, allowSparse, trimWhitespace, groupColumns));
 }