/// <summary> /// Select the best run from an enumeration of experiment runs. /// </summary> /// <param name="results">Enumeration of AutoML experiment run results.</param> /// <param name="metric">Metric to consider when selecting the best run.</param> /// <param name="groupIdColumnName">Name for the GroupId column.</param> /// <returns>The best experiment run.</returns> public static RunDetail <RankingMetrics> Best(this IEnumerable <RunDetail <RankingMetrics> > results, RankingMetric metric = RankingMetric.Ndcg, string groupIdColumnName = "GroupId") { var metricsAgent = new RankingMetricsAgent(null, metric, groupIdColumnName); var isMetricMaximizing = new OptimizingMetricInfo(metric).IsMaximizing; return(BestResultUtil.GetBestRun(results, metricsAgent, isMetricMaximizing)); }
/// <summary> /// Select the best run from an enumeration of experiment runs. /// </summary> /// <param name="results">Enumeration of AutoML experiment run results.</param> /// <param name="metric">Metric to consider when selecting the best run.</param> /// <param name="optimizationMetricTruncationLevel">Maximum truncation level for computing (N)DCG. Defaults to 10.</param> /// <returns>The best experiment run.</returns> public static RunDetail <RankingMetrics> Best(this IEnumerable <RunDetail <RankingMetrics> > results, RankingMetric metric = RankingMetric.Ndcg, uint optimizationMetricTruncationLevel = 10) { var metricsAgent = new RankingMetricsAgent(null, metric, optimizationMetricTruncationLevel); var isMetricMaximizing = new OptimizingMetricInfo(metric).IsMaximizing; return(BestResultUtil.GetBestRun(results, metricsAgent, isMetricMaximizing)); }
/// <summary> /// Select the best run from an enumeration of experiment cross validation runs. /// </summary> /// <param name="results">Enumeration of AutoML experiment cross validation run results.</param> /// <param name="metric">Metric to consider when selecting the best run.</param> /// <returns>The best experiment run.</returns> public static CrossValidationRunDetail <RegressionMetrics> Best(this IEnumerable <CrossValidationRunDetail <RegressionMetrics> > results, RegressionMetric metric = RegressionMetric.RSquared) { var metricsAgent = new RegressionMetricsAgent(null, metric); var isMetricMaximizing = new OptimizingMetricInfo(metric).IsMaximizing; return(BestResultUtil.GetBestRun(results, metricsAgent, isMetricMaximizing)); }
public Experiment(MLContext context, TaskKind task, OptimizingMetricInfo metricInfo, IProgress <TRunDetail> progressCallback, ExperimentSettings experimentSettings, IMetricsAgent <TMetrics> metricsAgent, IEnumerable <TrainerName> trainerAllowList, DatasetColumnInfo[] datasetColumnInfo, IRunner <TRunDetail> runner, IChannel logger) { _context = context; _history = new List <SuggestedPipelineRunDetail>(); _optimizingMetricInfo = metricInfo; _task = task; _progressCallback = progressCallback; _experimentSettings = experimentSettings; _metricsAgent = metricsAgent; _trainerAllowList = trainerAllowList; _modelDirectory = GetModelDirectory(_context.TempFilePath, _experimentSettings.CacheDirectoryName); _datasetColumnInfo = datasetColumnInfo; _runner = runner; _logger = logger; _experimentTimerExpired = false; }
public static RunDetail <MulticlassClassificationMetrics> GetBestRun(IEnumerable <RunDetail <MulticlassClassificationMetrics> > results, MulticlassClassificationMetric metric) { var metricsAgent = new MultiMetricsAgent(null, metric); var metricInfo = new OptimizingMetricInfo(metric); return(GetBestRun(results, metricsAgent, metricInfo.IsMaximizing)); }
public static RunDetail <RankingMetrics> GetBestRun(IEnumerable <RunDetail <RankingMetrics> > results, RankingMetric metric, string groupIdColumnName) { var metricsAgent = new RankingMetricsAgent(null, metric, groupIdColumnName); var metricInfo = new OptimizingMetricInfo(metric); return(GetBestRun(results, metricsAgent, metricInfo.IsMaximizing)); }
public static RunDetail <RankingMetrics> GetBestRun(IEnumerable <RunDetail <RankingMetrics> > results, RankingMetric metric, uint dcgTruncationLevel) { var metricsAgent = new RankingMetricsAgent(null, metric, dcgTruncationLevel); var metricInfo = new OptimizingMetricInfo(metric); return(GetBestRun(results, metricsAgent, metricInfo.IsMaximizing)); }
internal ExperimentBase(MLContext context, IMetricsAgent <TMetrics> metricsAgent, OptimizingMetricInfo optimizingMetricInfo, TExperimentSettings settings, TaskKind task, IEnumerable <TrainerName> trainerAllowList) { Context = context; MetricsAgent = metricsAgent; OptimizingMetricInfo = optimizingMetricInfo; Settings = settings; _logger = ((IChannelProvider)context).Start("AutoML"); _task = task; _trainerAllowList = trainerAllowList; }
public CrossValSummaryRunner(MLContext context, IDataView[] trainDatasets, IDataView[] validDatasets, IMetricsAgent <TMetrics> metricsAgent, IEstimator <ITransformer> preFeaturizer, ITransformer[] preprocessorTransforms, string labelColumn, OptimizingMetricInfo optimizingMetricInfo, IChannel logger) { _context = context; _trainDatasets = trainDatasets; _validDatasets = validDatasets; _metricsAgent = metricsAgent; _preFeaturizer = preFeaturizer; _preprocessorTransforms = preprocessorTransforms; _labelColumn = labelColumn; _optimizingMetricInfo = optimizingMetricInfo; _logger = logger; _modelInputSchema = trainDatasets[0].Schema; }