public ParameterSet[] ProposeSweeps(int maxSweeps, IEnumerable <IRunResult> previousRuns = null) { int numOfCandidates = maxSweeps; // Initialization: Will enter here on first iteration and use the default (random) // sweeper to generate initial candidates. int numRuns = previousRuns == null ? 0 : previousRuns.Count(); if (numRuns < _args.NumberInitialPopulation) { return(_randomSweeper.ProposeSweeps(Math.Min(numOfCandidates, _args.NumberInitialPopulation - numRuns), previousRuns)); } // Only retain viable runs List <IRunResult> viableRuns = new List <IRunResult>(); foreach (RunResult run in previousRuns) { if (run != null && run.HasMetricValue) { viableRuns.Add(run); } } // Fit Random Forest Model on previous run data. FastForestRegressionModelParameters forestPredictor = FitModel(viableRuns); // Using acquisition function and current best, get candidate configuration(s). return(GenerateCandidateConfigurations(numOfCandidates, viableRuns, forestPredictor)); }
/// <summary> /// Performs a local one-mutation neighborhood greedy search. /// </summary> /// <param name="parent">Starting parameter set configuration.</param> /// <param name="forest">Trained forest, for evaluation of points.</param> /// <param name="bestVal">Best performance seen thus far.</param> /// <param name="epsilon">Threshold for when to stop the local search.</param> /// <param name="isMetricMaximizing">Whether SMAC should aim to maximize (vs minimize) metric.</param> /// <returns></returns> private Tuple <double, ParameterSet> LocalSearch(ParameterSet parent, FastForestRegressionModelParameters forest, double bestVal, double epsilon, bool isMetricMaximizing) { try { double currentBestEI = EvaluateConfigurationsByEI(forest, bestVal, new ParameterSet[] { parent }, isMetricMaximizing)[0]; ParameterSet currentBestConfig = parent; for (; ;) { ParameterSet[] neighborhood = GetOneMutationNeighborhood(currentBestConfig); double[] eis = EvaluateConfigurationsByEI(forest, bestVal, neighborhood, isMetricMaximizing); int bestIndex = eis.ArgMax(); if (eis[bestIndex] - currentBestEI < _args.Epsilon) { break; } else { currentBestConfig = neighborhood[bestIndex]; currentBestEI = eis[bestIndex]; } } return(new Tuple <double, ParameterSet>(currentBestEI, currentBestConfig)); } catch (Exception e) { throw new InvalidOperationException("SMAC sweeper localSearch threw exception", e); } }
/// <summary> /// Goes through forest to extract the set of leaf values associated with filtering each configuration. /// </summary> /// <param name="forest">Trained forest predictor, used for filtering configs.</param> /// <param name="configs">Parameter configurations.</param> /// <returns>2D array where rows correspond to configurations, and columns to the predicted leaf values.</returns> private double[][] GetForestRegressionLeafValues(FastForestRegressionModelParameters forest, ParameterSet[] configs) { List <double[]> datasetLeafValues = new List <double[]>(); var e = forest.TrainedEnsemble; foreach (ParameterSet config in configs) { List <double> leafValues = new List <double>(); foreach (InternalRegressionTree t in e.Trees) { float[] transformedParams = SweeperProbabilityUtils.ParameterSetAsFloatArray(_host, _sweepParameters, config, true); VBuffer <float> features = new VBuffer <float>(transformedParams.Length, transformedParams); leafValues.Add((float)t.LeafValues[t.GetLeaf(in features)]);
/// <summary> /// Goes through forest to extract the set of leaf values associated with filtering each configuration. /// </summary> /// <param name="forest">Trained forest predictor, used for filtering configs.</param> /// <param name="configs">Parameter configurations.</param> /// <returns>2D array where rows correspond to configurations, and columns to the predicted leaf values.</returns> private double[][] GetForestRegressionLeafValues(FastForestRegressionModelParameters forest, ParameterSet[] configs) { List <double[]> datasetLeafValues = new List <double[]>(); foreach (ParameterSet config in configs) { List <double> leafValues = new List <double>(); for (var treeId = 0; treeId < forest.TrainedTreeEnsemble.Trees.Count; treeId++) { Float[] transformedParams = SweeperProbabilityUtils.ParameterSetAsFloatArray(_sweepParameters, config, true); VBuffer <Float> features = new VBuffer <Float>(transformedParams.Length, transformedParams); var leafId = GetLeaf(forest, treeId, features); var leafValue = GetLeafValue(forest, treeId, leafId); leafValues.Add(leafValue); } datasetLeafValues.Add(leafValues.ToArray()); } return(datasetLeafValues.ToArray()); }
/// <summary> /// Goes through forest to extract the set of leaf values associated with filtering each configuration. /// </summary> /// <param name="forest">Trained forest predictor, used for filtering configs.</param> /// <param name="configs">Parameter configurations.</param> /// <returns>2D array where rows correspond to configurations, and columns to the predicted leaf values.</returns> private double[][] GetForestRegressionLeafValues(FastForestRegressionModelParameters forest, ParameterSet[] configs) { List <double[]> datasetLeafValues = new List <double[]>(); foreach (ParameterSet config in configs) { List <double> leafValues = new List <double>(); for (var treeId = 0; treeId < _args.NumOfTrees; treeId++) { Float[] transformedParams = SweeperProbabilityUtils.ParameterSetAsFloatArray(_sweepParameters, config, true); VBuffer <Float> features = new VBuffer <Float>(transformedParams.Length, transformedParams); List <int> path = null; var leafId = forest.GetLeaf(treeId, features, ref path); var leafValue = forest.GetLeafValue(treeId, leafId); leafValues.Add(leafValue); } datasetLeafValues.Add(leafValues.ToArray()); } return(datasetLeafValues.ToArray()); }
/// <summary> /// Does a mix of greedy local search around best performing parameter sets, while throwing random parameter sets into the mix. /// </summary> /// <param name="parents">Beginning locations for local greedy search.</param> /// <param name="forest">Trained random forest, used later for evaluating parameters.</param> /// <param name="numOfCandidates">Number of candidate configurations returned by the method (top K).</param> /// <param name="previousRuns">Historical run results.</param> /// <returns>Array of parameter sets, which will then be evaluated.</returns> private ParameterSet[] GreedyPlusRandomSearch(ParameterSet[] parents, FastForestRegressionModelParameters forest, int numOfCandidates, IEnumerable <IRunResult> previousRuns) { // REVIEW: The IsMetricMaximizing flag affects the comparator, so that // performing Max() should get the best, regardless of if it is maximizing or // minimizing. RunResult bestRun = (RunResult)previousRuns.Max(); RunResult worstRun = (RunResult)previousRuns.Min(); double bestVal = bestRun.IsMetricMaximizing ? bestRun.MetricValue : worstRun.MetricValue - bestRun.MetricValue; HashSet <Tuple <double, ParameterSet> > configurations = new HashSet <Tuple <double, ParameterSet> >(); // Perform local search. foreach (ParameterSet c in parents) { Tuple <double, ParameterSet> bestChildKvp = LocalSearch(c, forest, bestVal, _args.Epsilon); configurations.Add(bestChildKvp); } // Additional set of random configurations to choose from during local search. ParameterSet[] randomConfigs = _randomSweeper.ProposeSweeps(_args.NumRandomEISearchConfigurations, previousRuns); double[] randomEIs = EvaluateConfigurationsByEI(forest, bestVal, randomConfigs); _host.Assert(randomConfigs.Length == randomEIs.Length); for (int i = 0; i < randomConfigs.Length; i++) { configurations.Add(new Tuple <double, ParameterSet>(randomEIs[i], randomConfigs[i])); } HashSet <ParameterSet> retainedConfigs = new HashSet <ParameterSet>(); IOrderedEnumerable <Tuple <double, ParameterSet> > bestConfigurations = configurations.OrderByDescending(x => x.Item1); foreach (Tuple <double, ParameterSet> t in bestConfigurations.Take(numOfCandidates)) { retainedConfigs.Add(t.Item2); } return(retainedConfigs.ToArray()); }
/// <summary> /// Does a mix of greedy local search around best performing parameter sets, while throwing random parameter sets into the mix. /// </summary> /// <param name="parents">Beginning locations for local greedy search.</param> /// <param name="forest">Trained random forest, used later for evaluating parameters.</param> /// <param name="numOfCandidates">Number of candidate configurations returned by the method (top K).</param> /// <param name="previousRuns">Historical run results.</param> /// <returns>Array of parameter sets, which will then be evaluated.</returns> private ParameterSet[] GreedyPlusRandomSearch(ParameterSet[] parents, FastForestRegressionModelParameters forest, int numOfCandidates, IEnumerable <IRunResult> previousRuns) { RunResult bestRun = (RunResult)previousRuns.Max(); RunResult worstRun = (RunResult)previousRuns.Min(); double bestVal = bestRun.MetricValue; HashSet <Tuple <double, ParameterSet> > configurations = new HashSet <Tuple <double, ParameterSet> >(); // Perform local search. foreach (ParameterSet c in parents) { Tuple <double, ParameterSet> bestChildKvp = LocalSearch(c, forest, bestVal, _args.Epsilon, bestRun.IsMetricMaximizing); configurations.Add(bestChildKvp); } // Additional set of random configurations to choose from during local search. ParameterSet[] randomConfigs = _randomSweeper.ProposeSweeps(_args.NumRandomEISearchConfigurations, previousRuns); double[] randomEIs = EvaluateConfigurationsByEI(forest, bestVal, randomConfigs, bestRun.IsMetricMaximizing); Runtime.Contracts.Assert(randomConfigs.Length == randomEIs.Length); for (int i = 0; i < randomConfigs.Length; i++) { configurations.Add(new Tuple <double, ParameterSet>(randomEIs[i], randomConfigs[i])); } IOrderedEnumerable <Tuple <double, ParameterSet> > bestConfigurations = configurations.OrderByDescending(x => x.Item1); var retainedConfigs = new HashSet <ParameterSet>(bestConfigurations.Select(x => x.Item2)); // remove configurations matching previous run foreach (var previousRun in previousRuns) { retainedConfigs.Remove(previousRun.ParameterSet); } return(retainedConfigs.Take(numOfCandidates).ToArray()); }
private double[] EvaluateConfigurationsByEI(FastForestRegressionModelParameters forest, double bestVal, ParameterSet[] configs, bool isMetricMaximizing) { double[][] leafPredictions = GetForestRegressionLeafValues(forest, configs); double[][] forestStatistics = ComputeForestStats(leafPredictions); return(ComputeEIs(bestVal, forestStatistics, isMetricMaximizing)); }
/// <summary> /// Generates a set of candidate configurations to sweep through, based on a combination of random and local /// search, as outlined in Hutter et al. - Sequential Model-Based Optimization for General Algorithm Configuration. /// Makes use of class private members which determine how many candidates are returned. This number will include /// random configurations interleaved (per the paper), and thus will be double the specified value. /// </summary> /// <param name="numOfCandidates">Number of candidate solutions to return.</param> /// <param name="previousRuns">History of previously evaluated points, with their empirical performance values.</param> /// <param name="forest">Trained random forest ensemble. Used in evaluating the candidates.</param> /// <returns>An array of ParamaterSets which are the candidate configurations to sweep.</returns> private ParameterSet[] GenerateCandidateConfigurations(int numOfCandidates, IEnumerable <IRunResult> previousRuns, FastForestRegressionModelParameters forest) { // Get k best previous runs ParameterSets. ParameterSet[] bestKParamSets = GetKBestConfigurations(previousRuns, _args.LocalSearchParentCount); // Perform local searches using the k best previous run configurations. ParameterSet[] eiChallengers = GreedyPlusRandomSearch(bestKParamSets, forest, (int)Math.Ceiling(numOfCandidates / 2.0F), previousRuns); // Generate another set of random configurations to interleave. ParameterSet[] randomChallengers = _randomSweeper.ProposeSweeps(numOfCandidates - eiChallengers.Length, previousRuns); // Return interleaved challenger candidates with random candidates. Since the number of candidates from either can be less than // the number asked for, since we only generate unique candidates, and the number from either method may vary considerably. ParameterSet[] configs = new ParameterSet[eiChallengers.Length + randomChallengers.Length]; Array.Copy(eiChallengers, 0, configs, 0, eiChallengers.Length); Array.Copy(randomChallengers, 0, configs, eiChallengers.Length, randomChallengers.Length); return(configs); }