public ParameterSet[] ProposeSweeps(int maxSweeps, IEnumerable <IRunResult> previousRuns = null)
{
int numOfCandidates = maxSweeps;
// Initialization: Will enter here on first iteration and use the default (random)
// sweeper to generate initial candidates.
int numRuns = previousRuns == null ? 0 : previousRuns.Count();
if (numRuns < _args.NumberInitialPopulation)
{
return(_randomSweeper.ProposeSweeps(Math.Min(numOfCandidates, _args.NumberInitialPopulation - numRuns), previousRuns));
}
// Only retain viable runs
List <IRunResult> viableRuns = new List <IRunResult>();
foreach (RunResult run in previousRuns)
{
if (run != null && run.HasMetricValue)
{
viableRuns.Add(run);
}
}
// Fit Random Forest Model on previous run data.
FastForestRegressionModelParameters forestPredictor = FitModel(viableRuns);
// Using acquisition function and current best, get candidate configuration(s).
return(GenerateCandidateConfigurations(numOfCandidates, viableRuns, forestPredictor));
}
/// <summary>
/// Performs a local one-mutation neighborhood greedy search.
/// </summary>
/// <param name="parent">Starting parameter set configuration.</param>
/// <param name="forest">Trained forest, for evaluation of points.</param>
/// <param name="bestVal">Best performance seen thus far.</param>
/// <param name="epsilon">Threshold for when to stop the local search.</param>
/// <param name="isMetricMaximizing">Whether SMAC should aim to maximize (vs minimize) metric.</param>
/// <returns></returns>
private Tuple <double, ParameterSet> LocalSearch(ParameterSet parent, FastForestRegressionModelParameters forest, double bestVal, double epsilon, bool isMetricMaximizing)
{
try
{
double currentBestEI = EvaluateConfigurationsByEI(forest, bestVal, new ParameterSet[] { parent }, isMetricMaximizing)[0];
ParameterSet currentBestConfig = parent;
for (; ;)
{
ParameterSet[] neighborhood = GetOneMutationNeighborhood(currentBestConfig);
double[] eis = EvaluateConfigurationsByEI(forest, bestVal, neighborhood, isMetricMaximizing);
int bestIndex = eis.ArgMax();
if (eis[bestIndex] - currentBestEI < _args.Epsilon)
{
break;
}
else
{
currentBestConfig = neighborhood[bestIndex];
currentBestEI = eis[bestIndex];
}
}
return(new Tuple <double, ParameterSet>(currentBestEI, currentBestConfig));
}
catch (Exception e)
{
throw new InvalidOperationException("SMAC sweeper localSearch threw exception", e);
}
}
/// <summary>
/// Goes through forest to extract the set of leaf values associated with filtering each configuration.
/// </summary>
/// <param name="forest">Trained forest predictor, used for filtering configs.</param>
/// <param name="configs">Parameter configurations.</param>
/// <returns>2D array where rows correspond to configurations, and columns to the predicted leaf values.</returns>
private double[][] GetForestRegressionLeafValues(FastForestRegressionModelParameters forest, ParameterSet[] configs)
{
List <double[]> datasetLeafValues = new List <double[]>();
var e = forest.TrainedEnsemble;
foreach (ParameterSet config in configs)
{
List <double> leafValues = new List <double>();
foreach (InternalRegressionTree t in e.Trees)
{
float[] transformedParams = SweeperProbabilityUtils.ParameterSetAsFloatArray(_host, _sweepParameters, config, true);
VBuffer <float> features = new VBuffer <float>(transformedParams.Length, transformedParams);
leafValues.Add((float)t.LeafValues[t.GetLeaf(in features)]);
/// <summary>
/// Goes through forest to extract the set of leaf values associated with filtering each configuration.
/// </summary>
/// <param name="forest">Trained forest predictor, used for filtering configs.</param>
/// <param name="configs">Parameter configurations.</param>
/// <returns>2D array where rows correspond to configurations, and columns to the predicted leaf values.</returns>
private double[][] GetForestRegressionLeafValues(FastForestRegressionModelParameters forest, ParameterSet[] configs)
{
List <double[]> datasetLeafValues = new List <double[]>();
foreach (ParameterSet config in configs)
{
List <double> leafValues = new List <double>();
for (var treeId = 0; treeId < forest.TrainedTreeEnsemble.Trees.Count; treeId++)
{
Float[] transformedParams = SweeperProbabilityUtils.ParameterSetAsFloatArray(_sweepParameters, config, true);
VBuffer <Float> features = new VBuffer <Float>(transformedParams.Length, transformedParams);
var leafId = GetLeaf(forest, treeId, features);
var leafValue = GetLeafValue(forest, treeId, leafId);
leafValues.Add(leafValue);
}
datasetLeafValues.Add(leafValues.ToArray());
}
return(datasetLeafValues.ToArray());
}
/// <summary>
/// Goes through forest to extract the set of leaf values associated with filtering each configuration.
/// </summary>
/// <param name="forest">Trained forest predictor, used for filtering configs.</param>
/// <param name="configs">Parameter configurations.</param>
/// <returns>2D array where rows correspond to configurations, and columns to the predicted leaf values.</returns>
private double[][] GetForestRegressionLeafValues(FastForestRegressionModelParameters forest, ParameterSet[] configs)
{
List <double[]> datasetLeafValues = new List <double[]>();
foreach (ParameterSet config in configs)
{
List <double> leafValues = new List <double>();
for (var treeId = 0; treeId < _args.NumOfTrees; treeId++)
{
Float[] transformedParams = SweeperProbabilityUtils.ParameterSetAsFloatArray(_sweepParameters, config, true);
VBuffer <Float> features = new VBuffer <Float>(transformedParams.Length, transformedParams);
List <int> path = null;
var leafId = forest.GetLeaf(treeId, features, ref path);
var leafValue = forest.GetLeafValue(treeId, leafId);
leafValues.Add(leafValue);
}
datasetLeafValues.Add(leafValues.ToArray());
}
return(datasetLeafValues.ToArray());
}
/// <summary>
/// Does a mix of greedy local search around best performing parameter sets, while throwing random parameter sets into the mix.
/// </summary>
/// <param name="parents">Beginning locations for local greedy search.</param>
/// <param name="forest">Trained random forest, used later for evaluating parameters.</param>
/// <param name="numOfCandidates">Number of candidate configurations returned by the method (top K).</param>
/// <param name="previousRuns">Historical run results.</param>
/// <returns>Array of parameter sets, which will then be evaluated.</returns>
private ParameterSet[] GreedyPlusRandomSearch(ParameterSet[] parents, FastForestRegressionModelParameters forest, int numOfCandidates, IEnumerable <IRunResult> previousRuns)
{
// REVIEW: The IsMetricMaximizing flag affects the comparator, so that
// performing Max() should get the best, regardless of if it is maximizing or
// minimizing.
RunResult bestRun = (RunResult)previousRuns.Max();
RunResult worstRun = (RunResult)previousRuns.Min();
double bestVal = bestRun.IsMetricMaximizing ? bestRun.MetricValue : worstRun.MetricValue - bestRun.MetricValue;
HashSet <Tuple <double, ParameterSet> > configurations = new HashSet <Tuple <double, ParameterSet> >();
// Perform local search.
foreach (ParameterSet c in parents)
{
Tuple <double, ParameterSet> bestChildKvp = LocalSearch(c, forest, bestVal, _args.Epsilon);
configurations.Add(bestChildKvp);
}
// Additional set of random configurations to choose from during local search.
ParameterSet[] randomConfigs = _randomSweeper.ProposeSweeps(_args.NumRandomEISearchConfigurations, previousRuns);
double[] randomEIs = EvaluateConfigurationsByEI(forest, bestVal, randomConfigs);
_host.Assert(randomConfigs.Length == randomEIs.Length);
for (int i = 0; i < randomConfigs.Length; i++)
{
configurations.Add(new Tuple <double, ParameterSet>(randomEIs[i], randomConfigs[i]));
}
HashSet <ParameterSet> retainedConfigs = new HashSet <ParameterSet>();
IOrderedEnumerable <Tuple <double, ParameterSet> > bestConfigurations = configurations.OrderByDescending(x => x.Item1);
foreach (Tuple <double, ParameterSet> t in bestConfigurations.Take(numOfCandidates))
{
retainedConfigs.Add(t.Item2);
}
return(retainedConfigs.ToArray());
}
/// <summary>
/// Does a mix of greedy local search around best performing parameter sets, while throwing random parameter sets into the mix.
/// </summary>
/// <param name="parents">Beginning locations for local greedy search.</param>
/// <param name="forest">Trained random forest, used later for evaluating parameters.</param>
/// <param name="numOfCandidates">Number of candidate configurations returned by the method (top K).</param>
/// <param name="previousRuns">Historical run results.</param>
/// <returns>Array of parameter sets, which will then be evaluated.</returns>
private ParameterSet[] GreedyPlusRandomSearch(ParameterSet[] parents, FastForestRegressionModelParameters forest, int numOfCandidates, IEnumerable <IRunResult> previousRuns)
{
RunResult bestRun = (RunResult)previousRuns.Max();
RunResult worstRun = (RunResult)previousRuns.Min();
double bestVal = bestRun.MetricValue;
HashSet <Tuple <double, ParameterSet> > configurations = new HashSet <Tuple <double, ParameterSet> >();
// Perform local search.
foreach (ParameterSet c in parents)
{
Tuple <double, ParameterSet> bestChildKvp = LocalSearch(c, forest, bestVal, _args.Epsilon, bestRun.IsMetricMaximizing);
configurations.Add(bestChildKvp);
}
// Additional set of random configurations to choose from during local search.
ParameterSet[] randomConfigs = _randomSweeper.ProposeSweeps(_args.NumRandomEISearchConfigurations, previousRuns);
double[] randomEIs = EvaluateConfigurationsByEI(forest, bestVal, randomConfigs, bestRun.IsMetricMaximizing);
Runtime.Contracts.Assert(randomConfigs.Length == randomEIs.Length);
for (int i = 0; i < randomConfigs.Length; i++)
{
configurations.Add(new Tuple <double, ParameterSet>(randomEIs[i], randomConfigs[i]));
}
IOrderedEnumerable <Tuple <double, ParameterSet> > bestConfigurations = configurations.OrderByDescending(x => x.Item1);
var retainedConfigs = new HashSet <ParameterSet>(bestConfigurations.Select(x => x.Item2));
// remove configurations matching previous run
foreach (var previousRun in previousRuns)
{
retainedConfigs.Remove(previousRun.ParameterSet);
}
return(retainedConfigs.Take(numOfCandidates).ToArray());
}
private double[] EvaluateConfigurationsByEI(FastForestRegressionModelParameters forest, double bestVal, ParameterSet[] configs, bool isMetricMaximizing)
{
double[][] leafPredictions = GetForestRegressionLeafValues(forest, configs);
double[][] forestStatistics = ComputeForestStats(leafPredictions);
return(ComputeEIs(bestVal, forestStatistics, isMetricMaximizing));
}
/// <summary>
/// Generates a set of candidate configurations to sweep through, based on a combination of random and local
/// search, as outlined in Hutter et al. - Sequential Model-Based Optimization for General Algorithm Configuration.
/// Makes use of class private members which determine how many candidates are returned. This number will include
/// random configurations interleaved (per the paper), and thus will be double the specified value.
/// </summary>
/// <param name="numOfCandidates">Number of candidate solutions to return.</param>
/// <param name="previousRuns">History of previously evaluated points, with their empirical performance values.</param>
/// <param name="forest">Trained random forest ensemble. Used in evaluating the candidates.</param>
/// <returns>An array of ParamaterSets which are the candidate configurations to sweep.</returns>
private ParameterSet[] GenerateCandidateConfigurations(int numOfCandidates, IEnumerable <IRunResult> previousRuns, FastForestRegressionModelParameters forest)
{
// Get k best previous runs ParameterSets.
ParameterSet[] bestKParamSets = GetKBestConfigurations(previousRuns, _args.LocalSearchParentCount);
// Perform local searches using the k best previous run configurations.
ParameterSet[] eiChallengers = GreedyPlusRandomSearch(bestKParamSets, forest, (int)Math.Ceiling(numOfCandidates / 2.0F), previousRuns);
// Generate another set of random configurations to interleave.
ParameterSet[] randomChallengers = _randomSweeper.ProposeSweeps(numOfCandidates - eiChallengers.Length, previousRuns);
// Return interleaved challenger candidates with random candidates. Since the number of candidates from either can be less than
// the number asked for, since we only generate unique candidates, and the number from either method may vary considerably.
ParameterSet[] configs = new ParameterSet[eiChallengers.Length + randomChallengers.Length];
Array.Copy(eiChallengers, 0, configs, 0, eiChallengers.Length);
Array.Copy(randomChallengers, 0, configs, eiChallengers.Length, randomChallengers.Length);
return(configs);
}
