/// <summary>
/// REVIEW: This was the original CategoriesToWeights function. Should be deprecated once we can validate the new function works
/// better. It contains a subtle issue, such that categories with poor performance but which are seen a lot will have
/// high weight. New function addresses this issue, while also improving exploration capability of algorithm.
/// </summary>
/// <param name="param"></param>
/// <param name="previousRuns"></param>
/// <returns></returns>
private double[] CategoriesToWeightsOld(DiscreteValueGenerator param, IEnumerable <IRunResult> previousRuns)
{
double[] weights = new double[param.Count];
Dictionary <string, int> labelToIndex = new Dictionary <string, int>();
// Map categorical values to their index.
for (int j = 0; j < param.Count; j++)
{
labelToIndex[param[j].ValueText] = j;
}
// Add pseudo-observations, to account for unobserved parameter settings.
for (int i = 0; i < weights.Length; i++)
{
weights[i] = 0.1;
}
// Sum up the results for each category value.
bool isMaximizing = true;
foreach (RunResult r in previousRuns)
{
weights[labelToIndex[r.ParameterSet[param.Name].ValueText]] += r.MetricValue;
isMaximizing = r.IsMetricMaximizing;
}
// Normalize weights to sum to one and return
return(isMaximizing ? SweeperProbabilityUtils.Normalize(weights) : SweeperProbabilityUtils.InverseNormalize(weights));
}
/// <summary>
/// Converts a set of history into a set of weights, one for each run in the history.
/// </summary>
/// <param name="history">Input set of historical runs.</param>
/// <param name="n">Number of total runs (history may be truncated)</param>
/// <param name="rMean">Mean metric value of previous random runs.</param>
/// <param name="rVar">Metric value empirical variance of previous random runs.</param>
/// <returns>Array of weights.</returns>
private double[] HistoryToWeights(IRunResult[] history, int n, double rMean, double rVar)
{
// Extract weights and normalize.
double[] weights = new double[history.Length];
for (int i = 0; i < history.Length; i++)
{
weights[i] = (double)history[i].MetricValue;
}
// Fitness proportional scaling constant.
bool isMinimizing = history.Length > 0 && !history[0].IsMetricMaximizing;
double currentMaxPerf = isMinimizing ? SweeperProbabilityUtils.NormalCdf(2 * rMean - weights.Min(), rMean, rVar) : SweeperProbabilityUtils.NormalCdf(weights.Max(), rMean, rVar);
// Normalize weights to sum to one. Automatically Takes care of case where all are equal to zero.
weights = isMinimizing ? SweeperProbabilityUtils.InverseNormalize(weights) : SweeperProbabilityUtils.Normalize(weights);
// Scale weights. (Concentrates mass on good points, depending on how good the best currently is.)
for (int i = 0; i < weights.Length; i++)
{
weights[i] = _args.Simple ? Math.Pow(weights[i], Math.Min(Math.Sqrt(n), 100)) : Math.Pow(weights[i], _args.WeightRescalingPower * currentMaxPerf);
}
weights = SweeperProbabilityUtils.Normalize(weights);
return(weights);
}
private double ComputeEI(double bestVal, double[] forestStatistics)
{
double empMean = forestStatistics[0];
double empStdDev = forestStatistics[1];
double centered = empMean - bestVal;
double ztrans = centered / empStdDev;
return(centered * SweeperProbabilityUtils.StdNormalCdf(ztrans) + empStdDev * SweeperProbabilityUtils.StdNormalPdf(ztrans));
}
public KdoSweeper(Arguments args)
{
_args = args;
_sweepParameters = args.SweptParameters.ToArray();
_randomSweeper = new UniformRandomSweeper(new SweeperBase.ArgumentsBase(), _sweepParameters);
_redundantSweeper = new UniformRandomSweeper(new SweeperBase.ArgumentsBase {
Retries = 0
}, _sweepParameters);
_spu = new SweeperProbabilityUtils();
_alreadySeenConfigs = new SortedSet <Float[]>(new FloatArrayComparer());
_randomParamSets = new List <ParameterSet>();
}
/// <summary>
/// New version of CategoryToWeights method, which fixes an issue where we could
/// potentially assign a lot of mass to bad categories.
/// </summary>
private double[] CategoriesToWeights(DiscreteValueGenerator param, IRunResult[] previousRuns)
{
double[] weights = new double[param.Count];
Dictionary <string, int> labelToIndex = new Dictionary <string, int>();
int[] counts = new int[param.Count];
// Map categorical values to their index.
for (int j = 0; j < param.Count; j++)
{
labelToIndex[param[j].ValueText] = j;
}
// Add mass according to performance
bool isMaximizing = true;
foreach (RunResult r in previousRuns)
{
weights[labelToIndex[r.ParameterSet[param.Name].ValueText]] += r.MetricValue;
counts[labelToIndex[r.ParameterSet[param.Name].ValueText]]++;
isMaximizing = r.IsMetricMaximizing;
}
// Take average mass for each category
for (int i = 0; i < weights.Length; i++)
{
weights[i] /= (counts[i] > 0 ? counts[i] : 1);
}
// If any learner has not been seen, default its average to
// best value to encourage exploration of untried algorithms.
double bestVal = isMaximizing ?
previousRuns.Cast <RunResult>().Where(r => r.HasMetricValue).Max(r => r.MetricValue) :
previousRuns.Cast <RunResult>().Where(r => r.HasMetricValue).Min(r => r.MetricValue);
for (int i = 0; i < weights.Length; i++)
{
weights[i] += counts[i] == 0 ? bestVal : 0;
}
// Normalize weights to sum to one and return
return(isMaximizing ? SweeperProbabilityUtils.Normalize(weights) : SweeperProbabilityUtils.InverseNormalize(weights));
}
/// <summary>
/// Goes through forest to extract the set of leaf values associated with filtering each configuration.
/// </summary>
/// <param name="forest">Trained forest predictor, used for filtering configs.</param>
/// <param name="configs">Parameter configurations.</param>
/// <returns>2D array where rows correspond to configurations, and columns to the predicted leaf values.</returns>
private double[][] GetForestRegressionLeafValues(FastForestRegressionModelParameters forest, ParameterSet[] configs)
{
List <double[]> datasetLeafValues = new List <double[]>();
foreach (ParameterSet config in configs)
{
List <double> leafValues = new List <double>();
for (var treeId = 0; treeId < _args.NumOfTrees; treeId++)
{
Float[] transformedParams = SweeperProbabilityUtils.ParameterSetAsFloatArray(_sweepParameters, config, true);
VBuffer <Float> features = new VBuffer <Float>(transformedParams.Length, transformedParams);
List <int> path = null;
var leafId = forest.GetLeaf(treeId, features, ref path);
var leafValue = forest.GetLeafValue(treeId, leafId);
leafValues.Add(leafValue);
}
datasetLeafValues.Add(leafValues.ToArray());
}
return(datasetLeafValues.ToArray());
}
private FastForestRegressionModelParameters FitModel(IEnumerable <IRunResult> previousRuns)
{
Single[] targets = new Single[previousRuns.Count()];
Single[][] features = new Single[previousRuns.Count()][];
int i = 0;
foreach (RunResult r in previousRuns)
{
features[i] = SweeperProbabilityUtils.ParameterSetAsFloatArray(_sweepParameters, r.ParameterSet, true);
targets[i] = (Float)r.MetricValue;
i++;
}
ArrayDataViewBuilder dvBuilder = new ArrayDataViewBuilder(_context);
dvBuilder.AddColumn(DefaultColumnNames.Label, NumberType.Float, targets);
dvBuilder.AddColumn(DefaultColumnNames.Features, NumberType.Float, features);
IDataView data = dvBuilder.GetDataView();
AutoMlUtils.Assert(data.GetRowCount() == targets.Length, "This data view will have as many rows as there have been evaluations");
// Set relevant random forest arguments.
// Train random forest.
var trainer = new FastForestRegression(_context, DefaultColumnNames.Label, DefaultColumnNames.Features, advancedSettings: s =>
{
s.FeatureFraction = _args.SplitRatio;
s.NumTrees = _args.NumOfTrees;
s.MinDocumentsInLeafs = _args.NMinForSplit;
});
var predictor = trainer.Train(data).Model;
// Return random forest predictor.
return(predictor);
}
/// <summary>
/// Computes a single-mutation neighborhood (one param at a time) for a given configuration. For
/// numeric parameters, samples K mutations (i.e., creates K neighbors based on that paramater).
/// </summary>
/// <param name="parent">Starting configuration.</param>
/// <returns>A set of configurations that each differ from parent in exactly one parameter.</returns>
private ParameterSet[] GetOneMutationNeighborhood(ParameterSet parent)
{
List <ParameterSet> neighbors = new List <ParameterSet>();
SweeperProbabilityUtils spu = new SweeperProbabilityUtils();
for (int i = 0; i < _sweepParameters.Length; i++)
{
// This allows us to query possible values of this parameter.
IValueGenerator sweepParam = _sweepParameters[i];
// This holds the actual value for this parameter, chosen in this parameter set.
IParameterValue pset = parent[sweepParam.Name];
AutoMlUtils.Assert(pset != null);
DiscreteValueGenerator parameterDiscrete = sweepParam as DiscreteValueGenerator;
if (parameterDiscrete != null)
{
// Create one neighbor for every discrete parameter.
Float[] neighbor = SweeperProbabilityUtils.ParameterSetAsFloatArray(_sweepParameters, parent, false);
int hotIndex = -1;
for (int j = 0; j < parameterDiscrete.Count; j++)
{
if (parameterDiscrete[j].Equals(pset))
{
hotIndex = j;
break;
}
}
AutoMlUtils.Assert(hotIndex >= 0);
Random r = new Random();
int randomIndex = r.Next(0, parameterDiscrete.Count - 1);
randomIndex += randomIndex >= hotIndex ? 1 : 0;
neighbor[i] = randomIndex;
neighbors.Add(SweeperProbabilityUtils.FloatArrayAsParameterSet(_sweepParameters, neighbor, false));
}
else
{
INumericValueGenerator parameterNumeric = sweepParam as INumericValueGenerator;
AutoMlUtils.Assert(parameterNumeric != null, "SMAC sweeper can only sweep over discrete and numeric parameters");
// Create k neighbors (typically 4) for every numerical parameter.
for (int j = 0; j < _args.NumNeighborsForNumericalParams; j++)
{
Float[] neigh = SweeperProbabilityUtils.ParameterSetAsFloatArray(_sweepParameters, parent, false);
double newVal = spu.NormalRVs(1, neigh[i], 0.2)[0];
while (newVal <= 0.0 || newVal >= 1.0)
{
newVal = spu.NormalRVs(1, neigh[i], 0.2)[0];
}
neigh[i] = (Float)newVal;
ParameterSet neighbor = SweeperProbabilityUtils.FloatArrayAsParameterSet(_sweepParameters, neigh, false);
neighbors.Add(neighbor);
}
}
}
return(neighbors.ToArray());
}
/// <summary>
/// Sample child configuration from configuration centered at parent, using fitness proportional mutation.
/// </summary>
/// <param name="parent">Starting parent configuration (used as mean in multivariate Gaussian).</param>
/// <param name="fitness">Numeric value indicating how good a configuration parent is.</param>
/// <param name="n">Count of how many items currently in history.</param>
/// <param name="previousRuns">Run history.</param>
/// <param name="rMean">Mean metric value of previous random runs.</param>
/// <param name="rVar">Metric value empirical variance of previous random runs.</param>
/// <param name="isMetricMaximizing">Flag for if we are minimizing or maximizing values.</param>
/// <returns>A mutated version of parent (i.e., point sampled near parent).</returns>
private ParameterSet SampleChild(ParameterSet parent, double fitness, int n, IRunResult[] previousRuns, double rMean, double rVar, bool isMetricMaximizing)
{
Float[] child = SweeperProbabilityUtils.ParameterSetAsFloatArray(_sweepParameters, parent, false);
List <int> numericParamIndices = new List <int>();
List <double> numericParamValues = new List <double>();
int loopCount = 0;
// Interleave uniform random samples, according to proportion defined.
if (_spu.SampleUniform() <= _args.ProportionRandom)
{
ParameterSet ps = _randomSweeper.ProposeSweeps(1)[0];
_randomParamSets.Add(ps);
return(ps);
}
do
{
for (int i = 0; i < _sweepParameters.Length; i++)
{
// This allows us to query possible values of this parameter.
var sweepParam = _sweepParameters[i];
if (sweepParam is DiscreteValueGenerator parameterDiscrete)
{
// Sample categorical parameter.
double[] categoryWeights = _args.LegacyDpBehavior
? CategoriesToWeightsOld(parameterDiscrete, previousRuns)
: CategoriesToWeights(parameterDiscrete, previousRuns);
child[i] = SampleCategoricalDist(1, categoryWeights)[0];
}
else
{
var parameterNumeric = sweepParam as INumericValueGenerator;
numericParamIndices.Add(i);
numericParamValues.Add(child[i]);
}
}
if (numericParamIndices.Count > 0)
{
if (!_args.Beta)
{
// Sample point from multivariate Gaussian, centered on parent values, with mutation proportional to fitness.
double[] mu = numericParamValues.ToArray();
double correctedVal = isMetricMaximizing
? 1.0 - SweeperProbabilityUtils.NormalCdf(fitness, rMean, rVar)
: 1.0 - SweeperProbabilityUtils.NormalCdf(2 * rMean - fitness, rMean, rVar);
double bandwidthScale = Math.Max(_args.MinimumMutationSpread, correctedVal);
double[] stddevs = Enumerable.Repeat(_args.Simple ? 0.2 : bandwidthScale, mu.Length).ToArray();
double[][] bandwidthMatrix = BuildBandwidthMatrix(n, stddevs);
double[] sampledPoint = SampleDiagonalCovMultivariateGaussian(1, mu, bandwidthMatrix)[0];
for (int j = 0; j < sampledPoint.Length; j++)
{
child[numericParamIndices[j]] = (Float)Corral(sampledPoint[j]);
}
}
else
{
// If Beta flag set, sample from independent Beta distributions instead.
double alpha = 1 + 15 * fitness;
foreach (int index in numericParamIndices)
{
const double epsCutoff = 1e-10;
double eps = Math.Min(Math.Max(child[index], epsCutoff), 1 - epsCutoff);
double beta = alpha / eps - alpha;
child[index] = (Float)Stats.SampleFromBeta(alpha, beta);
}
}
}
// Don't get stuck at local point.
loopCount++;
if (loopCount > 10)
{
return(_randomSweeper.ProposeSweeps(1, null)[0]);
}
} while (_alreadySeenConfigs.Contains(child));
_alreadySeenConfigs.Add(child);
return(SweeperProbabilityUtils.FloatArrayAsParameterSet(_sweepParameters, child, false));
}