/// <summary>
/// REVIEW: This was the original CategoriesToWeights function. Should be deprecated once we can validate the new function works
/// better. It contains a subtle issue, such that categories with poor performance but which are seen a lot will have
/// high weight. New function addresses this issue, while also improving exploration capability of algorithm.
/// </summary>
/// <param name="param"></param>
/// <param name="previousRuns"></param>
/// <returns></returns>
private double[] CategoriesToWeightsOld(DiscreteValueGenerator param, IEnumerable <IRunResult> previousRuns)
{
double[] weights = new double[param.Count];
Dictionary <string, int> labelToIndex = new Dictionary <string, int>();
// Map categorical values to their index.
for (int j = 0; j < param.Count; j++)
{
labelToIndex[param[j].ValueText] = j;
}
// Add pseudo-observations, to account for unobserved parameter settings.
for (int i = 0; i < weights.Length; i++)
{
weights[i] = 0.1;
}
// Sum up the results for each category value.
bool isMaximizing = true;
foreach (RunResult r in previousRuns)
{
weights[labelToIndex[r.ParameterSet[param.Name].ValueText]] += r.MetricValue;
isMaximizing = r.IsMetricMaximizing;
}
// Normalize weights to sum to one and return
return(isMaximizing ? SweeperProbabilityUtils.Normalize(weights) : SweeperProbabilityUtils.InverseNormalize(weights));
}
private static IValueGenerator[] ConvertToValueGenerators(IEnumerable <SweepableParam> hps)
{
var results = new IValueGenerator[hps.Count()];
for (int i = 0; i < hps.Count(); i++)
{
switch (hps.ElementAt(i))
{
case SweepableDiscreteParam dp:
var dpArgs = new DiscreteParamArguments()
{
Name = dp.Name,
Values = dp.Options.Select(o => o.ToString()).ToArray()
};
results[i] = new DiscreteValueGenerator(dpArgs);
break;
case SweepableFloatParam fp:
var fpArgs = new FloatParamArguments()
{
Name = fp.Name,
Min = fp.Min,
Max = fp.Max,
LogBase = fp.IsLogScale,
};
if (fp.NumSteps.HasValue)
{
fpArgs.NumSteps = fp.NumSteps.Value;
}
if (fp.StepSize.HasValue)
{
fpArgs.StepSize = fp.StepSize.Value;
}
results[i] = new FloatValueGenerator(fpArgs);
break;
case SweepableLongParam lp:
var lpArgs = new LongParamArguments()
{
Name = lp.Name,
Min = lp.Min,
Max = lp.Max,
LogBase = lp.IsLogScale
};
if (lp.NumSteps.HasValue)
{
lpArgs.NumSteps = lp.NumSteps.Value;
}
if (lp.StepSize.HasValue)
{
lpArgs.StepSize = lp.StepSize.Value;
}
results[i] = new LongValueGenerator(lpArgs);
break;
}
}
return(results);
}
/// <summary>
/// New version of CategoryToWeights method, which fixes an issue where we could
/// potentially assign a lot of mass to bad categories.
/// </summary>
private double[] CategoriesToWeights(DiscreteValueGenerator param, IRunResult[] previousRuns)
{
double[] weights = new double[param.Count];
Dictionary <string, int> labelToIndex = new Dictionary <string, int>();
int[] counts = new int[param.Count];
// Map categorical values to their index.
for (int j = 0; j < param.Count; j++)
{
labelToIndex[param[j].ValueText] = j;
}
// Add mass according to performance
bool isMaximizing = true;
foreach (RunResult r in previousRuns)
{
weights[labelToIndex[r.ParameterSet[param.Name].ValueText]] += r.MetricValue;
counts[labelToIndex[r.ParameterSet[param.Name].ValueText]]++;
isMaximizing = r.IsMetricMaximizing;
}
// Take average mass for each category
for (int i = 0; i < weights.Length; i++)
{
weights[i] /= (counts[i] > 0 ? counts[i] : 1);
}
// If any learner has not been seen, default its average to
// best value to encourage exploration of untried algorithms.
double bestVal = isMaximizing ?
previousRuns.Cast <RunResult>().Where(r => r.HasMetricValue).Max(r => r.MetricValue) :
previousRuns.Cast <RunResult>().Where(r => r.HasMetricValue).Min(r => r.MetricValue);
for (int i = 0; i < weights.Length; i++)
{
weights[i] += counts[i] == 0 ? bestVal : 0;
}
// Normalize weights to sum to one and return
return(isMaximizing ? SweeperProbabilityUtils.Normalize(weights) : SweeperProbabilityUtils.InverseNormalize(weights));
}
/// <summary>
/// Computes a single-mutation neighborhood (one param at a time) for a given configuration. For
/// numeric parameters, samples K mutations (i.e., creates K neighbors based on that paramater).
/// </summary>
/// <param name="parent">Starting configuration.</param>
/// <returns>A set of configurations that each differ from parent in exactly one parameter.</returns>
private ParameterSet[] GetOneMutationNeighborhood(ParameterSet parent)
{
List <ParameterSet> neighbors = new List <ParameterSet>();
SweeperProbabilityUtils spu = new SweeperProbabilityUtils();
for (int i = 0; i < _sweepParameters.Length; i++)
{
// This allows us to query possible values of this parameter.
IValueGenerator sweepParam = _sweepParameters[i];
// This holds the actual value for this parameter, chosen in this parameter set.
IParameterValue pset = parent[sweepParam.Name];
AutoMlUtils.Assert(pset != null);
DiscreteValueGenerator parameterDiscrete = sweepParam as DiscreteValueGenerator;
if (parameterDiscrete != null)
{
// Create one neighbor for every discrete parameter.
Float[] neighbor = SweeperProbabilityUtils.ParameterSetAsFloatArray(_sweepParameters, parent, false);
int hotIndex = -1;
for (int j = 0; j < parameterDiscrete.Count; j++)
{
if (parameterDiscrete[j].Equals(pset))
{
hotIndex = j;
break;
}
}
AutoMlUtils.Assert(hotIndex >= 0);
Random r = new Random();
int randomIndex = r.Next(0, parameterDiscrete.Count - 1);
randomIndex += randomIndex >= hotIndex ? 1 : 0;
neighbor[i] = randomIndex;
neighbors.Add(SweeperProbabilityUtils.FloatArrayAsParameterSet(_sweepParameters, neighbor, false));
}
else
{
INumericValueGenerator parameterNumeric = sweepParam as INumericValueGenerator;
AutoMlUtils.Assert(parameterNumeric != null, "SMAC sweeper can only sweep over discrete and numeric parameters");
// Create k neighbors (typically 4) for every numerical parameter.
for (int j = 0; j < _args.NumNeighborsForNumericalParams; j++)
{
Float[] neigh = SweeperProbabilityUtils.ParameterSetAsFloatArray(_sweepParameters, parent, false);
double newVal = spu.NormalRVs(1, neigh[i], 0.2)[0];
while (newVal <= 0.0 || newVal >= 1.0)
{
newVal = spu.NormalRVs(1, neigh[i], 0.2)[0];
}
neigh[i] = (Float)newVal;
ParameterSet neighbor = SweeperProbabilityUtils.FloatArrayAsParameterSet(_sweepParameters, neigh, false);
neighbors.Add(neighbor);
}
}
}
return(neighbors.ToArray());
}
