public void HyperbandOptimizer_OptimizeBest()
{
var parameters = new IParameterSpec[]
{
new MinMaxParameterSpec(min: 80, max: 300, transform: Transform.Linear), // iterations
new MinMaxParameterSpec(min: 0.02, max: 0.2, transform: Transform.Log10), // learning rate
new MinMaxParameterSpec(min: 8, max: 15, transform: Transform.Linear), // maximumTreeDepth
};
var random = new Random(343);
HyperbandObjectiveFunction minimize = (p, r) =>
{
var error = random.NextDouble();
return(new OptimizerResult(p, error));
};
var sut = new HyperbandOptimizer(
parameters,
maximumUnitsOfCompute: 81,
eta: 5,
skipLastIterationOfEachRound: false,
seed: 34);
var actual = sut.OptimizeBest(minimize);
var expected = new OptimizerResult(new[] { 278.337940, 0.098931, 13.177449 }, 0.009549);
AssertOptimizerResult(expected, actual);
}
public void HyperbandOptimizer_Optimize()
{
var parameters = new IParameterSpec[]
{
new MinMaxParameterSpec(min: 80, max: 300, transform: Transform.Linear),
new MinMaxParameterSpec(min: 0.02, max: 0.2, transform: Transform.Log10),
new MinMaxParameterSpec(min: 8, max: 15, transform: Transform.Linear),
};
var random = new Random(343);
HyperbandObjectiveFunction minimize = (p, r) =>
{
var error = random.NextDouble();
return(new OptimizerResult(p, error));
};
var sut = new HyperbandOptimizer(
parameters,
maximumBudget: 81,
eta: 5,
skipLastIterationOfEachRound: false,
seed: 34);
var actual = sut.Optimize(minimize);
AssertOptimizerResults(Expected, actual);
}
/// <summary>
/// Optimization using Hyberband.
/// Returns all results, chronologically ordered.
/// </summary>
/// <param name="functionToMinimize"></param>
/// <returns></returns>
public OptimizerResult[] Optimize(HyperbandObjectiveFunction functionToMinimize)
{
var allResults = new List <OptimizerResult>();
for (int rounds = m_numberOfRounds; rounds >= 0; rounds--)
{
// Initial configurations count.
var initialConfigurationCount = (int)Math.Ceiling((m_totalBudgetPerRound / m_maximumBudget)
* (Math.Pow(m_eta, rounds) / (rounds + 1)));
// Initial budget per parameter set.
var initialBudget = m_maximumBudget * Math.Pow(m_eta, -rounds);
var parameterSets = RandomSearchOptimizer.SampleRandomParameterSets(initialConfigurationCount,
m_parameters, m_sampler);
var results = new ConcurrentBag <OptimizerResult>();
var iterations = m_skipLastIterationOfEachRound ? rounds : (rounds + 1);
for (int iteration = 0; iteration < iterations; iteration++)
{
// Run each of the parameter sets with budget
// and keep the best (configurationCount / m_eta) configurations
var configurationCount = initialConfigurationCount * Math.Pow(m_eta, -iteration);
var budget = initialBudget * Math.Pow(m_eta, iteration);
//Trace.WriteLine($"{(int)Math.Round(configurationCount)} configurations x {budget:F1} budget each");
foreach (var parameterSet in parameterSets)
{
var result = functionToMinimize(parameterSet, budget);
results.Add(result);
}
// Select a number of best configurations for the next loop
var configurationsToKeep = (int)Math.Round(configurationCount / m_eta);
parameterSets = results.OrderBy(v => v.Error)
.Take(configurationsToKeep)
.Select(v => v.ParameterSet)
.ToArray();
}
allResults.AddRange(results);
//Trace.WriteLine($" Lowest loss so far: {allResults.OrderBy(v => v.Error).First().Error:F4}");
}
return(allResults.ToArray());
}
/// <summary> /// Optimization using Hyberband. /// Returns the result which best minimizes the provided function. /// </summary> /// <param name="functionToMinimize"></param> /// <returns></returns> public OptimizerResult OptimizeBest(HyperbandObjectiveFunction functionToMinimize) => // Return the best model found. Optimize(functionToMinimize).Where(v => !double.IsNaN(v.Error)).OrderBy(r => r.Error).First();
