public void CFO_should_find_maximum_value_when_function_is_convex()
{
var searchSpace = new SearchSpace <LSE3DSearchSpace>();
var initValues = searchSpace.SampleFromFeatureSpace(searchSpace.Default);
var cfo = new CostFrugalTuner(searchSpace, Parameter.FromObject(initValues), false);
double bestMetric = 0;
for (int i = 0; i != 100; ++i)
{
var trialSettings = new TrialSettings()
{
TrialId = 0,
};
var param = cfo.Propose(trialSettings).AsType <LSE3DSearchSpace>();
var x = param.X;
var y = param.Y;
var z = param.Z;
var metric = LSE3D(x, y, z);
bestMetric = Math.Max(bestMetric, metric);
Output.WriteLine($"{i} x: {x} y: {y} z: {z}");
if (x == 10 && y == 10 && z == 10)
{
break;
}
cfo.Update(new TrialResult()
{
DurationInMilliseconds = 1 * 1000,
Metric = metric,
TrialSettings = trialSettings,
});
}
bestMetric.Should().BeGreaterThan(LSE3D(10, 10, 10) - 2);
}
public void GridSearchTuner_should_search_entire_naive_search_space()
{
// the behavior of a grid search tuner is it will exhaustively generate
// candidates from a grid of parameter value. And once exhausted, it will start
// over again.
// So we use the following way to test a grid search tuner: we go through the grid search
// twice, and check if all candidates are repeated twice. By doing so we know if grid search
// tuner either search grid of parameter value exhaustively or or generate candidates exactly one
// time in each search.
// default step is used when an option is continous, like double option or single option.
var defaultStep = 10;
var searchSpace = new SearchSpace <NaiveSearchSpace>();
var tuner = new GridSearchTuner(searchSpace, defaultStep);
var parameters = new List <Parameter>();
// calculate the total steps, which is the sum of all grid points.
var steps = 1;
foreach (var step in searchSpace.Step)
{
steps *= step ?? defaultStep;
}
foreach (var i in Enumerable.Range(0, steps * 2))
{
var settings = new TrialSettings();
parameters.Add(tuner.Propose(settings));
}
steps.Should().Be(600);
parameters.Distinct().Count().Should().Be(steps);
}
public void CFO_e2e_test()
{
var searchSpace = new SearchSpace <LbfgsOption>();
var initValues = searchSpace.SampleFromFeatureSpace(searchSpace.Default);
var cfo = new CostFrugalTuner(searchSpace, Parameter.FromObject(initValues));
for (int i = 0; i != 1000; ++i)
{
var trialSettings = new TrialSettings()
{
TrialId = i,
};
var param = cfo.Propose(trialSettings);
var option = param.AsType <CodeGen.LbfgsOption>();
option.L1Regularization.Should().BeInRange(0.03125f, 32768.0f);
option.L2Regularization.Should().BeInRange(0.03125f, 32768.0f);
cfo.Update(new TrialResult()
{
DurationInMilliseconds = i * 1000,
Metric = i,
TrialSettings = trialSettings,
});
}
}
public TrialResult Run(TrialSettings settings, IServiceProvider provider = null)
{
Task.Delay(_finishAfterNSeconds * 1000).Wait();
settings.ExperimentSettings.CancellationToken.ThrowIfCancellationRequested();
return(new TrialResult
{
TrialSettings = settings,
DurationInMilliseconds = _finishAfterNSeconds * 1000,
Metric = 1.000 + 0.01 * settings.TrialId,
});
}
public void CFO_should_start_from_init_point_if_provided()
{
var trialSettings = new TrialSettings()
{
TrialId = 0,
};
var searchSpace = new SearchSpace <LSE3DSearchSpace>();
var initValues = searchSpace.SampleFromFeatureSpace(searchSpace.Default);
var cfo = new CostFrugalTuner(searchSpace, Parameter.FromObject(initValues), true);
var param = cfo.Propose(trialSettings).AsType <LSE3DSearchSpace>();
var x = param.X;
var y = param.Y;
var z = param.Z;
(x * x + y * y + z * z).Should().Be(0);
}
public void Hyper_parameters_from_CFO_should_be_culture_invariant_string()
{
var searchSpace = new SearchSpace <LSE3DSearchSpace>();
var initValues = searchSpace.SampleFromFeatureSpace(searchSpace.Default);
var cfo = new CostFrugalTuner(searchSpace, Parameter.FromObject(initValues), true);
var originalCuture = Thread.CurrentThread.CurrentCulture;
var usCulture = new CultureInfo("en-US", false);
Thread.CurrentThread.CurrentCulture = usCulture;
Thread.CurrentThread.CurrentCulture.NumberFormat.NumberDecimalSeparator.Should().Be(".");
for (int i = 0; i != 100; ++i)
{
var trialSettings = new TrialSettings()
{
TrialId = i,
};
var param = cfo.Propose(trialSettings).AsType <LSE3DSearchSpace>();
param.X.Should().BeInRange(-10, 10);
}
var frCulture = new CultureInfo("fr-FR", false);
Thread.CurrentThread.CurrentCulture = frCulture;
Thread.CurrentThread.CurrentCulture.NumberFormat.NumberDecimalSeparator.Should().Be(",");
for (int i = 0; i != 100; ++i)
{
var trialSettings = new TrialSettings()
{
TrialId = i,
};
var param = cfo.Propose(trialSettings).AsType <LSE3DSearchSpace>();
param.X.Should().BeInRange(-10, 10);
}
Thread.CurrentThread.CurrentCulture = originalCuture;
}
public void CFO_should_find_minimum_value_when_function_is_convex()
{
var searchSpace = new SearchSpace <LSE3DSearchSpace>();
var initValues = searchSpace.SampleFromFeatureSpace(searchSpace.Default);
var cfo = new CostFrugalTuner(searchSpace, Parameter.FromObject(initValues), true);
double loss = 0;
for (int i = 0; i != 100; ++i)
{
var trialSettings = new TrialSettings()
{
TrialId = i,
};
var param = cfo.Propose(trialSettings).AsType <LSE3DSearchSpace>();
var x = param.X;
var y = param.Y;
var z = param.Z;
loss = LSE3D(x, y, z);
Output.WriteLine(loss.ToString());
Output.WriteLine($"{i} x: {x} y: {y} z: {z}");
if (x == -10 && y == -10 && z == -10)
{
break;
}
cfo.Update(new TrialResult()
{
DurationInMilliseconds = 1000,
Metric = loss,
TrialSettings = trialSettings,
});
}
loss.Should().BeLessThan(LSE3D(-10, -10, -10) + 2);
}
private void updateTrialSettings()
{
// update stim
globalSettings.numberOfTrials++;
globalSettings.globalAmbOrder = globalAmbOrder;
TrialSettings tmp = new TrialSettings();
tmp.trialNumber = globalSettings.numberOfTrials;
tmp.trialType = trialType;
for (int i = 0; i < teststimuli.Count; i++)
{
tmp.testStimuli.Add(teststimuli[i]);
}
tmp.numberOfStimuli = teststimuli.Count;
globalSettings.Save_trials_Json.Add(tmp);
}