public void Nest_search_space_mapping_to_feature_space_test()
{
var ss = new SearchSpace <NestSearchSpace>();
var param = ss.SampleFromFeatureSpace(new[] { 0.0, 0, 0, 0, 0, 0, 0, 0 });
var features = ss.MappingToFeatureSpace(param);
features.Should().Equal(0, 0, 0, 0, 0, 0, 0, 0);
param = ss.SampleFromFeatureSpace(new[] { 0.5, 0.5, 0, 0.5, 0.5, 0.5, 0.5, 0.5 });
features = ss.MappingToFeatureSpace(param);
features.Should().Equal(0.5, 0.5, 0, 0.5, 0.5, 0.5, 0.5, 0.5);
}
public void SearchSpace_sample_from_feature_space_test()
{
var ss = new SearchSpace <BasicSearchSpace>();
var param = ss.SampleFromFeatureSpace(new[] { 0.0, 0, 0, 0, 0, 0 });
param.ChoiceStr.Should().Be("a");
param.UniformDouble.Should().Be(-1000);
param.UniformFloat.Should().Be(-1000);
param.UniformInt.Should().Be(-1000);
param = ss.SampleFromFeatureSpace(new[] { 0.5, 0.5, 0.5, 0.5, 0.5, 0.5 });
param.ChoiceStr.Should().Be("c");
param.UniformDouble.Should().Be(0);
param.UniformFloat.Should().Be(0);
param.UniformInt.Should().Be(0);
}
public void CFO_e2e_test()
{
var searchSpace = new SearchSpace <LbfgsOption>();
var initValues = searchSpace.SampleFromFeatureSpace(searchSpace.Default);
var cfo = new CostFrugalTuner(searchSpace, Parameter.FromObject(initValues));
for (int i = 0; i != 1000; ++i)
{
var trialSettings = new TrialSettings()
{
TrialId = i,
};
var param = cfo.Propose(trialSettings);
var option = param.AsType <CodeGen.LbfgsOption>();
option.L1Regularization.Should().BeInRange(0.03125f, 32768.0f);
option.L2Regularization.Should().BeInRange(0.03125f, 32768.0f);
cfo.Update(new TrialResult()
{
DurationInMilliseconds = i * 1000,
Metric = i,
TrialSettings = trialSettings,
});
}
}
public void CFO_should_find_maximum_value_when_function_is_convex()
{
var searchSpace = new SearchSpace <LSE3DSearchSpace>();
var initValues = searchSpace.SampleFromFeatureSpace(searchSpace.Default);
var cfo = new CostFrugalTuner(searchSpace, Parameter.FromObject(initValues), false);
double bestMetric = 0;
for (int i = 0; i != 100; ++i)
{
var trialSettings = new TrialSettings()
{
TrialId = 0,
};
var param = cfo.Propose(trialSettings).AsType <LSE3DSearchSpace>();
var x = param.X;
var y = param.Y;
var z = param.Z;
var metric = LSE3D(x, y, z);
bestMetric = Math.Max(bestMetric, metric);
Output.WriteLine($"{i} x: {x} y: {y} z: {z}");
if (x == 10 && y == 10 && z == 10)
{
break;
}
cfo.Update(new TrialResult()
{
DurationInMilliseconds = 1 * 1000,
Metric = metric,
TrialSettings = trialSettings,
});
}
bestMetric.Should().BeGreaterThan(LSE3D(10, 10, 10) - 2);
}
public Parameter Propose(SearchSpace searchSpace)
{
var d = searchSpace.FeatureSpaceDim;
var featureVec = Enumerable.Repeat(0, d).Select(i => _rnd.NextDouble()).ToArray();
return(searchSpace.SampleFromFeatureSpace(featureVec));
}
public IEnumerable <Parameter> Propose()
{
var steps = _searchSpace.Step.Select(x => x ?? _stepSize)
.Select(x => Enumerable.Range(0, x).Select(i => i * 1.0 / x).ToArray());
foreach (var featureVec in CartesianProduct(steps))
{
yield return(_searchSpace.SampleFromFeatureSpace(featureVec));
}
}
public void Nest_searchSpace_sample_from_feature_space_test()
{
var option = new NestSearchSpace()
{
BasicSS = new BasicSearchSpace()
{
DefaultSearchSpace = new DefaultSearchSpace()
{
Strings = new[] { "B", "C", "D" },
},
},
};
var ss = new SearchSpace <NestSearchSpace>(option);
ss.FeatureSpaceDim.Should().Be(8);
var param = ss.SampleFromFeatureSpace(new[] { 0.0, 0, 0, 0, 0, 0, 0, 0 });
param.UniformDouble.Should().Be(-1000);
param.UniformFloat.Should().Be(-1000);
param.BasicSS.UniformInt.Should().Be(-1000);
param.BasicSS.UniformDouble.Should().Be(-1000);
param.BasicSS.UniformFloat.Should().Be(-1000);
param.BasicSS.ChoiceStr.Should().Be("a");
param.BasicSS.DefaultSearchSpace.Strings.Should().BeEquivalentTo("B", "C", "D");
param.BasicSS.ChoiceBoolean.Should().BeFalse();
param.BasicSS.JTokenType.Should().Be(JsonTokenType.EndObject);
param = ss.SampleFromFeatureSpace(new[] { 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5 });
param.UniformDouble.Should().Be(0);
param.UniformFloat.Should().Be(0);
param.BasicSS.UniformInt.Should().Be(0);
param.BasicSS.UniformDouble.Should().Be(0);
param.BasicSS.UniformFloat.Should().Be(0);
param.BasicSS.ChoiceStr.Should().Be("c");
param.BasicSS.DefaultSearchSpace.Strings.Should().BeEquivalentTo("B", "C", "D");
param.BasicSS.ChoiceBoolean.Should().BeTrue();
param.BasicSS.JTokenType.Should().Be(JsonTokenType.Null);
}
public void Search_space_add_option_test()
{
var ss = new SearchSpace();
ss.FeatureSpaceDim.Should().Be(0);
ss.Add("A", new UniformIntOption(-1000, 1000));
ss.FeatureSpaceDim.Should().Be(1);
var param = ss.SampleFromFeatureSpace(new[] { 0.5 });
param["A"].AsType <int>().Should().Be(0);
}
public void Parameter_AsType_should_be_culture_invariant()
{
var originalCuture = Thread.CurrentThread.CurrentCulture;
var culture = new CultureInfo("ru", false);
Thread.CurrentThread.CurrentCulture = culture;
var ss = new SearchSpace();
ss.Add("_SampleSize", new UniformDoubleOption(10000, 20000));
var parameter = ss.SampleFromFeatureSpace(new[] { 0.5 });
parameter["_SampleSize"].AsType <double>().Should().Be(15000.0);
Thread.CurrentThread.CurrentCulture = originalCuture;
}
public void CFO_should_start_from_init_point_if_provided()
{
var trialSettings = new TrialSettings()
{
TrialId = 0,
};
var searchSpace = new SearchSpace <LSE3DSearchSpace>();
var initValues = searchSpace.SampleFromFeatureSpace(searchSpace.Default);
var cfo = new CostFrugalTuner(searchSpace, Parameter.FromObject(initValues), true);
var param = cfo.Propose(trialSettings).AsType <LSE3DSearchSpace>();
var x = param.X;
var y = param.Y;
var z = param.Z;
(x * x + y * y + z * z).Should().Be(0);
}
public void Search_space_remove_option_test()
{
var option = new BasicSearchSpace();
var ss = new SearchSpace <BasicSearchSpace>(option);
ss.FeatureSpaceDim.Should().Be(6);
ss.Remove("UniformInt").Should().BeTrue();
ss.FeatureSpaceDim.Should().Be(5);
ss.Keys.Should().BeEquivalentTo("ChoiceStr", "UniformDouble", "UniformFloat", "ChoiceBoolean", "JTokenType");
var parameter = ss.SampleFromFeatureSpace(new double[] { 0, 0, 0, 0, 0 });
parameter.DefaultSearchSpace.Strings.Should().BeEquivalentTo("A", "B", "C");
parameter.DefaultSearchSpace.String.Should().BeNullOrEmpty();
parameter.ChoiceStr.Should().Be("a");
parameter.ChoiceBoolean.Should().BeFalse();
parameter.JTokenType.Should().Be(JsonTokenType.EndObject);
}
public void Hyper_parameters_from_CFO_should_be_culture_invariant_string()
{
var searchSpace = new SearchSpace <LSE3DSearchSpace>();
var initValues = searchSpace.SampleFromFeatureSpace(searchSpace.Default);
var cfo = new CostFrugalTuner(searchSpace, Parameter.FromObject(initValues), true);
var originalCuture = Thread.CurrentThread.CurrentCulture;
var usCulture = new CultureInfo("en-US", false);
Thread.CurrentThread.CurrentCulture = usCulture;
Thread.CurrentThread.CurrentCulture.NumberFormat.NumberDecimalSeparator.Should().Be(".");
for (int i = 0; i != 100; ++i)
{
var trialSettings = new TrialSettings()
{
TrialId = i,
};
var param = cfo.Propose(trialSettings).AsType <LSE3DSearchSpace>();
param.X.Should().BeInRange(-10, 10);
}
var frCulture = new CultureInfo("fr-FR", false);
Thread.CurrentThread.CurrentCulture = frCulture;
Thread.CurrentThread.CurrentCulture.NumberFormat.NumberDecimalSeparator.Should().Be(",");
for (int i = 0; i != 100; ++i)
{
var trialSettings = new TrialSettings()
{
TrialId = i,
};
var param = cfo.Propose(trialSettings).AsType <LSE3DSearchSpace>();
param.X.Should().BeInRange(-10, 10);
}
Thread.CurrentThread.CurrentCulture = originalCuture;
}
public void CFO_should_find_minimum_value_when_function_is_convex()
{
var searchSpace = new SearchSpace <LSE3DSearchSpace>();
var initValues = searchSpace.SampleFromFeatureSpace(searchSpace.Default);
var cfo = new CostFrugalTuner(searchSpace, Parameter.FromObject(initValues), true);
double loss = 0;
for (int i = 0; i != 100; ++i)
{
var trialSettings = new TrialSettings()
{
TrialId = i,
};
var param = cfo.Propose(trialSettings).AsType <LSE3DSearchSpace>();
var x = param.X;
var y = param.Y;
var z = param.Z;
loss = LSE3D(x, y, z);
Output.WriteLine(loss.ToString());
Output.WriteLine($"{i} x: {x} y: {y} z: {z}");
if (x == -10 && y == -10 && z == -10)
{
break;
}
cfo.Update(new TrialResult()
{
DurationInMilliseconds = 1000,
Metric = loss,
TrialSettings = trialSettings,
});
}
loss.Should().BeLessThan(LSE3D(-10, -10, -10) + 2);
}
public void Search_space_default_search_space_test()
{
var defaultSearchSpace = new DefaultSearchSpace()
{
String = "String",
Int = 10,
Bool = true,
JTokenType = JsonTokenType.Null,
};
var ss = new SearchSpace <DefaultSearchSpace>(defaultSearchSpace);
var param = ss.SampleFromFeatureSpace(new double[0]);
param.Int.Should().Be(10);
param.Float.Should().Be(0f);
param.Double.Should().Be(0);
param.Bool.Should().BeTrue();
param.String.Should().Be("String");
param.Strings.Should().BeEquivalentTo("A", "B", "C");
param.JTokenType.Should().Be(JsonTokenType.Null);
param.NullString.Should().BeNull();
ss.FeatureSpaceDim.Should().Be(0);
ss.MappingToFeatureSpace(param).Should().HaveCount(0);
}
public Parameter Propose()
{
var defaultFeatureVec = _searchSpace.Default;
return(_searchSpace.SampleFromFeatureSpace(defaultFeatureVec));
}