public void MemoryLimitWorks()
{
// Check, if current OS is Linux.
Skip.IfNot(Environment.OSVersion.Platform == PlatformID.Unix);
var timeout = TimeSpan.FromSeconds(10);
var memoryLimitMegabyte = 1;
var tunerConfig = new AlgorithmTunerConfiguration.AlgorithmTunerConfigurationBuilder().SetCpuTimeout(timeout).Build(1);
var timer = new Stopwatch();
timer.Start();
var lingelingRunner = new LingelingRunner(
new Dictionary <string, IAllele>(),
LingelingRunnerTests.PathToExecutable,
tunerConfig,
memoryLimitMegabyte);
var runner = lingelingRunner.Run(
new InstanceSeedFile(LingelingRunnerTests.PathToTestInstance, LingelingRunnerTests.TestInstanceSeed),
this._cancellationTokenSource.Token);
runner.Wait();
timer.Stop();
runner.Result.ShouldNotBeNull();
runner.Result.IsCancelled.ShouldBeTrue();
runner.Result.Runtime.ShouldBe(timeout);
timer.Elapsed.ShouldBeLessThan(timeout);
}
/// <summary>
/// Creates a simple <see cref="GenomeBuilder"/> that has a mutation rate of 0 and uses the parameter tree from
/// <see cref="GenomeBuilderTest.BuildParameterTree"/>.
/// </summary>
/// <returns>The genome builder.</returns>
private static GenomeBuilder CreateGenomeBuilderWithoutRandomMutation()
{
var configuration =
new AlgorithmTunerConfiguration.AlgorithmTunerConfigurationBuilder()
.SetMutationRate(0)
.Build(maximumNumberParallelEvaluations: 1);
return(new GenomeBuilder(GenomeBuilderTest.BuildParameterTree(), configuration));
}
public void ConstructorThrowsForMissingRandomForestConfiguration()
{
var incompleteConfig = new AlgorithmTunerConfiguration.AlgorithmTunerConfigurationBuilder()
.SetInstanceNumbers(1, 1)
.SetEngineeredProportion(1)
.SetMaximumNumberParallelEvaluations(1)
.Build();
Assert.Throws <KeyNotFoundException>(() => this.InitializeAndPopulateProperties(this.GetDefaultParameterTree(), incompleteConfig));
}
/// <summary>
/// Creates a population with the given configuration.
/// </summary>
/// <param name="populationSize">The population's size.</param>
/// <param name="maxAge">Maximum age of a genome.</param>
/// <param name="populationMutantRatio">The ratio of non competitive genomes to replace each population.
/// </param>
/// <returns>The population.</returns>
private static Population CreatePopulation(int populationSize, int maxAge, double populationMutantRatio)
{
var configuration = new AlgorithmTunerConfiguration.AlgorithmTunerConfigurationBuilder()
.SetMaxGenomeAge(maxAge)
.SetPopulationSize(populationSize)
.SetPopulationMutantRatio(populationMutantRatio)
.Build(maximumNumberParallelEvaluations: 1);
return(new Population(configuration));
}
public void ConstructorThrowsExceptionOnMissingParameterTree()
{
var configuration =
new AlgorithmTunerConfiguration.AlgorithmTunerConfigurationBuilder().Build(maximumNumberParallelEvaluations: 1);
Assert.Throws <ArgumentNullException>(
() => new GenomeBuilder(
parameterTree: null,
configuration: configuration));
}
public void ExtractRunStatisticsCreatesCancelledResult(string consoleOutput)
{
var timeout = TimeSpan.FromSeconds(5);
var tunerConfig = new AlgorithmTunerConfiguration.AlgorithmTunerConfigurationBuilder().SetCpuTimeout(timeout).Build(1);
var lingelingRunner = new LingelingRunner(new Dictionary <string, IAllele>(), "ling", tunerConfig, 4000);
var runResult = lingelingRunner.ExtractRunStatistics(consoleOutput);
runResult.IsCancelled.ShouldBeTrue();
runResult.Runtime.ShouldBe(timeout);
}
public void FinishPhaseKeepsIncumbentForMaximumReplacementRate()
{
// Create population along with an incumbent.
var basePopulation = this.CreatePopulation();
var incumbent = this.CreateIncumbentGenomeWrapper();
basePopulation.AddGenome(new Genome(incumbent.IncumbentGenome), isCompetitive: true);
// Create strategy with a replacement rate of 1.
var configuration = new AlgorithmTunerConfiguration.AlgorithmTunerConfigurationBuilder()
.SetMaximumNumberParallelEvaluations(1)
.AddDetailedConfigurationBuilder(
RegressionForestArgumentParser.Identifier,
new GenomePredictionRandomForestConfig.GenomePredictionRandomForestConfigBuilder())
.AddDetailedConfigurationBuilder(
CovarianceMatrixAdaptationStrategyArgumentParser.Identifier,
new CovarianceMatrixAdaptationStrategyConfiguration.CovarianceMatrixAdaptationStrategyConfigurationBuilder()
.SetFixInstances(this.FixInstances)
.SetReplacementRate(1)
.SetMaximumNumberGenerations(30));
this.Strategy = new LocalCovarianceMatrixAdaptationStrategy <TestInstance, IntegerResult>(
configuration.Build(),
this.ParameterTree,
this.GenomeBuilder,
this.GenomeSorter,
this.ResultStorageActor);
// Perform an iteration and finish the phase.
this.Strategy.Initialize(basePopulation, incumbent, this.SingleTestInstance);
this.Strategy.PerformIteration(0, this.SingleTestInstance);
this.Strategy.DumpStatus();
var updatedPopulation = this.Strategy.FinishPhase(basePopulation);
// Incumbent should still exist.
Assert.True(
updatedPopulation.GetCompetitiveIndividuals().Contains(incumbent.IncumbentGenome, new Genome.GeneValueComparator()),
"Updated population should contain incumbent, but does not.");
// Age structure should be correct.
for (int age = 0; age < 3; age++)
{
Assert.True(
basePopulation.GetCompetitiveIndividuals().Count(individual => individual.Age == age)
== updatedPopulation.GetCompetitiveIndividuals().Count(individual => individual.Age == age),
$"Different number of genomes with age {age}.");
}
Assert.False(
updatedPopulation.GetCompetitiveIndividuals().Any(individual => individual.Age < 0 || individual.Age > 3),
"There exists a genome with age not in age range!");
}
public void ConstructorThrowsForWrongTypeRandomForestConfiguration()
{
var wrongConfig = new AlgorithmTunerConfiguration.AlgorithmTunerConfigurationBuilder()
.SetInstanceNumbers(1, 1)
.SetEngineeredProportion(1)
.SetMaximumNumberParallelEvaluations(1)
.AddDetailedConfigurationBuilder(
RegressionForestArgumentParser.Identifier,
new AlgorithmTunerConfiguration.AlgorithmTunerConfigurationBuilder().SetMaximumNumberParallelEvaluations(1))
.Build();
Assert.Throws <InvalidCastException>(() => this.InitializeAndPopulateProperties(this.GetDefaultParameterTree(), wrongConfig));
}
/// <summary>
/// Creates a new <see cref="TournamentSelector{NoOperation, TestInstance, TestResult}"/> in the
/// actor system.
/// </summary>
/// <param name="winnerPercentage">The percentage of winners in a mini tournament.</param>
/// <param name="miniTournamentSize">The size of a mini tournament.</param>
/// <returns>An <see cref="IActorRef"/> to the created tournament selector.</returns>
private IActorRef CreateTournamentSelector(double winnerPercentage, int miniTournamentSize)
{
// Build configuration.
AlgorithmTunerConfiguration config = new AlgorithmTunerConfiguration.AlgorithmTunerConfigurationBuilder()
.SetTournamentWinnerPercentage(winnerPercentage)
.SetMaximumMiniTournamentSize(miniTournamentSize)
.Build(maximumNumberParallelEvaluations: 1);
// Build tournament selector itself.
return(this.Sys.ActorOf(
props: new TournamentSelectorPropsBuilder().SetConfiguration(config).Build(this._resultStorageActorRef),
name: TournamentSelectorTest.SpecialTournamentSelectorName));
}
/// <summary>
/// Builds an <see cref="InstanceSelector{I}"/>.
/// </summary>
/// <param name="numInstancesAtStart">Number instances returned for first generation.</param>
/// <param name="numInstancesAtEnd">Number instances returned all generations between goal generation and last
/// generation.</param>
/// <param name="goalGeneration">First generation for which the maximum number of instances should be returned.
/// </param>
/// <param name="generations">The number of generations.</param>
/// <returns>The built <see cref="InstanceSelector{I}"/>.</returns>
private InstanceSelector <TestInstance> BuildSelector(
int numInstancesAtStart,
int numInstancesAtEnd,
int goalGeneration,
int generations)
{
AlgorithmTunerConfiguration configuration = new AlgorithmTunerConfiguration.AlgorithmTunerConfigurationBuilder()
.SetInstanceNumbers(numInstancesAtStart, numInstancesAtEnd)
.SetGenerations(generations)
.SetGoalGeneration(goalGeneration)
.Build(maximumNumberParallelEvaluations: 4);
return(new InstanceSelector <TestInstance>(this._instanceList, configuration));
}
public void WorstGenomesAreNeverWinners()
{
// Create some genomes.
int numberGenomes = 20;
var genomes = this.CreateGenomesAdheringToParameterTree(numberGenomes);
// Create a configuration using a certain winner percentage and mini tournament size.
double winnerPercentage = 0.1;
int miniTournamentSize = 10;
AlgorithmTunerConfiguration configuration = new AlgorithmTunerConfiguration.AlgorithmTunerConfigurationBuilder()
.SetTournamentWinnerPercentage(winnerPercentage)
.SetMaximumMiniTournamentSize(miniTournamentSize)
.Build(maximumNumberParallelEvaluations: 1);
// Create tournament selector using an evaluator which sorts by a certain parameter, highest first.
var tournamentSelectorProps = Props.Create(
() =>
new TournamentSelector <ExtractIntegerValue, TestInstance, IntegerResult>(
new ExtractIntegerValueCreator(),
new SortByValue(),
configuration,
this._resultStorageActorRef,
TournamentSelectorTest.CreateParameterTree()));
var tournamentSelector = this.Sys.ActorOf(tournamentSelectorProps, name: TournamentSelectorTest.SpecialTournamentSelectorName);
// Try selecting the best genomes via mini tournaments multiple times and make sure the worst genomes never
// get chosen:
int worstPossibleWinnerValue = miniTournamentSize - (int)(winnerPercentage * miniTournamentSize);
for (int i = 0; i < 15; i++)
{
// Send a select command.
tournamentSelector.Tell(new SelectCommand <TestInstance>(genomes, this._singleTestInstance, 0));
// Wait for results.
var results = this.ExpectMsg <SelectionResultMessage <IntegerResult> >();
// Make sure none belongs to the worst genomes.
foreach (var genome in results.CompetitiveParents)
{
int geneValue =
(int)genome.CreateMutableGenome().GetGeneValue(ExtractIntegerValue.ParameterName).GetValue();
Assert.True(
geneValue >= worstPossibleWinnerValue,
$"Found a winner with value {geneValue}, but expected at least {worstPossibleWinnerValue}.");
}
}
}
public void MutateMayChangeEveryParameter()
{
// Create genome builder with a mutation rate of 1.
var configuration = new AlgorithmTunerConfiguration.AlgorithmTunerConfigurationBuilder()
.SetMutationRate(1)
.Build(maximumNumberParallelEvaluations: 1);
var genomeBuilder = new GenomeBuilder(GenomeBuilderTest.BuildParameterTree(), configuration);
// Create genome.
var genome = GenomeBuilderTest.BuildFittingGenome();
// Prepare structure to store which values have been mutated.
string[] identifiers =
{
GenomeBuilderTest.DecisionParameter,
GenomeBuilderTest.SmallValueParameter,
GenomeBuilderTest.DiscreteParameter,
GenomeBuilderTest.ContinuousParameter,
};
Dictionary <string, bool> hasBeenMutated = identifiers.ToDictionary(
identifier => identifier,
identifier => false);
Dictionary <string, IAllele> originalValues = identifiers.ToDictionary(
identifier => identifier,
identifier => genome.GetGeneValue(identifier));
// Do a lot of mutations.
for (int i = 0; i < GenomeBuilderTest.loopCountForRandomTests; i++)
{
genomeBuilder.Mutate(genome);
// After each one, check which genes have changed...
foreach (string identifier in identifiers)
{
bool valueChanged = !object.Equals(originalValues[identifier], genome.GetGeneValue(identifier));
if (valueChanged)
{
// ...and store them.
hasBeenMutated[identifier] = true;
}
}
}
// Finally check that everything has been changed at least once.
Assert.True(
hasBeenMutated.All(keyValuePair => keyValuePair.Value),
$"Genes {TestUtils.PrintList(hasBeenMutated.Where(keyValuePair => !keyValuePair.Value).Select(keyValuePair => keyValuePair.Key))} have not been mutated.");
}
public void CancellationWorks()
{
// Check, if current OS is Linux.
Skip.IfNot(Environment.OSVersion.Platform == PlatformID.Unix);
var timeout = TimeSpan.FromSeconds(30);
var memoryLimitMegabyte = 4000;
var tunerConfig = new AlgorithmTunerConfiguration.AlgorithmTunerConfigurationBuilder().SetCpuTimeout(timeout).Build(1);
var timer = new Stopwatch();
timer.Start();
// Run Lingeling.
var lingelingRunner = new LingelingRunner(
new Dictionary <string, IAllele>(),
LingelingRunnerTests.PathToExecutable,
tunerConfig,
memoryLimitMegabyte);
var runner = lingelingRunner.Run(
new InstanceSeedFile(LingelingRunnerTests.PathToTestInstance, LingelingRunnerTests.TestInstanceSeed),
this._cancellationTokenSource.Token);
// Cancel task and expect it to be cancelled.
try
{
Thread.Sleep(100);
this._cancellationTokenSource.Cancel();
runner.Wait();
Assert.True(false, "Expected a task cancelled exception.");
}
catch (AggregateException aggregateException)
{
timer.Stop();
aggregateException.InnerExceptions.Count.ShouldBe(1);
var innerException = aggregateException.InnerExceptions.Single();
innerException.ShouldBeOfType <TaskCanceledException>();
runner.IsCanceled.ShouldBeTrue();
timer.Elapsed.ShouldBeLessThan(TimeSpan.FromSeconds(1));
}
catch (Exception)
{
Assert.True(false, "Expected a task cancelled exception.");
}
}
public void AllDummyParametersAreFiltered()
{
Randomizer.Reset();
Randomizer.Configure(0);
var parameterTree = GurobiUtils.CreateParameterTree();
var config = new AlgorithmTunerConfiguration.AlgorithmTunerConfigurationBuilder().Build(1);
var genomeBuilder = new GenomeBuilder(parameterTree, config);
var genome = genomeBuilder.CreateRandomGenome(0);
var filteredGenes = genome.GetFilteredGenes(parameterTree);
foreach (var filteredGenesKey in filteredGenes.Keys)
{
filteredGenesKey.ShouldNotContain("Indicator");
}
}
/// <summary>
/// Initializes a <see cref="TournamentSelector{TTargetAlgorithm,TInstance,TResult}"/>.
/// It creates <see cref="MiniTournamentActor{TTargetAlgorithm,TInstance,TResult}"/>s which create the
/// required <see cref="EvaluationActor{TTargetAlgorithm,TInstance,TResult}"/>s.
/// The hierarchy must be as in OPTANO Algorithm Tuner to ensure that <see cref="_genomeSorter"/> finds the
/// actors.
/// </summary>
private void BuildTournamentSelector()
{
var configuration = new AlgorithmTunerConfiguration.AlgorithmTunerConfigurationBuilder()
.SetEnableRacing(true)
.Build(GenomeSorterTest.MaximumNumberParallelEvaluations);
var parameterTree = new ParameterTree(
root: new ValueNode <int>(ExtractIntegerValue.ParameterName, new IntegerDomain()));
var tournamentSelector = this.Sys.ActorOf(
Props.Create(
() => new TournamentSelector <ExtractIntegerValue, TestInstance, IntegerResult>(
new ExtractIntegerValueCreator(),
this._runEvaluator,
configuration,
this._resultStorageActor,
parameterTree)),
AkkaNames.TournamentSelector);
}
public void LogFinishedGenerationAndLogFinalIncumbentGenerationThrowForUnknownDirectory(bool useLogFinishedGeneration)
{
var configuration = new AlgorithmTunerConfiguration.AlgorithmTunerConfigurationBuilder()
.SetLogFilePath("foo/bar.txt")
.Build(1);
var writer = new LogWriter <InstanceFile, RuntimeResult>(this._parameterTree, configuration);
if (useLogFinishedGeneration)
{
Assert.Throws <DirectoryNotFoundException>(
() => writer.LogFinishedGeneration(1, 1, this._testGenomeResults, true));
}
else
{
Assert.Throws <DirectoryNotFoundException>(
() => writer.LogFinalIncumbentGeneration(1, this._testGenomeResults, 1, 2, true));
}
}
public void TryToGetResultFromStringArrayWorksForGurobiResults()
{
var timeout = TimeSpan.FromSeconds(30);
var gurobiResult = new GurobiResult(0.5, timeout, TargetAlgorithmStatus.CancelledByTimeout, true);
var gurobiConfiguration = new GurobiRunnerConfiguration.GurobiRunnerConfigBuilder().Build(timeout);
var tunerConfiguration = new AlgorithmTunerConfiguration.AlgorithmTunerConfigurationBuilder().SetCpuTimeout(timeout)
.Build(1);
var targetAlgorithmFactory =
new GurobiRunnerFactory(gurobiConfiguration, tunerConfiguration) as
ITargetAlgorithmFactory <GurobiRunner, InstanceSeedFile, GurobiResult>;
targetAlgorithmFactory.TryToGetResultFromStringArray(gurobiResult.ToStringArray(), out var result).ShouldBeTrue();
result.TargetAlgorithmStatus.ShouldBe(gurobiResult.TargetAlgorithmStatus);
result.IsCancelled.ShouldBe(gurobiResult.IsCancelled);
result.Runtime.ShouldBe(gurobiResult.Runtime);
result.Gap.ShouldBe(gurobiResult.Gap);
result.HasValidSolution.ShouldBe(gurobiResult.HasValidSolution);
}
public void CorrectInstancesAreUsedForSelectCommand()
{
// Build tournament selector with a small tournament size to test using several workers
var config = new AlgorithmTunerConfiguration.AlgorithmTunerConfigurationBuilder()
.SetMaximumMiniTournamentSize(1)
.Build(maximumNumberParallelEvaluations: 1);
var tournamentSelector = this.Sys.ActorOf(
props: new TournamentSelectorPropsBuilder().SetConfiguration(config).Build(this._resultStorageActorRef),
name: TournamentSelectorTest.SpecialTournamentSelectorName);
// Send a simple select command and wait for completion.
var originalCommand = new SelectCommand <TestInstance>(
this.CreateGenomesAdheringToParameterTree(2),
new List <TestInstance> {
new TestInstance("0")
},
0);
tournamentSelector.Tell(originalCommand);
this.ExpectMsg <SelectionResultMessage <TestResult> >();
// Build a new select command with several new instances.
var instances = new List <TestInstance>()
{
new TestInstance("1"),
new TestInstance("2"),
};
var genomes = this.CreateGenomesAdheringToParameterTree(2);
var selectCommand = new SelectCommand <TestInstance>(genomes, instances, 1);
// Send it & wait for completion.
tournamentSelector.Tell(selectCommand);
this.ExpectMsg <SelectionResultMessage <TestResult> >();
// Check that storage actor has knowledge about both new instances for both genomes.
foreach (var genome in genomes)
{
foreach (var instance in instances)
{
this._resultStorageActorRef.Tell(new ResultRequest <TestInstance>(genome, instance));
this.ExpectMsg <ResultMessage <TestInstance, TestResult> >();
}
}
}
public void AkkaConfigurationIsTakenFromNewParametersOnContinue()
{
// Specify parameters for simple original configuration
var originalConfig =
new AlgorithmTunerConfiguration.AlgorithmTunerConfigurationBuilder()
.SetMaximumNumberParallelEvaluations(2)
.SetGenerations(1)
.SetGoalGeneration(0)
.SetInstanceNumbers(1, 1)
.SetEngineeredProportion(0)
.AddDetailedConfigurationBuilder(
RegressionForestArgumentParser.Identifier,
new GenomePredictionRandomForestConfig.GenomePredictionRandomForestConfigBuilder())
.Build();
this.WriteConfigurationToStatusFile(originalConfig, this._statusFilePath);
// Call run with continue option and different Akka configuraiton.
var args = new[]
{
"--continue",
// Akka config parameters
"--port=1234",
"--ownHostName=foo",
"--maxParallelEvaluations=1",
};
Master <NoOperation, TestInstance, TestResult> .Run(
args,
(config, pathToTrainingInstances, pathToTestInstances) =>
{
// Check new parameters get used for Akka config.
Assert.Equal(
1234,
config.AkkaConfiguration.GetInt("akka.remote.dot-netty.tcp.port"));
Assert.Equal(
"foo",
config.AkkaConfiguration.GetString("akka.remote.dot-netty.tcp.hostname"));
// Return an algorithm tuner that quickly terminates.
return(this.BuildSimpleAlgorithmTuner(originalConfig, pathToTrainingInstances, pathToTestInstances));
});
}
public void MutateChangesGivenGenomeIfMutationRateIs1()
{
// Create genome builder with a mutation rate of 1.
var configuration =
new AlgorithmTunerConfiguration.AlgorithmTunerConfigurationBuilder()
.SetMutationRate(1)
.Build(maximumNumberParallelEvaluations: 1);
var genomeBuilder = new GenomeBuilder(GenomeBuilderTest.BuildParameterTree(), configuration);
// Build a fitting genome and store original description.
var genome = GenomeBuilderTest.BuildFittingGenome();
string originalDescription = genome.ToString();
// Call mutate and check genome changed.
genomeBuilder.Mutate(genome);
Assert.NotEqual(
originalDescription,
genome.ToString());
}
public void InstanceFolderIsTakenFromNewParametersOnContinue()
{
// Specify parameters for simple original configuration
var originalConfig =
new AlgorithmTunerConfiguration.AlgorithmTunerConfigurationBuilder()
.SetMaximumNumberParallelEvaluations(2)
.SetGenerations(1)
.SetGoalGeneration(0)
.SetInstanceNumbers(1, 1)
.SetEngineeredProportion(0)
.AddDetailedConfigurationBuilder(
RegressionForestArgumentParser.Identifier,
new GenomePredictionRandomForestConfig.GenomePredictionRandomForestConfigBuilder())
.Build();
this.WriteConfigurationToStatusFile(originalConfig, this._statusFilePath);
// Call run with continue option and different instance folder.
var args = new[]
{
"--continue",
// Instance folder parameters
"--trainingInstanceFolder=foo",
"--testInstanceFolder=bar",
};
Master <NoOperation, TestInstance, TestResult> .Run(
args,
(config, pathToTrainingInstances, pathToTestInstances) =>
{
// Check new parameter get used for instance folders.
Assert.Equal(
"foo",
pathToTrainingInstances);
Assert.Equal(
"bar",
pathToTestInstances);
// Return an algorithm tuner that quickly terminates.
return(this.BuildSimpleAlgorithmTuner(originalConfig, pathToTrainingInstances, pathToTestInstances));
});
}
public void MutateRespectsMutationRate()
{
// Create genome builder with a mutation rate of 0.35.
double mutationRate = 0.35;
var configuration =
new AlgorithmTunerConfiguration.AlgorithmTunerConfigurationBuilder()
.SetMutationRate(mutationRate)
.Build(maximumNumberParallelEvaluations: 1);
var genomeBuilder = new GenomeBuilder(GenomeBuilderTest.BuildParameterTree(), configuration);
// Create genome.
var genome = GenomeBuilderTest.BuildFittingGenome();
// For a lot of iterations:
IAllele oldGeneValue = genome.GetGeneValue(GenomeBuilderTest.ContinuousParameter);
int changeCount = 0;
int numberLoops = 1000;
for (int i = 0; i < numberLoops; i++)
{
// Mutate the genome ...
genomeBuilder.Mutate(genome);
// ... compare the continuous parameter gene...
IAllele newGeneValue = genome.GetGeneValue(GenomeBuilderTest.ContinuousParameter);
if (!object.Equals(newGeneValue, oldGeneValue))
{
changeCount++;
}
// ... and count the number of times it changes.
oldGeneValue = newGeneValue;
}
// Finally compare the number of mutations with the expected number.
double expectedNumberMutations = mutationRate * numberLoops;
Assert.True(
Math.Abs(expectedNumberMutations - changeCount) <= 0.1 * expectedNumberMutations,
"Number of mutations was not as expected.");
}
public void WinnersAreSortedCorrectly()
{
// Build configuration with a specific winner percentage and tournament size.
AlgorithmTunerConfiguration config = new AlgorithmTunerConfiguration.AlgorithmTunerConfigurationBuilder()
.SetTournamentWinnerPercentage(0.5)
.SetMaximumMiniTournamentSize(2)
.Build(maximumNumberParallelEvaluations: 1);
// Build tournament selector selecting winners by a specific parameter.
var tournamentSelector = this.Sys.ActorOf(
props: Props.Create(
() => new TournamentSelector <ExtractIntegerValue, TestInstance, IntegerResult>(
new ExtractIntegerValueCreator(),
new SortByValue(),
config,
this._resultStorageActorRef,
TournamentSelectorTest.CreateParameterTree())),
name: TournamentSelectorTest.SpecialTournamentSelectorName);
// Send selection command.
int numberGenomes = 4;
var genomes = this.CreateGenomesAdheringToParameterTree(numberGenomes);
tournamentSelector.Tell(new SelectCommand <TestInstance>(genomes, this._singleTestInstance, 0));
// Wait for results.
var results = this.ExpectMsg <SelectionResultMessage <IntegerResult> >();
// Check that they are sorted correctly.
var sortedResults = results.CompetitiveParents
.OrderByDescending(
genome =>
(int)genome.CreateMutableGenome().GetGeneValue(ExtractIntegerValue.ParameterName).GetValue());
Assert.True(
sortedResults.SequenceEqual(results.CompetitiveParents, new ImmutableGenome.GeneValueComparer()),
$"Expected winners sorted like {TestUtils.PrintList(sortedResults)}, but were {TestUtils.PrintList(results.CompetitiveParents)}.");
}
public void StatusFileGetsIgnoredWithoutContinueParameter()
{
// Make sure a status file specifying a certain number of generations exist.
var configuration =
new AlgorithmTunerConfiguration.AlgorithmTunerConfigurationBuilder()
.SetGenerations(99)
.AddDetailedConfigurationBuilder(
RegressionForestArgumentParser.Identifier,
new GenomePredictionRandomForestConfig.GenomePredictionRandomForestConfigBuilder())
.Build(maximumNumberParallelEvaluations: 1);
this.WriteConfigurationToStatusFile(configuration, this._statusFilePath);
// Make sure it does not get used:
string[] args =
{
"--cores=1",
"--verbose=0",
"--numGens=1",
"--goalGen=0",
"--instanceNumbers=1:1",
"--engineeredProportion=0.0",
};
Master <NoOperation, TestInstance, TestResult> .Run(
args,
(config, pathToTrainingInstances, pathToTestInstances) =>
{
// Check the number of generations is different.
Assert.True(
configuration.Generations != config.Generations,
"Generation number should not be taken from status file without --continue.");
// Return an algorithm tuner quickly terminates.
return(this.BuildSimpleAlgorithmTuner(config, pathToTrainingInstances, pathToTestInstances));
});
}
public void LogFinishedGenerationAndLogFinalIncumbentGenerationLogTotalNumberOfEvaluationsIfDesired(bool useLogFinishedGeneration)
{
var configuration = new AlgorithmTunerConfiguration.AlgorithmTunerConfigurationBuilder()
.SetLogFilePath(LogWriterTest.LogFilePath)
.SetEvaluationLimit(25)
.Build(1);
var writer = new LogWriter <InstanceFile, RuntimeResult>(this._parameterTree, configuration);
if (useLogFinishedGeneration)
{
writer.LogFinishedGeneration(1, 3, this._testGenomeResults, true);
}
else
{
writer.LogFinalIncumbentGeneration(3, this._testGenomeResults, 1, 2, true);
}
var linesInFile = File.ReadLines(LogWriterTest.LogFilePath);
Assert.Equal(
"Evaluations: 3 / 25",
linesInFile.ElementAt(1));
}
public void OriginalConfigurationGetsOverwrittenWithNewParametersOnContinue()
{
// Specify parameters for original configuration
var originalConfig =
new AlgorithmTunerConfiguration.AlgorithmTunerConfigurationBuilder()
.SetEnableRacing(false)
.SetPopulationSize(8)
.SetGenerations(3)
.SetMaxGenomeAge(2)
.SetMaximumMiniTournamentSize(7)
.SetTournamentWinnerPercentage(0.34)
.SetCpuTimeout(TimeSpan.FromMilliseconds(30000))
.SetCrossoverSwitchProbability(0.05)
.SetMutationRate(0.2)
.SetMutationVariancePercentage(0.3)
.SetMaxRepairAttempts(11)
.SetMaximumNumberConsecutiveFailuresPerEvaluation(4)
.SetInstanceNumbers(1, 1)
.SetGoalGeneration(2)
.SetAkkaConfiguration(Config.Empty)
.SetVerbosity(VerbosityLevel.Debug)
.SetStatusFileDirectory(Path.GetDirectoryName(this._alternativeStatusFilePath))
.SetLogFilePath(this._alternativeLogFilePath)
.SetTrainModel(true)
.SetEngineeredProportion(0.21)
.SetTopPerformerThreshold(0.22)
.SetStartEngineeringAtIteration(1)
.SetPopulationMutantRatio(0.23)
.SetEnableSexualSelection(true)
.SetStrictCompatibilityCheck(false)
.SetCrossoverProbabilityCompetitive(0.24)
.SetHammingDistanceRelativeThreshold(0.25)
.SetTargetSampleSize(1)
.SetMaxRanksCompensatedByDistance(23)
.SetFeatureSubsetRatioForDistance(0.26)
.SetTrackConvergenceBehavior(true)
.SetDistanceMetric(DistanceMetric.L1Average.ToString())
.SetMaximumNumberParallelEvaluations(2)
.SetMaximumNumberParallelThreads(5)
.AddDetailedConfigurationBuilder(
RegressionForestArgumentParser.Identifier,
new GenomePredictionRandomForestConfig.GenomePredictionRandomForestConfigBuilder())
.Build();
this.WriteConfigurationToStatusFile(originalConfig, this._alternativeStatusFilePath);
// Call run with continue option and some additional parameters.
var args = new[]
{
// required parameters
"--continue",
$"--statusFileDir={originalConfig.StatusFileDirectory}",
// new parameter value
"--faultTolerance=750",
// equal to original
$"--maxRepair={originalConfig.MaxRepairAttempts}",
// default parameter value
$"--logFile={PathUtils.GetAbsolutePathFromCurrentDirectory("tunerLog.txt")}",
};
Master <NoOperation, TestInstance, TestResult> .Run(
args,
(config, pathToTrainingInstances, pathToTestInstances) =>
{
// Check new parameters get used.
Assert.Equal(
750,
config.MaximumNumberConsecutiveFailuresPerEvaluation);
Assert.Equal(
originalConfig.MaxRepairAttempts,
config.MaxRepairAttempts);
Assert.Equal(
PathUtils.GetAbsolutePathFromCurrentDirectory("tunerLog.txt"),
config.LogFilePath);
// Check original paramters are still in use where they did not get overwritten.
Assert.Equal(
originalConfig.EnableRacing,
config.EnableRacing);
Assert.Equal(
originalConfig.PopulationSize,
config.PopulationSize);
Assert.Equal(
originalConfig.Generations,
config.Generations);
Assert.Equal(
originalConfig.MaxGenomeAge,
config.MaxGenomeAge);
Assert.Equal(
originalConfig.MaximumMiniTournamentSize,
config.MaximumMiniTournamentSize);
Assert.Equal(
originalConfig.TournamentWinnerPercentage,
config.TournamentWinnerPercentage);
Assert.Equal(
originalConfig.CpuTimeout,
config.CpuTimeout);
Assert.Equal(
originalConfig.CrossoverSwitchProbability,
config.CrossoverSwitchProbability);
Assert.Equal(
originalConfig.MutationRate,
config.MutationRate);
Assert.Equal(
originalConfig.MutationVariancePercentage,
config.MutationVariancePercentage);
Assert.Equal(
originalConfig.StartNumInstances,
config.StartNumInstances);
Assert.Equal(
originalConfig.EndNumInstances,
config.EndNumInstances);
Assert.Equal(
originalConfig.GoalGeneration,
config.GoalGeneration);
Assert.Equal(
originalConfig.Verbosity,
config.Verbosity);
Assert.Equal(
originalConfig.StatusFileDirectory,
config.StatusFileDirectory);
Assert.Equal(
originalConfig.TrainModel,
config.TrainModel);
Assert.Equal(
originalConfig.TopPerformerThreshold,
config.TopPerformerThreshold);
Assert.Equal(
originalConfig.StartEngineeringAtIteration,
config.StartEngineeringAtIteration);
Assert.Equal(
originalConfig.PopulationMutantRatio,
config.PopulationMutantRatio);
Assert.Equal(
originalConfig.EnableSexualSelection,
config.EnableSexualSelection);
Assert.Equal(
originalConfig.StrictCompatibilityCheck,
config.StrictCompatibilityCheck);
Assert.Equal(
originalConfig.CrossoverProbabilityCompetitive,
config.CrossoverProbabilityCompetitive);
Assert.Equal(
originalConfig.HammingDistanceRelativeThreshold,
config.HammingDistanceRelativeThreshold);
Assert.Equal(
originalConfig.TargetSamplingSize,
config.TargetSamplingSize);
Assert.Equal(
originalConfig.MaxRanksCompensatedByDistance,
config.MaxRanksCompensatedByDistance);
Assert.Equal(
originalConfig.FeatureSubsetRatioForDistanceComputation,
config.FeatureSubsetRatioForDistanceComputation);
Assert.Equal(
originalConfig.TrackConvergenceBehavior,
config.TrackConvergenceBehavior);
Assert.Equal(
originalConfig.DistanceMetric,
config.DistanceMetric);
Assert.Equal(
originalConfig.MaximumNumberParallelEvaluations,
config.MaximumNumberParallelEvaluations);
Assert.Equal(
originalConfig.MaximumNumberParallelThreads,
config.MaximumNumberParallelThreads);
Assert.Equal(
originalConfig.EngineeredPopulationRatio,
config.EngineeredPopulationRatio);
// Return an algorithm tuner that quickly terminates.
return(this.BuildSimpleAlgorithmTuner(config, pathToTrainingInstances, pathToTestInstances));
});
}
public void CrossoverRespectsSwitchProbability()
{
// Build parameter tree that consists of two dependent continuous parameters.
string rootParameterName = "parameterRoot";
string childParameterName = "parameterChild";
var rootParameter = new ValueNode <double>(rootParameterName, new ContinuousDomain());
var childParameter = new ValueNode <double>(childParameterName, new ContinuousDomain());
rootParameter.SetChild(childParameter);
// Build genome builder with that parameter tree and a specific crossover switch probability.
double crossoverSwitchParameter = 0.25;
AlgorithmTunerConfiguration config = new AlgorithmTunerConfiguration.AlgorithmTunerConfigurationBuilder()
.SetCrossoverSwitchProbability(crossoverSwitchParameter)
.Build(maximumNumberParallelEvaluations: 1);
var genomeBuilder = new GenomeBuilder(new ParameterTree(rootParameter), config);
// Build parents.
var parent1 = new Genome();
var parent2 = new Genome();
var parent1RootAllele = new Allele <double>(1);
var parent1ChildAllele = new Allele <double>(2);
var parent2RootAllele = new Allele <double>(3);
var parent2ChildAllele = new Allele <double>(4);
parent1.SetGene(rootParameterName, parent1RootAllele);
parent1.SetGene(childParameterName, parent1ChildAllele);
parent2.SetGene(rootParameterName, parent2RootAllele);
parent2.SetGene(childParameterName, parent2ChildAllele);
// Observe if children's genes come from the same parent or not.
int numberLoops = 1000;
int genesCameFromSameParent = 0;
for (int i = 0; i < numberLoops; i++)
{
var child = genomeBuilder.Crossover(parent1, parent2);
IAllele rootAllele = child.GetGeneValue(rootParameterName);
IAllele childAllele = child.GetGeneValue(childParameterName);
bool geneValuesInheritedFromSameParent =
(object.Equals(rootAllele, parent1RootAllele) && object.Equals(childAllele, parent1ChildAllele)) ||
(object.Equals(rootAllele, parent2RootAllele) && object.Equals(childAllele, parent2ChildAllele));
if (geneValuesInheritedFromSameParent)
{
genesCameFromSameParent++;
}
}
double[] observed = { genesCameFromSameParent, numberLoops - genesCameFromSameParent };
// We would expect each case according to switch probability:
int expectedSwitches = (int)(crossoverSwitchParameter * numberLoops);
double[] expected = { numberLoops - expectedSwitches, expectedSwitches };
// Use Chi-Squared Test.
var matchesSwitchProbabilityTest = new ChiSquareTest(expected, observed, degreesOfFreedom: numberLoops - 1);
Assert.False(
matchesSwitchProbabilityTest.Significant,
$"Crossover was found not to respect the switch probability by the Chi-Squared test with significance level of {matchesSwitchProbabilityTest.Size}.");
}
