public void MemoryLimitWorks() { // Check, if current OS is Linux. Skip.IfNot(Environment.OSVersion.Platform == PlatformID.Unix); var timeout = TimeSpan.FromSeconds(10); var memoryLimitMegabyte = 1; var tunerConfig = new AlgorithmTunerConfiguration.AlgorithmTunerConfigurationBuilder().SetCpuTimeout(timeout).Build(1); var timer = new Stopwatch(); timer.Start(); var lingelingRunner = new LingelingRunner( new Dictionary <string, IAllele>(), LingelingRunnerTests.PathToExecutable, tunerConfig, memoryLimitMegabyte); var runner = lingelingRunner.Run( new InstanceSeedFile(LingelingRunnerTests.PathToTestInstance, LingelingRunnerTests.TestInstanceSeed), this._cancellationTokenSource.Token); runner.Wait(); timer.Stop(); runner.Result.ShouldNotBeNull(); runner.Result.IsCancelled.ShouldBeTrue(); runner.Result.Runtime.ShouldBe(timeout); timer.Elapsed.ShouldBeLessThan(timeout); }
/// <summary> /// Creates a simple <see cref="GenomeBuilder"/> that has a mutation rate of 0 and uses the parameter tree from /// <see cref="GenomeBuilderTest.BuildParameterTree"/>. /// </summary> /// <returns>The genome builder.</returns> private static GenomeBuilder CreateGenomeBuilderWithoutRandomMutation() { var configuration = new AlgorithmTunerConfiguration.AlgorithmTunerConfigurationBuilder() .SetMutationRate(0) .Build(maximumNumberParallelEvaluations: 1); return(new GenomeBuilder(GenomeBuilderTest.BuildParameterTree(), configuration)); }
public void ConstructorThrowsForMissingRandomForestConfiguration() { var incompleteConfig = new AlgorithmTunerConfiguration.AlgorithmTunerConfigurationBuilder() .SetInstanceNumbers(1, 1) .SetEngineeredProportion(1) .SetMaximumNumberParallelEvaluations(1) .Build(); Assert.Throws <KeyNotFoundException>(() => this.InitializeAndPopulateProperties(this.GetDefaultParameterTree(), incompleteConfig)); }
/// <summary> /// Creates a population with the given configuration. /// </summary> /// <param name="populationSize">The population's size.</param> /// <param name="maxAge">Maximum age of a genome.</param> /// <param name="populationMutantRatio">The ratio of non competitive genomes to replace each population. /// </param> /// <returns>The population.</returns> private static Population CreatePopulation(int populationSize, int maxAge, double populationMutantRatio) { var configuration = new AlgorithmTunerConfiguration.AlgorithmTunerConfigurationBuilder() .SetMaxGenomeAge(maxAge) .SetPopulationSize(populationSize) .SetPopulationMutantRatio(populationMutantRatio) .Build(maximumNumberParallelEvaluations: 1); return(new Population(configuration)); }
public void ConstructorThrowsExceptionOnMissingParameterTree() { var configuration = new AlgorithmTunerConfiguration.AlgorithmTunerConfigurationBuilder().Build(maximumNumberParallelEvaluations: 1); Assert.Throws <ArgumentNullException>( () => new GenomeBuilder( parameterTree: null, configuration: configuration)); }
public void ExtractRunStatisticsCreatesCancelledResult(string consoleOutput) { var timeout = TimeSpan.FromSeconds(5); var tunerConfig = new AlgorithmTunerConfiguration.AlgorithmTunerConfigurationBuilder().SetCpuTimeout(timeout).Build(1); var lingelingRunner = new LingelingRunner(new Dictionary <string, IAllele>(), "ling", tunerConfig, 4000); var runResult = lingelingRunner.ExtractRunStatistics(consoleOutput); runResult.IsCancelled.ShouldBeTrue(); runResult.Runtime.ShouldBe(timeout); }
public void FinishPhaseKeepsIncumbentForMaximumReplacementRate() { // Create population along with an incumbent. var basePopulation = this.CreatePopulation(); var incumbent = this.CreateIncumbentGenomeWrapper(); basePopulation.AddGenome(new Genome(incumbent.IncumbentGenome), isCompetitive: true); // Create strategy with a replacement rate of 1. var configuration = new AlgorithmTunerConfiguration.AlgorithmTunerConfigurationBuilder() .SetMaximumNumberParallelEvaluations(1) .AddDetailedConfigurationBuilder( RegressionForestArgumentParser.Identifier, new GenomePredictionRandomForestConfig.GenomePredictionRandomForestConfigBuilder()) .AddDetailedConfigurationBuilder( CovarianceMatrixAdaptationStrategyArgumentParser.Identifier, new CovarianceMatrixAdaptationStrategyConfiguration.CovarianceMatrixAdaptationStrategyConfigurationBuilder() .SetFixInstances(this.FixInstances) .SetReplacementRate(1) .SetMaximumNumberGenerations(30)); this.Strategy = new LocalCovarianceMatrixAdaptationStrategy <TestInstance, IntegerResult>( configuration.Build(), this.ParameterTree, this.GenomeBuilder, this.GenomeSorter, this.ResultStorageActor); // Perform an iteration and finish the phase. this.Strategy.Initialize(basePopulation, incumbent, this.SingleTestInstance); this.Strategy.PerformIteration(0, this.SingleTestInstance); this.Strategy.DumpStatus(); var updatedPopulation = this.Strategy.FinishPhase(basePopulation); // Incumbent should still exist. Assert.True( updatedPopulation.GetCompetitiveIndividuals().Contains(incumbent.IncumbentGenome, new Genome.GeneValueComparator()), "Updated population should contain incumbent, but does not."); // Age structure should be correct. for (int age = 0; age < 3; age++) { Assert.True( basePopulation.GetCompetitiveIndividuals().Count(individual => individual.Age == age) == updatedPopulation.GetCompetitiveIndividuals().Count(individual => individual.Age == age), $"Different number of genomes with age {age}."); } Assert.False( updatedPopulation.GetCompetitiveIndividuals().Any(individual => individual.Age < 0 || individual.Age > 3), "There exists a genome with age not in age range!"); }
public void ConstructorThrowsForWrongTypeRandomForestConfiguration() { var wrongConfig = new AlgorithmTunerConfiguration.AlgorithmTunerConfigurationBuilder() .SetInstanceNumbers(1, 1) .SetEngineeredProportion(1) .SetMaximumNumberParallelEvaluations(1) .AddDetailedConfigurationBuilder( RegressionForestArgumentParser.Identifier, new AlgorithmTunerConfiguration.AlgorithmTunerConfigurationBuilder().SetMaximumNumberParallelEvaluations(1)) .Build(); Assert.Throws <InvalidCastException>(() => this.InitializeAndPopulateProperties(this.GetDefaultParameterTree(), wrongConfig)); }
/// <summary> /// Creates a new <see cref="TournamentSelector{NoOperation, TestInstance, TestResult}"/> in the /// actor system. /// </summary> /// <param name="winnerPercentage">The percentage of winners in a mini tournament.</param> /// <param name="miniTournamentSize">The size of a mini tournament.</param> /// <returns>An <see cref="IActorRef"/> to the created tournament selector.</returns> private IActorRef CreateTournamentSelector(double winnerPercentage, int miniTournamentSize) { // Build configuration. AlgorithmTunerConfiguration config = new AlgorithmTunerConfiguration.AlgorithmTunerConfigurationBuilder() .SetTournamentWinnerPercentage(winnerPercentage) .SetMaximumMiniTournamentSize(miniTournamentSize) .Build(maximumNumberParallelEvaluations: 1); // Build tournament selector itself. return(this.Sys.ActorOf( props: new TournamentSelectorPropsBuilder().SetConfiguration(config).Build(this._resultStorageActorRef), name: TournamentSelectorTest.SpecialTournamentSelectorName)); }
/// <summary> /// Builds an <see cref="InstanceSelector{I}"/>. /// </summary> /// <param name="numInstancesAtStart">Number instances returned for first generation.</param> /// <param name="numInstancesAtEnd">Number instances returned all generations between goal generation and last /// generation.</param> /// <param name="goalGeneration">First generation for which the maximum number of instances should be returned. /// </param> /// <param name="generations">The number of generations.</param> /// <returns>The built <see cref="InstanceSelector{I}"/>.</returns> private InstanceSelector <TestInstance> BuildSelector( int numInstancesAtStart, int numInstancesAtEnd, int goalGeneration, int generations) { AlgorithmTunerConfiguration configuration = new AlgorithmTunerConfiguration.AlgorithmTunerConfigurationBuilder() .SetInstanceNumbers(numInstancesAtStart, numInstancesAtEnd) .SetGenerations(generations) .SetGoalGeneration(goalGeneration) .Build(maximumNumberParallelEvaluations: 4); return(new InstanceSelector <TestInstance>(this._instanceList, configuration)); }
public void WorstGenomesAreNeverWinners() { // Create some genomes. int numberGenomes = 20; var genomes = this.CreateGenomesAdheringToParameterTree(numberGenomes); // Create a configuration using a certain winner percentage and mini tournament size. double winnerPercentage = 0.1; int miniTournamentSize = 10; AlgorithmTunerConfiguration configuration = new AlgorithmTunerConfiguration.AlgorithmTunerConfigurationBuilder() .SetTournamentWinnerPercentage(winnerPercentage) .SetMaximumMiniTournamentSize(miniTournamentSize) .Build(maximumNumberParallelEvaluations: 1); // Create tournament selector using an evaluator which sorts by a certain parameter, highest first. var tournamentSelectorProps = Props.Create( () => new TournamentSelector <ExtractIntegerValue, TestInstance, IntegerResult>( new ExtractIntegerValueCreator(), new SortByValue(), configuration, this._resultStorageActorRef, TournamentSelectorTest.CreateParameterTree())); var tournamentSelector = this.Sys.ActorOf(tournamentSelectorProps, name: TournamentSelectorTest.SpecialTournamentSelectorName); // Try selecting the best genomes via mini tournaments multiple times and make sure the worst genomes never // get chosen: int worstPossibleWinnerValue = miniTournamentSize - (int)(winnerPercentage * miniTournamentSize); for (int i = 0; i < 15; i++) { // Send a select command. tournamentSelector.Tell(new SelectCommand <TestInstance>(genomes, this._singleTestInstance, 0)); // Wait for results. var results = this.ExpectMsg <SelectionResultMessage <IntegerResult> >(); // Make sure none belongs to the worst genomes. foreach (var genome in results.CompetitiveParents) { int geneValue = (int)genome.CreateMutableGenome().GetGeneValue(ExtractIntegerValue.ParameterName).GetValue(); Assert.True( geneValue >= worstPossibleWinnerValue, $"Found a winner with value {geneValue}, but expected at least {worstPossibleWinnerValue}."); } } }
public void MutateMayChangeEveryParameter() { // Create genome builder with a mutation rate of 1. var configuration = new AlgorithmTunerConfiguration.AlgorithmTunerConfigurationBuilder() .SetMutationRate(1) .Build(maximumNumberParallelEvaluations: 1); var genomeBuilder = new GenomeBuilder(GenomeBuilderTest.BuildParameterTree(), configuration); // Create genome. var genome = GenomeBuilderTest.BuildFittingGenome(); // Prepare structure to store which values have been mutated. string[] identifiers = { GenomeBuilderTest.DecisionParameter, GenomeBuilderTest.SmallValueParameter, GenomeBuilderTest.DiscreteParameter, GenomeBuilderTest.ContinuousParameter, }; Dictionary <string, bool> hasBeenMutated = identifiers.ToDictionary( identifier => identifier, identifier => false); Dictionary <string, IAllele> originalValues = identifiers.ToDictionary( identifier => identifier, identifier => genome.GetGeneValue(identifier)); // Do a lot of mutations. for (int i = 0; i < GenomeBuilderTest.loopCountForRandomTests; i++) { genomeBuilder.Mutate(genome); // After each one, check which genes have changed... foreach (string identifier in identifiers) { bool valueChanged = !object.Equals(originalValues[identifier], genome.GetGeneValue(identifier)); if (valueChanged) { // ...and store them. hasBeenMutated[identifier] = true; } } } // Finally check that everything has been changed at least once. Assert.True( hasBeenMutated.All(keyValuePair => keyValuePair.Value), $"Genes {TestUtils.PrintList(hasBeenMutated.Where(keyValuePair => !keyValuePair.Value).Select(keyValuePair => keyValuePair.Key))} have not been mutated."); }
public void CancellationWorks() { // Check, if current OS is Linux. Skip.IfNot(Environment.OSVersion.Platform == PlatformID.Unix); var timeout = TimeSpan.FromSeconds(30); var memoryLimitMegabyte = 4000; var tunerConfig = new AlgorithmTunerConfiguration.AlgorithmTunerConfigurationBuilder().SetCpuTimeout(timeout).Build(1); var timer = new Stopwatch(); timer.Start(); // Run Lingeling. var lingelingRunner = new LingelingRunner( new Dictionary <string, IAllele>(), LingelingRunnerTests.PathToExecutable, tunerConfig, memoryLimitMegabyte); var runner = lingelingRunner.Run( new InstanceSeedFile(LingelingRunnerTests.PathToTestInstance, LingelingRunnerTests.TestInstanceSeed), this._cancellationTokenSource.Token); // Cancel task and expect it to be cancelled. try { Thread.Sleep(100); this._cancellationTokenSource.Cancel(); runner.Wait(); Assert.True(false, "Expected a task cancelled exception."); } catch (AggregateException aggregateException) { timer.Stop(); aggregateException.InnerExceptions.Count.ShouldBe(1); var innerException = aggregateException.InnerExceptions.Single(); innerException.ShouldBeOfType <TaskCanceledException>(); runner.IsCanceled.ShouldBeTrue(); timer.Elapsed.ShouldBeLessThan(TimeSpan.FromSeconds(1)); } catch (Exception) { Assert.True(false, "Expected a task cancelled exception."); } }
public void AllDummyParametersAreFiltered() { Randomizer.Reset(); Randomizer.Configure(0); var parameterTree = GurobiUtils.CreateParameterTree(); var config = new AlgorithmTunerConfiguration.AlgorithmTunerConfigurationBuilder().Build(1); var genomeBuilder = new GenomeBuilder(parameterTree, config); var genome = genomeBuilder.CreateRandomGenome(0); var filteredGenes = genome.GetFilteredGenes(parameterTree); foreach (var filteredGenesKey in filteredGenes.Keys) { filteredGenesKey.ShouldNotContain("Indicator"); } }
/// <summary> /// Initializes a <see cref="TournamentSelector{TTargetAlgorithm,TInstance,TResult}"/>. /// It creates <see cref="MiniTournamentActor{TTargetAlgorithm,TInstance,TResult}"/>s which create the /// required <see cref="EvaluationActor{TTargetAlgorithm,TInstance,TResult}"/>s. /// The hierarchy must be as in OPTANO Algorithm Tuner to ensure that <see cref="_genomeSorter"/> finds the /// actors. /// </summary> private void BuildTournamentSelector() { var configuration = new AlgorithmTunerConfiguration.AlgorithmTunerConfigurationBuilder() .SetEnableRacing(true) .Build(GenomeSorterTest.MaximumNumberParallelEvaluations); var parameterTree = new ParameterTree( root: new ValueNode <int>(ExtractIntegerValue.ParameterName, new IntegerDomain())); var tournamentSelector = this.Sys.ActorOf( Props.Create( () => new TournamentSelector <ExtractIntegerValue, TestInstance, IntegerResult>( new ExtractIntegerValueCreator(), this._runEvaluator, configuration, this._resultStorageActor, parameterTree)), AkkaNames.TournamentSelector); }
public void LogFinishedGenerationAndLogFinalIncumbentGenerationThrowForUnknownDirectory(bool useLogFinishedGeneration) { var configuration = new AlgorithmTunerConfiguration.AlgorithmTunerConfigurationBuilder() .SetLogFilePath("foo/bar.txt") .Build(1); var writer = new LogWriter <InstanceFile, RuntimeResult>(this._parameterTree, configuration); if (useLogFinishedGeneration) { Assert.Throws <DirectoryNotFoundException>( () => writer.LogFinishedGeneration(1, 1, this._testGenomeResults, true)); } else { Assert.Throws <DirectoryNotFoundException>( () => writer.LogFinalIncumbentGeneration(1, this._testGenomeResults, 1, 2, true)); } }
public void TryToGetResultFromStringArrayWorksForGurobiResults() { var timeout = TimeSpan.FromSeconds(30); var gurobiResult = new GurobiResult(0.5, timeout, TargetAlgorithmStatus.CancelledByTimeout, true); var gurobiConfiguration = new GurobiRunnerConfiguration.GurobiRunnerConfigBuilder().Build(timeout); var tunerConfiguration = new AlgorithmTunerConfiguration.AlgorithmTunerConfigurationBuilder().SetCpuTimeout(timeout) .Build(1); var targetAlgorithmFactory = new GurobiRunnerFactory(gurobiConfiguration, tunerConfiguration) as ITargetAlgorithmFactory <GurobiRunner, InstanceSeedFile, GurobiResult>; targetAlgorithmFactory.TryToGetResultFromStringArray(gurobiResult.ToStringArray(), out var result).ShouldBeTrue(); result.TargetAlgorithmStatus.ShouldBe(gurobiResult.TargetAlgorithmStatus); result.IsCancelled.ShouldBe(gurobiResult.IsCancelled); result.Runtime.ShouldBe(gurobiResult.Runtime); result.Gap.ShouldBe(gurobiResult.Gap); result.HasValidSolution.ShouldBe(gurobiResult.HasValidSolution); }
public void CorrectInstancesAreUsedForSelectCommand() { // Build tournament selector with a small tournament size to test using several workers var config = new AlgorithmTunerConfiguration.AlgorithmTunerConfigurationBuilder() .SetMaximumMiniTournamentSize(1) .Build(maximumNumberParallelEvaluations: 1); var tournamentSelector = this.Sys.ActorOf( props: new TournamentSelectorPropsBuilder().SetConfiguration(config).Build(this._resultStorageActorRef), name: TournamentSelectorTest.SpecialTournamentSelectorName); // Send a simple select command and wait for completion. var originalCommand = new SelectCommand <TestInstance>( this.CreateGenomesAdheringToParameterTree(2), new List <TestInstance> { new TestInstance("0") }, 0); tournamentSelector.Tell(originalCommand); this.ExpectMsg <SelectionResultMessage <TestResult> >(); // Build a new select command with several new instances. var instances = new List <TestInstance>() { new TestInstance("1"), new TestInstance("2"), }; var genomes = this.CreateGenomesAdheringToParameterTree(2); var selectCommand = new SelectCommand <TestInstance>(genomes, instances, 1); // Send it & wait for completion. tournamentSelector.Tell(selectCommand); this.ExpectMsg <SelectionResultMessage <TestResult> >(); // Check that storage actor has knowledge about both new instances for both genomes. foreach (var genome in genomes) { foreach (var instance in instances) { this._resultStorageActorRef.Tell(new ResultRequest <TestInstance>(genome, instance)); this.ExpectMsg <ResultMessage <TestInstance, TestResult> >(); } } }
public void AkkaConfigurationIsTakenFromNewParametersOnContinue() { // Specify parameters for simple original configuration var originalConfig = new AlgorithmTunerConfiguration.AlgorithmTunerConfigurationBuilder() .SetMaximumNumberParallelEvaluations(2) .SetGenerations(1) .SetGoalGeneration(0) .SetInstanceNumbers(1, 1) .SetEngineeredProportion(0) .AddDetailedConfigurationBuilder( RegressionForestArgumentParser.Identifier, new GenomePredictionRandomForestConfig.GenomePredictionRandomForestConfigBuilder()) .Build(); this.WriteConfigurationToStatusFile(originalConfig, this._statusFilePath); // Call run with continue option and different Akka configuraiton. var args = new[] { "--continue", // Akka config parameters "--port=1234", "--ownHostName=foo", "--maxParallelEvaluations=1", }; Master <NoOperation, TestInstance, TestResult> .Run( args, (config, pathToTrainingInstances, pathToTestInstances) => { // Check new parameters get used for Akka config. Assert.Equal( 1234, config.AkkaConfiguration.GetInt("akka.remote.dot-netty.tcp.port")); Assert.Equal( "foo", config.AkkaConfiguration.GetString("akka.remote.dot-netty.tcp.hostname")); // Return an algorithm tuner that quickly terminates. return(this.BuildSimpleAlgorithmTuner(originalConfig, pathToTrainingInstances, pathToTestInstances)); }); }
public void MutateChangesGivenGenomeIfMutationRateIs1() { // Create genome builder with a mutation rate of 1. var configuration = new AlgorithmTunerConfiguration.AlgorithmTunerConfigurationBuilder() .SetMutationRate(1) .Build(maximumNumberParallelEvaluations: 1); var genomeBuilder = new GenomeBuilder(GenomeBuilderTest.BuildParameterTree(), configuration); // Build a fitting genome and store original description. var genome = GenomeBuilderTest.BuildFittingGenome(); string originalDescription = genome.ToString(); // Call mutate and check genome changed. genomeBuilder.Mutate(genome); Assert.NotEqual( originalDescription, genome.ToString()); }
public void InstanceFolderIsTakenFromNewParametersOnContinue() { // Specify parameters for simple original configuration var originalConfig = new AlgorithmTunerConfiguration.AlgorithmTunerConfigurationBuilder() .SetMaximumNumberParallelEvaluations(2) .SetGenerations(1) .SetGoalGeneration(0) .SetInstanceNumbers(1, 1) .SetEngineeredProportion(0) .AddDetailedConfigurationBuilder( RegressionForestArgumentParser.Identifier, new GenomePredictionRandomForestConfig.GenomePredictionRandomForestConfigBuilder()) .Build(); this.WriteConfigurationToStatusFile(originalConfig, this._statusFilePath); // Call run with continue option and different instance folder. var args = new[] { "--continue", // Instance folder parameters "--trainingInstanceFolder=foo", "--testInstanceFolder=bar", }; Master <NoOperation, TestInstance, TestResult> .Run( args, (config, pathToTrainingInstances, pathToTestInstances) => { // Check new parameter get used for instance folders. Assert.Equal( "foo", pathToTrainingInstances); Assert.Equal( "bar", pathToTestInstances); // Return an algorithm tuner that quickly terminates. return(this.BuildSimpleAlgorithmTuner(originalConfig, pathToTrainingInstances, pathToTestInstances)); }); }
public void MutateRespectsMutationRate() { // Create genome builder with a mutation rate of 0.35. double mutationRate = 0.35; var configuration = new AlgorithmTunerConfiguration.AlgorithmTunerConfigurationBuilder() .SetMutationRate(mutationRate) .Build(maximumNumberParallelEvaluations: 1); var genomeBuilder = new GenomeBuilder(GenomeBuilderTest.BuildParameterTree(), configuration); // Create genome. var genome = GenomeBuilderTest.BuildFittingGenome(); // For a lot of iterations: IAllele oldGeneValue = genome.GetGeneValue(GenomeBuilderTest.ContinuousParameter); int changeCount = 0; int numberLoops = 1000; for (int i = 0; i < numberLoops; i++) { // Mutate the genome ... genomeBuilder.Mutate(genome); // ... compare the continuous parameter gene... IAllele newGeneValue = genome.GetGeneValue(GenomeBuilderTest.ContinuousParameter); if (!object.Equals(newGeneValue, oldGeneValue)) { changeCount++; } // ... and count the number of times it changes. oldGeneValue = newGeneValue; } // Finally compare the number of mutations with the expected number. double expectedNumberMutations = mutationRate * numberLoops; Assert.True( Math.Abs(expectedNumberMutations - changeCount) <= 0.1 * expectedNumberMutations, "Number of mutations was not as expected."); }
public void WinnersAreSortedCorrectly() { // Build configuration with a specific winner percentage and tournament size. AlgorithmTunerConfiguration config = new AlgorithmTunerConfiguration.AlgorithmTunerConfigurationBuilder() .SetTournamentWinnerPercentage(0.5) .SetMaximumMiniTournamentSize(2) .Build(maximumNumberParallelEvaluations: 1); // Build tournament selector selecting winners by a specific parameter. var tournamentSelector = this.Sys.ActorOf( props: Props.Create( () => new TournamentSelector <ExtractIntegerValue, TestInstance, IntegerResult>( new ExtractIntegerValueCreator(), new SortByValue(), config, this._resultStorageActorRef, TournamentSelectorTest.CreateParameterTree())), name: TournamentSelectorTest.SpecialTournamentSelectorName); // Send selection command. int numberGenomes = 4; var genomes = this.CreateGenomesAdheringToParameterTree(numberGenomes); tournamentSelector.Tell(new SelectCommand <TestInstance>(genomes, this._singleTestInstance, 0)); // Wait for results. var results = this.ExpectMsg <SelectionResultMessage <IntegerResult> >(); // Check that they are sorted correctly. var sortedResults = results.CompetitiveParents .OrderByDescending( genome => (int)genome.CreateMutableGenome().GetGeneValue(ExtractIntegerValue.ParameterName).GetValue()); Assert.True( sortedResults.SequenceEqual(results.CompetitiveParents, new ImmutableGenome.GeneValueComparer()), $"Expected winners sorted like {TestUtils.PrintList(sortedResults)}, but were {TestUtils.PrintList(results.CompetitiveParents)}."); }
public void StatusFileGetsIgnoredWithoutContinueParameter() { // Make sure a status file specifying a certain number of generations exist. var configuration = new AlgorithmTunerConfiguration.AlgorithmTunerConfigurationBuilder() .SetGenerations(99) .AddDetailedConfigurationBuilder( RegressionForestArgumentParser.Identifier, new GenomePredictionRandomForestConfig.GenomePredictionRandomForestConfigBuilder()) .Build(maximumNumberParallelEvaluations: 1); this.WriteConfigurationToStatusFile(configuration, this._statusFilePath); // Make sure it does not get used: string[] args = { "--cores=1", "--verbose=0", "--numGens=1", "--goalGen=0", "--instanceNumbers=1:1", "--engineeredProportion=0.0", }; Master <NoOperation, TestInstance, TestResult> .Run( args, (config, pathToTrainingInstances, pathToTestInstances) => { // Check the number of generations is different. Assert.True( configuration.Generations != config.Generations, "Generation number should not be taken from status file without --continue."); // Return an algorithm tuner quickly terminates. return(this.BuildSimpleAlgorithmTuner(config, pathToTrainingInstances, pathToTestInstances)); }); }
public void LogFinishedGenerationAndLogFinalIncumbentGenerationLogTotalNumberOfEvaluationsIfDesired(bool useLogFinishedGeneration) { var configuration = new AlgorithmTunerConfiguration.AlgorithmTunerConfigurationBuilder() .SetLogFilePath(LogWriterTest.LogFilePath) .SetEvaluationLimit(25) .Build(1); var writer = new LogWriter <InstanceFile, RuntimeResult>(this._parameterTree, configuration); if (useLogFinishedGeneration) { writer.LogFinishedGeneration(1, 3, this._testGenomeResults, true); } else { writer.LogFinalIncumbentGeneration(3, this._testGenomeResults, 1, 2, true); } var linesInFile = File.ReadLines(LogWriterTest.LogFilePath); Assert.Equal( "Evaluations: 3 / 25", linesInFile.ElementAt(1)); }
public void OriginalConfigurationGetsOverwrittenWithNewParametersOnContinue() { // Specify parameters for original configuration var originalConfig = new AlgorithmTunerConfiguration.AlgorithmTunerConfigurationBuilder() .SetEnableRacing(false) .SetPopulationSize(8) .SetGenerations(3) .SetMaxGenomeAge(2) .SetMaximumMiniTournamentSize(7) .SetTournamentWinnerPercentage(0.34) .SetCpuTimeout(TimeSpan.FromMilliseconds(30000)) .SetCrossoverSwitchProbability(0.05) .SetMutationRate(0.2) .SetMutationVariancePercentage(0.3) .SetMaxRepairAttempts(11) .SetMaximumNumberConsecutiveFailuresPerEvaluation(4) .SetInstanceNumbers(1, 1) .SetGoalGeneration(2) .SetAkkaConfiguration(Config.Empty) .SetVerbosity(VerbosityLevel.Debug) .SetStatusFileDirectory(Path.GetDirectoryName(this._alternativeStatusFilePath)) .SetLogFilePath(this._alternativeLogFilePath) .SetTrainModel(true) .SetEngineeredProportion(0.21) .SetTopPerformerThreshold(0.22) .SetStartEngineeringAtIteration(1) .SetPopulationMutantRatio(0.23) .SetEnableSexualSelection(true) .SetStrictCompatibilityCheck(false) .SetCrossoverProbabilityCompetitive(0.24) .SetHammingDistanceRelativeThreshold(0.25) .SetTargetSampleSize(1) .SetMaxRanksCompensatedByDistance(23) .SetFeatureSubsetRatioForDistance(0.26) .SetTrackConvergenceBehavior(true) .SetDistanceMetric(DistanceMetric.L1Average.ToString()) .SetMaximumNumberParallelEvaluations(2) .SetMaximumNumberParallelThreads(5) .AddDetailedConfigurationBuilder( RegressionForestArgumentParser.Identifier, new GenomePredictionRandomForestConfig.GenomePredictionRandomForestConfigBuilder()) .Build(); this.WriteConfigurationToStatusFile(originalConfig, this._alternativeStatusFilePath); // Call run with continue option and some additional parameters. var args = new[] { // required parameters "--continue", $"--statusFileDir={originalConfig.StatusFileDirectory}", // new parameter value "--faultTolerance=750", // equal to original $"--maxRepair={originalConfig.MaxRepairAttempts}", // default parameter value $"--logFile={PathUtils.GetAbsolutePathFromCurrentDirectory("tunerLog.txt")}", }; Master <NoOperation, TestInstance, TestResult> .Run( args, (config, pathToTrainingInstances, pathToTestInstances) => { // Check new parameters get used. Assert.Equal( 750, config.MaximumNumberConsecutiveFailuresPerEvaluation); Assert.Equal( originalConfig.MaxRepairAttempts, config.MaxRepairAttempts); Assert.Equal( PathUtils.GetAbsolutePathFromCurrentDirectory("tunerLog.txt"), config.LogFilePath); // Check original paramters are still in use where they did not get overwritten. Assert.Equal( originalConfig.EnableRacing, config.EnableRacing); Assert.Equal( originalConfig.PopulationSize, config.PopulationSize); Assert.Equal( originalConfig.Generations, config.Generations); Assert.Equal( originalConfig.MaxGenomeAge, config.MaxGenomeAge); Assert.Equal( originalConfig.MaximumMiniTournamentSize, config.MaximumMiniTournamentSize); Assert.Equal( originalConfig.TournamentWinnerPercentage, config.TournamentWinnerPercentage); Assert.Equal( originalConfig.CpuTimeout, config.CpuTimeout); Assert.Equal( originalConfig.CrossoverSwitchProbability, config.CrossoverSwitchProbability); Assert.Equal( originalConfig.MutationRate, config.MutationRate); Assert.Equal( originalConfig.MutationVariancePercentage, config.MutationVariancePercentage); Assert.Equal( originalConfig.StartNumInstances, config.StartNumInstances); Assert.Equal( originalConfig.EndNumInstances, config.EndNumInstances); Assert.Equal( originalConfig.GoalGeneration, config.GoalGeneration); Assert.Equal( originalConfig.Verbosity, config.Verbosity); Assert.Equal( originalConfig.StatusFileDirectory, config.StatusFileDirectory); Assert.Equal( originalConfig.TrainModel, config.TrainModel); Assert.Equal( originalConfig.TopPerformerThreshold, config.TopPerformerThreshold); Assert.Equal( originalConfig.StartEngineeringAtIteration, config.StartEngineeringAtIteration); Assert.Equal( originalConfig.PopulationMutantRatio, config.PopulationMutantRatio); Assert.Equal( originalConfig.EnableSexualSelection, config.EnableSexualSelection); Assert.Equal( originalConfig.StrictCompatibilityCheck, config.StrictCompatibilityCheck); Assert.Equal( originalConfig.CrossoverProbabilityCompetitive, config.CrossoverProbabilityCompetitive); Assert.Equal( originalConfig.HammingDistanceRelativeThreshold, config.HammingDistanceRelativeThreshold); Assert.Equal( originalConfig.TargetSamplingSize, config.TargetSamplingSize); Assert.Equal( originalConfig.MaxRanksCompensatedByDistance, config.MaxRanksCompensatedByDistance); Assert.Equal( originalConfig.FeatureSubsetRatioForDistanceComputation, config.FeatureSubsetRatioForDistanceComputation); Assert.Equal( originalConfig.TrackConvergenceBehavior, config.TrackConvergenceBehavior); Assert.Equal( originalConfig.DistanceMetric, config.DistanceMetric); Assert.Equal( originalConfig.MaximumNumberParallelEvaluations, config.MaximumNumberParallelEvaluations); Assert.Equal( originalConfig.MaximumNumberParallelThreads, config.MaximumNumberParallelThreads); Assert.Equal( originalConfig.EngineeredPopulationRatio, config.EngineeredPopulationRatio); // Return an algorithm tuner that quickly terminates. return(this.BuildSimpleAlgorithmTuner(config, pathToTrainingInstances, pathToTestInstances)); }); }
public void CrossoverRespectsSwitchProbability() { // Build parameter tree that consists of two dependent continuous parameters. string rootParameterName = "parameterRoot"; string childParameterName = "parameterChild"; var rootParameter = new ValueNode <double>(rootParameterName, new ContinuousDomain()); var childParameter = new ValueNode <double>(childParameterName, new ContinuousDomain()); rootParameter.SetChild(childParameter); // Build genome builder with that parameter tree and a specific crossover switch probability. double crossoverSwitchParameter = 0.25; AlgorithmTunerConfiguration config = new AlgorithmTunerConfiguration.AlgorithmTunerConfigurationBuilder() .SetCrossoverSwitchProbability(crossoverSwitchParameter) .Build(maximumNumberParallelEvaluations: 1); var genomeBuilder = new GenomeBuilder(new ParameterTree(rootParameter), config); // Build parents. var parent1 = new Genome(); var parent2 = new Genome(); var parent1RootAllele = new Allele <double>(1); var parent1ChildAllele = new Allele <double>(2); var parent2RootAllele = new Allele <double>(3); var parent2ChildAllele = new Allele <double>(4); parent1.SetGene(rootParameterName, parent1RootAllele); parent1.SetGene(childParameterName, parent1ChildAllele); parent2.SetGene(rootParameterName, parent2RootAllele); parent2.SetGene(childParameterName, parent2ChildAllele); // Observe if children's genes come from the same parent or not. int numberLoops = 1000; int genesCameFromSameParent = 0; for (int i = 0; i < numberLoops; i++) { var child = genomeBuilder.Crossover(parent1, parent2); IAllele rootAllele = child.GetGeneValue(rootParameterName); IAllele childAllele = child.GetGeneValue(childParameterName); bool geneValuesInheritedFromSameParent = (object.Equals(rootAllele, parent1RootAllele) && object.Equals(childAllele, parent1ChildAllele)) || (object.Equals(rootAllele, parent2RootAllele) && object.Equals(childAllele, parent2ChildAllele)); if (geneValuesInheritedFromSameParent) { genesCameFromSameParent++; } } double[] observed = { genesCameFromSameParent, numberLoops - genesCameFromSameParent }; // We would expect each case according to switch probability: int expectedSwitches = (int)(crossoverSwitchParameter * numberLoops); double[] expected = { numberLoops - expectedSwitches, expectedSwitches }; // Use Chi-Squared Test. var matchesSwitchProbabilityTest = new ChiSquareTest(expected, observed, degreesOfFreedom: numberLoops - 1); Assert.False( matchesSwitchProbabilityTest.Significant, $"Crossover was found not to respect the switch probability by the Chi-Squared test with significance level of {matchesSwitchProbabilityTest.Size}."); }