private void CalculateValues(GenerationInformation entries, out float minValue, out float averageValue, out float[] best)
        {
            minValue     = float.MaxValue;
            averageValue = 0;
            var list            = entries.Entries;
            var numberOfEntries = list.Count;
            int minIndex        = -1;

            for (int i = 0; i < numberOfEntries; i++)
            {
                var value = list[i][0];
                if (value < minValue)
                {
                    minValue = value;
                    minIndex = i;
                }
                averageValue += value;
            }
            if (minIndex >= 0)
            {
                best = new float[list[minIndex].Length - 1];
                Array.Copy(list[minIndex], 1, best, 0, best.Length);
            }
            else
            {
                best = new float[0];
            }
            averageValue /= (numberOfEntries * numberOfEntries - numberOfEntries) / 2.0f;
        }
        private void LoadInformation(Dictionary <int, GenerationInformation> generationDictionary)
        {
            using (CsvReader reader = new CsvReader(this.EvaluationFileName))
            {
                LoadHeader(reader);

                int numberOfParameters = this.Headers.Length;
                while (reader.LoadLine() != 0)
                {
                    int generation;
                    reader.Get(out generation, 0);
                    var parameters = new float[numberOfParameters + 1];
                    for (int i = 0; i < numberOfParameters + 1; i++)
                    {
                        // offset 1 for the generation i == 0 is the value
                        reader.Get(out parameters[i], i + 1);
                    }
                    GenerationInformation info;
                    if (!generationDictionary.TryGetValue(generation, out info))
                    {
                        info         = new GenerationInformation();
                        info.Entries = new List <float[]>(250);
                        generationDictionary.Add(generation, info);
                    }
                    info.Entries.Add(parameters);
                }
            }
        }
Esempio n. 3
0
        public void ScoreInformationHistoryScoresGenerationInformationObjectsWithoutTestInstances()
        {
            var incumbent1 = new Genome();

            incumbent1.SetGene(ExtractIntegerValue.ParameterName, new Allele <int>(-2));
            var generationInformation = new GenerationInformation(0, TimeSpan.Zero, 0, typeof(int), new ImmutableGenome(incumbent1), "id");

            var scorer = new GenerationInformationScorer <InstanceSeedFile, IntegerResult>(
                this._generationEvaluationActor,
                this._resultStorageActor,
                this._runEvaluator);

            scorer.ScoreInformationHistory(
                new List <GenerationInformation> {
                generationInformation
            },
                trainingInstances: GenerationInformationScorerTest.CreateInstances(startSeed: 50, number: 4),
                testInstances: new List <InstanceSeedFile>());

            Assert.Equal(
                -103,
                generationInformation.IncumbentTrainingScore);
            Assert.Null(
                generationInformation.IncumbentTestScore);
        }
Esempio n. 4
0
        public void ToStringWorksForMissingInformation()
        {
            var information = new GenerationInformation(
                GenerationInformationTest.Generation,
                GenerationInformationTest.TotalNumberOfEvaluations,
                this._strategy,
                this._incumbent);

            Assert.Equal(
                "12;3468;;;DifferentialEvolutionStrategy`2;[a: -23](Age: 2)[Engineered: yes]",
                information.ToString());
        }
Esempio n. 5
0
        public void ToStringWorksForMissingInformation()
        {
            var information = new GenerationInformation(
                GenerationInformationTest.Generation,
                totalElapsedTime: TimeSpan.FromSeconds(30),
                GenerationInformationTest.TotalNumberOfEvaluations,
                this._strategy,
                this._incumbent,
                "id");

            Assert.Equal(
                "12;0:00:00:30.0000000;3468;;;DifferentialEvolutionStrategy`2;[a: -23](Age: 2)[Engineered: yes];id",
                information.ToString());
        }
Esempio n. 6
0
        public void ToStringContainsAllInformation()
        {
            var information = new GenerationInformation(
                GenerationInformationTest.Generation,
                GenerationInformationTest.TotalNumberOfEvaluations,
                this._strategy,
                this._incumbent);

            information.IncumbentTrainingScore = -3.4;
            information.IncumbentTestScore     = 1234.8;
            Assert.Equal(
                "12;3468;-3.4;1234.8;DifferentialEvolutionStrategy`2;[a: -23](Age: 2)[Engineered: yes]",
                information.ToString());
        }
        /// <summary>
        /// Updates <see cref="_informationHistory"/> with the current (finished) generation.
        /// </summary>
        /// <param name="currentStrategy">The current strategy.</param>
        private void UpdateGenerationHistory(IPopulationUpdateStrategy <TInstance, TResult> currentStrategy)
        {
            var evaluationCountRequest =
                this._targetRunResultStorage.Ask <EvaluationStatistic>(new EvaluationStatisticRequest());

            evaluationCountRequest.Wait();

            var generationInformation = new GenerationInformation(
                generation: this._currGeneration,
                totalNumberOfEvaluations: evaluationCountRequest.Result.TotalEvaluationCount,
                strategy: currentStrategy.GetType(),
                incumbent: new ImmutableGenome(this._incumbentGenomeWrapper.IncumbentGenome));

            this._informationHistory.Add(generationInformation);
        }
        private float CalculateOverallAverage(GenerationInformation entries)
        {
            var    list    = entries.Entries;
            var    length  = list.Count;
            double average = 0;

            for (int i = 0; i < length - 1; i++)
            {
                for (int j = i + 1; j < length; j++)
                {
                    average += this.Distance(list[i], list[j]);
                }
            }
            return((float)(average / ((length * length - length) / 2.0)));
        }
        private float CalculateOverallSTD(GenerationInformation entries, float average)
        {
            var    list   = entries.Entries;
            var    length = list.Count;
            double std    = 0;

            for (int i = 0; i < length - 1; i++)
            {
                for (int j = i + 1; j < length; j++)
                {
                    std += (float)Math.Abs(this.Distance(list[i], list[j]) - average);
                }
            }
            return((float)(std / ((length * length - length) / 2.0)));
        }
Esempio n. 10
0
        public void ToStringContainsAllInformation()
        {
            var information = new GenerationInformation(
                GenerationInformationTest.Generation,
                totalElapsedTime: TimeSpan.FromSeconds(30),
                GenerationInformationTest.TotalNumberOfEvaluations,
                this._strategy,
                this._incumbent,
                "id");

            information.IncumbentTrainingScore = -3.4;
            information.IncumbentTestScore     = 1234.8;
            Assert.Equal(
                "12;0:00:00:30.0000000;3468;-3.4;1234.8;DifferentialEvolutionStrategy`2;[a: -23](Age: 2)[Engineered: yes];id",
                information.ToString());
        }
Esempio n. 11
0
        public void ScoreInformationHistoryThrowsForMissingTestInstancesSet()
        {
            var scorer = new GenerationInformationScorer <InstanceSeedFile, IntegerResult>(
                this._generationEvaluationActor,
                this._resultStorageActor,
                this._runEvaluator);

            var dummyInformation = new GenerationInformation(0, TimeSpan.Zero, 0, typeof(int), new ImmutableGenome(new Genome()), "id");

            Assert.Throws <ArgumentNullException>(
                () => scorer.ScoreInformationHistory(
                    informationHistory: new List <GenerationInformation> {
                dummyInformation
            },
                    trainingInstances: GenerationInformationScorerTest.CreateInstances(0, 1),
                    testInstances: null));
        }
Esempio n. 12
0
        public void ScoreInformationHistoryThrowsForEmptySetOfTrainingInstances()
        {
            var scorer = new GenerationInformationScorer <InstanceSeedFile, IntegerResult>(
                this._genomeSorter,
                this._resultStorageActor,
                this._runEvaluator);

            var dummyInformation = new GenerationInformation(0, 0, typeof(int), new ImmutableGenome(new Genome()));

            Assert.Throws <ArgumentOutOfRangeException>(
                () => scorer.ScoreInformationHistory(
                    informationHistory: new List <GenerationInformation> {
                dummyInformation
            },
                    trainingInstances: new List <InstanceSeedFile>(),
                    testInstances: GenerationInformationScorerTest.CreateInstances(0, 1)));
        }
Esempio n. 13
0
        public void ExportGenerationHistoryWritesOutAllInformation()
        {
            var firstGeneration = new GenerationInformation(
                0,
                TimeSpan.FromSeconds(30),
                34,
                typeof(GgaStrategy <TestInstance, TestResult>),
                new ImmutableGenome(new Genome()),
                "id");

            firstGeneration.IncumbentTrainingScore = -34.5;
            firstGeneration.IncumbentTestScore     = -20;
            var secondGeneration = new GenerationInformation(
                1,
                TimeSpan.FromSeconds(60),
                2587,
                typeof(DifferentialEvolutionStrategy <TestInstance, TestResult>),
                new ImmutableGenome(new Genome()),
                "id");

            secondGeneration.IncumbentTrainingScore = -104;
            secondGeneration.IncumbentTestScore     = -100;

            RunStatisticTracker.ExportGenerationHistory(new List <GenerationInformation> {
                firstGeneration, secondGeneration
            });

            var exported = File.ReadAllLines("generationHistory.csv");

            Assert.True(3 == exported.Length, "Expected three lines: One legend and two generations.");
            Assert.True(
                "Generation;Elapsed(d:hh:mm:ss);Total # Evaluations;Average Train Incumbent;Average Test Incumbent;Strategy;Incumbent;IncumbentID"
                == exported[0],
                "Legend is not as expected.");
            exported[1].ShouldBe(
                "0;0:00:00:30.0000000;34;-34.5;-20;GgaStrategy`2;[](Age: 0)[Engineered: no];id");
            exported[2].ShouldBe(
                "1;0:00:01:00.0000000;2587;-104;-100;DifferentialEvolutionStrategy`2;[](Age: 0)[Engineered: no];id");
        }
Esempio n. 14
0
        public void ExportGenerationHistoryWritesOutAllInformation()
        {
            var firstGeneration = new GenerationInformation(
                0,
                34,
                typeof(GgaStrategy <TestInstance, TestResult>),
                new ImmutableGenome(new Genome()));

            firstGeneration.IncumbentTrainingScore = -34.5;
            firstGeneration.IncumbentTestScore     = -20;
            var secondGeneration = new GenerationInformation(
                1,
                2587,
                typeof(DifferentialEvolutionStrategy <TestInstance, TestResult>),
                new ImmutableGenome(new Genome()));

            secondGeneration.IncumbentTrainingScore = -104;
            secondGeneration.IncumbentTestScore     = -100;

            RunStatisticTracker.ExportGenerationHistory(new List <GenerationInformation> {
                firstGeneration, secondGeneration
            });

            var exported = File.ReadAllLines("generationHistory.csv");

            Assert.True(3 == exported.Length, "Expected three lines: One legend and two generations.");
            Assert.True(
                "Generation;Total # Evaluations;Average Train Incumbent;Average Test Incumbent;Strategy;Incumbent" == exported[0],
                "Legend is not as expected.");
            Assert.True(
                "0;34;-34.5;-20;GgaStrategy`2;[](Age: 0)[Engineered: no]" == exported[1],
                "First generation information is not as expected.");
            Assert.True(
                "1;2587;-104;-100;DifferentialEvolutionStrategy`2;[](Age: 0)[Engineered: no]" == exported[2],
                "Second generation information is not as expected.");
        }
Esempio n. 15
0
        public void ExportAverageIncumbentScoresDeterminesScoresCorrectly()
        {
            var incumbent = new ImmutableGenome(new Genome());
            var strategy  = typeof(GgaStrategy <TestInstance, TestResult>);

            // Check what happens if the first generation takes more than 100 evaluations.
            var generation0 = new GenerationInformation(0, TimeSpan.Zero, 150, strategy, incumbent, "id");

            generation0.IncumbentTrainingScore = -34.5;
            generation0.IncumbentTestScore     = -20;

            // Check what happens for multiple information objects within one evaluation level.
            var generation1 = new GenerationInformation(1, TimeSpan.Zero, 199, strategy, incumbent, "id");

            generation1.IncumbentTrainingScore = 12.34;
            generation1.IncumbentTestScore     = 28.6;

            // Check what happens for an evaluation number equal to a bound.
            var generation2 = new GenerationInformation(2, TimeSpan.Zero, 300, strategy, incumbent, "id");

            generation2.IncumbentTrainingScore = 12.01;
            generation2.IncumbentTestScore     = 29;

            // Check what happens if there is no information object in a certain level (301-400).
            var generation3 = new GenerationInformation(2, TimeSpan.Zero, 401, strategy, incumbent, "id");

            generation3.IncumbentTrainingScore = 14;
            generation3.IncumbentTestScore     = 286;

            // Make sure to try an evaluation limit higher than the last total number of evaluations.
            RunStatisticTracker.ExportAverageIncumbentScores(
                new List <GenerationInformation> {
                generation0, generation1, generation2, generation3
            },
                600);

            var exported = File.ReadAllLines("scores.csv");

            Assert.True(7 == exported.Length, "Expected seven lines: One legend and six evaluation levels.");
            Assert.True(
                "# Evaluations;Average Train Incumbent;Average Test Incumbent" == exported[0],
                "Legend is not as expected.");
            Assert.True(
                "100;;" == exported[1],
                "There should be an empty line as first information is only gathered at 150 evaluations.");
            Assert.True(
                "200;12.34;28.6" == exported[2],
                "First score line should use latest information.");
            Assert.True(
                "300;12.01;29" == exported[3],
                "Second score line should use information with evaluation number equal to the bound.");
            Assert.True(
                "400;12.01;29" == exported[4],
                "Third score line should not change scores.");
            Assert.True(
                "500;14;286" == exported[5],
                "Fourth score line should use the newest data again.");
            Assert.True(
                "600;14;286" == exported[6],
                "Fifth score line should be written to have scores until the limit.");
        }
        public override void ReadFromFileDeserializesCorrectly()
        {
            /* Create status. */
            /* (1) population */
            var competitiveGenome = new Genome(2);

            competitiveGenome.SetGene("a", new Allele <int>(6));
            var nonCompetitiveGenome = new Genome(1);

            nonCompetitiveGenome.SetGene("b", new Allele <string>("oh"));
            this._population.AddGenome(competitiveGenome, isCompetitive: true);
            this._population.AddGenome(nonCompetitiveGenome, isCompetitive: false);
            /* (2) generation number, strategy index */
            var generation    = 2;
            var strategyIndex = 17;
            /* (3) information history */
            var generation0 =
                new GenerationInformation(0, 100, typeof(int), new ImmutableGenome(competitiveGenome));
            var generation1 =
                new GenerationInformation(1, 234, typeof(string), new ImmutableGenome(competitiveGenome));

            generation0.IncumbentTrainingScore = 12;
            generation0.IncumbentTestScore     = 345.8;
            this._informationHistory.Add(generation0);
            this._informationHistory.Add(generation1);
            var status = this.CreateStatus(
                generation,
                this._population,
                this._configuration,
                strategyIndex,
                this._informationHistory);
            /* (4) run results */
            var instance = new TestInstance("1");
            var result   = new TestResult(TimeSpan.FromMilliseconds(3));
            var results  = new Dictionary <TestInstance, TestResult> {
                { instance, result }
            };
            var runResults = new Dictionary <ImmutableGenome, ImmutableDictionary <TestInstance, TestResult> >();

            runResults.Add(new ImmutableGenome(new Genome()), results.ToImmutableDictionary());
            status.SetRunResults(runResults.ToImmutableDictionary());

            /* Write and read it from file. */
            status.WriteToFile(this.StatusFilePath);
            var deserializedStatus = StatusBase
                                     .ReadFromFile <Status <TestInstance, TestResult, GenomePredictionRandomForest <ReuseOldTreesStrategy>,
                                                            GenomePredictionForestModel <GenomePredictionTree>,
                                                            ReuseOldTreesStrategy> >(this.StatusFilePath);

            /* Check it's still the same. */
            /* (a) generation number, strategy index */
            Assert.Equal(generation, deserializedStatus.Generation);
            Assert.Equal(
                strategyIndex,
                deserializedStatus.CurrentUpdateStrategyIndex);
            /* (b) population */
            Assert.Equal(
                1,
                deserializedStatus.Population.GetCompetitiveIndividuals().Count);
            var deserializedCompetitiveGenome = deserializedStatus.Population.GetCompetitiveIndividuals().First();

            Assert.True(
                new Genome.GeneValueComparator().Equals(competitiveGenome, deserializedCompetitiveGenome),
                "Expected different competive genome.");
            Assert.Equal(
                competitiveGenome.Age,
                deserializedCompetitiveGenome.Age);
            Assert.Equal(
                1,
                deserializedStatus.Population.GetNonCompetitiveMates().Count);
            var deserializedNonCompetitiveGenome = deserializedStatus.Population.GetNonCompetitiveMates().First();

            Assert.True(
                new Genome.GeneValueComparator().Equals(nonCompetitiveGenome, deserializedNonCompetitiveGenome),
                "Expected different non-competive genome.");
            Assert.Equal(
                nonCompetitiveGenome.Age,
                deserializedNonCompetitiveGenome.Age);
            /* (c) configuration */
            Assert.True(
                deserializedStatus.Configuration.IsCompatible(this._configuration),
                "Expected different configuration.");
            /* (d) generation history */
            Assert.Equal(
                this._informationHistory.Select(information => information.ToString()).ToArray(),
                deserializedStatus.InformationHistory.Select(information => information.ToString()).ToArray());
            /* (e) run results */
            var instanceToResult = deserializedStatus.RunResults.Single().Value.Single();

            Assert.Equal(instance.ToString(), instanceToResult.Key.ToString());
            Assert.Equal(result.Runtime, instanceToResult.Value.Runtime);
        }
Esempio n. 17
0
 private float CalculateOverallAverage(GenerationInformation entries)
 {
     var list = entries.Entries;
     var length = list.Count;
     double average = 0;
     for(int i = 0; i < length - 1; i++)
     {
         for(int j = i + 1; j < length; j++)
         {
             average += this.Distance(list[i], list[j]);
         }
     }
     return (float)(average / ((length * length - length) / 2.0));
 }
Esempio n. 18
0
 private float CalculateOverallSTD(GenerationInformation entries, float average)
 {
     var list = entries.Entries;
     var length = list.Count;
     double std = 0;
     for(int i = 0; i < length - 1; i++)
     {
         for(int j = i + 1; j < length; j++)
         {
             std += (float)Math.Abs(this.Distance(list[i], list[j]) - average);
         }
     }
     return (float)(std / ((length * length - length) / 2.0));
 }
Esempio n. 19
0
 private void CalculateParameterValues(GenerationInformation entries, out float avgDistance, out float stdDistance, int pos)
 {
     var list = entries.Entries;
     CalculateDirectDistance(pos, list, out avgDistance, out stdDistance);
 }
Esempio n. 20
0
        private void CalculateParameterValues(GenerationInformation entries, out float avgDistance, out float stdDistance, int pos)
        {
            var list = entries.Entries;

            CalculateDirectDistance(pos, list, out avgDistance, out stdDistance);
        }
Esempio n. 21
0
 private void CalculateValues(GenerationInformation entries, out float minValue, out float averageValue, out float[] best)
 {
     minValue = float.MaxValue;
     averageValue = 0;
     var list = entries.Entries;
     var numberOfEntries = list.Count;
     int minIndex = -1;
     for(int i = 0; i < numberOfEntries; i++)
     {
         var value = list[i][0];
         if(value < minValue)
         {
             minValue = value;
             minIndex = i;
         }
         averageValue += value;
     }
     if(minIndex >= 0)
     {
         best = new float[list[minIndex].Length - 1];
         Array.Copy(list[minIndex], 1, best, 0, best.Length);
     }
     else
     {
         best = new float[0];
     }
     averageValue /= (numberOfEntries * numberOfEntries - numberOfEntries) / 2.0f;
 }
Esempio n. 22
0
        private void LoadInformation(Dictionary<int, GenerationInformation> generationDictionary)
        {
            using (CsvReader reader = new CsvReader(this.EvaluationFileName))
            {
                LoadHeader(reader);

                int numberOfParameters = this.Headers.Length;
                while(reader.LoadLine() != 0)
                {
                    int generation;
                    reader.Get(out generation, 0);
                    var parameters = new float[numberOfParameters + 1];
                    for(int i = 0; i < numberOfParameters + 1; i++)
                    {
                        // offset 1 for the generation i == 0 is the value
                        reader.Get(out parameters[i], i + 1);
                    }
                    GenerationInformation info;
                    if(!generationDictionary.TryGetValue(generation, out info))
                    {
                        info = new GenerationInformation();
                        info.Entries = new List<float[]>(250);
                        generationDictionary.Add(generation, info);
                    }
                    info.Entries.Add(parameters);
                }
            }
        }