private void CalculateValues(GenerationInformation entries, out float minValue, out float averageValue, out float[] best)
{
minValue = float.MaxValue;
averageValue = 0;
var list = entries.Entries;
var numberOfEntries = list.Count;
int minIndex = -1;
for (int i = 0; i < numberOfEntries; i++)
{
var value = list[i][0];
if (value < minValue)
{
minValue = value;
minIndex = i;
}
averageValue += value;
}
if (minIndex >= 0)
{
best = new float[list[minIndex].Length - 1];
Array.Copy(list[minIndex], 1, best, 0, best.Length);
}
else
{
best = new float[0];
}
averageValue /= (numberOfEntries * numberOfEntries - numberOfEntries) / 2.0f;
}
private void LoadInformation(Dictionary <int, GenerationInformation> generationDictionary)
{
using (CsvReader reader = new CsvReader(this.EvaluationFileName))
{
LoadHeader(reader);
int numberOfParameters = this.Headers.Length;
while (reader.LoadLine() != 0)
{
int generation;
reader.Get(out generation, 0);
var parameters = new float[numberOfParameters + 1];
for (int i = 0; i < numberOfParameters + 1; i++)
{
// offset 1 for the generation i == 0 is the value
reader.Get(out parameters[i], i + 1);
}
GenerationInformation info;
if (!generationDictionary.TryGetValue(generation, out info))
{
info = new GenerationInformation();
info.Entries = new List <float[]>(250);
generationDictionary.Add(generation, info);
}
info.Entries.Add(parameters);
}
}
}
public void ScoreInformationHistoryScoresGenerationInformationObjectsWithoutTestInstances()
{
var incumbent1 = new Genome();
incumbent1.SetGene(ExtractIntegerValue.ParameterName, new Allele <int>(-2));
var generationInformation = new GenerationInformation(0, TimeSpan.Zero, 0, typeof(int), new ImmutableGenome(incumbent1), "id");
var scorer = new GenerationInformationScorer <InstanceSeedFile, IntegerResult>(
this._generationEvaluationActor,
this._resultStorageActor,
this._runEvaluator);
scorer.ScoreInformationHistory(
new List <GenerationInformation> {
generationInformation
},
trainingInstances: GenerationInformationScorerTest.CreateInstances(startSeed: 50, number: 4),
testInstances: new List <InstanceSeedFile>());
Assert.Equal(
-103,
generationInformation.IncumbentTrainingScore);
Assert.Null(
generationInformation.IncumbentTestScore);
}
public void ToStringWorksForMissingInformation()
{
var information = new GenerationInformation(
GenerationInformationTest.Generation,
GenerationInformationTest.TotalNumberOfEvaluations,
this._strategy,
this._incumbent);
Assert.Equal(
"12;3468;;;DifferentialEvolutionStrategy`2;[a: -23](Age: 2)[Engineered: yes]",
information.ToString());
}
public void ToStringWorksForMissingInformation()
{
var information = new GenerationInformation(
GenerationInformationTest.Generation,
totalElapsedTime: TimeSpan.FromSeconds(30),
GenerationInformationTest.TotalNumberOfEvaluations,
this._strategy,
this._incumbent,
"id");
Assert.Equal(
"12;0:00:00:30.0000000;3468;;;DifferentialEvolutionStrategy`2;[a: -23](Age: 2)[Engineered: yes];id",
information.ToString());
}
public void ToStringContainsAllInformation()
{
var information = new GenerationInformation(
GenerationInformationTest.Generation,
GenerationInformationTest.TotalNumberOfEvaluations,
this._strategy,
this._incumbent);
information.IncumbentTrainingScore = -3.4;
information.IncumbentTestScore = 1234.8;
Assert.Equal(
"12;3468;-3.4;1234.8;DifferentialEvolutionStrategy`2;[a: -23](Age: 2)[Engineered: yes]",
information.ToString());
}
/// <summary>
/// Updates <see cref="_informationHistory"/> with the current (finished) generation.
/// </summary>
/// <param name="currentStrategy">The current strategy.</param>
private void UpdateGenerationHistory(IPopulationUpdateStrategy <TInstance, TResult> currentStrategy)
{
var evaluationCountRequest =
this._targetRunResultStorage.Ask <EvaluationStatistic>(new EvaluationStatisticRequest());
evaluationCountRequest.Wait();
var generationInformation = new GenerationInformation(
generation: this._currGeneration,
totalNumberOfEvaluations: evaluationCountRequest.Result.TotalEvaluationCount,
strategy: currentStrategy.GetType(),
incumbent: new ImmutableGenome(this._incumbentGenomeWrapper.IncumbentGenome));
this._informationHistory.Add(generationInformation);
}
private float CalculateOverallAverage(GenerationInformation entries)
{
var list = entries.Entries;
var length = list.Count;
double average = 0;
for (int i = 0; i < length - 1; i++)
{
for (int j = i + 1; j < length; j++)
{
average += this.Distance(list[i], list[j]);
}
}
return((float)(average / ((length * length - length) / 2.0)));
}
private float CalculateOverallSTD(GenerationInformation entries, float average)
{
var list = entries.Entries;
var length = list.Count;
double std = 0;
for (int i = 0; i < length - 1; i++)
{
for (int j = i + 1; j < length; j++)
{
std += (float)Math.Abs(this.Distance(list[i], list[j]) - average);
}
}
return((float)(std / ((length * length - length) / 2.0)));
}
public void ToStringContainsAllInformation()
{
var information = new GenerationInformation(
GenerationInformationTest.Generation,
totalElapsedTime: TimeSpan.FromSeconds(30),
GenerationInformationTest.TotalNumberOfEvaluations,
this._strategy,
this._incumbent,
"id");
information.IncumbentTrainingScore = -3.4;
information.IncumbentTestScore = 1234.8;
Assert.Equal(
"12;0:00:00:30.0000000;3468;-3.4;1234.8;DifferentialEvolutionStrategy`2;[a: -23](Age: 2)[Engineered: yes];id",
information.ToString());
}
public void ScoreInformationHistoryThrowsForMissingTestInstancesSet()
{
var scorer = new GenerationInformationScorer <InstanceSeedFile, IntegerResult>(
this._generationEvaluationActor,
this._resultStorageActor,
this._runEvaluator);
var dummyInformation = new GenerationInformation(0, TimeSpan.Zero, 0, typeof(int), new ImmutableGenome(new Genome()), "id");
Assert.Throws <ArgumentNullException>(
() => scorer.ScoreInformationHistory(
informationHistory: new List <GenerationInformation> {
dummyInformation
},
trainingInstances: GenerationInformationScorerTest.CreateInstances(0, 1),
testInstances: null));
}
public void ScoreInformationHistoryThrowsForEmptySetOfTrainingInstances()
{
var scorer = new GenerationInformationScorer <InstanceSeedFile, IntegerResult>(
this._genomeSorter,
this._resultStorageActor,
this._runEvaluator);
var dummyInformation = new GenerationInformation(0, 0, typeof(int), new ImmutableGenome(new Genome()));
Assert.Throws <ArgumentOutOfRangeException>(
() => scorer.ScoreInformationHistory(
informationHistory: new List <GenerationInformation> {
dummyInformation
},
trainingInstances: new List <InstanceSeedFile>(),
testInstances: GenerationInformationScorerTest.CreateInstances(0, 1)));
}
public void ExportGenerationHistoryWritesOutAllInformation()
{
var firstGeneration = new GenerationInformation(
0,
TimeSpan.FromSeconds(30),
34,
typeof(GgaStrategy <TestInstance, TestResult>),
new ImmutableGenome(new Genome()),
"id");
firstGeneration.IncumbentTrainingScore = -34.5;
firstGeneration.IncumbentTestScore = -20;
var secondGeneration = new GenerationInformation(
1,
TimeSpan.FromSeconds(60),
2587,
typeof(DifferentialEvolutionStrategy <TestInstance, TestResult>),
new ImmutableGenome(new Genome()),
"id");
secondGeneration.IncumbentTrainingScore = -104;
secondGeneration.IncumbentTestScore = -100;
RunStatisticTracker.ExportGenerationHistory(new List <GenerationInformation> {
firstGeneration, secondGeneration
});
var exported = File.ReadAllLines("generationHistory.csv");
Assert.True(3 == exported.Length, "Expected three lines: One legend and two generations.");
Assert.True(
"Generation;Elapsed(d:hh:mm:ss);Total # Evaluations;Average Train Incumbent;Average Test Incumbent;Strategy;Incumbent;IncumbentID"
== exported[0],
"Legend is not as expected.");
exported[1].ShouldBe(
"0;0:00:00:30.0000000;34;-34.5;-20;GgaStrategy`2;[](Age: 0)[Engineered: no];id");
exported[2].ShouldBe(
"1;0:00:01:00.0000000;2587;-104;-100;DifferentialEvolutionStrategy`2;[](Age: 0)[Engineered: no];id");
}
public void ExportGenerationHistoryWritesOutAllInformation()
{
var firstGeneration = new GenerationInformation(
0,
34,
typeof(GgaStrategy <TestInstance, TestResult>),
new ImmutableGenome(new Genome()));
firstGeneration.IncumbentTrainingScore = -34.5;
firstGeneration.IncumbentTestScore = -20;
var secondGeneration = new GenerationInformation(
1,
2587,
typeof(DifferentialEvolutionStrategy <TestInstance, TestResult>),
new ImmutableGenome(new Genome()));
secondGeneration.IncumbentTrainingScore = -104;
secondGeneration.IncumbentTestScore = -100;
RunStatisticTracker.ExportGenerationHistory(new List <GenerationInformation> {
firstGeneration, secondGeneration
});
var exported = File.ReadAllLines("generationHistory.csv");
Assert.True(3 == exported.Length, "Expected three lines: One legend and two generations.");
Assert.True(
"Generation;Total # Evaluations;Average Train Incumbent;Average Test Incumbent;Strategy;Incumbent" == exported[0],
"Legend is not as expected.");
Assert.True(
"0;34;-34.5;-20;GgaStrategy`2;[](Age: 0)[Engineered: no]" == exported[1],
"First generation information is not as expected.");
Assert.True(
"1;2587;-104;-100;DifferentialEvolutionStrategy`2;[](Age: 0)[Engineered: no]" == exported[2],
"Second generation information is not as expected.");
}
public void ExportAverageIncumbentScoresDeterminesScoresCorrectly()
{
var incumbent = new ImmutableGenome(new Genome());
var strategy = typeof(GgaStrategy <TestInstance, TestResult>);
// Check what happens if the first generation takes more than 100 evaluations.
var generation0 = new GenerationInformation(0, TimeSpan.Zero, 150, strategy, incumbent, "id");
generation0.IncumbentTrainingScore = -34.5;
generation0.IncumbentTestScore = -20;
// Check what happens for multiple information objects within one evaluation level.
var generation1 = new GenerationInformation(1, TimeSpan.Zero, 199, strategy, incumbent, "id");
generation1.IncumbentTrainingScore = 12.34;
generation1.IncumbentTestScore = 28.6;
// Check what happens for an evaluation number equal to a bound.
var generation2 = new GenerationInformation(2, TimeSpan.Zero, 300, strategy, incumbent, "id");
generation2.IncumbentTrainingScore = 12.01;
generation2.IncumbentTestScore = 29;
// Check what happens if there is no information object in a certain level (301-400).
var generation3 = new GenerationInformation(2, TimeSpan.Zero, 401, strategy, incumbent, "id");
generation3.IncumbentTrainingScore = 14;
generation3.IncumbentTestScore = 286;
// Make sure to try an evaluation limit higher than the last total number of evaluations.
RunStatisticTracker.ExportAverageIncumbentScores(
new List <GenerationInformation> {
generation0, generation1, generation2, generation3
},
600);
var exported = File.ReadAllLines("scores.csv");
Assert.True(7 == exported.Length, "Expected seven lines: One legend and six evaluation levels.");
Assert.True(
"# Evaluations;Average Train Incumbent;Average Test Incumbent" == exported[0],
"Legend is not as expected.");
Assert.True(
"100;;" == exported[1],
"There should be an empty line as first information is only gathered at 150 evaluations.");
Assert.True(
"200;12.34;28.6" == exported[2],
"First score line should use latest information.");
Assert.True(
"300;12.01;29" == exported[3],
"Second score line should use information with evaluation number equal to the bound.");
Assert.True(
"400;12.01;29" == exported[4],
"Third score line should not change scores.");
Assert.True(
"500;14;286" == exported[5],
"Fourth score line should use the newest data again.");
Assert.True(
"600;14;286" == exported[6],
"Fifth score line should be written to have scores until the limit.");
}
public override void ReadFromFileDeserializesCorrectly()
{
/* Create status. */
/* (1) population */
var competitiveGenome = new Genome(2);
competitiveGenome.SetGene("a", new Allele <int>(6));
var nonCompetitiveGenome = new Genome(1);
nonCompetitiveGenome.SetGene("b", new Allele <string>("oh"));
this._population.AddGenome(competitiveGenome, isCompetitive: true);
this._population.AddGenome(nonCompetitiveGenome, isCompetitive: false);
/* (2) generation number, strategy index */
var generation = 2;
var strategyIndex = 17;
/* (3) information history */
var generation0 =
new GenerationInformation(0, 100, typeof(int), new ImmutableGenome(competitiveGenome));
var generation1 =
new GenerationInformation(1, 234, typeof(string), new ImmutableGenome(competitiveGenome));
generation0.IncumbentTrainingScore = 12;
generation0.IncumbentTestScore = 345.8;
this._informationHistory.Add(generation0);
this._informationHistory.Add(generation1);
var status = this.CreateStatus(
generation,
this._population,
this._configuration,
strategyIndex,
this._informationHistory);
/* (4) run results */
var instance = new TestInstance("1");
var result = new TestResult(TimeSpan.FromMilliseconds(3));
var results = new Dictionary <TestInstance, TestResult> {
{ instance, result }
};
var runResults = new Dictionary <ImmutableGenome, ImmutableDictionary <TestInstance, TestResult> >();
runResults.Add(new ImmutableGenome(new Genome()), results.ToImmutableDictionary());
status.SetRunResults(runResults.ToImmutableDictionary());
/* Write and read it from file. */
status.WriteToFile(this.StatusFilePath);
var deserializedStatus = StatusBase
.ReadFromFile <Status <TestInstance, TestResult, GenomePredictionRandomForest <ReuseOldTreesStrategy>,
GenomePredictionForestModel <GenomePredictionTree>,
ReuseOldTreesStrategy> >(this.StatusFilePath);
/* Check it's still the same. */
/* (a) generation number, strategy index */
Assert.Equal(generation, deserializedStatus.Generation);
Assert.Equal(
strategyIndex,
deserializedStatus.CurrentUpdateStrategyIndex);
/* (b) population */
Assert.Equal(
1,
deserializedStatus.Population.GetCompetitiveIndividuals().Count);
var deserializedCompetitiveGenome = deserializedStatus.Population.GetCompetitiveIndividuals().First();
Assert.True(
new Genome.GeneValueComparator().Equals(competitiveGenome, deserializedCompetitiveGenome),
"Expected different competive genome.");
Assert.Equal(
competitiveGenome.Age,
deserializedCompetitiveGenome.Age);
Assert.Equal(
1,
deserializedStatus.Population.GetNonCompetitiveMates().Count);
var deserializedNonCompetitiveGenome = deserializedStatus.Population.GetNonCompetitiveMates().First();
Assert.True(
new Genome.GeneValueComparator().Equals(nonCompetitiveGenome, deserializedNonCompetitiveGenome),
"Expected different non-competive genome.");
Assert.Equal(
nonCompetitiveGenome.Age,
deserializedNonCompetitiveGenome.Age);
/* (c) configuration */
Assert.True(
deserializedStatus.Configuration.IsCompatible(this._configuration),
"Expected different configuration.");
/* (d) generation history */
Assert.Equal(
this._informationHistory.Select(information => information.ToString()).ToArray(),
deserializedStatus.InformationHistory.Select(information => information.ToString()).ToArray());
/* (e) run results */
var instanceToResult = deserializedStatus.RunResults.Single().Value.Single();
Assert.Equal(instance.ToString(), instanceToResult.Key.ToString());
Assert.Equal(result.Runtime, instanceToResult.Value.Runtime);
}
private float CalculateOverallAverage(GenerationInformation entries)
{
var list = entries.Entries;
var length = list.Count;
double average = 0;
for(int i = 0; i < length - 1; i++)
{
for(int j = i + 1; j < length; j++)
{
average += this.Distance(list[i], list[j]);
}
}
return (float)(average / ((length * length - length) / 2.0));
}
private float CalculateOverallSTD(GenerationInformation entries, float average)
{
var list = entries.Entries;
var length = list.Count;
double std = 0;
for(int i = 0; i < length - 1; i++)
{
for(int j = i + 1; j < length; j++)
{
std += (float)Math.Abs(this.Distance(list[i], list[j]) - average);
}
}
return (float)(std / ((length * length - length) / 2.0));
}
private void CalculateParameterValues(GenerationInformation entries, out float avgDistance, out float stdDistance, int pos)
{
var list = entries.Entries;
CalculateDirectDistance(pos, list, out avgDistance, out stdDistance);
}
private void CalculateParameterValues(GenerationInformation entries, out float avgDistance, out float stdDistance, int pos)
{
var list = entries.Entries;
CalculateDirectDistance(pos, list, out avgDistance, out stdDistance);
}
private void CalculateValues(GenerationInformation entries, out float minValue, out float averageValue, out float[] best)
{
minValue = float.MaxValue;
averageValue = 0;
var list = entries.Entries;
var numberOfEntries = list.Count;
int minIndex = -1;
for(int i = 0; i < numberOfEntries; i++)
{
var value = list[i][0];
if(value < minValue)
{
minValue = value;
minIndex = i;
}
averageValue += value;
}
if(minIndex >= 0)
{
best = new float[list[minIndex].Length - 1];
Array.Copy(list[minIndex], 1, best, 0, best.Length);
}
else
{
best = new float[0];
}
averageValue /= (numberOfEntries * numberOfEntries - numberOfEntries) / 2.0f;
}
private void LoadInformation(Dictionary<int, GenerationInformation> generationDictionary)
{
using (CsvReader reader = new CsvReader(this.EvaluationFileName))
{
LoadHeader(reader);
int numberOfParameters = this.Headers.Length;
while(reader.LoadLine() != 0)
{
int generation;
reader.Get(out generation, 0);
var parameters = new float[numberOfParameters + 1];
for(int i = 0; i < numberOfParameters + 1; i++)
{
// offset 1 for the generation i == 0 is the value
reader.Get(out parameters[i], i + 1);
}
GenerationInformation info;
if(!generationDictionary.TryGetValue(generation, out info))
{
info = new GenerationInformation();
info.Entries = new List<float[]>(250);
generationDictionary.Add(generation, info);
}
info.Entries.Add(parameters);
}
}
}
