/// <summary>
/// Runs post-hoc analysis for all batches in the given experiment/run. This can be part of either the initialization
/// or primary run phase.
/// </summary>
/// <param name="batchesWithGenomeData">The total number of batches containing genome data.</param>
/// <param name="runPhase">
/// Indicates whether this is part of the initialization or primary run phase.
/// </param>
/// <param name="experimentParameters">Experiment configuration parameters.</param>
/// <param name="inputNeuronCount">Count of neurons in controller input layer.</param>
/// <param name="outputNeuronCount">Count of neurons in controller output layer.</param>
/// <param name="curRun">The run number.</param>
/// <param name="numRuns">The total number of runs.</param>
/// <param name="curExperimentConfiguration">The experiment configuration parameters.</param>
/// <param name="generateSimulationResults">Indicates whether to write out the results of the batch simulation.</param>
/// <param name="writeResultsToDatabase">
/// Indicates whether to write results directly into a database (if not, results are
/// written to a flat file).
/// </param>
/// <param name="generateTrajectoryData">Indicates whether the full navigator trajectory should be simulated and persisted.</param>
/// <param name="generateTrajectoryDiversityScore">
/// Indicates whether quantification of navigator trajectory diversity
/// should be written out.
/// </param>
/// <param name="generateNaturalClustering">Indicates whether the natural population clusters should be analyzed.</param>
/// <param name="generateMazeBitmaps">Indicates whether bitmap files of the distinct mazes should be written out.</param>
/// <param name="generateTrajectoryBitmaps">
/// Indicates whether bitmap files depicting the navigator trajectory should be
/// written out.
/// </param>
/// <param name="baseImageOutputDirectory">The path to the output directory for the trajectory images.</param>
/// <param name="clusterImprovementThreshold">
/// The number of cluster additions that are permitted without further maximization of silhouette width.
/// </param>
/// <param name="sampleSize">The number of genomes sampled from the extant species for clustering analysis.</param>
/// <param name="sampleClusterObservationsFromSpecies">
/// Indicates whether to sample observations used in clustering analysis
/// from species or from the population as a whole.
/// </param>
private static void ProcessAndLogPerBatchResults(IList<int> batchesWithGenomeData, RunPhase runPhase,
ExperimentParameters experimentParameters, int inputNeuronCount, int outputNeuronCount, int curRun,
int numRuns, ExperimentDictionary curExperimentConfiguration, bool generateSimulationResults,
bool generateTrajectoryData, bool generateTrajectoryDiversityScore, bool generateNaturalClustering,
bool writeResultsToDatabase, bool generateMazeBitmaps, bool generateTrajectoryBitmaps,
string baseImageOutputDirectory, int clusterImprovementThreshold, int sampleSize,
bool sampleClusterObservationsFromSpecies)
{
IList<CoevolutionMCSMazeExperimentGenome> staticInitializationMazes = null;
// If this invocation is processing initialization results, just get the maze up front as it will remain
// the same throughout the initialization process
if (runPhase == RunPhase.Initialization)
{
staticInitializationMazes =
ExperimentDataHandler.GetMazeGenomeData(curExperimentConfiguration.ExperimentDictionaryID,
curRun, 0);
}
// Iterate through each batch and evaluate maze/navigator combinations
foreach (int curBatch in batchesWithGenomeData)
{
// Create the maze/navigator map
MapEvaluator mapEvaluator =
new MapEvaluator(experimentParameters, inputNeuronCount, outputNeuronCount);
_executionLogger.Info(string.Format("Executing {0} run phase analysis for batch [{1}] of run [{2}/{3}]",
runPhase, curBatch, curRun, numRuns));
// TODO: Check first to see if trajectory evaluations already exist
// Get any existing navigation results (this avoids rerunning failed combinations)
var successfulNavigations =
ExperimentDataHandler.GetSuccessfulNavigations(curExperimentConfiguration.ExperimentDictionaryID,
curRun, curBatch);
// If successful navigation results were found and we're not re-running the simulations,
// initialize the map evaluator with only those combinations known to be successful
if (generateSimulationResults == false && successfulNavigations != null &&
successfulNavigations.Count > 0)
{
List<Tuple<CoevolutionMCSMazeExperimentGenome, CoevolutionMCSNavigatorExperimentGenome>>
successfulGenomeCombos =
new List<Tuple<CoevolutionMCSMazeExperimentGenome, CoevolutionMCSNavigatorExperimentGenome>>
(successfulNavigations.Count());
// Get distinct maze and navigator genomes
var mazeGenomeData =
ExperimentDataHandler.GetMazeGenomeData(curExperimentConfiguration.ExperimentDictionaryID,
curRun, curBatch, successfulNavigations.Select(n => n.MazeGenomeID).Distinct().ToList());
var navigatorGenomeData =
ExperimentDataHandler.GetNavigatorGenomeData(curExperimentConfiguration.ExperimentDictionaryID,
curRun, curBatch, runPhase,
successfulNavigations.Select(n => n.NavigatorGenomeID).Distinct().ToList());
// Build list of successful maze/navigator combinations
successfulGenomeCombos.AddRange(
successfulNavigations.Select(
successfulNav =>
new Tuple<CoevolutionMCSMazeExperimentGenome, CoevolutionMCSNavigatorExperimentGenome>(
mazeGenomeData.First(gd => successfulNav.MazeGenomeID == gd.GenomeID),
navigatorGenomeData.First(gd => successfulNav.NavigatorGenomeID == gd.GenomeID))));
// Initialize the maze/navigator map with combinations that are known to be successful
mapEvaluator.Initialize(successfulGenomeCombos);
}
// Otherwise, just initialize with all combinations
else
{
// Initialize the maze/navigator map with the serialized maze and navigator data (this does the parsing)
mapEvaluator.Initialize(
runPhase == RunPhase.Initialization
? staticInitializationMazes
: ExperimentDataHandler.GetMazeGenomeData(curExperimentConfiguration.ExperimentDictionaryID,
curRun, curBatch), ExperimentDataHandler.GetNavigatorGenomeData(
curExperimentConfiguration.ExperimentDictionaryID, curRun, curBatch, runPhase));
}
// Evaluate all of the maze/navigator combinations in the batch
mapEvaluator.RunTrajectoryEvaluations();
if (generateSimulationResults)
{
// Save the evaluation results
ExperimentDataHandler.WriteNavigatorMazeEvaluationData(
curExperimentConfiguration.ExperimentDictionaryID, curRun, curBatch, runPhase,
mapEvaluator.EvaluationUnits, CommitPageSize, writeResultsToDatabase);
}
if (generateMazeBitmaps)
{
// Generate bitmaps of distinct mazes extant at the current point in time
ImageGenerationHandler.GenerateMazeBitmaps(baseImageOutputDirectory,
curExperimentConfiguration.ExperimentName, curExperimentConfiguration.ExperimentDictionaryID,
curRun, curBatch, mapEvaluator.EvaluationUnits);
}
if (generateTrajectoryBitmaps)
{
// Generate bitmaps of trajectory for all successful trials
ImageGenerationHandler.GenerateBitmapsForSuccessfulTrials(
baseImageOutputDirectory, curExperimentConfiguration.ExperimentName,
curExperimentConfiguration.ExperimentDictionaryID,
curRun, curBatch, mapEvaluator.EvaluationUnits, runPhase);
}
if (generateTrajectoryData && runPhase != RunPhase.Initialization)
{
// Write out the full trajectory of all agents through all solved mazes
ExperimentDataHandler.WriteTrajectoryData(curExperimentConfiguration.ExperimentDictionaryID, curRun,
curBatch, mapEvaluator.EvaluationUnits, CommitPageSize, writeResultsToDatabase);
}
// Compare trajectories of agents through maze to get quantitative sense of solution diversity
// Mean euclidean distance will be calculated for selected trajectory against:
// 1. Other agent trajectories in the current maze only
// 2. Other agent trajectories on *another* maze only
// 3. All other agent trajectories (regardless of maze)
if (generateTrajectoryDiversityScore && runPhase != RunPhase.Initialization)
{
ExperimentDataHandler.WriteTrajectoryDiversityData(
curExperimentConfiguration.ExperimentDictionaryID, curRun, curBatch,
EvaluationHandler.CalculateTrajectoryDiversity(mapEvaluator.EvaluationUnits),
writeResultsToDatabase);
}
// Only write clustering results for primary runs when the number of trajectories have surpassed
// the minimum cluster count of 3
if (generateNaturalClustering && runPhase != RunPhase.Initialization)
{
// Extract uniform samples of maze and navigator genomes from each extant specie
// on which to run clustering analysis
var evaluationSamples = sampleClusterObservationsFromSpecies
? DataManipulationUtil.ExtractEvaluationUnitSamplesFromSpecies(
curExperimentConfiguration.ExperimentDictionaryID, curRun, curBatch,
mapEvaluator.EvaluationUnits.Where(eu => eu.IsMazeSolved).ToList(), sampleSize)
: DataManipulationUtil.ExtractEvaluationUnitSamplesFromPopulation(
mapEvaluator.EvaluationUnits.Where(eu => eu.IsMazeSolved).ToList(),
sampleSize);
// Calculate natural clustering of the population trajectories at each point in time and persist
ExperimentDataHandler.WriteClusteringDiversityData(
curExperimentConfiguration.ExperimentDictionaryID, curRun, curBatch,
EvaluationHandler.CalculateNaturalClustering(evaluationSamples, clusterImprovementThreshold),
writeResultsToDatabase);
}
}
}
/// <summary>
/// Runs post-hoc analysis for the entire experiment run in aggregate (i.e. not per batch). This analyzes the primary
/// run phase only.
/// </summary>
/// <param name="batchesWithGenomeData">The total number of batches containing genome data.</param>
/// <param name="experimentParameters">Experiment configuration parameters.</param>
/// <param name="inputNeuronCount">Count of neurons in controller input layer.</param>
/// <param name="outputNeuronCount">Count of neurons in controller output layer.</param>
/// <param name="curRun">The run number.</param>
/// <param name="numRuns">The total number of runs.</param>
/// <param name="curExperimentConfiguration">The experiment configuration parameters.</param>
/// <param name="generateSimulationResults">Indicates whether to write out the results of the batch simulation.</param>
/// <param name="writeResultsToDatabase">
/// Indicates whether to write results directly into a database (if not, results are
/// written to a flat file).
/// </param>
/// <param name="generateTrajectoryData">Indicates whether the full navigator trajectory should be simulated and persisted.</param>
/// <param name="generateTrajectoryDiversityScore">
/// Indicates whether quantification of navigator trajectory diversity
/// should be written out.
/// </param>
/// <param name="generateNaturalClustering">Indicates whether the natural population clusters should be analyzed.</param>
/// <param name="generateMazeBitmaps">Indicates whether bitmap files of the distinct mazes should be written out.</param>
/// <param name="generateTrajectoryBitmaps">
/// Indicates whether bitmap files depicting the navigator trajectory should be
/// written out.
/// </param>
/// <param name="baseImageOutputDirectory">The path to the output directory for the trajectory images.</param>
/// <param name="clusterImprovementThreshold">
/// The number of cluster additions that are permitted without further maximization of silhouette width.
/// </param>
/// <param name="specieSampleSize">The number of genomes sampled from the extant species for clustering analysis.</param>
private static void ProcessAndLogAggregateRunResults(IList<int> batchesWithGenomeData,
ExperimentParameters experimentParameters,
int inputNeuronCount, int outputNeuronCount, int curRun,
int numRuns, ExperimentDictionary curExperimentConfiguration, bool generateSimulationResults,
bool generateTrajectoryData, bool generateTrajectoryDiversityScore, bool generateNaturalClustering,
bool writeResultsToDatabase, bool generateMazeBitmaps, bool generateTrajectoryBitmaps,
string baseImageOutputDirectory, int clusterImprovementThreshold, int specieSampleSize)
{
// Create the maze/navigator map
MapEvaluator mapEvaluator =
new MapEvaluator(experimentParameters, inputNeuronCount, outputNeuronCount);
_executionLogger.Info(string.Format("Executing aggregate run analysis for run [{0}/{1}]", curRun, numRuns));
// TODO: Might need to consider the option of pinning the mazes in memory and batching through the agents
// TODO: as pulling both into memory simultaneously is probably going to use too much memory
// Initialize the evaluator with the maze genomes discovered throughout the entire run
mapEvaluator.Initialize(
ExperimentDataHandler.GetMazeGenomeData(curExperimentConfiguration.ExperimentDictionaryID, curRun));
// Then evaluate each agent against each maze on a batch-by-batch basis
foreach (int curBatch in batchesWithGenomeData)
{
_executionLogger.Info(string.Format("Executing trajectory for batch [{0}] and run [{1}/{2}]", curBatch,
curRun, numRuns));
// Initialize with the agent genomes from the current batch
mapEvaluator.Initialize(
ExperimentDataHandler.GetNavigatorGenomeData(curExperimentConfiguration.ExperimentDictionaryID,
curRun, curBatch, RunPhase.Primary));
// Evaluate all of the maze/navigator combinations in the batch
mapEvaluator.RunTrajectoryEvaluations();
if (generateSimulationResults)
{
// Save the evaluation results
// TODO: Implement - not sure if this adds a whole lot of value beyond per-batch approach
}
if (generateMazeBitmaps)
{
// Generate bitmaps of distinct mazes extant throughout the run
// TODO: Implement - not sure if this adds a whole lot of value beyond per-batch approach
}
if (generateTrajectoryBitmaps)
{
// Generate bitmaps of trajectory for all successful trials
// TODO: Implement - not sure if this adds a whole lot of value beyond per-batch approach
}
if (generateTrajectoryData)
{
// Write out the full trajectory of all agents through all solved mazes
// TODO: Implement - may end up removing this as it takes up an enormous amount of space
}
// Compare trajectories of agents through maze to get quantitative sense of solution diversity
// Mean euclidean distance will be calculated for selected trajectory against:
// 1. Other agent trajectories in the current maze only
// 2. Other agent trajectories on *another* maze only
// 3. All other agent trajectories (regardless of maze)
if (generateTrajectoryDiversityScore)
{
// TODO: Implement
ExperimentDataHandler.WriteTrajectoryDiversityData(
curExperimentConfiguration.ExperimentDictionaryID, curRun,
EvaluationHandler.CalculateTrajectoryDiversity(mapEvaluator.EvaluationUnits),
writeResultsToDatabase);
}
if (generateNaturalClustering)
{
// Calculate natural clustering of the population trajectories throughout the run and persist
ExperimentDataHandler.WriteClusteringDiversityData(
curExperimentConfiguration.ExperimentDictionaryID, curRun,
EvaluationHandler.CalculateNaturalClustering(mapEvaluator.EvaluationUnits,
clusterImprovementThreshold), writeResultsToDatabase);
}
}
}
