/// <summary>
/// Retrieves the batches during a given run of a given experiment that have associated genome data.
/// </summary>
/// <param name="experimentId">The experiment that was executed.</param>
/// <param name="run">The run number of the given experiment.</param>
/// <param name="runPhase">The run phase (i.e. initialization or primary) for which to get the associated batches.</param>
/// <returns>The collection of batch numbers in the given experiment/run that have associated genome data.</returns>
public static IList<int> GetBatchesWithGenomeData(int experimentId, int run, RunPhase runPhase)
{
IList<int> batchesWithGenomeData = null;
bool querySuccess = false;
int retryCnt = 0;
while (querySuccess == false && retryCnt <= MaxQueryRetryCnt)
{
try
{
// Query for the distinct batches in the current run of the given experiment
using (ExperimentDataEntities context = new ExperimentDataEntities())
{
batchesWithGenomeData = context.CoevolutionMCSNavigatorExperimentGenomes.Where(
expData =>
expData.ExperimentDictionaryID == experimentId && expData.Run == run &&
expData.RunPhase.RunPhaseName == runPhase.ToString())
.Select(row => row.Generation)
.Distinct().OrderBy(row => row).ToList();
}
querySuccess = true;
}
catch (Exception e)
{
HandleQueryException(MethodBase.GetCurrentMethod().ToString(), retryCnt++, e);
}
}
return batchesWithGenomeData;
}
/// <summary>
/// Writes the given evaluation results to the experiment database.
/// </summary>
/// <param name="experimentId">The experiment that was executed.</param>
/// <param name="run">The run number of the given experiment.</param>
/// <param name="batch">The batch number of the given run.</param>
/// <param name="runPhase">
/// Indicates whether this is part of the initialization or primary experiment phase.
/// </param>
/// <param name="evaluationUnits">The evaluation results to persist.</param>
/// <param name="commitPageSize">The number of records that are committed within a single batch/context.</param>
private static void WriteNavigatorMazeEvaluationDataToDatabase(int experimentId, int run, int batch,
RunPhase runPhase,
IList<MazeNavigatorEvaluationUnit> evaluationUnits, int commitPageSize)
{
// Page through the result set, committing each in the specified batch size
for (int curPage = 0; curPage <= evaluationUnits.Count/commitPageSize; curPage++)
{
IList<CoevolutionMCSMazeNavigatorResult> serializedResults =
new List<CoevolutionMCSMazeNavigatorResult>(commitPageSize);
// Go ahead and lookup the run phase key for all of the records
// (instead of hitting the database on every iteration of the below loop)
int runPhaseKey = GetRunPhaseKey(runPhase);
// Build a list of serialized results
foreach (
MazeNavigatorEvaluationUnit evaluationUnit in
evaluationUnits.Skip(curPage*commitPageSize).Take(commitPageSize))
{
serializedResults.Add(new CoevolutionMCSMazeNavigatorResult
{
ExperimentDictionaryID = experimentId,
Run = run,
Generation = batch,
RunPhase_FK = runPhaseKey,
MazeGenomeID = evaluationUnit.MazeId,
NavigatorGenomeID = evaluationUnit.AgentId,
IsMazeSolved = evaluationUnit.IsMazeSolved,
NumTimesteps = evaluationUnit.NumTimesteps
});
}
// Create a new context and persist the batch
using (ExperimentDataEntities context = new ExperimentDataEntities())
{
// Auto-detect changes and save validation are switched off to speed things up
context.Configuration.AutoDetectChangesEnabled = false;
context.Configuration.ValidateOnSaveEnabled = false;
context.CoevolutionMCSMazeNavigatorResults.AddRange(serializedResults);
context.SaveChanges();
}
}
}
/// <summary>
/// Retrieves the primary key associated with the given run phase.
/// </summary>
/// <param name="runPhase">The run phase for which to lookup the key.</param>
/// <returns>The key (in the "RunPhase" database table) for the given run phase object.</returns>
private static int GetRunPhaseKey(RunPhase runPhase)
{
int runPhaseKey = 0;
bool querySuccess = false;
int retryCnt = 0;
while (querySuccess == false && retryCnt <= MaxQueryRetryCnt)
{
try
{
// Query for the run phase key
using (ExperimentDataEntities context = new ExperimentDataEntities())
{
runPhaseKey =
context.RunPhases.First(runPhaseData => runPhaseData.RunPhaseName == runPhase.ToString())
.RunPhaseID;
}
querySuccess = true;
}
catch (Exception e)
{
HandleQueryException(MethodBase.GetCurrentMethod().ToString(), retryCnt++, e);
}
}
return runPhaseKey;
}
/// <summary>
/// Writes the given evaluation results to the experiment database or to a flat file.
/// </summary>
/// <param name="experimentId">The experiment that was executed.</param>
/// <param name="run">The run number of the given experiment.</param>
/// <param name="batch">The batch number of the given run.</param>
/// <param name="runPhase">
/// Indicates whether this is part of the initialization or primary experiment phase.
/// </param>
/// <param name="evaluationUnits">The evaluation results to persist.</param>
/// <param name="commitPageSize">The number of records that are committed within a single batch/context.</param>
/// <param name="writeToDatabase">
/// Indicates whether evaluation results should be written directly to the database or to a
/// flat file.
/// </param>
public static void WriteNavigatorMazeEvaluationData(int experimentId, int run, int batch, RunPhase runPhase,
IList<MazeNavigatorEvaluationUnit> evaluationUnits, int commitPageSize, bool writeToDatabase)
{
// Write results to the database if the option has been specified
if (writeToDatabase)
{
WriteNavigatorMazeEvaluationDataToDatabase(experimentId, run, batch, runPhase, evaluationUnits,
commitPageSize);
}
// Otherwise, write to the flat file output
else
{
WriteNavigatorMazeEvaluationDataToFile(experimentId, run, batch, runPhase, evaluationUnits);
}
}
/// <summary>
/// Retrieves specie assignments for the given navigator genome IDs and experiment, run, and batch.
/// </summary>
/// <param name="experimentId">The experiment that was executed.</param>
/// <param name="run">The run number of the given experiment.</param>
/// <param name="batch">The batch number of the given run.</param>
/// <param name="runPhase">Indicates whether this is part of the initialization or primary experiment phase.</param>
/// <param name="navigatorGenomeIds">The navigator genome IDs to group into species.</param>
/// <returns>Specie assignments for the given navigator genome IDs and experiment, run, and batch.</returns>
public static List<SpecieGenomesGroup> GetSpecieAssignmentsForNavigatorGenomeIds(int experimentId, int run,
int batch, RunPhase runPhase, IList<int> navigatorGenomeIds)
{
var navigatorSpecieGenomesGroups = new List<SpecieGenomesGroup>();
bool querySuccess = false;
int retryCnt = 0;
while (querySuccess == false && retryCnt <= MaxQueryRetryCnt)
{
try
{
using (ExperimentDataEntities context = new ExperimentDataEntities())
{
// Get the species to which the navigators are assigned and group by species
var specieGroupedNavigators =
context.CoevolutionMCSNavigatorExperimentGenomes.Where(
nav =>
experimentId == nav.ExperimentDictionaryID && run == nav.Run &&
batch == nav.Generation && runPhase.ToString().Equals(nav.RunPhase.RunPhaseName) &&
navigatorGenomeIds.Contains(nav.GenomeID))
.Select(nav => new {nav.SpecieID, nav.GenomeID})
.GroupBy(nav => nav.SpecieID)
.ToList();
// Build list of navigator specie genome groups
navigatorSpecieGenomesGroups.AddRange(
specieGroupedNavigators.Select(
specieGenomesGroup =>
new SpecieGenomesGroup((int) specieGenomesGroup.Key,
specieGenomesGroup.Select(gg => gg.GenomeID).ToList())));
}
querySuccess = true;
}
catch (Exception e)
{
HandleQueryException(MethodBase.GetCurrentMethod().ToString(), retryCnt++, e);
}
}
return navigatorSpecieGenomesGroups;
}
/// <summary>
/// Retrieves the navigator genome data (i.e. evaluation statistics and XML) for a particular batch of a given
/// run/experiemnt.
/// </summary>
/// <param name="experimentId">The experiment that was executed.</param>
/// <param name="run">The run number of the given experiment.</param>
/// <param name="batch">The batch number of the given run.</param>
/// <param name="runPhase">
/// Indicates whether this is part of the initialization or primary experiment phase.
/// </param>
/// <param name="navigatorGenomeIds">The navigator genome IDs by which to filter the query result set (optional).</param>
/// <returns>The navigator genome data.</returns>
public static IList<CoevolutionMCSNavigatorExperimentGenome> GetNavigatorGenomeData(int experimentId, int run,
int batch, RunPhase runPhase, IList<int> navigatorGenomeIds = null)
{
IList<CoevolutionMCSNavigatorExperimentGenome> navigatorGenomes = null;
bool querySuccess = false;
int retryCnt = 0;
while (querySuccess == false && retryCnt <= MaxQueryRetryCnt)
{
try
{
using (ExperimentDataEntities context = new ExperimentDataEntities())
{
// Query for navigator genomes logged during the current batch and constrained by the given set of genome IDs
if (navigatorGenomeIds != null)
{
navigatorGenomes =
context.CoevolutionMCSNavigatorExperimentGenomes.Where(
expData =>
navigatorGenomeIds.Contains(expData.GenomeID) &&
expData.ExperimentDictionaryID == experimentId && expData.Run == run &&
expData.Generation == batch &&
expData.RunPhase.RunPhaseName == runPhase.ToString())
.ToList();
}
// Otherwise, query for all navigator genomes logged during the current batch
else
{
navigatorGenomes =
context.CoevolutionMCSNavigatorExperimentGenomes.Where(
expData =>
expData.ExperimentDictionaryID == experimentId && expData.Run == run &&
expData.Generation == batch &&
expData.RunPhase.RunPhaseName == runPhase.ToString())
.ToList();
}
}
querySuccess = true;
}
catch (Exception e)
{
HandleQueryException(MethodBase.GetCurrentMethod().ToString(), retryCnt++, e);
}
}
return navigatorGenomes;
}
/// <summary>
/// Retrieves the navigator genome data (i.e. evaluation statistics and XML) for the entirety of a given
/// run/experiment.
/// </summary>
/// <param name="experimentId">The experiment that was executed.</param>
/// <param name="run">The run number of the given experiment.</param>
/// <param name="runPhase">
/// Indicates whether this is part of the initialization or primary experiment phase.
/// </param>
/// <returns>The navigator genome data.</returns>
public static IList<CoevolutionMCSNavigatorExperimentGenome> GetNavigatorGenomeData(int experimentId, int run,
RunPhase runPhase)
{
IList<CoevolutionMCSNavigatorExperimentGenome> navigatorGenomes = null;
bool querySuccess = false;
int retryCnt = 0;
while (querySuccess == false && retryCnt <= MaxQueryRetryCnt)
{
try
{
// Query for the distinct navigator genomes logged during the run
using (ExperimentDataEntities context = new ExperimentDataEntities())
{
navigatorGenomes = context.CoevolutionMCSNavigatorExperimentGenomes.Where(
expData =>
expData.ExperimentDictionaryID == experimentId && expData.Run == run &&
expData.RunPhase.RunPhaseName == runPhase.ToString())
.GroupBy(expData => expData.GenomeID)
.Select(expDataGroup => expDataGroup.OrderBy(expData => expData.Generation).FirstOrDefault())
.ToList();
}
querySuccess = true;
}
catch (Exception e)
{
HandleQueryException(MethodBase.GetCurrentMethod().ToString(), retryCnt++, e);
}
}
return navigatorGenomes;
}
/// <summary>
/// Writes the given evaluation results to a flat file.
/// </summary>
/// <param name="experimentId">The experiment that was executed.</param>
/// <param name="run">The run number of the given experiment.</param>
/// <param name="batch">The batch number of the given run.</param>
/// <param name="runPhase">
/// Indicates whether this is part of the initialization or primary experiment phase.
/// </param>
/// <param name="evaluationUnits">The evaluation results to persist.</param>
private static void WriteNavigatorMazeEvaluationDataToFile(int experimentId, int run, int batch,
RunPhase runPhase, IList<MazeNavigatorEvaluationUnit> evaluationUnits)
{
// Make sure the file writer actually exists before attempting to write to it
if (FileWriters.ContainsKey(OutputFileType.NavigatorMazeEvaluationData) == false)
{
throw new Exception(
string.Format("Cannot write to output stream as no file writer of type {0} has been created.",
OutputFileType.NavigatorMazeEvaluationData));
}
// Loop through the evaluation units and write each row
foreach (MazeNavigatorEvaluationUnit evaluationUnit in evaluationUnits)
{
FileWriters[OutputFileType.NavigatorMazeEvaluationData].WriteLine(string.Join(FileDelimiter,
new List<string>
{
experimentId.ToString(),
run.ToString(),
batch.ToString(),
runPhase.ToString(),
evaluationUnit.MazeId.ToString(),
evaluationUnit.AgentId.ToString(),
evaluationUnit.IsMazeSolved.ToString(),
evaluationUnit.NumTimesteps.ToString()
}));
}
// Immediately flush to the output file
FileWriters[OutputFileType.NavigatorMazeEvaluationData].Flush();
}
/// <summary>
/// Runs post-hoc analysis for all batches in the given experiment/run. This can be part of either the initialization
/// or primary run phase.
/// </summary>
/// <param name="batchesWithGenomeData">The total number of batches containing genome data.</param>
/// <param name="runPhase">
/// Indicates whether this is part of the initialization or primary run phase.
/// </param>
/// <param name="experimentParameters">Experiment configuration parameters.</param>
/// <param name="inputNeuronCount">Count of neurons in controller input layer.</param>
/// <param name="outputNeuronCount">Count of neurons in controller output layer.</param>
/// <param name="curRun">The run number.</param>
/// <param name="numRuns">The total number of runs.</param>
/// <param name="curExperimentConfiguration">The experiment configuration parameters.</param>
/// <param name="generateSimulationResults">Indicates whether to write out the results of the batch simulation.</param>
/// <param name="writeResultsToDatabase">
/// Indicates whether to write results directly into a database (if not, results are
/// written to a flat file).
/// </param>
/// <param name="generateTrajectoryData">Indicates whether the full navigator trajectory should be simulated and persisted.</param>
/// <param name="generateTrajectoryDiversityScore">
/// Indicates whether quantification of navigator trajectory diversity
/// should be written out.
/// </param>
/// <param name="generateNaturalClustering">Indicates whether the natural population clusters should be analyzed.</param>
/// <param name="generateMazeBitmaps">Indicates whether bitmap files of the distinct mazes should be written out.</param>
/// <param name="generateTrajectoryBitmaps">
/// Indicates whether bitmap files depicting the navigator trajectory should be
/// written out.
/// </param>
/// <param name="baseImageOutputDirectory">The path to the output directory for the trajectory images.</param>
/// <param name="clusterImprovementThreshold">
/// The number of cluster additions that are permitted without further maximization of silhouette width.
/// </param>
/// <param name="sampleSize">The number of genomes sampled from the extant species for clustering analysis.</param>
/// <param name="sampleClusterObservationsFromSpecies">
/// Indicates whether to sample observations used in clustering analysis
/// from species or from the population as a whole.
/// </param>
private static void ProcessAndLogPerBatchResults(IList<int> batchesWithGenomeData, RunPhase runPhase,
ExperimentParameters experimentParameters, int inputNeuronCount, int outputNeuronCount, int curRun,
int numRuns, ExperimentDictionary curExperimentConfiguration, bool generateSimulationResults,
bool generateTrajectoryData, bool generateTrajectoryDiversityScore, bool generateNaturalClustering,
bool writeResultsToDatabase, bool generateMazeBitmaps, bool generateTrajectoryBitmaps,
string baseImageOutputDirectory, int clusterImprovementThreshold, int sampleSize,
bool sampleClusterObservationsFromSpecies)
{
IList<CoevolutionMCSMazeExperimentGenome> staticInitializationMazes = null;
// If this invocation is processing initialization results, just get the maze up front as it will remain
// the same throughout the initialization process
if (runPhase == RunPhase.Initialization)
{
staticInitializationMazes =
ExperimentDataHandler.GetMazeGenomeData(curExperimentConfiguration.ExperimentDictionaryID,
curRun, 0);
}
// Iterate through each batch and evaluate maze/navigator combinations
foreach (int curBatch in batchesWithGenomeData)
{
// Create the maze/navigator map
MapEvaluator mapEvaluator =
new MapEvaluator(experimentParameters, inputNeuronCount, outputNeuronCount);
_executionLogger.Info(string.Format("Executing {0} run phase analysis for batch [{1}] of run [{2}/{3}]",
runPhase, curBatch, curRun, numRuns));
// TODO: Check first to see if trajectory evaluations already exist
// Get any existing navigation results (this avoids rerunning failed combinations)
var successfulNavigations =
ExperimentDataHandler.GetSuccessfulNavigations(curExperimentConfiguration.ExperimentDictionaryID,
curRun, curBatch);
// If successful navigation results were found and we're not re-running the simulations,
// initialize the map evaluator with only those combinations known to be successful
if (generateSimulationResults == false && successfulNavigations != null &&
successfulNavigations.Count > 0)
{
List<Tuple<CoevolutionMCSMazeExperimentGenome, CoevolutionMCSNavigatorExperimentGenome>>
successfulGenomeCombos =
new List<Tuple<CoevolutionMCSMazeExperimentGenome, CoevolutionMCSNavigatorExperimentGenome>>
(successfulNavigations.Count());
// Get distinct maze and navigator genomes
var mazeGenomeData =
ExperimentDataHandler.GetMazeGenomeData(curExperimentConfiguration.ExperimentDictionaryID,
curRun, curBatch, successfulNavigations.Select(n => n.MazeGenomeID).Distinct().ToList());
var navigatorGenomeData =
ExperimentDataHandler.GetNavigatorGenomeData(curExperimentConfiguration.ExperimentDictionaryID,
curRun, curBatch, runPhase,
successfulNavigations.Select(n => n.NavigatorGenomeID).Distinct().ToList());
// Build list of successful maze/navigator combinations
successfulGenomeCombos.AddRange(
successfulNavigations.Select(
successfulNav =>
new Tuple<CoevolutionMCSMazeExperimentGenome, CoevolutionMCSNavigatorExperimentGenome>(
mazeGenomeData.First(gd => successfulNav.MazeGenomeID == gd.GenomeID),
navigatorGenomeData.First(gd => successfulNav.NavigatorGenomeID == gd.GenomeID))));
// Initialize the maze/navigator map with combinations that are known to be successful
mapEvaluator.Initialize(successfulGenomeCombos);
}
// Otherwise, just initialize with all combinations
else
{
// Initialize the maze/navigator map with the serialized maze and navigator data (this does the parsing)
mapEvaluator.Initialize(
runPhase == RunPhase.Initialization
? staticInitializationMazes
: ExperimentDataHandler.GetMazeGenomeData(curExperimentConfiguration.ExperimentDictionaryID,
curRun, curBatch), ExperimentDataHandler.GetNavigatorGenomeData(
curExperimentConfiguration.ExperimentDictionaryID, curRun, curBatch, runPhase));
}
// Evaluate all of the maze/navigator combinations in the batch
mapEvaluator.RunTrajectoryEvaluations();
if (generateSimulationResults)
{
// Save the evaluation results
ExperimentDataHandler.WriteNavigatorMazeEvaluationData(
curExperimentConfiguration.ExperimentDictionaryID, curRun, curBatch, runPhase,
mapEvaluator.EvaluationUnits, CommitPageSize, writeResultsToDatabase);
}
if (generateMazeBitmaps)
{
// Generate bitmaps of distinct mazes extant at the current point in time
ImageGenerationHandler.GenerateMazeBitmaps(baseImageOutputDirectory,
curExperimentConfiguration.ExperimentName, curExperimentConfiguration.ExperimentDictionaryID,
curRun, curBatch, mapEvaluator.EvaluationUnits);
}
if (generateTrajectoryBitmaps)
{
// Generate bitmaps of trajectory for all successful trials
ImageGenerationHandler.GenerateBitmapsForSuccessfulTrials(
baseImageOutputDirectory, curExperimentConfiguration.ExperimentName,
curExperimentConfiguration.ExperimentDictionaryID,
curRun, curBatch, mapEvaluator.EvaluationUnits, runPhase);
}
if (generateTrajectoryData && runPhase != RunPhase.Initialization)
{
// Write out the full trajectory of all agents through all solved mazes
ExperimentDataHandler.WriteTrajectoryData(curExperimentConfiguration.ExperimentDictionaryID, curRun,
curBatch, mapEvaluator.EvaluationUnits, CommitPageSize, writeResultsToDatabase);
}
// Compare trajectories of agents through maze to get quantitative sense of solution diversity
// Mean euclidean distance will be calculated for selected trajectory against:
// 1. Other agent trajectories in the current maze only
// 2. Other agent trajectories on *another* maze only
// 3. All other agent trajectories (regardless of maze)
if (generateTrajectoryDiversityScore && runPhase != RunPhase.Initialization)
{
ExperimentDataHandler.WriteTrajectoryDiversityData(
curExperimentConfiguration.ExperimentDictionaryID, curRun, curBatch,
EvaluationHandler.CalculateTrajectoryDiversity(mapEvaluator.EvaluationUnits),
writeResultsToDatabase);
}
// Only write clustering results for primary runs when the number of trajectories have surpassed
// the minimum cluster count of 3
if (generateNaturalClustering && runPhase != RunPhase.Initialization)
{
// Extract uniform samples of maze and navigator genomes from each extant specie
// on which to run clustering analysis
var evaluationSamples = sampleClusterObservationsFromSpecies
? DataManipulationUtil.ExtractEvaluationUnitSamplesFromSpecies(
curExperimentConfiguration.ExperimentDictionaryID, curRun, curBatch,
mapEvaluator.EvaluationUnits.Where(eu => eu.IsMazeSolved).ToList(), sampleSize)
: DataManipulationUtil.ExtractEvaluationUnitSamplesFromPopulation(
mapEvaluator.EvaluationUnits.Where(eu => eu.IsMazeSolved).ToList(),
sampleSize);
// Calculate natural clustering of the population trajectories at each point in time and persist
ExperimentDataHandler.WriteClusteringDiversityData(
curExperimentConfiguration.ExperimentDictionaryID, curRun, curBatch,
EvaluationHandler.CalculateNaturalClustering(evaluationSamples, clusterImprovementThreshold),
writeResultsToDatabase);
}
}
}
