private void SetNavParams()
{
// Get reference to the NavScanner
NavScanner nsInstance = FindObjectOfType <NavScanner>();
// Get NavScanner's class type
Type nsType = typeof(NavScanner);
// Assume no seed strategy for the nav scanner's PRNG
_navSeedStrategy = null;
// Loop through all nav parameter sets in the current experiment
foreach (KeyValuePair <string, object> settings in _experiment.NavParamSet[_navParamSet])
{
// Check what's the current parameter
if (settings.Key.Equals("seedStrategy"))
{
// If it's a seed strategy (not exactly a parameter), keep it
_navSeedStrategy = settings.Value as Func <int, int>;
}
else
{
// If we get here, it's a regular parameter, set it in the
// NavScanner using reflection
FieldInfo nsField = nsType.GetField(
settings.Key,
BindingFlags.NonPublic | BindingFlags.Instance);
if (nsField is null)
{
Debug.LogWarning($"Unknown {nameof(NavScanner)} field: '{settings.Key}'");
}
else
{
nsField.SetValue(nsInstance, settings.Value);
}
}
}
}
private void StartExperiment()
{
// Require at least 30 seconds before re-saving results
const long saveIntervalMillis = 30000;
// Time at which the results file was saved by the last time
long lastSaveTime = 0;
// Start a stopwatch to measure the experiment time (all scenarios)
Stopwatch stopwatch = Stopwatch.StartNew();
// This StringBuilder will contain the temporary results, before
// being flushed to a file
StringBuilder expResultPendingSave =
new StringBuilder("run,genset,navset,tg,tv,c,ar,nclu,genseed,navseed\n");
// Currently selected parameter sets, to be restored when the
// experiment finishes
string currentGenParamSet = _genParamSet;
string currentNavParamSet = _navParamSet;
// Obtain the existing instances of the generation manager and nav scanner
GenerationManager gmInstance = FindObjectOfType <GenerationManager>();
NavScanner nsInstance = FindObjectOfType <NavScanner>();
// Determine class types of the generation manager and nav scanner
Type gmType = typeof(GenerationManager);
Type nsType = typeof(NavScanner);
// Save the generation manager's current configuration, to be restored
// after the experiment finishes
string savedGm = JsonUtility.ToJson(gmInstance);
// Get reference to the selection method configuration field in the
// generation manager
FieldInfo gmFieldSmCfg = gmType.GetField(
"_selectionParams",
BindingFlags.NonPublic | BindingFlags.Instance);
// Save the current selection method configuration, to be restored
// after the experiment finishes
object gmSmConfig = gmFieldSmCfg.GetValue(gmInstance);
string savedSmCfg = JsonUtility.ToJson(gmSmConfig);
// Save the nav scanner's current configuration, to be restored
// after the experiment finishes
string savedNs = JsonUtility.ToJson(nsInstance);
// Get references to the map generation and map clear methods in the
// generation manager
MethodInfo genMeth = gmType.GetMethod(
"GenerateMap",
BindingFlags.NonPublic | BindingFlags.Instance);
MethodInfo clearMeth = gmType.GetMethod(
"ClearMap",
BindingFlags.NonPublic | BindingFlags.Instance);
// Get references to the seed fields in the generation manager and
// nav scanner
FieldInfo gmSeed = gmType.GetField(
"_seed",
BindingFlags.NonPublic | BindingFlags.Instance);
FieldInfo nsSeed = nsType.GetField(
"_seed", BindingFlags.NonPublic | BindingFlags.Instance);
// Determine folder where to place results, at the project's root
string expResultsFolder = Path.Combine(
Path.GetDirectoryName(Application.dataPath),
"experiments");
// Determine full path of file containing experiment results
string expResultsFile = Path.Combine(
expResultsFolder,
$"{_experimentName}-{DateTime.Now.ToString("yyyyMMddHHmmss", DateTimeFormatInfo.InvariantInfo)}.csv");
// Current step, total steps and experiment cancelled status
int step = 0;
float totalSteps = _genParamSets.Length * _navParamSets.Length * _runsPerGenNavCombo;
bool cancelled = false;
// Create the experiment results folder, if it's not already created
Directory.CreateDirectory(expResultsFolder);
// Let's start the experiment
Debug.Log($"==== Starting experiment '{_experimentName}' ====");
// Loop through all generation parameter sets
foreach (string genParamSet in _genParamSets)
{
// Set the current generation parameters and get the respective seed
_genParamSet = genParamSet;
SetGenParams();
int genInitSeed = (int)gmSeed.GetValue(gmInstance);
// Loop through all navigation parameter set
foreach (string navParamSet in _navParamSets)
{
// Set the current navigation parameters and get the respective seed
_navParamSet = navParamSet;
SetNavParams();
int navInitSeed = (int)nsSeed.GetValue(nsInstance);
// Perform the specified number of runs per scenario
for (int i = 0; i < _runsPerGenNavCombo; i++)
{
// Gen and nav seeds, to save in results
int genSeed, navSeed;
// Increment step
step++;
// Notify user of current experiment progress using a
// progress bar
if (EditorUtility.DisplayCancelableProgressBar(
$"Performing experiment '{_experimentName}'",
$"Running scenario {step}/{(int)totalSteps}...",
step / totalSteps))
{
// If user cancelled the experiment, bail out
cancelled = true;
break;
}
// Should the current scenario/run be skipped?
if (_skipFirstNScenarios >= step)
{
continue;
}
if (_genSeedStrategy is null)
{
// If a generation seed strategy wasn't specified,
// get seed from the generation manager and keep it
genSeed = (int)gmSeed.GetValue(gmInstance);
}
else
{
// Otherwise use strategy to obtain a seed and set
// it in the generation manager
genSeed = _genSeedStrategy.Invoke(genInitSeed + i);
gmSeed.SetValue(gmInstance, genSeed);
}
if (_navSeedStrategy is null)
{
// If a navigation seed strategy wasn't specified,
// get seed from the nav scanner and keep it
navSeed = (int)nsSeed.GetValue(nsInstance);
}
else
{
// Otherwise use strategy to obtain a seed and set
// it in the nav scanner
navSeed = _navSeedStrategy.Invoke(navInitSeed + i);
nsSeed.SetValue(nsInstance, navSeed);
}
// Generate map
genMeth.Invoke(gmInstance, null);
// Take note of results for current scenario/run
expResultPendingSave.AppendFormat(
CultureInfo.InvariantCulture,
"{0},{1},{2},{3},{4},{5},{6},{7},{8},{9}\n",
i,
$"\"{_genParamSet}\"",
$"\"{_navParamSet}\"",
gmInstance.GenTimeMillis,
nsInstance.ValidationTimeMillis,
nsInstance.MeanValidConnections,
nsInstance.RelAreaLargestCluster,
nsInstance.Clusters.Count,
genSeed,
navSeed);
// Is it time to save unsaved results to the results file?
if (stopwatch.ElapsedMilliseconds > lastSaveTime + saveIntervalMillis)
{
// Append unsaved results to the results file
File.AppendAllText(
expResultsFile,
expResultPendingSave.ToString());
// Clear string builder of unsaved results
expResultPendingSave.Clear();
// Take note of time results were saved to file
lastSaveTime = stopwatch.ElapsedMilliseconds;
}
}
// Bail out if experiment was cancelled by user
if (cancelled)
{
break;
}
}
// Bail out if experiment was cancelled by user
if (cancelled)
{
break;
}
}
// Clear progress bar
EditorUtility.ClearProgressBar();
// Save unsaved results
File.AppendAllText(expResultsFile, expResultPendingSave.ToString());
// Restore scene state to what it was before the experiment
JsonUtility.FromJsonOverwrite(savedNs, nsInstance);
if (savedSmCfg != null)
{
JsonUtility.FromJsonOverwrite(savedSmCfg, gmSmConfig);
}
JsonUtility.FromJsonOverwrite(savedGm, gmInstance);
_genParamSet = currentGenParamSet;
_navParamSet = currentNavParamSet;
// Since we can't restore the previously existing map, we clear the
// last map generated in the experiment
clearMeth.Invoke(gmInstance, null);
// Log experiment duration
Debug.Log(string.Format(
"==== Experiment '{0}' finished after {1} ms, results saved to {2} ====",
_experimentName,
stopwatch.ElapsedMilliseconds,
expResultsFile));
}
