private void SetNavParams() { // Get reference to the NavScanner NavScanner nsInstance = FindObjectOfType <NavScanner>(); // Get NavScanner's class type Type nsType = typeof(NavScanner); // Assume no seed strategy for the nav scanner's PRNG _navSeedStrategy = null; // Loop through all nav parameter sets in the current experiment foreach (KeyValuePair <string, object> settings in _experiment.NavParamSet[_navParamSet]) { // Check what's the current parameter if (settings.Key.Equals("seedStrategy")) { // If it's a seed strategy (not exactly a parameter), keep it _navSeedStrategy = settings.Value as Func <int, int>; } else { // If we get here, it's a regular parameter, set it in the // NavScanner using reflection FieldInfo nsField = nsType.GetField( settings.Key, BindingFlags.NonPublic | BindingFlags.Instance); if (nsField is null) { Debug.LogWarning($"Unknown {nameof(NavScanner)} field: '{settings.Key}'"); } else { nsField.SetValue(nsInstance, settings.Value); } } } }
private void StartExperiment() { // Require at least 30 seconds before re-saving results const long saveIntervalMillis = 30000; // Time at which the results file was saved by the last time long lastSaveTime = 0; // Start a stopwatch to measure the experiment time (all scenarios) Stopwatch stopwatch = Stopwatch.StartNew(); // This StringBuilder will contain the temporary results, before // being flushed to a file StringBuilder expResultPendingSave = new StringBuilder("run,genset,navset,tg,tv,c,ar,nclu,genseed,navseed\n"); // Currently selected parameter sets, to be restored when the // experiment finishes string currentGenParamSet = _genParamSet; string currentNavParamSet = _navParamSet; // Obtain the existing instances of the generation manager and nav scanner GenerationManager gmInstance = FindObjectOfType <GenerationManager>(); NavScanner nsInstance = FindObjectOfType <NavScanner>(); // Determine class types of the generation manager and nav scanner Type gmType = typeof(GenerationManager); Type nsType = typeof(NavScanner); // Save the generation manager's current configuration, to be restored // after the experiment finishes string savedGm = JsonUtility.ToJson(gmInstance); // Get reference to the selection method configuration field in the // generation manager FieldInfo gmFieldSmCfg = gmType.GetField( "_selectionParams", BindingFlags.NonPublic | BindingFlags.Instance); // Save the current selection method configuration, to be restored // after the experiment finishes object gmSmConfig = gmFieldSmCfg.GetValue(gmInstance); string savedSmCfg = JsonUtility.ToJson(gmSmConfig); // Save the nav scanner's current configuration, to be restored // after the experiment finishes string savedNs = JsonUtility.ToJson(nsInstance); // Get references to the map generation and map clear methods in the // generation manager MethodInfo genMeth = gmType.GetMethod( "GenerateMap", BindingFlags.NonPublic | BindingFlags.Instance); MethodInfo clearMeth = gmType.GetMethod( "ClearMap", BindingFlags.NonPublic | BindingFlags.Instance); // Get references to the seed fields in the generation manager and // nav scanner FieldInfo gmSeed = gmType.GetField( "_seed", BindingFlags.NonPublic | BindingFlags.Instance); FieldInfo nsSeed = nsType.GetField( "_seed", BindingFlags.NonPublic | BindingFlags.Instance); // Determine folder where to place results, at the project's root string expResultsFolder = Path.Combine( Path.GetDirectoryName(Application.dataPath), "experiments"); // Determine full path of file containing experiment results string expResultsFile = Path.Combine( expResultsFolder, $"{_experimentName}-{DateTime.Now.ToString("yyyyMMddHHmmss", DateTimeFormatInfo.InvariantInfo)}.csv"); // Current step, total steps and experiment cancelled status int step = 0; float totalSteps = _genParamSets.Length * _navParamSets.Length * _runsPerGenNavCombo; bool cancelled = false; // Create the experiment results folder, if it's not already created Directory.CreateDirectory(expResultsFolder); // Let's start the experiment Debug.Log($"==== Starting experiment '{_experimentName}' ===="); // Loop through all generation parameter sets foreach (string genParamSet in _genParamSets) { // Set the current generation parameters and get the respective seed _genParamSet = genParamSet; SetGenParams(); int genInitSeed = (int)gmSeed.GetValue(gmInstance); // Loop through all navigation parameter set foreach (string navParamSet in _navParamSets) { // Set the current navigation parameters and get the respective seed _navParamSet = navParamSet; SetNavParams(); int navInitSeed = (int)nsSeed.GetValue(nsInstance); // Perform the specified number of runs per scenario for (int i = 0; i < _runsPerGenNavCombo; i++) { // Gen and nav seeds, to save in results int genSeed, navSeed; // Increment step step++; // Notify user of current experiment progress using a // progress bar if (EditorUtility.DisplayCancelableProgressBar( $"Performing experiment '{_experimentName}'", $"Running scenario {step}/{(int)totalSteps}...", step / totalSteps)) { // If user cancelled the experiment, bail out cancelled = true; break; } // Should the current scenario/run be skipped? if (_skipFirstNScenarios >= step) { continue; } if (_genSeedStrategy is null) { // If a generation seed strategy wasn't specified, // get seed from the generation manager and keep it genSeed = (int)gmSeed.GetValue(gmInstance); } else { // Otherwise use strategy to obtain a seed and set // it in the generation manager genSeed = _genSeedStrategy.Invoke(genInitSeed + i); gmSeed.SetValue(gmInstance, genSeed); } if (_navSeedStrategy is null) { // If a navigation seed strategy wasn't specified, // get seed from the nav scanner and keep it navSeed = (int)nsSeed.GetValue(nsInstance); } else { // Otherwise use strategy to obtain a seed and set // it in the nav scanner navSeed = _navSeedStrategy.Invoke(navInitSeed + i); nsSeed.SetValue(nsInstance, navSeed); } // Generate map genMeth.Invoke(gmInstance, null); // Take note of results for current scenario/run expResultPendingSave.AppendFormat( CultureInfo.InvariantCulture, "{0},{1},{2},{3},{4},{5},{6},{7},{8},{9}\n", i, $"\"{_genParamSet}\"", $"\"{_navParamSet}\"", gmInstance.GenTimeMillis, nsInstance.ValidationTimeMillis, nsInstance.MeanValidConnections, nsInstance.RelAreaLargestCluster, nsInstance.Clusters.Count, genSeed, navSeed); // Is it time to save unsaved results to the results file? if (stopwatch.ElapsedMilliseconds > lastSaveTime + saveIntervalMillis) { // Append unsaved results to the results file File.AppendAllText( expResultsFile, expResultPendingSave.ToString()); // Clear string builder of unsaved results expResultPendingSave.Clear(); // Take note of time results were saved to file lastSaveTime = stopwatch.ElapsedMilliseconds; } } // Bail out if experiment was cancelled by user if (cancelled) { break; } } // Bail out if experiment was cancelled by user if (cancelled) { break; } } // Clear progress bar EditorUtility.ClearProgressBar(); // Save unsaved results File.AppendAllText(expResultsFile, expResultPendingSave.ToString()); // Restore scene state to what it was before the experiment JsonUtility.FromJsonOverwrite(savedNs, nsInstance); if (savedSmCfg != null) { JsonUtility.FromJsonOverwrite(savedSmCfg, gmSmConfig); } JsonUtility.FromJsonOverwrite(savedGm, gmInstance); _genParamSet = currentGenParamSet; _navParamSet = currentNavParamSet; // Since we can't restore the previously existing map, we clear the // last map generated in the experiment clearMeth.Invoke(gmInstance, null); // Log experiment duration Debug.Log(string.Format( "==== Experiment '{0}' finished after {1} ms, results saved to {2} ====", _experimentName, stopwatch.ElapsedMilliseconds, expResultsFile)); }