protected virtual BenchmarkScenarioResult RunBenchmark <TNode>(BenchmarkScenario <TNode> scenario, ILevelGeneratorFactory <TNode> generator, int iterations) { var scenarioResult = BenchmarkRunner.Run(scenario, generator, iterations, new Legacy.Benchmarks.BenchmarkOptions() { WithConsolePreview = Options.WithConsolePreview, MultiThreaded = Options.MaxThreads > 1, MaxDegreeOfParallelism = Options.MaxThreads, WithFileOutput = false, }); var resultSaver = new BenchmarkResultSaver(); resultSaver.SaveResultDefaultLocation(scenarioResult, directory: DirectoryFullPath, name: $"{Directory}_{scenario.Name}_{generator.Name}", withDatetime: false); return(scenarioResult); }
protected virtual BenchmarkScenarioResult RunBenchmark(IEnumerable <DungeonGeneratorInput <int> > inputs, int iterations, string name) { var benchmarkRunner = new BenchmarkRunner <IMapDescription <int> >(); var benchmarkScenario = new BenchmarkScenario <IMapDescription <int> >(name, GetGeneratorRunnerFactory); var resultSaver = new BenchmarkResultSaver(); var scenarioResult = benchmarkRunner.Run(benchmarkScenario, inputs, iterations, new BenchmarkOptions() { WithConsolePreview = Options.WithConsolePreview, MultiThreaded = Options.MaxThreads > 1, MaxDegreeOfParallelism = Options.MaxThreads, WithFileOutput = false, }); resultSaver.SaveResultDefaultLocation(scenarioResult, directory: DirectoryFullPath, name: $"{Directory}_{name}", withDatetime: false); return(scenarioResult); }
public static async Task SaveAndUpload(this BenchmarkResultSaver resultSaver, BenchmarkScenarioResult scenarioResult, string name, string group) { if (resultSaver == null) { throw new ArgumentNullException(nameof(resultSaver)); } if (name == null) { throw new ArgumentNullException(nameof(name)); } if (group == null) { throw new ArgumentNullException(nameof(group)); } resultSaver.SaveResultDefaultLocation(scenarioResult, $"{group}_{name}"); var uploadConfig = GetDefaultUploadConfig(); await resultSaver.UploadManualResult(scenarioResult, uploadConfig, new ManualInfo() { Group = group, Name = name, }); }
public static void RunOld(Options options) { // TODO: make better Directory = Path.Combine("DungeonGeneratorEvolutions", FileNamesHelper.PrefixWithTimestamp(options.Name)); System.IO.Directory.CreateDirectory(Directory); Logger = new Logger(new ConsoleLoggerHandler(), new FileLoggerHandler(Path.Combine(Directory, "log.txt"))); var allGraphs = new Dictionary <string, Tuple <string, IGraph <int> > >() { { "1", Tuple.Create("Example 1 (fig. 1)", GraphsDatabase.GetExample1()) }, { "2", Tuple.Create("Example 2 (fig. 7 top)", GraphsDatabase.GetExample2()) }, { "3", Tuple.Create("Example 3 (fig. 7 bottom)", GraphsDatabase.GetExample3()) }, { "4", Tuple.Create("Example 4 (fig. 8)", GraphsDatabase.GetExample4()) }, { "5", Tuple.Create("Example 5 (fig. 9)", GraphsDatabase.GetExample5()) }, }; var mapDescriptions = new Dictionary <string, Tuple <string, MapDescription <int> > >() { { "gungeon_1_1", Tuple.Create("Gungeon 1_1", LoadMapDescription("gungeon_1_1")) }, { "gungeon_1_2", Tuple.Create("Gungeon 1_2", LoadMapDescription("gungeon_1_2")) }, { "gungeon_2_1", Tuple.Create("Gungeon 2_1", LoadMapDescription("gungeon_2_1")) }, { "gungeon_2_2", Tuple.Create("Gungeon 2_2", LoadMapDescription("gungeon_2_2")) }, // { "gungeon_2_4", Tuple.Create("Gungeon 2_4", LoadMapDescription("gungeon_2_4")) }, }; var allAnalyzers = new Dictionary <string, Func <IMapDescription <int>, IPerformanceAnalyzer <DungeonGeneratorConfiguration <int>, Individual <int> > > >() { { "MaxStageTwoFailures", (_) => new MaxStageTwoFailuresAnalyzer <DungeonGeneratorConfiguration <int>, GeneratorData>() }, { "MaxIterations", (_) => new MaxIterationsAnalyzer <DungeonGeneratorConfiguration <int>, GeneratorData>() }, { "ChainMerge", (_) => new ChainMergeAnalyzer <DungeonGeneratorConfiguration <int>, int, GeneratorData>() }, { "ChainOrder", (_) => new ChainOrderAnalyzer <DungeonGeneratorConfiguration <int>, int, GeneratorData>() }, { "MaxBranching", (_) => new MaxBranchingAnalyzer <DungeonGeneratorConfiguration <int>, GeneratorData>() }, { "ChainDecomposition", (mapDescription) => new ChainDecompositionAnalyzer <DungeonGeneratorConfiguration <int>, int, GeneratorData>(mapDescription) }, }; // Select graphs var graphs = options.Graphs.Count() != 0 ? options .Graphs .Select(x => allGraphs[x]) .ToList() : allGraphs.Values.ToList(); graphs = options .Graphs .Select(x => allGraphs[x]) .ToList(); // Select analyzers var analyzers = options.Mutations.Count() != 0 ? options .Mutations .Select(x => allAnalyzers[x]) .ToList() : allAnalyzers.Values.ToList(); var inputs = new List <Input>(); foreach (var graphPair in graphs) { foreach (var corridorOffset in options.CorridorOffsets) { var corridorOffsets = corridorOffset.Split(",").Select(x => int.Parse(x)).ToList(); var withCorridors = corridorOffsets[0] != 0; var canTouch = options.CanTouch || !withCorridors; var name = MapDescriptionUtils.GetInputName(graphPair.Item1, options.Scale, withCorridors, corridorOffsets, canTouch); var graph = graphPair.Item2; var basicRoomTemplates = MapDescriptionUtils.GetBasicRoomTemplates(options.Scale); var basicRoomDescription = new BasicRoomDescription(basicRoomTemplates); var corridorRoomTemplates = withCorridors ? MapDescriptionUtils.GetCorridorRoomTemplates(corridorOffsets) : null; var corridorRoomDescription = withCorridors ? new CorridorRoomDescription(corridorRoomTemplates) : null; var mapDescription = MapDescriptionUtils.GetBasicMapDescription(graph, basicRoomDescription, corridorRoomDescription, withCorridors); inputs.Add(new Input() { Name = name, MapDescription = mapDescription, Configuration = new DungeonGeneratorConfiguration <int>() { RoomsCanTouch = canTouch, } }); } } foreach (var mapDescriptionKey in options.MapDescriptions) { var mapDescription = mapDescriptions[mapDescriptionKey]; inputs.Add(new Input() { Name = mapDescription.Item1, MapDescription = mapDescription.Item2, Configuration = new DungeonGeneratorConfiguration <int>() { RoomsCanTouch = options.CanTouch, } }); } var resultsDict = new Dictionary <Input, Result>(); var partitioner = Partitioner.Create(inputs, EnumerablePartitionerOptions.NoBuffering); Parallel.ForEach(partitioner, new ParallelOptions { MaxDegreeOfParallelism = options.MaxThreads }, input => { lock (Logger) { Logger.WriteLine($"Started {input.Name}"); } var result = RunEvolution(input, options, analyzers.Select(x => x(input.MapDescription)).ToList()); lock (Logger) { Logger.WriteLine($"Ended {input.Name}"); } lock (resultsDict) { resultsDict[input] = result; } }); var results = inputs.Select(x => resultsDict[x]).ToList(); Logger.WriteLine(); AnalyzeMutations(results.ToList()); var inputsNewConfigurations = results.Select(x => new DungeonGeneratorInput <int>(x.Input.Name, x.Input.MapDescription, x.NewConfiguration)); var inputsOldConfigurations = results.Select(x => new DungeonGeneratorInput <int>(x.Input.Name, x.Input.MapDescription, x.Input.Configuration)); var benchmarkRunner = new BenchmarkRunner <IMapDescription <int> >(); var benchmarkScenarioNew = new BenchmarkScenario <IMapDescription <int> >("NewConfigurations", GetGeneratorRunnerFactory); var benchmarkScenarioOld = new BenchmarkScenario <IMapDescription <int> >("OldConfigurations", GetGeneratorRunnerFactory); var resultSaver = new BenchmarkResultSaver(); var scenarioResultNew = benchmarkRunner.Run(benchmarkScenarioNew, inputsNewConfigurations, options.FinalEvaluationIterations, new BenchmarkOptions() { WithConsolePreview = false, }); resultSaver.SaveResultDefaultLocation(scenarioResultNew, directory: Directory); var scenarioResultOld = benchmarkRunner.Run(benchmarkScenarioOld, inputsOldConfigurations, options.FinalEvaluationIterations, new BenchmarkOptions() { WithConsolePreview = false, }); resultSaver.SaveResultDefaultLocation(scenarioResultOld, directory: Directory); }
protected override Individual <TNode> EvaluateIndividual(Individual <TNode> individual) { Logger.WriteLine($"Evaluating individual {individual}"); var scenario = new BenchmarkScenario <IMapDescription <TNode> >("SimulatedAnnealingParameters", input => { // Setup early stopping if (individual.Parent != null) { individual.Configuration.EarlyStopIfIterationsExceeded = Math.Min(50 * (int)individual.Parent.Iterations, InitialIndividual.Configuration.EarlyStopIfIterationsExceeded ?? int.MaxValue); } else { // TODO: how to do this? // individual.Configuration.EarlyStopIfIterationsExceeded = null; } var layoutGenerator = new DungeonGenerator <TNode>(input.MapDescription, individual.Configuration); layoutGenerator.InjectRandomGenerator(new Random(0)); var generatorRunner = new LambdaGeneratorRunner(() => { var simulatedAnnealingArgsContainer = new List <SimulatedAnnealingEventArgs>(); void SimulatedAnnealingEventHandler(object sender, SimulatedAnnealingEventArgs eventArgs) { simulatedAnnealingArgsContainer.Add(eventArgs); } layoutGenerator.OnSimulatedAnnealingEvent += SimulatedAnnealingEventHandler; var layout = layoutGenerator.GenerateLayout(); layoutGenerator.OnSimulatedAnnealingEvent -= SimulatedAnnealingEventHandler; var additionalData = new AdditionalRunData <TNode>() { GeneratedLayout = layout, SimulatedAnnealingEventArgs = simulatedAnnealingArgsContainer, }; var generatorRun = new GeneratorRun <AdditionalRunData <TNode> >(layout != null, layoutGenerator.TimeTotal, layoutGenerator.IterationsCount, additionalData); return(generatorRun); }); if (individual.Parent != null) { return(new EarlyStoppingGeneratorRunner(generatorRunner, individual.Parent.Iterations, (successful, time, iterations) => new GeneratorRun <AdditionalRunData <TNode> >(successful, time, iterations, null))); } else { return(generatorRunner); } }); var scenarioResult = benchmarkRunner.Run(scenario, new List <GeneratorInput <IMapDescription <TNode> > >() { new GeneratorInput <IMapDescription <TNode> >("DungeonGeneratorEvolution", mapDescription) }, Options.EvaluationIterations, new BenchmarkOptions() { WithConsoleOutput = false, WithFileOutput = false, }); var generatorRuns = scenarioResult .BenchmarkResults .First() .Runs .Cast <IGeneratorRun <AdditionalRunData <TNode> > >() .ToList(); var generatorEvaluation = new GeneratorEvaluation <AdditionalRunData <TNode> >(generatorRuns); // TODO: ugly individual.ConfigurationEvaluation = generatorEvaluation; individual.Iterations = generatorRuns.Average(x => x.Iterations); individual.Time = generatorRuns.Average(x => x.Time); individual.Fitness = Options.FitnessType == FitnessType.Iterations ? individual.Iterations : individual.Time; individual.SuccessRate = generatorRuns.Count(x => x.IsSuccessful) / (double)generatorRuns.Count; var layouts = generatorRuns .Where(x => x.IsSuccessful) .Select(x => x.AdditionalData.GeneratedLayout) .ToList(); var layoutsForClustering = layouts.Take(Math.Min(layouts.Count, 250)).ToList(); var roomTemplatesEntropy = entropyCalculator.ComputeAverageRoomTemplatesEntropy(mapDescription, layouts); var averageRoomTemplateSize = LayoutsDistance.GetAverageRoomTemplateSize(mapDescription); var positionOnlyClusters = layoutsClustering.GetClusters(layoutsForClustering, LayoutsDistance.PositionOnlyDistance, averageRoomTemplateSize); var positionAndShapeClusters = layoutsClustering.GetClusters(layoutsForClustering, (x1, x2) => LayoutsDistance.PositionAndShapeDistance(x1, x2, averageRoomTemplateSize), averageRoomTemplateSize); var summary = ""; var fitnessDifferenceParent = individual.Parent != null? StatisticsUtils.DifferenceToReference(individual, individual.Parent, x => x.Fitness) : 0; var fitnessDifferenceTotal = StatisticsUtils.DifferenceToReference(individual, InitialIndividual, x => x.Fitness); var iterationsDifferenceParent = individual.Parent != null? StatisticsUtils.DifferenceToReference(individual, individual.Parent, x => x.Iterations) : 0; var iterationsDifferenceTotal = StatisticsUtils.DifferenceToReference(individual, InitialIndividual, x => x.Iterations); var timeDifferenceParent = individual.Parent != null? StatisticsUtils.DifferenceToReference(individual, individual.Parent, x => x.Time) : 0; var timeDifferenceTotal = StatisticsUtils.DifferenceToReference(individual, InitialIndividual, x => x.Time); summary += $" fitness {individual.Fitness:F}, {(fitnessDifferenceParent > 0 ? "+" : "")}{fitnessDifferenceParent:F}%, {(fitnessDifferenceTotal > 0 ? "+" : "")}{fitnessDifferenceTotal:F}% total \n"; summary += $" iterations {individual.Iterations:F}, {(iterationsDifferenceParent > 0 ? "+" : "")}{iterationsDifferenceParent:F}%, {(iterationsDifferenceTotal > 0 ? "+" : "")}{iterationsDifferenceTotal:F}% total \n"; summary += $" time {individual.Time:F}, {(timeDifferenceParent > 0 ? "+" : "")}{timeDifferenceParent:F}%, {(timeDifferenceTotal > 0 ? "+" : "")}{timeDifferenceTotal:F}% total \n"; summary += $" entropy {roomTemplatesEntropy:F}, clusters {positionOnlyClusters.Count}/{positionAndShapeClusters.Count} \n"; summary += $" success rate {individual.SuccessRate * 100:F}%\n"; Logger.WriteLine(summary); // Directory.CreateDirectory($"{ResultsDirectory}/{individual.Id}"); for (int i = 0; i < generatorRuns.Count; i++) { var generatorRun = generatorRuns[i]; if (generatorRun.IsSuccessful) { var layout = generatorRun.AdditionalData.GeneratedLayout; var svg = layoutDrawer.DrawLayout(layout, 800, forceSquare: true); // File.WriteAllText($"{ResultsDirectory}/{individual.Id}/{i}.svg", svg); generatorRun.AdditionalData.GeneratedLayout = null; generatorRun.AdditionalData.GeneratedLayoutSvg = svg; } } var resultSaver = new BenchmarkResultSaver(); resultSaver.SaveResultDefaultLocation(scenarioResult, $"{individual.Id}_benchmarkResults", ResultsDirectory, withDatetime: false); using (var file = new StreamWriter(Path.Combine(ResultsDirectory, $"{individual.Id}_visualization.txt"))) { file.WriteLine(summary); if (generatorRuns.Any(x => x.IsSuccessful)) { var dataVisualization = new ChainStatsVisualization <GeneratorData>(); dataVisualization.Visualize(generatorEvaluation, file); } } return(individual); }