Пример #1
0
        protected virtual BenchmarkScenarioResult RunBenchmark <TNode>(BenchmarkScenario <TNode> scenario, ILevelGeneratorFactory <TNode> generator, int iterations)
        {
            var scenarioResult = BenchmarkRunner.Run(scenario, generator, iterations, new Legacy.Benchmarks.BenchmarkOptions()
            {
                WithConsolePreview     = Options.WithConsolePreview,
                MultiThreaded          = Options.MaxThreads > 1,
                MaxDegreeOfParallelism = Options.MaxThreads,
                WithFileOutput         = false,
            });

            var resultSaver = new BenchmarkResultSaver();

            resultSaver.SaveResultDefaultLocation(scenarioResult, directory: DirectoryFullPath, name: $"{Directory}_{scenario.Name}_{generator.Name}", withDatetime: false);

            return(scenarioResult);
        }
Пример #2
0
        protected virtual BenchmarkScenarioResult RunBenchmark(IEnumerable <DungeonGeneratorInput <int> > inputs, int iterations, string name)
        {
            var benchmarkRunner   = new BenchmarkRunner <IMapDescription <int> >();
            var benchmarkScenario = new BenchmarkScenario <IMapDescription <int> >(name, GetGeneratorRunnerFactory);

            var resultSaver = new BenchmarkResultSaver();

            var scenarioResult = benchmarkRunner.Run(benchmarkScenario, inputs, iterations, new BenchmarkOptions()
            {
                WithConsolePreview     = Options.WithConsolePreview,
                MultiThreaded          = Options.MaxThreads > 1,
                MaxDegreeOfParallelism = Options.MaxThreads,
                WithFileOutput         = false,
            });

            resultSaver.SaveResultDefaultLocation(scenarioResult, directory: DirectoryFullPath, name: $"{Directory}_{name}", withDatetime: false);

            return(scenarioResult);
        }
Пример #3
0
        public static async Task SaveAndUpload(this BenchmarkResultSaver resultSaver, BenchmarkScenarioResult scenarioResult, string name, string group)
        {
            if (resultSaver == null)
            {
                throw new ArgumentNullException(nameof(resultSaver));
            }
            if (name == null)
            {
                throw new ArgumentNullException(nameof(name));
            }
            if (group == null)
            {
                throw new ArgumentNullException(nameof(group));
            }

            resultSaver.SaveResultDefaultLocation(scenarioResult, $"{group}_{name}");

            var uploadConfig = GetDefaultUploadConfig();
            await resultSaver.UploadManualResult(scenarioResult, uploadConfig, new ManualInfo()
            {
                Group = group,
                Name  = name,
            });
        }
Пример #4
0
        public static void RunOld(Options options)
        {
            // TODO: make better
            Directory = Path.Combine("DungeonGeneratorEvolutions", FileNamesHelper.PrefixWithTimestamp(options.Name));
            System.IO.Directory.CreateDirectory(Directory);
            Logger = new Logger(new ConsoleLoggerHandler(), new FileLoggerHandler(Path.Combine(Directory, "log.txt")));

            var allGraphs = new Dictionary <string, Tuple <string, IGraph <int> > >()
            {
                { "1", Tuple.Create("Example 1 (fig. 1)", GraphsDatabase.GetExample1()) },
                { "2", Tuple.Create("Example 2 (fig. 7 top)", GraphsDatabase.GetExample2()) },
                { "3", Tuple.Create("Example 3 (fig. 7 bottom)", GraphsDatabase.GetExample3()) },
                { "4", Tuple.Create("Example 4 (fig. 8)", GraphsDatabase.GetExample4()) },
                { "5", Tuple.Create("Example 5 (fig. 9)", GraphsDatabase.GetExample5()) },
            };

            var mapDescriptions = new Dictionary <string, Tuple <string, MapDescription <int> > >()
            {
                { "gungeon_1_1", Tuple.Create("Gungeon 1_1", LoadMapDescription("gungeon_1_1")) },
                { "gungeon_1_2", Tuple.Create("Gungeon 1_2", LoadMapDescription("gungeon_1_2")) },
                { "gungeon_2_1", Tuple.Create("Gungeon 2_1", LoadMapDescription("gungeon_2_1")) },
                { "gungeon_2_2", Tuple.Create("Gungeon 2_2", LoadMapDescription("gungeon_2_2")) },
                // { "gungeon_2_4", Tuple.Create("Gungeon 2_4", LoadMapDescription("gungeon_2_4")) },
            };

            var allAnalyzers = new Dictionary <string, Func <IMapDescription <int>, IPerformanceAnalyzer <DungeonGeneratorConfiguration <int>, Individual <int> > > >()
            {
                { "MaxStageTwoFailures", (_) => new MaxStageTwoFailuresAnalyzer <DungeonGeneratorConfiguration <int>, GeneratorData>() },
                { "MaxIterations", (_) => new MaxIterationsAnalyzer <DungeonGeneratorConfiguration <int>, GeneratorData>() },
                { "ChainMerge", (_) => new ChainMergeAnalyzer <DungeonGeneratorConfiguration <int>, int, GeneratorData>() },
                { "ChainOrder", (_) => new ChainOrderAnalyzer <DungeonGeneratorConfiguration <int>, int, GeneratorData>() },
                { "MaxBranching", (_) => new MaxBranchingAnalyzer <DungeonGeneratorConfiguration <int>, GeneratorData>() },
                { "ChainDecomposition", (mapDescription) => new ChainDecompositionAnalyzer <DungeonGeneratorConfiguration <int>, int, GeneratorData>(mapDescription) },
            };

            // Select graphs
            var graphs =
                options.Graphs.Count() != 0
                    ? options
                .Graphs
                .Select(x => allGraphs[x])
                .ToList()
                    : allGraphs.Values.ToList();

            graphs = options
                     .Graphs
                     .Select(x => allGraphs[x])
                     .ToList();

            // Select analyzers
            var analyzers =
                options.Mutations.Count() != 0
                    ? options
                .Mutations
                .Select(x => allAnalyzers[x])
                .ToList()
                    : allAnalyzers.Values.ToList();

            var inputs = new List <Input>();

            foreach (var graphPair in graphs)
            {
                foreach (var corridorOffset in options.CorridorOffsets)
                {
                    var corridorOffsets = corridorOffset.Split(",").Select(x => int.Parse(x)).ToList();
                    var withCorridors   = corridorOffsets[0] != 0;
                    var canTouch        = options.CanTouch || !withCorridors;

                    var name = MapDescriptionUtils.GetInputName(graphPair.Item1, options.Scale, withCorridors,
                                                                corridorOffsets, canTouch);
                    var graph = graphPair.Item2;

                    var basicRoomTemplates   = MapDescriptionUtils.GetBasicRoomTemplates(options.Scale);
                    var basicRoomDescription = new BasicRoomDescription(basicRoomTemplates);

                    var corridorRoomTemplates   = withCorridors ? MapDescriptionUtils.GetCorridorRoomTemplates(corridorOffsets) : null;
                    var corridorRoomDescription = withCorridors ? new CorridorRoomDescription(corridorRoomTemplates) : null;

                    var mapDescription = MapDescriptionUtils.GetBasicMapDescription(graph, basicRoomDescription,
                                                                                    corridorRoomDescription, withCorridors);

                    inputs.Add(new Input()
                    {
                        Name           = name,
                        MapDescription = mapDescription,
                        Configuration  = new DungeonGeneratorConfiguration <int>()
                        {
                            RoomsCanTouch = canTouch,
                        }
                    });
                }
            }

            foreach (var mapDescriptionKey in options.MapDescriptions)
            {
                var mapDescription = mapDescriptions[mapDescriptionKey];

                inputs.Add(new Input()
                {
                    Name           = mapDescription.Item1,
                    MapDescription = mapDescription.Item2,
                    Configuration  = new DungeonGeneratorConfiguration <int>()
                    {
                        RoomsCanTouch = options.CanTouch,
                    }
                });
            }

            var resultsDict = new Dictionary <Input, Result>();
            var partitioner = Partitioner.Create(inputs, EnumerablePartitionerOptions.NoBuffering);

            Parallel.ForEach(partitioner, new ParallelOptions {
                MaxDegreeOfParallelism = options.MaxThreads
            }, input =>
            {
                lock (Logger)
                {
                    Logger.WriteLine($"Started {input.Name}");
                }

                var result = RunEvolution(input, options, analyzers.Select(x => x(input.MapDescription)).ToList());

                lock (Logger)
                {
                    Logger.WriteLine($"Ended {input.Name}");
                }


                lock (resultsDict)
                {
                    resultsDict[input] = result;
                }
            });

            var results = inputs.Select(x => resultsDict[x]).ToList();

            Logger.WriteLine();
            AnalyzeMutations(results.ToList());

            var inputsNewConfigurations = results.Select(x =>
                                                         new DungeonGeneratorInput <int>(x.Input.Name, x.Input.MapDescription, x.NewConfiguration));
            var inputsOldConfigurations = results.Select(x =>
                                                         new DungeonGeneratorInput <int>(x.Input.Name, x.Input.MapDescription, x.Input.Configuration));

            var benchmarkRunner = new BenchmarkRunner <IMapDescription <int> >();

            var benchmarkScenarioNew = new BenchmarkScenario <IMapDescription <int> >("NewConfigurations", GetGeneratorRunnerFactory);
            var benchmarkScenarioOld = new BenchmarkScenario <IMapDescription <int> >("OldConfigurations", GetGeneratorRunnerFactory);

            var resultSaver = new BenchmarkResultSaver();

            var scenarioResultNew = benchmarkRunner.Run(benchmarkScenarioNew, inputsNewConfigurations, options.FinalEvaluationIterations, new BenchmarkOptions()
            {
                WithConsolePreview = false,
            });

            resultSaver.SaveResultDefaultLocation(scenarioResultNew, directory: Directory);

            var scenarioResultOld = benchmarkRunner.Run(benchmarkScenarioOld, inputsOldConfigurations, options.FinalEvaluationIterations, new BenchmarkOptions()
            {
                WithConsolePreview = false,
            });

            resultSaver.SaveResultDefaultLocation(scenarioResultOld, directory: Directory);
        }
        protected override Individual <TNode> EvaluateIndividual(Individual <TNode> individual)
        {
            Logger.WriteLine($"Evaluating individual {individual}");

            var scenario = new BenchmarkScenario <IMapDescription <TNode> >("SimulatedAnnealingParameters",
                                                                            input =>
            {
                // Setup early stopping
                if (individual.Parent != null)
                {
                    individual.Configuration.EarlyStopIfIterationsExceeded = Math.Min(50 * (int)individual.Parent.Iterations, InitialIndividual.Configuration.EarlyStopIfIterationsExceeded ?? int.MaxValue);
                }
                else
                {
                    // TODO: how to do this?
                    // individual.Configuration.EarlyStopIfIterationsExceeded = null;
                }

                var layoutGenerator = new DungeonGenerator <TNode>(input.MapDescription, individual.Configuration);
                layoutGenerator.InjectRandomGenerator(new Random(0));

                var generatorRunner = new LambdaGeneratorRunner(() =>
                {
                    var simulatedAnnealingArgsContainer = new List <SimulatedAnnealingEventArgs>();
                    void SimulatedAnnealingEventHandler(object sender, SimulatedAnnealingEventArgs eventArgs)
                    {
                        simulatedAnnealingArgsContainer.Add(eventArgs);
                    }

                    layoutGenerator.OnSimulatedAnnealingEvent += SimulatedAnnealingEventHandler;
                    var layout = layoutGenerator.GenerateLayout();
                    layoutGenerator.OnSimulatedAnnealingEvent -= SimulatedAnnealingEventHandler;

                    var additionalData = new AdditionalRunData <TNode>()
                    {
                        GeneratedLayout             = layout,
                        SimulatedAnnealingEventArgs = simulatedAnnealingArgsContainer,
                    };

                    var generatorRun = new GeneratorRun <AdditionalRunData <TNode> >(layout != null, layoutGenerator.TimeTotal, layoutGenerator.IterationsCount, additionalData);

                    return(generatorRun);
                });

                if (individual.Parent != null)
                {
                    return(new EarlyStoppingGeneratorRunner(generatorRunner, individual.Parent.Iterations, (successful, time, iterations) => new GeneratorRun <AdditionalRunData <TNode> >(successful, time, iterations, null)));
                }
                else
                {
                    return(generatorRunner);
                }
            });

            var scenarioResult = benchmarkRunner.Run(scenario, new List <GeneratorInput <IMapDescription <TNode> > >()
            {
                new GeneratorInput <IMapDescription <TNode> >("DungeonGeneratorEvolution", mapDescription)
            }, Options.EvaluationIterations, new BenchmarkOptions()
            {
                WithConsoleOutput = false,
                WithFileOutput    = false,
            });
            var generatorRuns = scenarioResult
                                .BenchmarkResults
                                .First()
                                .Runs
                                .Cast <IGeneratorRun <AdditionalRunData <TNode> > >()
                                .ToList();

            var generatorEvaluation = new GeneratorEvaluation <AdditionalRunData <TNode> >(generatorRuns); // TODO: ugly

            individual.ConfigurationEvaluation = generatorEvaluation;

            individual.Iterations = generatorRuns.Average(x => x.Iterations);
            individual.Time       = generatorRuns.Average(x => x.Time);
            individual.Fitness    = Options.FitnessType == FitnessType.Iterations ? individual.Iterations : individual.Time;

            individual.SuccessRate = generatorRuns.Count(x => x.IsSuccessful) / (double)generatorRuns.Count;

            var layouts = generatorRuns
                          .Where(x => x.IsSuccessful)
                          .Select(x => x.AdditionalData.GeneratedLayout)
                          .ToList();

            var layoutsForClustering = layouts.Take(Math.Min(layouts.Count, 250)).ToList();

            var roomTemplatesEntropy     = entropyCalculator.ComputeAverageRoomTemplatesEntropy(mapDescription, layouts);
            var averageRoomTemplateSize  = LayoutsDistance.GetAverageRoomTemplateSize(mapDescription);
            var positionOnlyClusters     = layoutsClustering.GetClusters(layoutsForClustering, LayoutsDistance.PositionOnlyDistance, averageRoomTemplateSize);
            var positionAndShapeClusters = layoutsClustering.GetClusters(layoutsForClustering, (x1, x2) => LayoutsDistance.PositionAndShapeDistance(x1, x2, averageRoomTemplateSize), averageRoomTemplateSize);

            var summary = "";

            var fitnessDifferenceParent = individual.Parent != null?
                                          StatisticsUtils.DifferenceToReference(individual, individual.Parent, x => x.Fitness) : 0;

            var fitnessDifferenceTotal =
                StatisticsUtils.DifferenceToReference(individual, InitialIndividual, x => x.Fitness);

            var iterationsDifferenceParent = individual.Parent != null?
                                             StatisticsUtils.DifferenceToReference(individual, individual.Parent, x => x.Iterations) : 0;

            var iterationsDifferenceTotal =
                StatisticsUtils.DifferenceToReference(individual, InitialIndividual, x => x.Iterations);

            var timeDifferenceParent = individual.Parent != null?
                                       StatisticsUtils.DifferenceToReference(individual, individual.Parent, x => x.Time) : 0;

            var timeDifferenceTotal =
                StatisticsUtils.DifferenceToReference(individual, InitialIndividual, x => x.Time);

            summary += $"  fitness {individual.Fitness:F}, {(fitnessDifferenceParent > 0 ? "+" : "")}{fitnessDifferenceParent:F}%, {(fitnessDifferenceTotal > 0 ? "+" : "")}{fitnessDifferenceTotal:F}% total  \n";
            summary += $"  iterations {individual.Iterations:F}, {(iterationsDifferenceParent > 0 ? "+" : "")}{iterationsDifferenceParent:F}%, {(iterationsDifferenceTotal > 0 ? "+" : "")}{iterationsDifferenceTotal:F}% total  \n";
            summary += $"  time {individual.Time:F}, {(timeDifferenceParent > 0 ? "+" : "")}{timeDifferenceParent:F}%, {(timeDifferenceTotal > 0 ? "+" : "")}{timeDifferenceTotal:F}% total  \n";
            summary += $"  entropy {roomTemplatesEntropy:F}, clusters {positionOnlyClusters.Count}/{positionAndShapeClusters.Count} \n";
            summary += $"  success rate {individual.SuccessRate * 100:F}%\n";


            Logger.WriteLine(summary);

            // Directory.CreateDirectory($"{ResultsDirectory}/{individual.Id}");

            for (int i = 0; i < generatorRuns.Count; i++)
            {
                var generatorRun = generatorRuns[i];

                if (generatorRun.IsSuccessful)
                {
                    var layout = generatorRun.AdditionalData.GeneratedLayout;
                    var svg    = layoutDrawer.DrawLayout(layout, 800, forceSquare: true);
                    // File.WriteAllText($"{ResultsDirectory}/{individual.Id}/{i}.svg", svg);
                    generatorRun.AdditionalData.GeneratedLayout    = null;
                    generatorRun.AdditionalData.GeneratedLayoutSvg = svg;
                }
            }

            var resultSaver = new BenchmarkResultSaver();

            resultSaver.SaveResultDefaultLocation(scenarioResult, $"{individual.Id}_benchmarkResults", ResultsDirectory, withDatetime: false);

            using (var file =
                       new StreamWriter(Path.Combine(ResultsDirectory, $"{individual.Id}_visualization.txt")))
            {
                file.WriteLine(summary);

                if (generatorRuns.Any(x => x.IsSuccessful))
                {
                    var dataVisualization = new ChainStatsVisualization <GeneratorData>();
                    dataVisualization.Visualize(generatorEvaluation, file);
                }
            }

            return(individual);
        }