protected virtual BenchmarkScenarioResult RunBenchmark <TNode>(BenchmarkScenario <TNode> scenario, ILevelGeneratorFactory <TNode> generator, int iterations)
{
var scenarioResult = BenchmarkRunner.Run(scenario, generator, iterations, new Legacy.Benchmarks.BenchmarkOptions()
{
WithConsolePreview = Options.WithConsolePreview,
MultiThreaded = Options.MaxThreads > 1,
MaxDegreeOfParallelism = Options.MaxThreads,
WithFileOutput = false,
});
var resultSaver = new BenchmarkResultSaver();
resultSaver.SaveResultDefaultLocation(scenarioResult, directory: DirectoryFullPath, name: $"{Directory}_{scenario.Name}_{generator.Name}", withDatetime: false);
return(scenarioResult);
}
protected virtual BenchmarkScenarioResult RunBenchmark(IEnumerable <DungeonGeneratorInput <int> > inputs, int iterations, string name)
{
var benchmarkRunner = new BenchmarkRunner <IMapDescription <int> >();
var benchmarkScenario = new BenchmarkScenario <IMapDescription <int> >(name, GetGeneratorRunnerFactory);
var resultSaver = new BenchmarkResultSaver();
var scenarioResult = benchmarkRunner.Run(benchmarkScenario, inputs, iterations, new BenchmarkOptions()
{
WithConsolePreview = Options.WithConsolePreview,
MultiThreaded = Options.MaxThreads > 1,
MaxDegreeOfParallelism = Options.MaxThreads,
WithFileOutput = false,
});
resultSaver.SaveResultDefaultLocation(scenarioResult, directory: DirectoryFullPath, name: $"{Directory}_{name}", withDatetime: false);
return(scenarioResult);
}
public static async Task SaveAndUpload(this BenchmarkResultSaver resultSaver, BenchmarkScenarioResult scenarioResult, string name, string group)
{
if (resultSaver == null)
{
throw new ArgumentNullException(nameof(resultSaver));
}
if (name == null)
{
throw new ArgumentNullException(nameof(name));
}
if (group == null)
{
throw new ArgumentNullException(nameof(group));
}
resultSaver.SaveResultDefaultLocation(scenarioResult, $"{group}_{name}");
var uploadConfig = GetDefaultUploadConfig();
await resultSaver.UploadManualResult(scenarioResult, uploadConfig, new ManualInfo()
{
Group = group,
Name = name,
});
}
public static void RunOld(Options options)
{
// TODO: make better
Directory = Path.Combine("DungeonGeneratorEvolutions", FileNamesHelper.PrefixWithTimestamp(options.Name));
System.IO.Directory.CreateDirectory(Directory);
Logger = new Logger(new ConsoleLoggerHandler(), new FileLoggerHandler(Path.Combine(Directory, "log.txt")));
var allGraphs = new Dictionary <string, Tuple <string, IGraph <int> > >()
{
{ "1", Tuple.Create("Example 1 (fig. 1)", GraphsDatabase.GetExample1()) },
{ "2", Tuple.Create("Example 2 (fig. 7 top)", GraphsDatabase.GetExample2()) },
{ "3", Tuple.Create("Example 3 (fig. 7 bottom)", GraphsDatabase.GetExample3()) },
{ "4", Tuple.Create("Example 4 (fig. 8)", GraphsDatabase.GetExample4()) },
{ "5", Tuple.Create("Example 5 (fig. 9)", GraphsDatabase.GetExample5()) },
};
var mapDescriptions = new Dictionary <string, Tuple <string, MapDescription <int> > >()
{
{ "gungeon_1_1", Tuple.Create("Gungeon 1_1", LoadMapDescription("gungeon_1_1")) },
{ "gungeon_1_2", Tuple.Create("Gungeon 1_2", LoadMapDescription("gungeon_1_2")) },
{ "gungeon_2_1", Tuple.Create("Gungeon 2_1", LoadMapDescription("gungeon_2_1")) },
{ "gungeon_2_2", Tuple.Create("Gungeon 2_2", LoadMapDescription("gungeon_2_2")) },
// { "gungeon_2_4", Tuple.Create("Gungeon 2_4", LoadMapDescription("gungeon_2_4")) },
};
var allAnalyzers = new Dictionary <string, Func <IMapDescription <int>, IPerformanceAnalyzer <DungeonGeneratorConfiguration <int>, Individual <int> > > >()
{
{ "MaxStageTwoFailures", (_) => new MaxStageTwoFailuresAnalyzer <DungeonGeneratorConfiguration <int>, GeneratorData>() },
{ "MaxIterations", (_) => new MaxIterationsAnalyzer <DungeonGeneratorConfiguration <int>, GeneratorData>() },
{ "ChainMerge", (_) => new ChainMergeAnalyzer <DungeonGeneratorConfiguration <int>, int, GeneratorData>() },
{ "ChainOrder", (_) => new ChainOrderAnalyzer <DungeonGeneratorConfiguration <int>, int, GeneratorData>() },
{ "MaxBranching", (_) => new MaxBranchingAnalyzer <DungeonGeneratorConfiguration <int>, GeneratorData>() },
{ "ChainDecomposition", (mapDescription) => new ChainDecompositionAnalyzer <DungeonGeneratorConfiguration <int>, int, GeneratorData>(mapDescription) },
};
// Select graphs
var graphs =
options.Graphs.Count() != 0
? options
.Graphs
.Select(x => allGraphs[x])
.ToList()
: allGraphs.Values.ToList();
graphs = options
.Graphs
.Select(x => allGraphs[x])
.ToList();
// Select analyzers
var analyzers =
options.Mutations.Count() != 0
? options
.Mutations
.Select(x => allAnalyzers[x])
.ToList()
: allAnalyzers.Values.ToList();
var inputs = new List <Input>();
foreach (var graphPair in graphs)
{
foreach (var corridorOffset in options.CorridorOffsets)
{
var corridorOffsets = corridorOffset.Split(",").Select(x => int.Parse(x)).ToList();
var withCorridors = corridorOffsets[0] != 0;
var canTouch = options.CanTouch || !withCorridors;
var name = MapDescriptionUtils.GetInputName(graphPair.Item1, options.Scale, withCorridors,
corridorOffsets, canTouch);
var graph = graphPair.Item2;
var basicRoomTemplates = MapDescriptionUtils.GetBasicRoomTemplates(options.Scale);
var basicRoomDescription = new BasicRoomDescription(basicRoomTemplates);
var corridorRoomTemplates = withCorridors ? MapDescriptionUtils.GetCorridorRoomTemplates(corridorOffsets) : null;
var corridorRoomDescription = withCorridors ? new CorridorRoomDescription(corridorRoomTemplates) : null;
var mapDescription = MapDescriptionUtils.GetBasicMapDescription(graph, basicRoomDescription,
corridorRoomDescription, withCorridors);
inputs.Add(new Input()
{
Name = name,
MapDescription = mapDescription,
Configuration = new DungeonGeneratorConfiguration <int>()
{
RoomsCanTouch = canTouch,
}
});
}
}
foreach (var mapDescriptionKey in options.MapDescriptions)
{
var mapDescription = mapDescriptions[mapDescriptionKey];
inputs.Add(new Input()
{
Name = mapDescription.Item1,
MapDescription = mapDescription.Item2,
Configuration = new DungeonGeneratorConfiguration <int>()
{
RoomsCanTouch = options.CanTouch,
}
});
}
var resultsDict = new Dictionary <Input, Result>();
var partitioner = Partitioner.Create(inputs, EnumerablePartitionerOptions.NoBuffering);
Parallel.ForEach(partitioner, new ParallelOptions {
MaxDegreeOfParallelism = options.MaxThreads
}, input =>
{
lock (Logger)
{
Logger.WriteLine($"Started {input.Name}");
}
var result = RunEvolution(input, options, analyzers.Select(x => x(input.MapDescription)).ToList());
lock (Logger)
{
Logger.WriteLine($"Ended {input.Name}");
}
lock (resultsDict)
{
resultsDict[input] = result;
}
});
var results = inputs.Select(x => resultsDict[x]).ToList();
Logger.WriteLine();
AnalyzeMutations(results.ToList());
var inputsNewConfigurations = results.Select(x =>
new DungeonGeneratorInput <int>(x.Input.Name, x.Input.MapDescription, x.NewConfiguration));
var inputsOldConfigurations = results.Select(x =>
new DungeonGeneratorInput <int>(x.Input.Name, x.Input.MapDescription, x.Input.Configuration));
var benchmarkRunner = new BenchmarkRunner <IMapDescription <int> >();
var benchmarkScenarioNew = new BenchmarkScenario <IMapDescription <int> >("NewConfigurations", GetGeneratorRunnerFactory);
var benchmarkScenarioOld = new BenchmarkScenario <IMapDescription <int> >("OldConfigurations", GetGeneratorRunnerFactory);
var resultSaver = new BenchmarkResultSaver();
var scenarioResultNew = benchmarkRunner.Run(benchmarkScenarioNew, inputsNewConfigurations, options.FinalEvaluationIterations, new BenchmarkOptions()
{
WithConsolePreview = false,
});
resultSaver.SaveResultDefaultLocation(scenarioResultNew, directory: Directory);
var scenarioResultOld = benchmarkRunner.Run(benchmarkScenarioOld, inputsOldConfigurations, options.FinalEvaluationIterations, new BenchmarkOptions()
{
WithConsolePreview = false,
});
resultSaver.SaveResultDefaultLocation(scenarioResultOld, directory: Directory);
}
protected override Individual <TNode> EvaluateIndividual(Individual <TNode> individual)
{
Logger.WriteLine($"Evaluating individual {individual}");
var scenario = new BenchmarkScenario <IMapDescription <TNode> >("SimulatedAnnealingParameters",
input =>
{
// Setup early stopping
if (individual.Parent != null)
{
individual.Configuration.EarlyStopIfIterationsExceeded = Math.Min(50 * (int)individual.Parent.Iterations, InitialIndividual.Configuration.EarlyStopIfIterationsExceeded ?? int.MaxValue);
}
else
{
// TODO: how to do this?
// individual.Configuration.EarlyStopIfIterationsExceeded = null;
}
var layoutGenerator = new DungeonGenerator <TNode>(input.MapDescription, individual.Configuration);
layoutGenerator.InjectRandomGenerator(new Random(0));
var generatorRunner = new LambdaGeneratorRunner(() =>
{
var simulatedAnnealingArgsContainer = new List <SimulatedAnnealingEventArgs>();
void SimulatedAnnealingEventHandler(object sender, SimulatedAnnealingEventArgs eventArgs)
{
simulatedAnnealingArgsContainer.Add(eventArgs);
}
layoutGenerator.OnSimulatedAnnealingEvent += SimulatedAnnealingEventHandler;
var layout = layoutGenerator.GenerateLayout();
layoutGenerator.OnSimulatedAnnealingEvent -= SimulatedAnnealingEventHandler;
var additionalData = new AdditionalRunData <TNode>()
{
GeneratedLayout = layout,
SimulatedAnnealingEventArgs = simulatedAnnealingArgsContainer,
};
var generatorRun = new GeneratorRun <AdditionalRunData <TNode> >(layout != null, layoutGenerator.TimeTotal, layoutGenerator.IterationsCount, additionalData);
return(generatorRun);
});
if (individual.Parent != null)
{
return(new EarlyStoppingGeneratorRunner(generatorRunner, individual.Parent.Iterations, (successful, time, iterations) => new GeneratorRun <AdditionalRunData <TNode> >(successful, time, iterations, null)));
}
else
{
return(generatorRunner);
}
});
var scenarioResult = benchmarkRunner.Run(scenario, new List <GeneratorInput <IMapDescription <TNode> > >()
{
new GeneratorInput <IMapDescription <TNode> >("DungeonGeneratorEvolution", mapDescription)
}, Options.EvaluationIterations, new BenchmarkOptions()
{
WithConsoleOutput = false,
WithFileOutput = false,
});
var generatorRuns = scenarioResult
.BenchmarkResults
.First()
.Runs
.Cast <IGeneratorRun <AdditionalRunData <TNode> > >()
.ToList();
var generatorEvaluation = new GeneratorEvaluation <AdditionalRunData <TNode> >(generatorRuns); // TODO: ugly
individual.ConfigurationEvaluation = generatorEvaluation;
individual.Iterations = generatorRuns.Average(x => x.Iterations);
individual.Time = generatorRuns.Average(x => x.Time);
individual.Fitness = Options.FitnessType == FitnessType.Iterations ? individual.Iterations : individual.Time;
individual.SuccessRate = generatorRuns.Count(x => x.IsSuccessful) / (double)generatorRuns.Count;
var layouts = generatorRuns
.Where(x => x.IsSuccessful)
.Select(x => x.AdditionalData.GeneratedLayout)
.ToList();
var layoutsForClustering = layouts.Take(Math.Min(layouts.Count, 250)).ToList();
var roomTemplatesEntropy = entropyCalculator.ComputeAverageRoomTemplatesEntropy(mapDescription, layouts);
var averageRoomTemplateSize = LayoutsDistance.GetAverageRoomTemplateSize(mapDescription);
var positionOnlyClusters = layoutsClustering.GetClusters(layoutsForClustering, LayoutsDistance.PositionOnlyDistance, averageRoomTemplateSize);
var positionAndShapeClusters = layoutsClustering.GetClusters(layoutsForClustering, (x1, x2) => LayoutsDistance.PositionAndShapeDistance(x1, x2, averageRoomTemplateSize), averageRoomTemplateSize);
var summary = "";
var fitnessDifferenceParent = individual.Parent != null?
StatisticsUtils.DifferenceToReference(individual, individual.Parent, x => x.Fitness) : 0;
var fitnessDifferenceTotal =
StatisticsUtils.DifferenceToReference(individual, InitialIndividual, x => x.Fitness);
var iterationsDifferenceParent = individual.Parent != null?
StatisticsUtils.DifferenceToReference(individual, individual.Parent, x => x.Iterations) : 0;
var iterationsDifferenceTotal =
StatisticsUtils.DifferenceToReference(individual, InitialIndividual, x => x.Iterations);
var timeDifferenceParent = individual.Parent != null?
StatisticsUtils.DifferenceToReference(individual, individual.Parent, x => x.Time) : 0;
var timeDifferenceTotal =
StatisticsUtils.DifferenceToReference(individual, InitialIndividual, x => x.Time);
summary += $" fitness {individual.Fitness:F}, {(fitnessDifferenceParent > 0 ? "+" : "")}{fitnessDifferenceParent:F}%, {(fitnessDifferenceTotal > 0 ? "+" : "")}{fitnessDifferenceTotal:F}% total \n";
summary += $" iterations {individual.Iterations:F}, {(iterationsDifferenceParent > 0 ? "+" : "")}{iterationsDifferenceParent:F}%, {(iterationsDifferenceTotal > 0 ? "+" : "")}{iterationsDifferenceTotal:F}% total \n";
summary += $" time {individual.Time:F}, {(timeDifferenceParent > 0 ? "+" : "")}{timeDifferenceParent:F}%, {(timeDifferenceTotal > 0 ? "+" : "")}{timeDifferenceTotal:F}% total \n";
summary += $" entropy {roomTemplatesEntropy:F}, clusters {positionOnlyClusters.Count}/{positionAndShapeClusters.Count} \n";
summary += $" success rate {individual.SuccessRate * 100:F}%\n";
Logger.WriteLine(summary);
// Directory.CreateDirectory($"{ResultsDirectory}/{individual.Id}");
for (int i = 0; i < generatorRuns.Count; i++)
{
var generatorRun = generatorRuns[i];
if (generatorRun.IsSuccessful)
{
var layout = generatorRun.AdditionalData.GeneratedLayout;
var svg = layoutDrawer.DrawLayout(layout, 800, forceSquare: true);
// File.WriteAllText($"{ResultsDirectory}/{individual.Id}/{i}.svg", svg);
generatorRun.AdditionalData.GeneratedLayout = null;
generatorRun.AdditionalData.GeneratedLayoutSvg = svg;
}
}
var resultSaver = new BenchmarkResultSaver();
resultSaver.SaveResultDefaultLocation(scenarioResult, $"{individual.Id}_benchmarkResults", ResultsDirectory, withDatetime: false);
using (var file =
new StreamWriter(Path.Combine(ResultsDirectory, $"{individual.Id}_visualization.txt")))
{
file.WriteLine(summary);
if (generatorRuns.Any(x => x.IsSuccessful))
{
var dataVisualization = new ChainStatsVisualization <GeneratorData>();
dataVisualization.Visualize(generatorEvaluation, file);
}
}
return(individual);
}