public int Run() { if (!ValidateOptions()) { ShowHelpMessage(); return(1); } if (options.ShouldShowHelp) { ShowHelpMessage(); return(0); } var jobGenerator = new JobGenerator(fileSystem); var schemaFilesRaw = SchemaFiles.GetSchemaFilesRaw(options.JsonDirectory, fileSystem) .Where(f => string.IsNullOrEmpty(options.Package) || f.package == options.Package) .ToList(); var jobs = jobGenerator.GenerateJobs(schemaFilesRaw, options); var jobRunner = new JobRunner(fileSystem); jobRunner.Run(jobs, new List <string>() { options.OutputDir }); return(0); }
public static void a_schema_with_an_enum_generates_the_expected_files_for_unreal() { var schemaFileRaw = TestUtils.GetRawJsonDataFromResource <SchemaFileRaw>("Improbable.CodeGeneration.Resources.json.test.enum.json"); var schemaFilesRaw = new List <SchemaFileRaw>() { schemaFileRaw }; var jobGenerator = new JobGenerator(null); var jobs = jobGenerator.GenerateJobs(schemaFilesRaw, Options); Assert.That(jobs.Count == 8, "The wrong number of jobs were generated when a schema with an enum was processed"); Assert.That(jobs.OfType <UnrealEnumJob>().Count() == 1, "Incorrect number of enum jobs were generated for the schema"); var enumJob = jobs.OfType <UnrealEnumJob>().First(); Assert.That(enumJob.InputFiles.Contains(EnumSchemaPath), "The input file for the enum job did not match the schema path"); Assert.That(enumJob.OutputFiles.Count == 1, "The enum job has more output files than expected"); Assert.That(enumJob.OutputFiles.First() == "ExampleColour.h", "The output path did not contain the expected file"); Assert.That(jobs.OfType <UnrealComponentJob>().Count() == 1, "Incorrect number of component jobs were generated"); Assert.That(jobs.OfType <UnrealTypeJob>().Count() == 1, "Incorrect number of type jobs were generated"); Assert.That(jobs.OfType <UnrealCommanderJob>().Count() == 1, "Incorrect number of commander jobs were generated"); Assert.That(jobs.OfType <UnrealEntityTemplateJob>().Count() == 1, "Incorrect number of entity template jobs were generated"); Assert.That(jobs.OfType <UnrealEntityPipelineJob>().Count() == 1, "Incorrect number of entity pipeline jobs were generated"); Assert.That(jobs.OfType <UnrealCommonHeaderJob>().Count() == 1, "Incorrect number of common header jobs were generated"); Assert.That(jobs.OfType <UnrealCallbackDispatcherJob>().Count() == 1, "Incorrect number of callback dispatcher jobs were generated"); var componentJob = jobs.OfType <UnrealComponentJob>().First(); Assert.That(componentJob.InputFiles.Contains(EnumSchemaPath), string.Format("The input files {0} for the component job did not match that of the schema", string.Join(", ", componentJob.InputFiles.ToArray()))); Assert.That(componentJob.OutputFiles.Count == 5, "Incorrect number of output files were created for the component job"); Assert.That(componentJob.OutputFiles.Contains("ExampleEnumComponentComponent.h"), "The output files for the component job did not contain the expected file `ExampleEnumComponentComponent.h`"); Assert.That(componentJob.OutputFiles.Contains("ExampleEnumComponentComponent.cpp"), "The output files for the component job did not contain the expected file `ExampleEnumComponentComponent.cpp`"); Assert.That(componentJob.OutputFiles.Contains("ExampleEnumComponentComponentUpdate.h"), "The output files for the component job did not contain the expected file `ExampleEnumComponentComponentUpdate.h`"); Assert.That(componentJob.OutputFiles.Contains("ExampleEnumComponentComponentUpdate.cpp"), "The output files for the component job did not contain the expected file `ExampleEnumComponentComponentUpdate.cpp`"); Assert.That(componentJob.OutputFiles.Contains("ExampleEnumComponentAddComponentOp.h"), "The output files for the component job did not contain the expected file `ExampleEnumComponentAddComponentOp.h`"); var typeJob = jobs.OfType <UnrealTypeJob>().First(); Assert.That(typeJob.InputFiles.Contains(EnumSchemaPath), "The input file for the type job did not match that of the schema"); Assert.That(typeJob.OutputFiles.Count == 2, "Incorrect number of output files were created for the type job"); Assert.That(typeJob.OutputFiles.Contains("ExampleEnumComponentData.h"), "The output files for the type job did not contain the expected files"); Assert.That(typeJob.OutputFiles.Contains("ExampleEnumComponentData.cpp"), "The output files for the type job did not contain the expected files"); }
public static void two_schemas_with_a_commands_each_generates_a_callback_dispatcher_with_expected_inputs_and_outputs_for_unreal() { var jobGenerator = new JobGenerator(null); var firstSchemaFileRaw = TestUtils.GetRawJsonDataFromResource <SchemaFileRaw>("Improbable.CodeGeneration.Resources.json.test.command.json"); var secondSchemaFileRaw = TestUtils.GetRawJsonDataFromResource <SchemaFileRaw>("Improbable.CodeGeneration.Resources.json.test.commander.json"); var schemaFilesRaw = new List <SchemaFileRaw> { firstSchemaFileRaw, secondSchemaFileRaw }; var jobs = jobGenerator.GenerateJobs(schemaFilesRaw, Options); Assert.That(jobs.Count == 15, "The wrong number of jobs were generated when two schemas with a commands were processed"); Assert.That(jobs.OfType <UnrealComponentJob>().Count() == 2, "Incorrect number of component jobs were generated"); Assert.That(jobs.OfType <UnrealTypeJob>().Count() == 6, "Incorrect number of type jobs were generated"); Assert.That(jobs.OfType <UnrealCommandJob>().Count() == 2, "Incorrect number of command jobs were generated"); Assert.That(jobs.OfType <UnrealCommanderJob>().Count() == 1, "Incorrect number of commander jobs were generated"); Assert.That(jobs.OfType <UnrealEntityTemplateJob>().Count() == 1, "Incorrect number of entity template jobs were generated"); Assert.That(jobs.OfType <UnrealEntityPipelineJob>().Count() == 1, "Incorrect number of entity pipeline jobs were generated"); Assert.That(jobs.OfType <UnrealCallbackDispatcherJob>().Count() == 1, "Incorrect number of callback dispatcher jobs were generated"); var callbackDispatcherJobs = jobs.OfType <UnrealCallbackDispatcherJob>(); Assert.That(callbackDispatcherJobs.FirstOrDefault((job) => { var inputFiles = job.InputFiles; return(inputFiles.Contains(ExampleComponentSchemaPath) && inputFiles.Contains(CommanderSchemaPath) && inputFiles.Count == 2); }) != null, "The generated commander did not have the expected input files"); Assert.That(callbackDispatcherJobs.FirstOrDefault((job) => { var outputFiles = job.OutputFiles; return(outputFiles.Contains("CallbackDispatcher.h") && outputFiles.Contains("CallbackDispatcher.cpp") && outputFiles.Count == 2); }) != null, "The generated commander did not have the expected output files"); }
public static void a_schema_with_a_component_generates_the_expected_files_for_unreal() { var jobGenerator = new JobGenerator(null); var schemaFileRaw = TestUtils.GetRawJsonDataFromResource <SchemaFileRaw>("Improbable.CodeGeneration.Resources.json.test.empty_component.json"); var schemaFilesRaw = new List <SchemaFileRaw> { schemaFileRaw }; var jobs = jobGenerator.GenerateJobs(schemaFilesRaw, Options); Assert.That(jobs.Count() == 7, "The wrong number of jobs were generated when a schema with an empty component was processed"); Assert.That(jobs.OfType <UnrealComponentJob>().Count() == 1, "Incorrect number of component jobs were generated"); Assert.That(jobs.OfType <UnrealTypeJob>().Count() == 1, "Incorrect number of type jobs were generated"); Assert.That(jobs.OfType <UnrealCommanderJob>().Count() == 1, "Incorrect number of commander jobs were generated"); Assert.That(jobs.OfType <UnrealEntityTemplateJob>().Count() == 1, "Incorrect number of entity template jobs were generated"); Assert.That(jobs.OfType <UnrealEntityPipelineJob>().Count() == 1, "Incorrect number of entity pipeline jobs were generated"); Assert.That(jobs.OfType <UnrealCommonHeaderJob>().Count() == 1, "Incorrect number of common header jobs were generated"); Assert.That(jobs.OfType <UnrealCallbackDispatcherJob>().Count() == 1, "Incorrect number of callback dispatcher jobs were generated"); var componentJob = jobs.OfType <UnrealComponentJob>().First(); Assert.That(componentJob.InputFiles.Contains(EmptyComponentSchemaPath), "The input file for the component job did not match that of the schema"); Assert.That(componentJob.OutputFiles.Count == 5, "Incorrect number of output files were created for the component job"); Assert.That(componentJob.OutputFiles.Contains("EmptyComponentComponent.h"), "The output files for the component job did not contain the expected file `EmptyComponentComponent.h`"); Assert.That(componentJob.OutputFiles.Contains("EmptyComponentComponent.cpp"), "The output files for the component job did not contain the expected file `EmptyComponentComponent.cpp`"); Assert.That(componentJob.OutputFiles.Contains("EmptyComponentComponentUpdate.h"), "The output files for the component job did not contain the expected file `EmptyComponentComponentUpdate.h`"); Assert.That(componentJob.OutputFiles.Contains("EmptyComponentComponentUpdate.cpp"), "The output files for the component job did not contain the expected file `EmptyComponentComponentUpdate.cpp`"); Assert.That(componentJob.OutputFiles.Contains("EmptyComponentAddComponentOp.h"), "The output files for the component job did not contain the expected file `EmptyComponentAddComponentOp.h`"); var typeJob = jobs.OfType <UnrealTypeJob>().First(); Assert.That(typeJob.InputFiles.Contains(EmptyComponentSchemaPath), "The input file for the type job did not match that of the schema"); Assert.That(typeJob.OutputFiles.Count == 2, "Incorrect number of output files were created for the type job"); Assert.That(typeJob.OutputFiles.Contains("EmptyComponentData.h"), "The output files for the type job did not contain the expected files"); Assert.That(typeJob.OutputFiles.Contains("EmptyComponentData.cpp"), "The output files for the type job did not contain the expected files"); }
public static void two_schemas_with_a_component_each_generates_an_entity_template_job_with_expected_inputs_and_outputs_for_unreal() { var jobGenerator = new JobGenerator(null); var firstSchemaFileRaw = TestUtils.GetRawJsonDataFromResource <SchemaFileRaw>("Improbable.CodeGeneration.Resources.json.test.empty_component.json"); var secondSchemaFileRaw = TestUtils.GetRawJsonDataFromResource <SchemaFileRaw>("Improbable.CodeGeneration.Resources.json.test.entity_template.json"); var schemaFilesRaw = new List <SchemaFileRaw> { firstSchemaFileRaw, secondSchemaFileRaw }; var jobs = jobGenerator.GenerateJobs(schemaFilesRaw, Options); Assert.That(jobs.Count == 9, "The wrong number of jobs were generated when two schemas with a component were processed"); Assert.That(jobs.OfType <UnrealComponentJob>().Count() == 2, "Incorrect number of component jobs were generated"); Assert.That(jobs.OfType <UnrealTypeJob>().Count() == 2, "Incorrect number of type jobs were generated"); Assert.That(jobs.OfType <UnrealCommanderJob>().Count() == 1, "Incorrect number of commander jobs were generated"); Assert.That(jobs.OfType <UnrealCommonHeaderJob>().Count() == 1, "Incorrect number of common header jobs were generated"); Assert.That(jobs.OfType <UnrealEntityTemplateJob>().Count() == 1, "Incorrect number of entity template jobs were generated"); var entityTemplateJob = jobs.OfType <UnrealEntityTemplateJob>(); Assert.That(entityTemplateJob.FirstOrDefault((job) => { var inputFiles = job.InputFiles; return(inputFiles.Contains(EmptyComponentSchemaPath) && inputFiles.Contains(EntityTemplateSchemaPath) && inputFiles.Count == 2); }) != null, "The generated commander did not have the expected input files"); Assert.That(entityTemplateJob.FirstOrDefault((job) => { var outputFiles = job.OutputFiles; return(outputFiles.Contains("EntityTemplate.h") && outputFiles.Contains("EntityTemplate.cpp") && outputFiles.Count == 2); }) != null, "The generated commander did not have the expected output files"); }
public static void a_schema_with_a_command_generates_the_expected_files_for_unreal() { var jobGenerator = new JobGenerator(null); var schemaFileRaw = TestUtils.GetRawJsonDataFromResource <SchemaFileRaw>("Improbable.CodeGeneration.Resources.json.test.command.json"); var schemaFilesRaw = new List <SchemaFileRaw> { schemaFileRaw }; var jobs = jobGenerator.GenerateJobs(schemaFilesRaw, Options); Assert.That(jobs.Count == 10, "The wrong number of jobs were generated when a schema with a command was processed"); Assert.That(jobs.Count(job => { return(job.InputFiles.Count == 1); }) == 10, "Not all jobs had the expected number of input files"); Assert.That(jobs.Count(job => { return(job.InputFiles.Contains(ExampleComponentSchemaPath)); }) == 10, "Not all jobs had the expected input file"); Assert.That(jobs.OfType <UnrealComponentJob>().Count() == 1, "Incorrect number of component jobs were generated"); Assert.That(jobs.OfType <UnrealTypeJob>().Count() == 3, "Incorrect number of type jobs were generated"); Assert.That(jobs.OfType <UnrealCommandJob>().Count() == 1, "Incorrect number of type jobs were generated"); Assert.That(jobs.OfType <UnrealCommanderJob>().Count() == 1, "Incorrect number of commander jobs were generated"); Assert.That(jobs.OfType <UnrealEntityTemplateJob>().Count() == 1, "Incorrect number of entity template jobs were generated"); Assert.That(jobs.OfType <UnrealEntityPipelineJob>().Count() == 1, "Incorrect number of entity pipeline jobs were generated"); Assert.That(jobs.OfType <UnrealCommonHeaderJob>().Count() == 1, "Incorrect number of common header jobs were generated"); Assert.That(jobs.OfType <UnrealCallbackDispatcherJob>().Count() == 1, "Incorrect number of callback dispatcher jobs were generated"); var typeJobs = jobs.OfType <UnrealTypeJob>(); Assert.That(typeJobs.FirstOrDefault((job) => { var outputFiles = job.OutputFiles; return(outputFiles.Contains("ExampleRequest.h") && outputFiles.Contains("ExampleRequest.cpp") && outputFiles.Count == 2); }) != null, "The generated type jobs for a schema with a command did not produce the expected command request type output files"); Assert.That(typeJobs.FirstOrDefault((job) => { var outputFiles = job.OutputFiles; return(outputFiles.Contains("ExampleResponse.h") && outputFiles.Contains("ExampleResponse.cpp") && outputFiles.Count == 2); }) != null, "The generated type jobs for a schema with a command did not produce the expected command response type output files"); Assert.That(typeJobs.FirstOrDefault((job) => { var outputFiles = job.OutputFiles; return(outputFiles.Contains("ExampleComponentData.h") && outputFiles.Contains("ExampleComponentData.cpp") && outputFiles.Count == 2); }) != null, "The generated type jobs for a schema with a command did not produce the expected component type output files"); var componentJobs = jobs.OfType <UnrealComponentJob>(); Assert.That(componentJobs.FirstOrDefault((job) => { var outputFiles = job.OutputFiles; return(outputFiles.Contains("ExampleComponentComponent.h") && outputFiles.Contains("ExampleComponentComponent.cpp") && outputFiles.Contains("ExampleComponentComponentUpdate.h") && outputFiles.Contains("ExampleComponentComponentUpdate.cpp") && outputFiles.Contains("ExampleComponentAddComponentOp.h") && outputFiles.Count == 5); }) != null, "The generated type jobs for a schema with a command did not produce the expected component output files"); var commandJobs = jobs.OfType <UnrealCommandJob>(); Assert.That(commandJobs.FirstOrDefault((job) => { var outputFiles = job.OutputFiles; return(outputFiles.Contains("ExampleCommandCommandResponder.h") && outputFiles.Contains("ExampleCommandCommandResponder.cpp") && outputFiles.Count == 2); }) != null, "The generated type jobs for a schema with a command did not produce the expected command output files"); }
private static void ParseArgs(string[] args) { var structuredArgs = StructuredArguments.FromStrings(args); void BatchSimulation() { var fromSeed = structuredArgs.AsIntOrDefault("From", 1); var toSeed = structuredArgs.AsIntOrDefault("To", 10); var td = BatchSimulator.LinesFromSeedRange(fromSeed, toSeed); var outFilename = $"results_from_{fromSeed}_to_{toSeed}.bin"; TrainingDataPersistence.SaveToDisk(td, outFilename); } void BatchSimulationOptimizationBased() { var td = OptimizationBasedGenerator.BatchGenerateTrainingData(100, 100); TrainingDataPersistence.SaveToDisk(td, $"new_results.bin"); } void TrainNetwork() { var td = TrainingDataPersistence.LoadFromDisk(structuredArgs.AsString("Filename")); var tvd = MlUtils.Split(td, 0.5f, false); var model = NetworkTrainer.TrainNetworkWithData(tvd.Training, tvd.Validation); model.Save("model.hdf5"); } void TrainNetworkOrthogonalSampling() { var td = TrainingDataPersistence.ParseCsv(structuredArgs.AsString("Filename")); var tvd = MlUtils.Split(td, 0.999f, true); var model = NetworkTrainer.TrainNetworkWithData(tvd.Training, tvd.Validation); model.Save("modelOrtho.hdf5"); } void JobGeneration() { JobGenerator.GenerateJobs(); } void MergeResults() { var mergedData = MergeResultFiles.MergeDataInPath(".", ".bin"); TrainingDataPersistence.SaveToDisk(mergedData, "merged.bin"); } void PrintData() { var maxCount = structuredArgs.AsIntOrDefault("NumRows", int.MaxValue); var samples = TrainingDataPersistence.LoadFromDisk(structuredArgs.AsString("Filename")).Samples; Console.WriteLine( $"Overall number of samples is {samples.Count}. Now showing up to {maxCount} samples..."); var ctr = 0; foreach (var sample in samples) { Console.WriteLine(sample); if (ctr++ >= maxCount) { break; } } } void Optimize() { var methodName = structuredArgs.AsStringOrDefault("Method", "LocalSolver"); var problem = ProblemInstanceGenerator.Generate(23); BaseProductionRatePredictor predictor = null; //predictor = new KerasNeuralProductionRatePredictor(ModelPersistence.LoadFromDisk("model.hdf5")); predictor = new OnnxNeuralProductionRatePredictor("converted.onnx"); //predictor = new MlProductionRatePredictor("model.zip"); MilkrunBufferAllocationSolution sol = null; switch (methodName) { case "SimulatedAnnealing": sol = SimAnnealOptimizer.Solve(problem, predictor, 1000, 1.0f); break; case "LocalSolver": //var evaluator = new SimulationEvaluator(problem); var evaluator = new PredictorBasedEvaluator(problem, predictor); sol = LocalSolverOptimizer.Solve(problem, evaluator); break; } Console.WriteLine("Solution of optimization = {0}", sol); Console.WriteLine("Production rate from predictor = {0}", predictor.Predict(sol.ToSample(problem.ProcessingRates))); Console.WriteLine("Production rate from simulation = {0}", SimulationRunner.ProductionRateForConfiguration(sol.ToFlowlineConfiguration(problem.ProcessingRates))); Console.WriteLine("Minimum production rate = {0}", problem.MinProductionRate); } void TrainForest() { var td = TrainingDataPersistence.LoadFromDisk(structuredArgs.AsString("Filename")); var tvd = MlUtils.Split(td, 0.999f, true); MLContext context = new MLContext(23); var transformer = ModelTrainer.TrainModelWithData(context, tvd.Training, tvd.Validation, out var schema); context.Model.Save(transformer, schema, "model.zip"); } void AutoMl() { var td = TrainingDataPersistence.LoadFromDisk(structuredArgs.AsString("Filename")); var tvd = MlUtils.Split(td, 1.0f, true); MLContext context = new MLContext(23); ModelSearch.AutoMlOnDataset(context, tvd.Training, tvd.Validation); } void DumpPredictionErrors() { var td = TrainingDataPersistence.LoadFromDisk(structuredArgs.AsString("Filename")); var tvd = MlUtils.Split(td, 0.5f, false); var dnn = new OnnxNeuralProductionRatePredictor("converted.onnx"); PredictionSample Predict(Sample sample) { var predictedRate = dnn.Predict(sample); float deviation = predictedRate - sample.ProductionRate; return(new PredictionSample(sample, deviation)); } var psamples = tvd.Validation.Samples.Take(100).Select(Predict).ToList(); File.WriteAllText("deviations.csv", CsvSerializer.SerializeToCsv(psamples)); } void TestExhaustiveGenerator() { int numMachines = 6; int numBuffers = numMachines - 1; var features = new List <FeatureDescription> { new FeatureDescription { IsDiscrete = false, LowerBound = 30, UpperBound = 120, Name = DefaultFeatures.MilkRunCycleLength.ToString() } }; features.AddRange(Enumerable.Range(0, numMachines).Select(i => new FeatureDescription { IsDiscrete = false, LowerBound = 0.8, UpperBound = 1.2, Name = DefaultFeatures.ProcessingRate + $"{i+1}" })); features.AddRange(Enumerable.Range(0, numMachines).Select(i => new FeatureDescription { IsDiscrete = false, LowerBound = 0.5, UpperBound = 1.5, Name = DefaultFeatures.MaterialRatio + $"{i+1}" })); features.AddRange(Enumerable.Range(0, numBuffers).Select(i => new FeatureDescription { IsDiscrete = true, LowerBound = 0, UpperBound = 80, Name = DefaultFeatures.BufferSize + $"{i+1}" })); // Big ortho experiment //int targetSampleCount = 2000000; //int subCubeSplitFactor = 2; // Small latin only experiment int targetSampleCount = 2000000; int subCubeSplitFactor = 1; int numCubes = Utils.Pow(subCubeSplitFactor, features.Count); int numValues = (int)Math.Ceiling(targetSampleCount / (double)numCubes) * numCubes; var samples = OrthoLatinHyperCube.PickSamples(features.ToArray(), numValues, subCubeSplitFactor); var lines = new List <string> { string.Join(",", samples.First().ColumnNames()) }; lines.AddRange(samples.Select(sample => string.Join(",", sample.ToFloats()))); File.WriteAllText("ortholatinhypercube.csv", string.Join("\n", lines)); Console.WriteLine("\nDistinct values"); for (int i = 0; i < numMachines; i++) { if (i < numBuffers) { Console.WriteLine($"Distinct buffer {i+1} sizes = {samples.Select(s => s.BufferSizes[i]).Distinct().Count()}"); } Console.WriteLine($"Distinct order up to levels {i+1} = {samples.Select(s => s.OrderUpToLevels[i]).Distinct().Count()}"); Console.WriteLine($"Distinct processing rates {i+1} = {samples.Select(s => s.ProcessingRates[i]).Distinct().Count()}"); } Console.WriteLine($"Distinct milk run cycle lengths = {samples.Select(s => s.MilkrunCycleLength).Distinct().Count()}"); } void GenerateInstance() { int seed = structuredArgs.AsInt("Seed"); string filename = structuredArgs.AsStringOrDefault("Filename", "instance.json"); var flowLineConfig = InstanceGenerator.Generate(seed); Utils.SaveObjectAsJson(flowLineConfig, filename); } var availableActions = new List <Action> { BatchSimulation, TrainNetwork, TrainNetworkOrthogonalSampling, JobGeneration, MergeResults, PrintData, Optimize, TrainForest, AutoMl, BatchSimulationOptimizationBased, DumpPredictionErrors, TestExhaustiveGenerator, GenerateInstance }; var actionMappings = availableActions.ToDictionary(action => Utils.NameOfLocalActionFunction("ParseArgs", action), action => action); if (args.Length >= 1) { var action = structuredArgs.GetAction(); if (actionMappings.ContainsKey(action)) { actionMappings[action](); return; } } ShowUsage(actionMappings); }