private string PathToModels()
{
if (RDocker.UseDocker())
{
// This is the expected path to the models executable in the docker
// image. Nasty stuff.
return("/opt/apsim/Models");
}
return(typeof(IModel).Assembly.Location);
}
/// <summary>
/// Generates the R script which performs the optimization.
/// </summary>
/// <param name="fileName">File path to which the R code will be saved.</param>
/// <param name="outputPath">Directory/path to which output results will be saved. This is passed as a parameter to croptimizR.</param>
/// <param name="apsimxFileName">Name of the .apsimx file to be run by the optimisation.</param>
private void GenerateRScript(string fileName, string outputPath, string apsimxFileName)
{
// tbi: package installation. Need to test on a clean VM.
StringBuilder contents = new StringBuilder();
contents.AppendLine($"variable_names <- c({string.Join(", ", VariableNames.Select(x => $"'{x.Trim()}'").ToArray())})");
// In theory, it would be better to always use relative path to
// the input file, by setting the working directory of the R
// process appropriately. Unfortunately, this will require some
// refactoring of the R wrapper which I don't really want to do
// right now. So for now I'm going to just use relative path if
// using docker.
if (RDocker.UseDocker())
{
apsimxFileName = Path.GetFileName(apsimxFileName);
}
// If we're reading from the PredictedObserved table, need to fix
// Predicted./Observed. suffix for the observed variables.
string escapedOutputPath = outputPath.Replace(@"\", "/");
string[] sanitisedObservedVariables = GetObservedVariableName().Select(x => $"'{x.Trim()}'").ToArray();
string dateVariable = VariableNames.Any(v => v.StartsWith("Predicted.")) ? "Predicted.Clock.Today" : "Clock.Today";
contents.AppendLine($"observed_variable_names <- c({string.Join(", ", sanitisedObservedVariables)}, '{dateVariable}')");
contents.AppendLine($"apsimx_path <- '{PathToModels().Replace(@"\", "/")}'");
contents.AppendLine($"apsimx_file <- '{apsimxFileName.Replace(@"\", "/")}'");
contents.AppendLine($"simulation_names <- {GetSimulationNames()}");
contents.AppendLine($"predicted_table_name <- '{PredictedTableName}'");
contents.AppendLine($"observed_table_name <- '{ObservedTableName}'");
contents.AppendLine($"param_info <- {GetParamInfo()}");
contents.AppendLine();
contents.AppendLine(OptimizationMethod.GenerateOptimizationOptions("optim_options"));
contents.AppendLine($"optim_options$path_results <- '{escapedOutputPath}'");
if (RandomSeed != null)
{
contents.AppendLine($"optim_options$ranseed <- {RandomSeed}");
}
contents.AppendLine();
contents.AppendLine($"crit_function <- {OptimizationMethod.CritFunction}");
contents.AppendLine($"optim_method <- '{OptimizationMethod.ROptimizerName}'");
contents.AppendLine();
contents.AppendLine(ReflectionUtilities.GetResourceAsString("Models.Resources.RScripts.OptimizR.r"));
// Don't use Path.Combine() - as this may be running in a (linux) docker container.
// R will work with forward slashes for path separators on win and linux,
// but backslashes will not work on linux.
string rDataExpectedPath = $"{escapedOutputPath}/optim_results.Rdata";
contents.AppendLine(CreateReadRDataScript(rDataExpectedPath));
File.WriteAllText(fileName, contents.ToString());
}
/// <summary>
/// Run the optimization (and wait for it to finish).
/// </summary>
/// <param name="cancelToken">Cancellation token.</param>
public void Run(CancellationTokenSource cancelToken)
{
Progress = 0;
Status = "Generating R Script";
string fileName = GetTempFileName("parameter_estimation.r");
string outputPath = GetWorkingDirectory();
if (!Directory.Exists(outputPath))
{
Directory.CreateDirectory(outputPath);
}
string apsimxFileName = GenerateApsimXFile();
// If running with docker, all references to the output path must
// be relative (ie '.'). It would be nice to be able to do this with
// native R runner as well but that will require some refactoring work.
string outputReferencePath = RDocker.UseDocker() ? "." : outputPath;
GenerateRScript(fileName, outputReferencePath, apsimxFileName);
if (RDocker.UseDocker())
{
// todo: we should really be reporting warnings/errors here.
// However any summary file will not have its links connected,
// due to the way that CroptimizR is run. This needs further thought.
IR client = new RDocker(
outputHandler: OnOutputReceived
// warningHandler: w => FindInScope<ISummary>()?.WriteMessage(this, w, MessageType.Warning),
// errorHandler: e => FindInScope<ISummary>()?.WriteMessage(this, e, MessageType.Error)
);
Status = "Running Parameter Optimization";
try
{
client.RunScriptAsync(fileName, new List <string>(), cancelToken.Token).Wait();
}
catch (AggregateException errors)
{
// Don't propagate task canceled exceptions.
if (errors.InnerExceptions.Count == 1 && errors.InnerExceptions[0] is TaskCanceledException)
{
return;
}
throw;
}
}
else
{
Status = "Installing R Packages";
R r = new R(cancelToken.Token);
r.InstallPackages("remotes", "dplyr", "nloptr", "DiceDesign", "DBI", "cli");
r.InstallFromGithub("hol430/ApsimOnR", "SticsRPacks/CroptimizR");
// todo - capture stderr as well?
r.OutputReceived += OnOutputReceivedFromR;
string stdout = r.Run(fileName);
r.OutputReceived -= OnOutputReceivedFromR;
WriteMessage(stdout);
}
// Copy output files into appropriate output directory, if one is specified. Otherwise, delete them.
Status = "Reading Output";
DataTable output = null;
string apsimxFileDir = FindAncestor <Simulations>()?.FileName;
if (string.IsNullOrEmpty(apsimxFileDir))
{
apsimxFileDir = FindAncestor <Simulation>()?.FileName;
}
if (!string.IsNullOrEmpty(apsimxFileDir))
{
apsimxFileDir = Path.GetDirectoryName(apsimxFileDir);
}
IDataStore storage = FindInScope <IDataStore>();
bool firstFile = true;
foreach (string file in Directory.EnumerateFiles(outputPath))
{
if (Path.GetFileName(file) == outputCsvFileName)
{
if (storage != null && storage.Writer != null)
{
output = ReadRData(file);
storage.Writer.WriteTable(output, deleteAllData: firstFile);
firstFile = false;
}
}
if (!string.IsNullOrEmpty(apsimxFileDir))
{
File.Copy(file, Path.Combine(apsimxFileDir, Path.Combine($"{Name}-{Path.GetFileName(file)}")), true);
}
}
// Now, we run the simulations with the optimal values, and store
// the results in a checkpoint called 'After'. Checkpointing has
// not been implemented on the sockets storage implementation.
if (output != null && FindInScope <IDataStore>().Writer is DataStoreWriter)
{
Status = "Running simulations with optimised parameters";
IEnumerable <CompositeFactor> optimalValues = GetOptimalValues(output);
RunSimsWithOptimalValues(apsimxFileName, "Optimal", optimalValues);
// Now run sims without optimal values, to populate the 'Current' checkpoint.
Status = "Running simulations";
Runner runner = new Runner(Children);
List <Exception> errors = runner.Run();
if (errors != null && errors.Count > 0)
{
throw errors[0];
}
}
// Delete temp outputs.
Directory.Delete(outputPath, true);
}