/// <summary>
/// Starts the execution of an R script.
/// </summary>
/// <param name="fileName">Path to an R script. May be a file on disk, or an embedded resource.</param>
/// <param name="arguments">Command line arguments to pass to the script.</param>
public void RunAsync(string fileName, params string[] arguments)
{
string scriptName = fileName;
if (!File.Exists(scriptName) && Assembly.GetExecutingAssembly().GetManifestResourceInfo(scriptName) != null)
{
// If the file doesn't exist, we check the list of resources.
string script = string.Empty;
using (Stream s = Assembly.GetExecutingAssembly().GetManifestResourceStream(scriptName))
using (StreamReader reader = new StreamReader(s))
script = reader.ReadToEnd();
scriptName = Path.ChangeExtension(Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()), ".r");
File.WriteAllText(scriptName, script);
}
// Each command line argument must be quoted in case it contains spaces.
string args = "";
if (arguments.Length > 0)
{
args = arguments.Aggregate((x, y) => $"\"{x}\" \"{y}\"");
}
proc = new ProcessUtilities.ProcessWithRedirectedOutput();
proc.Exited += OnExited;
proc.Start(rScript, "\"" + scriptName + "\" " + args, workingDirectory, true);
}
/// <summary>
/// Main task thread for working with a single APSIMRunner.exe process using
/// an anonymous pipe.
/// </summary>
private void PipeServerTaskThread()
{
// Create 2 anonymous pipes (read and write) for duplex communications
// (each pipe is one-way)
using (var pipeRead = new AnonymousPipeServerStream(PipeDirection.In, HandleInheritability.Inheritable))
using (var pipeWrite = new AnonymousPipeServerStream(PipeDirection.Out, HandleInheritability.Inheritable))
{
var pipeHandles = pipeRead.GetClientHandleAsString() + " " + pipeWrite.GetClientHandleAsString();
// Run a APSIMRunner process passing the pipe read and write handles as arguments.
var runnerProcess = new ProcessUtilities.ProcessWithRedirectedOutput();
runnerProcess.Exited += OnExited;
var runnerExecutable = Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), "APSIMRunner.exe");
runnerProcess.Start(executable: runnerExecutable,
arguments: pipeHandles,
workingDirectory: Directory.GetCurrentDirectory(),
redirectOutput: true,
writeToConsole: false);
// Release the local handles that were created with the above GetClientHandleAsString calls
pipeRead.DisposeLocalCopyOfClientHandle();
pipeWrite.DisposeLocalCopyOfClientHandle();
try
{
// Get the next job to run.
var job = GetJobToRun();
while (job != null)
{
var startTime = DateTime.Now;
// Send the job to APSIMRunner.exe - this will run the simulation.
PipeUtilities.SendObjectToPipe(pipeWrite, job.JobSentToClient);
pipeWrite.WaitForPipeDrain();
// Get the output data back from APSIMRunner.exe
var endJob = PipeUtilities.GetObjectFromPipe(pipeRead) as JobOutput;
// Send the output data to storage.
endJob.WriteOutput(job.DataStore);
// Signal end of job.
InvokeJobCompleted(job.RunnableJob,
job.JobManager,
startTime,
endJob.ErrorMessage);
// Get the next job to run.
job = GetJobToRun();
}
}
catch (Exception err)
{
AddException(err);
}
}
}
/// <summary>
/// Gets the path to an executable (uses the Unix which utility).
/// Throws if the package does not exist. Obviously this will not
/// work on Windows.
/// </summary>
private string GetPathToPackage(string package)
{
ProcessUtilities.ProcessWithRedirectedOutput findR = new ProcessUtilities.ProcessWithRedirectedOutput();
findR.Start("/usr/bin/which", package, Path.GetTempPath(), true);
findR.WaitForExit();
if (string.IsNullOrEmpty(findR.StdOut) && !string.IsNullOrEmpty(findR.StdErr))
{
// If the shell command generated anything in StdErr, we display that message.
throw new Exception("Encountered an error while searching for " + package + " installation: " + findR.StdErr);
}
return(findR.StdOut);
}
/// <summary>A runner process has exited. Check for errors</summary>
/// <param name="sender">The sender of the event.</param>
/// <param name="e">The event arguments</param>
private void OnExited(object sender, EventArgs e)
{
ProcessUtilities.ProcessWithRedirectedOutput p = sender as ProcessUtilities.ProcessWithRedirectedOutput;
if (p.ExitCode != 0)
{
errors += p.StdOut + Environment.NewLine;
}
if (Process.GetProcessesByName("APSIMRunner").Length == 0)
{
Stop();
}
}
/// <summary>Create one job runner process for each CPU</summary>
/// <param name="numberOfProcessors">The maximum number of cores to use</param>
private void CreateRunners(int numberOfProcessors)
{
int numRunners = Process.GetProcessesByName("APSIMRunner").Length;
for (int i = numRunners; i < numberOfProcessors; i++)
{
string workingDirectory = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);
string runnerFileName = Path.Combine(workingDirectory, "APSIMRunner.exe");
ProcessUtilities.ProcessWithRedirectedOutput runnerProcess = new ProcessUtilities.ProcessWithRedirectedOutput();
runnerProcess.Exited += OnExited;
runnerProcess.Start(runnerFileName, null, workingDirectory, false);
}
}
/// <summary>A runner process has exited. Check for errors</summary>
/// <param name="sender">The sender of the event.</param>
/// <param name="e">The event arguments</param>
private void OnExited(object sender, EventArgs e)
{
ProcessUtilities.ProcessWithRedirectedOutput p = sender as ProcessUtilities.ProcessWithRedirectedOutput;
if (p.ExitCode != 0)
{
var exception = new Exception(p.StdOut + Environment.NewLine + p.StdErr);
AddException(exception);
}
if (Process.GetProcessesByName("APSIMRunner").Length == 0)
{
allStopped = true;
}
}
public static string RunModels(string arguments)
{
string pathToModels = typeof(IModel).Assembly.Location;
ProcessUtilities.ProcessWithRedirectedOutput proc = new ProcessUtilities.ProcessWithRedirectedOutput();
proc.Start(pathToModels, arguments, Path.GetTempPath(), true);
proc.WaitForExit();
if (proc.ExitCode != 0)
{
throw new Exception(proc.StdOut);
}
return(proc.StdOut);
}
public void OnCreatedShouldFailRun()
{
string json = ReflectionUtilities.GetResourceAsString("UnitTests.Core.ApsimFile.OnCreatedError.apsimx");
string fileName = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString() + ".apsimx");
File.WriteAllText(fileName, json);
Assembly models = typeof(IModel).Assembly;
string modelsExe = AppDomain.CurrentDomain.GetAssemblies().First(a => string.Equals(a.FullName, models.FullName, StringComparison.Ordinal)).Location;
var proc = new ProcessUtilities.ProcessWithRedirectedOutput();
proc.Start(modelsExe, fileName, Path.GetTempPath(), true);
proc.WaitForExit();
Assert.AreNotEqual(0, proc.ExitCode, "A file ran without error when an exception should have been thrown while opening the file.");
}
/// <summary>
/// Starts the execution of an R script.
/// </summary>
/// <param name="fileName">Path to an R script.</param>
public void RunAsync(string fileName)
{
if (File.Exists(fileName))
{
string rScript = GetRExePath();
// Create a temporary working directory.
workingDirectory = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString());
if (!Directory.Exists(workingDirectory)) // I would be very suprised if it did already exist
{
Directory.CreateDirectory(workingDirectory);
}
proc = new ProcessUtilities.ProcessWithRedirectedOutput();
proc.Exited += OnExited;
proc.Start(rScript, fileName + " ", workingDirectory, true);
}
}
/// <summary>
/// Starts the execution of an R script.
/// </summary>
/// <param name="fileName">Path to an R script. May be a file on disk, or an embedded resource.</param>
/// <param name="arguments">Command line arguments to pass to the script.</param>
public void RunAsync(string fileName, string arguments = "")
{
string scriptName = fileName;
if (!File.Exists(scriptName) && Assembly.GetExecutingAssembly().GetManifestResourceInfo(scriptName) != null)
{
// If the file doesn't exist, we check the list of resources.
string script = string.Empty;
using (Stream s = Assembly.GetExecutingAssembly().GetManifestResourceStream(scriptName))
using (StreamReader reader = new StreamReader(s))
script = reader.ReadToEnd();
scriptName = Path.ChangeExtension(Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()), ".r");
File.WriteAllText(scriptName, script);
}
proc = new ProcessUtilities.ProcessWithRedirectedOutput();
proc.Exited += OnExited;
proc.Start(rScript, "\"" + scriptName + "\" " + arguments, workingDirectory, true);
}
/// <summary>
/// Starts the execution of an R script.
/// </summary>
/// <param name="fileName">Path to an R script. May be a file on disk, or an embedded resource.</param>
/// <param name="arguments">Command line arguments to pass to the script.</param>
public void RunAsync(string fileName, params string[] arguments)
{
string scriptName = fileName;
if (!File.Exists(scriptName) && Assembly.GetExecutingAssembly().GetManifestResourceInfo(scriptName) != null)
{
// If the file doesn't exist, we check the list of resources.
string script = string.Empty;
using (Stream s = Assembly.GetExecutingAssembly().GetManifestResourceStream(scriptName))
using (StreamReader reader = new StreamReader(s))
script = reader.ReadToEnd();
scriptName = Path.ChangeExtension(Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()), ".r");
File.WriteAllText(scriptName, script);
}
// Each command line argument must be quoted in case it contains spaces.
string args = "";
if (arguments.Length > 0)
{
args = arguments.Aggregate((x, y) => $"\"{x}\" \"{y}\"");
}
// Don't load the user's .RProfile. Load a custom APSIM-specific
// one instead, by setting the R_PROFILE_USER environment variable.
Dictionary <string, string> environment = new Dictionary <string, string>();
environment["R_PROFILE_USER"] = startupFile;
proc = new ProcessUtilities.ProcessWithRedirectedOutput();
proc.Exited += OnExited;
if (OutputReceived != null)
{
proc.OutputReceived += OutputReceived;
}
if (ErrorReceived != null)
{
proc.ErrorReceived += ErrorReceived;
}
proc.Start(rScript, "\"" + scriptName + "\" " + args, workingDirectory, true, cancelToken, environment: environment);
}
/// <summary>
/// Runs the R installer.
/// </summary>
/// <param name="installerPath">Path to the installer.</param>
private void InstallR(string installerPath)
{
if (installerPath == null)
{
return;
}
// Setup a working directory.
string workingDirectory = Path.Combine(Path.GetTempPath(), "RSetup");
if (!Directory.Exists(workingDirectory))
{
Directory.CreateDirectory(workingDirectory);
}
// Copy installer to working directory.
try
{
string newInstallerPath = Path.Combine(workingDirectory, Path.GetFileNameWithoutExtension(installerPath));
File.Copy(installerPath, newInstallerPath);
installerPath = newInstallerPath;
}
catch
{
}
// Check to see if installer is already running for whatever reason.
// Kill them if found.
foreach (var process in Process.GetProcessesByName(Path.GetFileNameWithoutExtension(installerPath)))
{
process.Kill();
}
// Run the installer.
var installer = new ProcessUtilities.ProcessWithRedirectedOutput();
installer.Start(installerPath, "", workingDirectory, false);
installer.WaitForExit();
}
public void TestEditing()
{
string configFile = Path.GetTempFileName();
File.WriteAllLines(configFile, new[]
{
// Modify an array
"[Report].VariableNames = x,y,z",
// Modify a date - try a few different formats.
"[Clock].StartDate = 2000-01-01",
"[Clock].EndDate = 2000-01-10T00:00:00",
// Modify a string
"[Weather].FileName = fdsa.met",
@"[Weather2].FullFileName = .\jkl.met",
// Replace a model with a model from another file.
$"[Weather3] = {extFile}",
$"[Weather4] = {extFile};[w2]",
// Change a property of a resource model.
"[Wheat].Leaf.Photosynthesis.RUE.FixedValue = 0.4",
// Change a property of a manager script.
"[Manager].Script.Amount = 1234",
// Set an entire array.
"[Physical].BD = 1, 2, 3, 4, 5",
// Modify a single element of an array.
"[Physical].AirDry[2] = 6",
// Modify multiple elements of an array.
"[Physical].LL15[3:4] = 7",
});
string models = typeof(IModel).Assembly.Location;
string args = $"{fileName} /Edit {configFile}";
var proc = new ProcessUtilities.ProcessWithRedirectedOutput();
proc.Start(models, args, Path.GetTempPath(), true, writeToConsole: true);
proc.WaitForExit();
// Children of simulation are, in order:
// Clock, summary, zone, Weather, Weather2, w1, w2
Assert.AreEqual(null, proc.StdOut);
Assert.AreEqual(null, proc.StdErr);
Simulations file = FileFormat.ReadFromFile <Simulations>(fileName, out List <Exception> errors);
if (errors != null && errors.Count > 0)
{
throw errors[0];
}
var report = Apsim.Find(file, typeof(Models.Report)) as Models.Report;
string[] variableNames = new[] { "x", "y", "z" };
Assert.AreEqual(variableNames, report.VariableNames);
IModel sim = Apsim.Child(file, typeof(Simulation));
// Use an index-based lookup to locate child models.
// When we replace an entire model, we want to ensure
// that the replacement is inserted at the correct index.
Clock clock = sim.Children[0] as Clock;
Assert.AreEqual(new DateTime(2000, 1, 1), clock.StartDate);
Assert.AreEqual(new DateTime(2000, 1, 10), clock.EndDate);
var weather = sim.Children[3] as Models.Weather;
Assert.NotNull(weather);
Assert.AreEqual("Weather", weather.Name);
Assert.AreEqual("fdsa.met", weather.FileName);
var weather2 = sim.Children[4] as Models.Weather;
Assert.NotNull(weather2);
Assert.AreEqual("Weather2", weather2.Name);
Assert.AreEqual(@".\jkl.met", weather2.FileName);
// Weather3 and Weather4 should have been
// renamed to w1 and w2, respectively.
var weather3 = sim.Children[5] as Models.Weather;
Assert.NotNull(weather3);
Assert.AreEqual("w1", weather3.Name);
Assert.AreEqual("w1.met", weather3.FileName);
var weather4 = sim.Children[6] as Models.Weather;
Assert.NotNull(weather4);
Assert.AreEqual("w2", weather4.Name);
Assert.AreEqual("w2.met", weather4.FileName);
// The edit file operation should have changed RUE value to 0.4.
var wheat = sim.Children[2].Children[2] as Plant;
var rue = wheat.Children[6].Children[5].Children[0] as Constant;
Assert.AreEqual(0.4, rue.FixedValue);
double amount = (double)Apsim.Get(sim, "[Manager].Script.Amount");
Assert.AreEqual(1234, amount);
Physical physical = sim.Children[2].Children[4] as Physical;
Assert.AreEqual(new double[5] {
1, 2, 3, 4, 5
}, physical.BD);
Assert.AreEqual(new double[5] {
0, 6, 0, 0, 0
}, physical.AirDry);
Assert.AreEqual(new double[5] {
0, 0, 7, 7, 0
}, physical.LL15);
}
public void TestSimNameRegex()
{
string models = typeof(IModel).Assembly.Location;
IModel sim1 = Utilities.GetRunnableSim().Children[1];
sim1.Name = "sim1";
IModel sim2 = Utilities.GetRunnableSim().Children[1];
sim2.Name = "sim2";
IModel sim3 = Utilities.GetRunnableSim().Children[1];
sim3.Name = "simulation3";
IModel sim4 = Utilities.GetRunnableSim().Children[1];
sim4.Name = "Base";
Simulations sims = Simulations.Create(new[] { sim1, sim2, sim3, sim4, new DataStore() });
sims.ParentAllDescendants();
string apsimxFileName = Path.ChangeExtension(Path.GetTempFileName(), ".apsimx");
sims.Write(apsimxFileName);
// Need to quote the regex on unix systems.
string args;
if (ProcessUtilities.CurrentOS.IsWindows)
{
args = $@"{apsimxFileName} /Verbose /SimulationNameRegexPattern:sim\d";
}
else
{
args = $@"{apsimxFileName} /Verbose '/SimulationNameRegexPattern:sim\d'";
}
ProcessUtilities.ProcessWithRedirectedOutput proc = new ProcessUtilities.ProcessWithRedirectedOutput();
proc.Start(models, args, Directory.GetCurrentDirectory(), true);
proc.WaitForExit();
Assert.Null(proc.StdErr);
Assert.True(proc.StdOut.Contains("sim1"));
Assert.True(proc.StdOut.Contains("sim2"));
Assert.False(proc.StdOut.Contains("simulation3"));
Assert.False(proc.StdOut.Contains("Base"));
args = $@"{apsimxFileName} /Verbose /SimulationNameRegexPattern:sim1";
proc = new ProcessUtilities.ProcessWithRedirectedOutput();
proc.Start(models, args, Directory.GetCurrentDirectory(), true);
proc.WaitForExit();
Assert.Null(proc.StdErr);
Assert.True(proc.StdOut.Contains("sim1"));
Assert.False(proc.StdOut.Contains("sim2"));
Assert.False(proc.StdOut.Contains("simulation3"));
Assert.False(proc.StdOut.Contains("Base"));
args = $@"{apsimxFileName} /Verbose /SimulationNameRegexPattern:(simulation3)|(Base)";
proc = new ProcessUtilities.ProcessWithRedirectedOutput();
proc.Start(models, args, Directory.GetCurrentDirectory(), true);
proc.WaitForExit();
Assert.Null(proc.StdErr);
Assert.False(proc.StdOut.Contains("sim1"));
Assert.False(proc.StdOut.Contains("sim2"));
Assert.True(proc.StdOut.Contains("simulation3"));
Assert.True(proc.StdOut.Contains("Base"));
}
public void TestEditing()
{
string configFile = Path.GetTempFileName();
File.WriteAllLines(configFile, new[]
{
// Modify an array
"[Report].VariableNames = x,y,z",
// Modify a date - try a few different formats.
"[Clock].StartDate = 2000-01-01",
"[Clock].EndDate = 2000-01-10T00:00:00",
// Modify a string
"[Weather].FileName = fdsa.met",
@"[Weather2].FullFileName = .\jkl.met",
// Replace a model with a model from another file.
$"[Weather3] = {extFile}",
$"[Weather4] = {extFile};[w2]",
});
string models = typeof(IModel).Assembly.Location;
string args = $"{fileName} /Edit {configFile}";
var proc = new ProcessUtilities.ProcessWithRedirectedOutput();
proc.Start(models, args, Path.GetTempPath(), true, true);
proc.WaitForExit();
// Children of simulation are, in order:
// Clock, summary, zone, Weather, Weather2, w1, w2
Assert.AreEqual(null, proc.StdOut);
Assert.AreEqual(null, proc.StdErr);
Simulations file = FileFormat.ReadFromFile <Simulations>(fileName, out List <Exception> errors);
if (errors != null && errors.Count > 0)
{
throw errors[0];
}
var report = Apsim.Find(file, typeof(Models.Report)) as Models.Report;
string[] variableNames = new[] { "x", "y", "z" };
Assert.AreEqual(variableNames, report.VariableNames);
IModel sim = Apsim.Child(file, typeof(Simulation));
// Use an index-based lookup to locate child models.
// When we replace an entire model, we want to ensure
// that the replacement is inserted at the correct index.
Clock clock = sim.Children[0] as Clock;
Assert.AreEqual(new DateTime(2000, 1, 1), clock.StartDate);
Assert.AreEqual(new DateTime(2000, 1, 10), clock.EndDate);
var weather = sim.Children[3] as Models.Weather;
Assert.NotNull(weather);
Assert.AreEqual("Weather", weather.Name);
Assert.AreEqual("fdsa.met", weather.FileName);
var weather2 = sim.Children[4] as Models.Weather;
Assert.NotNull(weather2);
Assert.AreEqual("Weather2", weather2.Name);
Assert.AreEqual(@".\jkl.met", weather2.FileName);
// Weather3 and Weather4 should have been
// renamed to w1 and w2, respectively.
var weather3 = sim.Children[5] as Models.Weather;
Assert.NotNull(weather3);
Assert.AreEqual("w1", weather3.Name);
Assert.AreEqual("w1.met", weather3.FileName);
var weather4 = sim.Children[6] as Models.Weather;
Assert.NotNull(weather4);
Assert.AreEqual("w2", weather4.Name);
Assert.AreEqual("w2.met", weather4.FileName);
}
