/// <summary>Main DoWork method for the task thread.</summary>
private void JobRunnerThread()
{
IJobManager nextJob = null;
while (!cancel)
{
try
{
nextJob = GetJobFromServer();
if (nextJob != null)
{
// Run the job.
IJobRunner runner = new JobRunnerAsync();
runner.Run(nextJob,
wait: true,
numberOfProcessors: Convert.ToInt32(appSettings["MaximumNumberOfCores"]));
// Tell the server we've finished the job.
UpdateServerForCompletedJob(nextJob);
}
}
catch (Exception err)
{
WriteToLog(err.Message);
}
}
}
public void EnsureAllToolsRun()
{
string json = ReflectionUtilities.GetResourceAsString("UnitTests.Core.PostSimulationTool.apsimx");
List <Exception> errors;
Simulations sims = FileFormat.ReadFromString <Simulations>(json, out errors);
Assert.AreEqual(0, errors.Count);
IModel script = Apsim.Child(Apsim.Find(sims, "Tool2"), "Script");
Assert.NotNull(script);
Simulation sim = Apsim.Find(sims, typeof(Simulation)) as Simulation;
Assert.NotNull(sim);
bool hasBeenRun = (bool)ReflectionUtilities.GetValueOfFieldOrProperty("HasBeenRun", script);
Assert.False(hasBeenRun);
IJobManager jobManager = new RunOrganiser(sims, sim, false);
IJobRunner jobRunner = new JobRunnerAsync();
jobRunner.Run(jobManager, wait: true);
hasBeenRun = (bool)ReflectionUtilities.GetValueOfFieldOrProperty("HasBeenRun", script);
Assert.True(hasBeenRun, "Failure in a post simulation tool prevented another post simulation tool from running.");
}
public void RunAusfarmGetOutputs()
{
RuntimeEnvironment environment = new RuntimeEnvironment
{
AusfarmRevision = "AusFarm-1.4.12",
};
RunF4PJob job = new RunF4PJob(GetDefaultF4PSimulationSpec(), environment);
IJobRunner runner = new JobRunnerAsync();
runner.Run(job, wait: true);
// Make sure we don't have an error.
Assert.AreEqual(job.Errors.Count, 0);
// Make sure we have a daily output table.
//Assert.AreEqual(job.Outputs.Tables.Count, 3);
//Assert.AreEqual(job.Outputs.Tables[0].TableName, "Summary");
//
//Assert.AreEqual(job.Outputs.Tables[1].TableName, "YieldProphetDaily");
//Assert.AreEqual(job.Outputs.Tables[1].Rows.Count, 92);
//double[] biomass = DataTableUtilities.GetColumnAsDoubles(job.Outputs.Tables[1], "biomass");
//Assert.IsTrue(MathUtilities.Max(biomass) > 20.0); // make sure something is growing.
//
//// Make sure we have a depth table.
//Assert.AreEqual(job.Outputs.Tables[2].TableName, "YieldProphetDepth");
//Assert.AreEqual(job.Outputs.Tables[2].Rows.Count, 8);
//double[] sw = DataTableUtilities.GetColumnAsDoubles(job.Outputs.Tables[2], "sw");
//Assert.IsTrue(MathUtilities.Max(sw) > 0.0); // make sure there are sw values
}
public void RunAPSIMXGetOutputs()
{
APSIMSpecification simulation = GetDefaultSimulationSpec();
List <APSIMSpecification> simulations = new List <APSIMSpecification>();
simulations.Add(simulation);
RuntimeEnvironment environment = new RuntimeEnvironment
{
APSIMxBuildNumber = 2473 // issue number that was resovled.
};
RunYPJob job = new RunYPJob(simulations, environment);
IJobRunner runner = new JobRunnerAsync();
runner.Run(job, wait: true);
// Make sure we don't have an error.
Assert.AreEqual(job.Errors.Count, 0);
// Make sure we have a daily output table.
Assert.AreEqual(job.Outputs.Tables.Count, 1);
Assert.AreEqual(job.Outputs.Tables[0].TableName, "Daily");
double[] biomass = DataTableUtilities.GetColumnAsDoubles(job.Outputs.Tables[0], "Wheat.Aboveground.Wt");
Assert.IsTrue(MathUtilities.Max(biomass) > 5); // make sure something is growing.
}
/// <summary>Runs the job (.xml file) specified on the command line.</summary>
/// <returns>True if something was run.</returns>
private static bool RunJobFromCommandLine(Dictionary <string, string> appSettings)
{
if (appSettings.ContainsKey("FileName"))
{
string jobFileName = appSettings["FileName"];
if (File.Exists(jobFileName))
{
appSettings.TryGetValue("APSIMXExecutable", out string executable);
string jobXML = File.ReadAllText(jobFileName);
string jobName = Path.GetFileNameWithoutExtension(jobFileName);
var environment = new APSIM.Cloud.Shared.RuntimeEnvironment
{
APSIMRevision = appSettings["APSIMRevision"],
RuntimePackage = appSettings["RuntimePackage"],
};
RunYPJob job = new RunYPJob(jobXML, environment)
{
ApsimXExecutable = executable
};
if (job.Errors.Count == 0)
{
IJobRunner runner = new JobRunnerAsync();
runner.Run(job, wait: true);
}
if (job.AllFilesZipped != null)
{
string destZipFileName = Path.ChangeExtension(jobFileName, ".out.zip");
using (Stream s = File.Create(destZipFileName))
{
job.AllFilesZipped.Seek(0, SeekOrigin.Begin);
job.AllFilesZipped.CopyTo(s);
}
}
if (job.Errors.Count > 0)
{
string msg = string.Empty;
foreach (string error in job.Errors)
{
msg += error + Environment.NewLine;
}
throw new Exception(msg);
}
return(true);
}
}
return(false);
}
/// <summary>
/// Runs an <see cref="IJobManager"/> with all implementations of <see cref="IJobRunner"/>.
/// </summary>
/// <param name="jobManager">The job manager to run.</param>
/// <param name="onSimulationCompleted">Event handler which will be invoked by each job runner after it finishes running.</param>
private void TestWithAllJobRunners(IJobManager jobManager, EventHandler <AllCompletedArgs> onSimulationCompleted)
{
IJobRunner jobRunner = new JobRunnerSync();
jobRunner.AllJobsCompleted += onSimulationCompleted;
Assert.DoesNotThrow(() => jobRunner.Run(jobManager, true));
jobRunner.AllJobsCompleted -= onSimulationCompleted;
jobRunner = new JobRunnerAsync();
jobRunner.AllJobsCompleted += onSimulationCompleted;
Assert.DoesNotThrow(() => jobRunner.Run(jobManager, true));
jobRunner.AllJobsCompleted -= onSimulationCompleted;
jobRunner = new JobRunnerMultiProcess(false);
jobRunner.AllJobsCompleted += onSimulationCompleted;
Assert.DoesNotThrow(() => jobRunner.Run(jobManager, true));
jobRunner.AllJobsCompleted -= onSimulationCompleted;
}
public void RefreshCheckpointNames()
{
Simulations sims = Utilities.GetRunnableSim();
sims.FileName = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString() + ".apsimx");
sims.Write(sims.FileName);
Simulation sim = Apsim.Find(sims, typeof(Simulation)) as Simulation;
IDataStore storage = Apsim.Find(sims, typeof(IDataStore)) as IDataStore;
// Record checkpoint names before and after running the simulation,
// and ensure that they are not the same.
string[] checkpointNamesBeforeRun = storage.Reader.CheckpointNames.ToArray();
// Run the simulation
IJobManager jobManager = new RunOrganiser(sims, sim, false);
IJobRunner jobRunner = new JobRunnerAsync();
jobRunner.Run(jobManager, wait: true);
string[] checkpointNamesAfterRun = storage.Reader.CheckpointNames.ToArray();
Assert.AreNotEqual(checkpointNamesBeforeRun, checkpointNamesAfterRun, "Storage reader failed to update checkpoint names after simulation was run.");
}
public void RunAPSIMGetError()
{
APSIMSpecification simulation = GetDefaultSimulationSpec();
(simulation.Management[0] as Sow).Cultivar = string.Empty;
List <APSIMSpecification> simulations = new List <APSIMSpecification>();
simulations.Add(simulation);
RuntimeEnvironment environment = new RuntimeEnvironment
{
APSIMRevision = "Apsim7.8-R4013"
};
RunYPJob job = new RunYPJob(simulations, environment);
IJobRunner runner = new JobRunnerAsync();
runner.Run(job, wait: true);
// Make sure we have an error.
Assert.AreEqual(job.Errors.Count, 1);
Assert.AreEqual(job.Errors[0], "Cannot sow plant - : Cultivar not specified\r\n Component name: Paddock.Wheat");
}
public void RunAPSIMGetOutputs()
{
APSIMSpecification simulation = GetDefaultSimulationSpec();
List <APSIMSpecification> simulations = new List <APSIMSpecification>();
simulations.Add(simulation);
RuntimeEnvironment environment = new RuntimeEnvironment
{
APSIMRevision = "Apsim7.8-R4013"
};
RunYPJob job = new RunYPJob(simulations, environment);
IJobRunner runner = new JobRunnerAsync();
runner.Run(job, wait: true);
// Make sure we don't have an error.
Assert.AreEqual(job.Errors.Count, 0);
// Make sure we have a daily output table.
Assert.AreEqual(job.Outputs.Tables.Count, 3);
Assert.AreEqual(job.Outputs.Tables[0].TableName, "Summary");
Assert.AreEqual(job.Outputs.Tables[1].TableName, "YieldProphetDaily");
Assert.AreEqual(job.Outputs.Tables[1].Rows.Count, 92);
double[] biomass = DataTableUtilities.GetColumnAsDoubles(job.Outputs.Tables[1], "biomass");
Assert.IsTrue(MathUtilities.Max(biomass) > 20.0); // make sure something is growing.
// Make sure we have a depth table.
Assert.AreEqual(job.Outputs.Tables[2].TableName, "YieldProphetDepth");
Assert.AreEqual(job.Outputs.Tables[2].Rows.Count, 8);
double[] sw = DataTableUtilities.GetColumnAsDoubles(job.Outputs.Tables[2], "sw");
Assert.IsTrue(MathUtilities.Max(sw) > 0.0); // make sure there are sw values
}
/// <summary>
/// Main program entry point.
/// </summary>
/// <param name="args"> Command line arguments</param>
/// <returns> Program exit code (0 for success)</returns>
public static int Main(string[] args)
{
string tempFolder = Path.Combine(Path.GetTempPath(), "ApsimX");
Directory.CreateDirectory(tempFolder);
Environment.SetEnvironmentVariable("TMP", tempFolder, EnvironmentVariableTarget.Process);
AppDomain.CurrentDomain.AssemblyResolve += new ResolveEventHandler(Manager.ResolveManagerAssembliesEventHandler);
int exitCode = 0;
try
{
string fileName = null;
// Extract file name from command line arguments.
if (args.Length >= 1)
{
fileName = args[0];
}
if (args.Length < 1 || args.Length > 4)
{
throw new Exception("Usage: ApsimX ApsimXFileSpec [/Recurse] [/SingleThreaded] [/RunTests]");
}
Stopwatch timer = new Stopwatch();
timer.Start();
// If the filename argument has a wildcard then create a IJobManager to go look for matching files to run.
// Otherwise, create a JobManager to open the filename and run it in a separate, external process
IJobManager job;
if (fileName.Contains('*') || fileName.Contains('?'))
{
job = Runner.ForFolder(fileName, args.Contains("/Recurse"), args.Contains("/RunTests"));
}
else
{
job = Runner.ForFile(fileName, args.Contains("/RunTests"));
}
// Run the job created above using either a single thread or multi threaded (default)
IJobRunner jobRunner;
if (args.Contains("/SingleThreaded"))
{
jobRunner = new JobRunnerSync();
}
else
{
jobRunner = new JobRunnerAsync();
}
jobRunner.JobCompleted += OnJobCompleted;
jobRunner.Run(job, wait: true);
// If errors occurred, write them to the console.
if (errors != null)
{
Console.WriteLine("ERRORS FOUND!!");
Console.WriteLine(errors);
exitCode = 1;
}
else
{
exitCode = 0;
}
timer.Stop();
Console.WriteLine("Finished running simulations. Duration " + timer.Elapsed.TotalSeconds.ToString("#.00") + " sec.");
}
catch (Exception err)
{
Console.WriteLine(err.ToString());
exitCode = 1;
}
return(exitCode);
}
/// <summary>
/// Main program entry point.
/// </summary>
/// <param name="args"> Command line arguments</param>
/// <returns> Program exit code (0 for success)</returns>
public static int Main(string[] args)
{
string tempFolder = Path.Combine(Path.GetTempPath(), "ApsimX");
Directory.CreateDirectory(tempFolder);
Environment.SetEnvironmentVariable("TMP", tempFolder, EnvironmentVariableTarget.Process);
AppDomain.CurrentDomain.AssemblyResolve += new ResolveEventHandler(Manager.ResolveManagerAssembliesEventHandler);
int exitCode = 0;
try
{
string fileName = null;
// Extract file name from command line arguments.
if (args.Length >= 1)
{
fileName = args[0];
}
if (args.Contains("/?"))
{
string detailedHelpInfo = "Usage: Models ApsimXFileSpec [/Recurse] [/SingleThreaded] [/RunTests] [/Csv] [/?]";
detailedHelpInfo += Environment.NewLine + Environment.NewLine;
detailedHelpInfo += "ApsimXFileSpec: The path to an .apsimx file. May include wildcard.";
detailedHelpInfo += Environment.NewLine + Environment.NewLine + "Options:" + Environment.NewLine;
detailedHelpInfo += " /Recurse Recursively search subdirectories for files matching ApsimXFileSpec" + Environment.NewLine;
detailedHelpInfo += " /SingleThreaded Run all simulations in a single thread." + Environment.NewLine;
detailedHelpInfo += " /RunTests Run all tests." + Environment.NewLine;
detailedHelpInfo += " /Csv Export all reports to .csv files." + Environment.NewLine;
detailedHelpInfo += " /? Show detailed help information.";
Console.WriteLine(detailedHelpInfo);
return(1);
}
if (args.Length < 1 || args.Length > 6)
{
Console.WriteLine("Usage: Models ApsimXFileSpec [/Recurse] [/SingleThreaded] [/RunTests] [/Csv] [/?]");
return(1);
}
Stopwatch timer = new Stopwatch();
timer.Start();
// If the filename argument has a wildcard then create a IJobManager to go look for matching files to run.
// Otherwise, create a JobManager to open the filename and run it in a separate, external process
IJobManager job;
if (fileName.Contains('*') || fileName.Contains('?'))
{
job = Runner.ForFolder(fileName, args.Contains("/Recurse"), args.Contains("/RunTests"));
}
else
{
job = Runner.ForFile(fileName, args.Contains("/RunTests"));
}
// Run the job created above using either a single thread or multi threaded (default)
IJobRunner jobRunner;
if (args.Contains("/SingleThreaded"))
{
jobRunner = new JobRunnerSync();
}
else
{
jobRunner = new JobRunnerAsync();
}
if (args.Select(arg => arg.ToLower()).Contains("/csv"))
{
string dir = Path.GetDirectoryName(fileName);
if (dir == "")
{
dir = Directory.GetCurrentDirectory();
}
files = Directory.GetFiles(
dir,
Path.GetFileName(fileName),
args.Contains("/Recurse") ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly).ToList();
jobRunner.AllJobsCompleted += GenerateCsvFiles;
}
jobRunner.JobCompleted += OnJobCompleted;
jobRunner.AllJobsCompleted += OnAllJobsCompleted;
jobRunner.Run(job, wait: true);
// If errors occurred, write them to the console.
if (errors != string.Empty)
{
Console.WriteLine("ERRORS FOUND!!");
Console.WriteLine(errors);
exitCode = 1;
}
else
{
exitCode = 0;
}
timer.Stop();
Console.WriteLine("Finished running simulations. Duration " + timer.Elapsed.TotalSeconds.ToString("#.00") + " sec.");
}
catch (Exception err)
{
Console.WriteLine(err.ToString());
exitCode = 1;
}
return(exitCode);
}
/// <summary>
/// Main program entry point.
/// </summary>
/// <param name="args"> Command line arguments</param>
/// <returns> Program exit code (0 for success)</returns>
public static int Main(string[] args)
{
int exitCode = 0;
try
{
string fileName = null;
// Extract file name from command line arguments.
if (args.Length >= 1)
{
fileName = args[0];
}
string usageMessage = "Usage: Models ApsimXFileSpec [/Recurse] [/SingleThreaded] [/RunTests] [/Csv] [/Version] [/Verbose] [/Upgrade] [/m] [/?]";
if (args.Contains("/?"))
{
string detailedHelpInfo = usageMessage;
detailedHelpInfo += Environment.NewLine + Environment.NewLine;
detailedHelpInfo += "ApsimXFileSpec: The path to an .apsimx file. May include wildcard.";
detailedHelpInfo += Environment.NewLine + Environment.NewLine + "Options:" + Environment.NewLine;
detailedHelpInfo += " /Recurse Recursively search subdirectories for files matching ApsimXFileSpec" + Environment.NewLine;
detailedHelpInfo += " /SingleThreaded Run all simulations in a single thread." + Environment.NewLine;
detailedHelpInfo += " /RunTests Run all tests." + Environment.NewLine;
detailedHelpInfo += " /Csv Export all reports to .csv files." + Environment.NewLine;
detailedHelpInfo += " /Version Display the version number." + Environment.NewLine;
detailedHelpInfo += " /Verbose Write messages to StdOut when a simulation starts/finishes. Only has an effect when running a directory of .apsimx files (*.apsimx)." + Environment.NewLine;
detailedHelpInfo += " /Upgrade Upgrades a file to the latest version of the .apsimx file format. Does not run the file." + Environment.NewLine;
detailedHelpInfo += " /m Use the experimental multi-process job runner." + Environment.NewLine;
detailedHelpInfo += " /? Show detailed help information.";
Console.WriteLine(detailedHelpInfo);
return(1);
}
if (args.Length < 1 || args.Length > 10)
{
Console.WriteLine(usageMessage);
return(1);
}
if (args.Contains("/Version"))
{
Model m = new Model();
Console.WriteLine(m.ApsimVersion);
return(0);
}
if (args.Contains("/Upgrade"))
{
string dir = Path.GetDirectoryName(fileName);
if (string.IsNullOrWhiteSpace(dir))
{
dir = Directory.GetCurrentDirectory();
}
string[] files = Directory.EnumerateFiles(dir, Path.GetFileName(fileName), args.Contains("/Recurse") ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly).ToArray();
foreach (string file in files)
{
List <Exception> errors;
IModel sims = FileFormat.ReadFromFile <Model>(file, out errors);
if (errors != null && errors.Count > 0)
{
foreach (Exception error in errors)
{
Console.Error.WriteLine(error.ToString());
}
}
File.WriteAllText(file, FileFormat.WriteToString(sims));
Console.WriteLine("Successfully upgraded " + file);
}
return(0);
}
Stopwatch timer = new Stopwatch();
timer.Start();
// If the filename argument has a wildcard then create a IJobManager to go look for matching files to run.
// Otherwise, create a JobManager to open the filename and run it in a separate, external process
IJobManager job;
bool hasWildcard = fileName.Contains('*') || fileName.Contains('?');
if (hasWildcard)
{
job = Runner.ForFolder(fileName, args.Contains("/Recurse"), args.Contains("/RunTests"), args.Contains("/Verbose"), args.Contains("/m"));
}
else
{
job = Runner.ForFile(fileName, args.Contains("/RunTests"));
}
// Run the job created above using either a single thread or multi threaded (default)
IJobRunner jobRunner;
if (args.Contains("/SingleThreaded"))
{
jobRunner = new JobRunnerSync();
}
else if (args.Contains("/m"))
{
// If the multi-process switch has been provided as well as a wildcard in the filename,
// we want to use the single threaded job runner, but run each job in multi-process mode.
// TODO : might be useful to allow users to run a directory of apsimx files via the multi-
// process job runner. This could be faster if they have many small files.
if (hasWildcard)
{
jobRunner = new JobRunnerSync();
}
else
{
jobRunner = new JobRunnerMultiProcess(args.Contains("/Verbose"));
}
}
else
{
jobRunner = new JobRunnerAsync();
}
if (args.Select(arg => arg.ToLower()).Contains("/csv"))
{
string dir = Path.GetDirectoryName(fileName);
if (dir == "")
{
dir = Directory.GetCurrentDirectory();
}
files = Directory.GetFiles(
dir,
Path.GetFileName(fileName),
args.Contains("/Recurse") ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly).ToList();
jobRunner.AllJobsCompleted += GenerateCsvFiles;
}
jobRunner.JobCompleted += OnJobCompleted;
jobRunner.AllJobsCompleted += OnAllJobsCompleted;
jobRunner.Run(job, wait: true);
// If errors occurred, write them to the console.
if (errors != string.Empty)
{
Console.WriteLine("ERRORS FOUND!!");
Console.WriteLine(errors);
exitCode = 1;
}
else
{
exitCode = 0;
}
timer.Stop();
Console.WriteLine("Finished running simulations. Duration " + timer.Elapsed.TotalSeconds.ToString("#.00") + " sec.");
}
catch (Exception err)
{
Console.WriteLine(err.ToString());
exitCode = 1;
}
return(exitCode);
}
/// <summary>Runs the job (.xml file) specified on the command line.</summary>
/// <returns>True if something was run.</returns>
private static bool RunJobFromCommandLine(Dictionary <string, string> appSettings)
{
if (appSettings.ContainsKey("FileName"))
{
string jobFileName = appSettings["FileName"];
if (File.Exists(jobFileName))
{
var ypEnvironment = new APSIM.Cloud.Shared.RuntimeEnvironment
{
APSIMRevision = appSettings["APSIMRevision"],
RuntimePackage = appSettings["RuntimePackage"],
};
if (appSettings.ContainsKey("UpdateFile"))
{
YieldProphetOld.UpdateFile(jobFileName);
return(true);
}
else if (appSettings.ContainsKey("ConvertToAPSIM"))
{
string jobXML = File.ReadAllText(jobFileName);
RunYPJob job = new RunYPJob(jobXML, ypEnvironment, createSims: false);
AllocConsole();
Console.WriteLine(job.WorkingDirectory);
Console.WriteLine();
if (job.Errors != null)
{
string msg = null;
foreach (string error in job.Errors)
{
msg += error + Environment.NewLine;
}
throw new Exception(msg);
}
return(true);
}
else
{
appSettings.TryGetValue("APSIMXExecutable", out string executable);
string jobXML = File.ReadAllText(jobFileName);
string jobName = Path.GetFileNameWithoutExtension(jobFileName);
IYPJob job = null;
if (jobXML.Contains("Farm4Prophet"))
{
var environment = new APSIM.Cloud.Shared.RuntimeEnvironment
{
AusfarmRevision = appSettings["AusfarmRevision"],
};
job = new RunF4PJob(jobXML, environment);
}
else
{
job = new RunYPJob(jobXML, ypEnvironment)
{
ApsimXExecutable = executable
};
}
if (job.Errors == null || job.Errors.Count == 0)
{
IJobRunner runner = new JobRunnerAsync();
runner.Run(job as IJobManager, wait: true);
}
if (job.AllFilesZipped != null)
{
string destZipFileName = Path.ChangeExtension(jobFileName, ".out.zip");
using (Stream s = File.Create(destZipFileName))
{
job.AllFilesZipped.Seek(0, SeekOrigin.Begin);
job.AllFilesZipped.CopyTo(s);
}
}
if (job.Errors != null && job.Errors.Count > 0)
{
string msg = string.Empty;
foreach (string error in job.Errors)
{
msg += error + Environment.NewLine;
}
throw new Exception(msg);
}
return(true);
}
}
}
return(false);
}
