public override void Run()
{
_IsRunning = true;
// We need to run ConToSim first. When it has finished then we can
// let the base class run APSIM.
SimFileName = Path.Combine(Directory.GetCurrentDirectory(),
Path.GetDirectoryName(_ConFileName),
Path.GetFileNameWithoutExtension(_ConFileName) + "."
+ _SimulationName + ".sim");
_ConFileName = Path.Combine(Directory.GetCurrentDirectory(), _ConFileName);
_ConToSimProcess = new ProcessCaller(Application.OpenForms[0]);
if (Configuration.getArchitecture() == Configuration.architecture.unix)
{
_ConToSimProcess.FileName = "mono";
_ConToSimProcess.Arguments = StringManip.DQuote(Configuration.RemoveMacros(Path.Combine("%apsim%", "Model", "ConToSim.exe")))
+ " " + StringManip.DQuote(_ConFileName) + " " + StringManip.DQuote(_SimulationName);
}
else
{
_ConToSimProcess.FileName = Configuration.RemoveMacros(Path.Combine("%apsim%", "Model", "ConToSim.exe"));
_ConToSimProcess.Arguments = StringManip.DQuote(_ConFileName) + " " + StringManip.DQuote(_SimulationName);
}
_ConToSimProcess.AllFinished += OnConToSimExited;
_ConToSimProcess.StdOutReceived += OnStdOut;
_ConToSimProcess.StdErrReceived += OnStdError;
_ConToSimProcess.Start();
}
/// <summary>
/// Create a job for each simulation in the specified .con file.
/// </summary>
private void CreateJobsFromCON(string FileName, string[] SimulationPaths, ref List <IJob> jobs)
{
// Run ConToSim first.
string ConToSimExe = Path.Combine(Configuration.ApsimBinDirectory(), "ConToSim.exe");
Process ConToSim = Utility.RunProcess(ConToSimExe,
StringManip.DQuote(FileName),
Path.GetDirectoryName(FileName));
Utility.CheckProcessExitedProperly(ConToSim);
// If no paths were specified then get a list of all paths.
if (SimulationPaths == null || SimulationPaths.Length == 0)
{
List <String> AllPaths = new List <String> ();
AllPaths = ConFile.GetSimsInConFile(FileName);
SimulationPaths = AllPaths.ToArray();
}
// Create a series of jobs for each simulation in the .con file.
foreach (string SimulationPath in SimulationPaths)
{
string SimFileName = Path.Combine(Path.GetDirectoryName(FileName),
Path.GetFileNameWithoutExtension(FileName) + "." + SimulationPath + ".sim");
Job J = CreateJob(SimFileName, SimFileName.Replace(".sim", ".sum"));
jobs.Add(J);
J = CleanupJob(SimFileName, J.Name);
jobs.Add(J);
}
}
/// <summary>
/// User has clicked ok - create patch.
/// </summary>
private void OnOKButtonClick(object sender, EventArgs e)
{
if (SaveFileDialog.ShowDialog() == System.Windows.Forms.DialogResult.OK)
{
try
{
string DirectoryName = Directory.GetCurrentDirectory();
bool SomeAreOutOfDate = false;
string RevisionsFileName = Path.Combine(Directory.GetCurrentDirectory(), "patch.revisions");
StreamWriter Revisions = new StreamWriter(RevisionsFileName);
List <string> FileNames = new List <string>();
FileNames.Add(Path.GetFileName(RevisionsFileName));
foreach (ListViewItem Item in ListView.Items)
{
if (Item.Checked)
{
string Status = Item.SubItems[3].Text;
if (Status != "Deleted")
{
FileNames.Add(Item.Text);
}
Revisions.WriteLine(StringManip.DQuote(Item.Text) + " " + Status + " " + Item.SubItems[4].Text);
if (Item.SubItems[3].Text == "OutOfDate")
{
SomeAreOutOfDate = true;
}
}
}
Revisions.Close();
saveSelections();
if (SomeAreOutOfDate)
{
MessageBox.Show("Some of the selected files are out of date. You need to do an SVN update before submitting a patch to Bob.",
"Error", MessageBoxButtons.OK, MessageBoxIcon.Error);
}
else
{
// Zip all files.
Zip.ZipFilesWithDirectories(FileNames, SaveFileDialog.FileName, "");
if (MessageBox.Show("Patch file successfully created. Upload to Bob?", "Success", MessageBoxButtons.YesNo, MessageBoxIcon.Question) == System.Windows.Forms.DialogResult.Yes)
{
Process.Start("http://www.apsim.info/APSIM.Builds.Portal/Upload.aspx");
}
Close();
}
}
catch (Exception err)
{
MessageBox.Show(err.Message, "Error", MessageBoxButtons.OK, MessageBoxIcon.Error);
}
}
}
/// <summary>
/// delete a file after a job has finished
/// </summary>
private Job CleanupJob(string FileName, string JobName)
{
// create job and return it.
Job J = new Job();
J.CommandLine = "%ComSpec% /c del " + StringManip.DQuote(FileName);
J.CommandLineUnix = "/bin/rm -f " + StringManip.DQuote(FileName);
J.WorkingDirectory = Path.GetDirectoryName(FileName);
J.Name = "Delete " + FileName;
J.DependsOn = new List <DependsOn> ();
J.DependsOn.Add(new DependsOn(JobName));
return(J);
}
/// <summary>
/// Create and return a job to run APSIM.
/// </summary>
private Job CreateJob(string FileName, string SumFileName, string SimulationPath = null)
{
NumApsimRuns++;
string Executable = Path.Combine(Configuration.ApsimBinDirectory(), "ApsimModel.exe");
// Create arguments
string Arguments = StringManip.DQuote(FileName);
if (SimulationPath != null)
{
Arguments += " " + StringManip.DQuote("Simulation=" + SimulationPath);
}
if (MaxLinesInSummaryFile > 0)
{
Arguments += " MaxOutputLines=" + MaxLinesInSummaryFile.ToString();
}
// create job and return it.
Job J = new Job();
J.CommandLine = StringManip.DQuote(Executable) + " " + Arguments;
J.WorkingDirectory = Path.GetDirectoryName(FileName);
// A UNC Path cannot be used as the working directory,
// so test for this and use the LocalApplicationData folder instead.
// This folder is reasonably likely to have a non-UNC path, since it is
// intended to be "local".
Uri URI = null;
bool isUNC = Uri.TryCreate(J.WorkingDirectory, UriKind.Absolute, out URI) && URI.IsUnc;
if (isUNC)
{
J.WorkingDirectory = Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData);
}
J.Name = FileName + ":";
if (SimulationPath == null)
{
J.Name += Path.GetFileNameWithoutExtension(SumFileName);
}
else
{
J.Name += SimulationPath;
}
J.IgnoreErrors = true;
J.maxLines = MaxLinesInSummaryFile;
J.StdOutFilename = SumFileName;
return(J);
}
public RunEntireApsimFileJob(string ApsimFileName, JobRunner JobRunner)
: base(Path.GetFileName(ApsimFileName), JobRunner)
{
_ApsimFileName = ApsimFileName;
if (Configuration.getArchitecture() == Configuration.architecture.unix)
{
_Executable = "mono";
_Arguments = StringManip.DQuote(_Executable) + " " + StringManip.DQuote(_ApsimFileName);
}
else
{
_Executable = Configuration.RemoveMacros(Path.Combine("%apsim%", "Model", "ApsimToSim.exe"));
_Arguments = StringManip.DQuote(_ApsimFileName);
}
}
public static void Plant2Documentation(BaseController Controller)
{
if ((!(BaseActions.FileSave(Controller))))
{
return;
}
string XmlFileName = Controller.ApsimData.FileName;
//Dim HtmlFileName As String = Path.GetTempPath() + Path.GetFileNameWithoutExtension(XmlFileName) + ".html" //removed from temp dir as Firefox can't handle abs dirs. JF
string HtmlFileName = Path.Combine("..", "Documentation", "Plant2Docs", Path.GetFileNameWithoutExtension(XmlFileName) + ".html");
string Arguments = StringManip.DQuote(XmlFileName) + " " + StringManip.DQuote(HtmlFileName);
//Dim P As Process = Process.Start(Configuration.ApsimBinDirectory + "\Plant2Documentation", Arguments)
Process P = Utility.RunProcess(Path.Combine(Configuration.ApsimBinDirectory(), "Plant2Documentation.exe"), Arguments, Path.GetDirectoryName(XmlFileName));
Utility.CheckProcessExitedProperly(P);
Process.Start(HtmlFileName);
}
private static void Go()
{
string ApsimDirectory = Directory.GetCurrentDirectory();
int JobID = Convert.ToInt32(System.Environment.GetEnvironmentVariable("JobID"));
string SVNFileName = CSGeneral.Utility.FindFileOnPath("svn.exe");
if (SVNFileName == "")
{
throw new Exception("Cannot find svn.exe on PATH");
}
string url = "https://apsimdev.apsim.info/APSIM.Builds.Service/BuildsClassic.svc/GetDoCommit" +
"?JobID=" + JobID;
int doCommit = Utils.REST.CallService <int>(url);
if (doCommit == 0)
{
Console.WriteLine("The commit option was unchecked on the upload patch form");
}
else
{
url = "https://apsimdev.apsim.info/APSIM.Builds.Service/BuildsClassic.svc/GetBugID" +
"?JobID=" + JobID;
string BugID = Utils.REST.CallService <string>(url);
url = "https://apsimdev.apsim.info/APSIM.Builds.Service/BuildsClassic.svc/GetDescription" +
"?JobID=" + JobID;
string Description = Utils.REST.CallService <string>(url);
Description += "\r\nbugid: " + BugID;
url = "https://apsimdev.apsim.info/APSIM.Builds.Service/BuildsClassic.svc/GetUserName" +
"?JobID=" + JobID;
string UserName = Utils.REST.CallService <string>(url);
url = "https://apsimdev.apsim.info/APSIM.Builds.Service/BuildsClassic.svc/GetPassword" +
"?JobID=" + JobID +
"&DbConnectPassword="******"commit --username " + UserName + " --password " + Password +
" -m " + StringManip.DQuote(Description);
Process SVNP = Utility.RunProcess(SVNFileName, SVNArguments, ApsimDirectory);
string StdOut = Utility.CheckProcessExitedProperly(SVNP);
Console.WriteLine(StdOut);
}
}
public override void Run()
{
_IsRunning = true;
// We need to run ApsimToSim first. When it has finished then we can
// let the base class run APSIM.
string SimulationName = _SimulationPath;
int PosLastSlash = _SimulationPath.LastIndexOf('/');
if (PosLastSlash != -1)
SimulationName = _SimulationPath.Substring(PosLastSlash + 1);
SimFileName = Path.Combine(Path.GetDirectoryName(_ApsimFileName),
SimulationName + ".sim");
_ApsimToSimProcess = new ProcessCaller(System.Windows.Forms.Application.OpenForms[0]);
if (Configuration.getArchitecture() == Configuration.architecture.unix)
_ApsimToSimProcess.FileName = "mono";
_ApsimToSimProcess.Arguments = StringManip.DQuote(Configuration.RemoveMacros(Path.Combine("%apsim%", "Model", "ApsimToSim.exe")))
_ + " " + StringManip.DQuote(_ApsimFileName) + " " + StringManip.DQuote(_SimulationPath);
} else {
/// <summary>
/// Call ConToSim to convert the .con file to a .sim file.
/// </summary>
private static void ConvertConToSims(string ConFileName)
{
Process PlugInProcess = new Process();
string Exe = Configuration.RemoveMacros(Path.Combine("%apsim%", "Model", "ConToSim.exe"));
if (Configuration.getArchitecture() == Configuration.architecture.unix)
{
PlugInProcess.StartInfo.FileName = "mono";
PlugInProcess.StartInfo.Arguments = StringManip.DQuote(Exe) + " " + Path.GetFileName(ConFileName);
}
else
{
PlugInProcess.StartInfo.FileName = Exe;
PlugInProcess.StartInfo.Arguments = Path.GetFileName(ConFileName);
}
// Determine whether or not the file is an executable; execute from the shell if it's not
PlugInProcess.StartInfo.UseShellExecute = false;
PlugInProcess.StartInfo.CreateNoWindow = true;
PlugInProcess.StartInfo.WorkingDirectory = Path.GetDirectoryName(ConFileName);
PlugInProcess.Start();
PlugInProcess.WaitForExit();
}
// Create a .sub file for condor
public void CreateSubmitFile(XmlNode jobDoc)
{
StreamWriter SubWriter = new StreamWriter(Path.Combine(WorkingFolder, "Apsim.sub"));
SubWriter.WriteLine("universe = vanilla");
SubWriter.WriteLine("should_transfer_files = YES");
SubWriter.WriteLine("when_to_transfer_output = ON_EXIT");
SubWriter.WriteLine("log = Apsim.condorlog");
SubWriter.WriteLine("skip_filechecks = true");
SubWriter.WriteLine("periodic_remove = (((CurrentTime - EnteredCurrentStatus) > 600) && JobStatus == 5)"); // Abort if held (missing files)
SubWriter.WriteLine("nice_user = "******"TRUE" : "FALSE"));
if (!useSingleCPU)
{
SubWriter.WriteLine("+RequiresWholeMachine = True");
}
SubWriter.Write("requirements = ");
if (arch == Configuration.architecture.unix)
{
SubWriter.Write(" OpSys == \"LINUX\"");
}
else if (arch == Configuration.architecture.win32)
{
SubWriter.Write(" Regexp(\"^WIN\", OpSys)");
}
else if (arch == (Configuration.architecture.win32 | Configuration.architecture.unix))
{
SubWriter.Write(" ((OpSys == \"LINUX\") || Regexp(\"^WIN\", OpSys))");
}
else
{
throw new Exception("Please select at least one Operating System to run on");
}
SubWriter.Write(" && ((Arch == \"INTEL\") || (Arch == \"X86_64\")) ");
// see https://htcondor-wiki.cs.wisc.edu/index.cgi/wiki?p=WholeMachineSlots
if (!useSingleCPU)
{
SubWriter.Write(" && (Target.CAN_RUN_WHOLE_MACHINE =?= True)");
}
SubWriter.WriteLine("");
SubWriter.WriteLine("executable = Apsim.$$(OpSys).$$(Arch).bat");
if (useSingleCPU)
{
SubWriter.WriteLine("environment = \"NUMBER_OF_PROCESSORS=1\"");
}
// Create a top level batch file.
StreamWriter ExeWriter;
ExeWriter = new StreamWriter(Path.Combine(WorkingFolder, "Apsim.LINUX.INTEL.bat"));
ExeWriter.NewLine = "\n";
ExeWriter.WriteLine("#!/bin/bash");
ExeWriter.WriteLine("export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin");
ExeWriter.WriteLine("for i in $@; do if [ ! -f $i ]; then wget -nd -q $i; if [ \"$?\" != \"0\" ]; then exit $?; fi; i=`basename $i`; fi ; chmod +x $i; ./$i; rm -f $i; done");
ExeWriter.WriteLine("if [ -d Temp ]; then rm -rf Temp; fi");
ExeWriter.Close();
File.Copy(Path.Combine(WorkingFolder, "Apsim.LINUX.INTEL.bat"), Path.Combine(WorkingFolder, "Apsim.LINUX.X86_64.bat"));
ExeWriter = new StreamWriter(Path.Combine(WorkingFolder, "Apsim.WINDOWS.INTEL.bat"));
ExeWriter.NewLine = "\r\n";
ExeWriter.WriteLine(":top");
ExeWriter.WriteLine("IF (%1) == () GOTO END");
ExeWriter.WriteLine("IF not exist %1 (");
ExeWriter.WriteLine(" wget -nd -q %1");
ExeWriter.WriteLine(" IF not ERRORLEVEL 0 (");
ExeWriter.WriteLine(" goto end");
ExeWriter.WriteLine(" )");
ExeWriter.WriteLine(")");
ExeWriter.WriteLine("set DirPath=%1");
ExeWriter.WriteLine("set filename=");
ExeWriter.WriteLine(":loop");
ExeWriter.WriteLine(" If \"%DirPath%\" == \"\" GoTo :done");
ExeWriter.WriteLine(" For /F \"tokens=1* delims=/\" %%a in (\"%DirPath%\") Do set filename=%%a");
ExeWriter.WriteLine(" For /F \"tokens=1* delims=/\" %%a in (\"%DirPath%\") Do Set DirPath=%%b");
ExeWriter.WriteLine(" GoTo :loop");
ExeWriter.WriteLine(":done");
ExeWriter.WriteLine("%filename%");
ExeWriter.WriteLine("DEL /s /q /f %filename%");
ExeWriter.WriteLine("SHIFT");
ExeWriter.WriteLine("GOTO TOP");
ExeWriter.WriteLine(":END");
ExeWriter.WriteLine("DEL /s /q /f Temp");
ExeWriter.Close();
StringBuilder PBSJobList = new StringBuilder();
File.Copy(Path.Combine(WorkingFolder, "Apsim.WINDOWS.INTEL.bat"), Path.Combine(WorkingFolder, "Apsim.WINDOWS.X86_64.bat"));
List <string> inputfiles = new List <string> ();
if (File.Exists(SelfExtractingExecutableLocation))
{
inputfiles.Add(Path.GetFileName(SelfExtractingExecutableLocation));
}
// Number of simulations in the current job
int numSims = 0;
int jobCounter = 0;
StreamWriter SimsWriter = null;
foreach (XmlNode simNode in jobDoc.SelectNodes("//simulation"))
{
string apsimFile = Path.GetFileName(XmlHelper.Attribute(simNode, "source"));
string simsFile = null;
if (numSims == 0)
{
SubWriter.WriteLine("output = " + "Apsim." + Convert.ToString(jobCounter) + ".stdout");
SubWriter.WriteLine("error = " + "Apsim." + Convert.ToString(jobCounter) + ".stderr");
SubWriter.WriteLine("arguments = " + SelfExtractingExecutableLocation + " " + "Apsim.$$(OpSys)." + Convert.ToString(jobCounter) + ".bat");
if (File.Exists(SelfExtractingExecutableLocation))
{
inputfiles.Add(Path.GetFileName(SelfExtractingExecutableLocation));
}
inputfiles.Add("Apsim.$$(OpSys)." + Convert.ToString(jobCounter) + ".bat");
simsFile = "Apsim." + Convert.ToString(jobCounter) + ".simulations";
SimsWriter = new StreamWriter(Path.Combine(WorkingFolder, simsFile));
inputfiles.Add(simsFile);
StreamWriter WinExeWriter = new StreamWriter(Path.Combine(WorkingFolder, "Apsim.WINDOWS." + Convert.ToString(jobCounter) + ".bat"));
WinExeWriter.NewLine = "\r\n";
WinExeWriter.WriteLine("echo Running on %COMPUTERNAME%");
WinExeWriter.WriteLine(".\\Temp\\Model\\Apsim.exe @" + simsFile);
WinExeWriter.Close();
StreamWriter LinuxExeWriter = new StreamWriter(Path.Combine(WorkingFolder, "Apsim.LINUX." + Convert.ToString(jobCounter) + ".bat"));
LinuxExeWriter.NewLine = "\n";
LinuxExeWriter.WriteLine("#!/bin/bash");
LinuxExeWriter.WriteLine("echo Running on `hostname -f` at `date`");
LinuxExeWriter.Write("mono ./Temp/Model/Apsim.exe @" + simsFile);
LinuxExeWriter.Close();
StreamWriter Linux2ExeWriter = new StreamWriter(Path.Combine(WorkingFolder, "Apsim.singularity." + Convert.ToString(jobCounter) + ".sh"));
Linux2ExeWriter.NewLine = "\n";
Linux2ExeWriter.WriteLine("#!/bin/bash");
Linux2ExeWriter.WriteLine("echo Running on `hostname -f` at `date`");
if (SelfExtractingExecutableLocation == "")
{
Linux2ExeWriter.WriteLine("singularity exec -B /30days/$USER:/30days -B /90days/$USER:/90days -B $TMPDIR:/TMPDIR --pwd /TMPDIR /home/uqpdevo1/Apsim.latest.sapp Apsim.exe @" + simsFile);
}
else
{
Linux2ExeWriter.WriteLine("singularity exec -B /30days/$USER:/30days -B /90days/$USER:/90days -B $TMPDIR:/TMPDIR --pwd /TMPDIR " + SelfExtractingExecutableLocation + " Apsim.exe @" + simsFile);
}
Linux2ExeWriter.Close();
}
SimsWriter.WriteLine(StringManip.DQuote(apsimFile) +
" Simulation=" + StringManip.DQuote(XmlHelper.Attribute(simNode, "name")));
if (!inputfiles.Contains(apsimFile))
{
inputfiles.Add(apsimFile);
}
foreach (XmlNode inputNode in simNode.SelectNodes(".//input"))
{
if (!inputfiles.Contains(XmlHelper.Attribute(inputNode, "name")))
{
inputfiles.Add(XmlHelper.Attribute(inputNode, "name"));
}
}
numSims++;
if (numSims >= numberSimsPerJob)
{
SubWriter.WriteLine("transfer_input_files = " + string.Join(",", inputfiles));
SubWriter.WriteLine("queue");
SubWriter.WriteLine();
List <string> pbsInputs = new List <string>();
pbsInputs.Add("Apsim.singularity." + Convert.ToString(jobCounter) + ".sh");
foreach (string input in inputfiles)
{
if (!new Regex("^Apsim.*.bat$").IsMatch(input))
{
pbsInputs.Add(input);
}
}
PBSJobList.Append("Apsim." + Convert.ToString(jobCounter) + "|" + // Jobname
string.Join(",", pbsInputs) + "|" + // input files
"Apsim.singularity." + Convert.ToString(jobCounter) + ".sh\n"); //command
SimsWriter.Close();
numSims = 0;
inputfiles.Clear();
jobCounter++;
}
}
if (numSims > 0)
{
SimsWriter.Close();
SubWriter.WriteLine("transfer_input_files = " + string.Join(",", inputfiles));
SubWriter.WriteLine("queue");
List <string> pbsInputs = new List <string>();
pbsInputs.Add("Apsim.singularity." + Convert.ToString(jobCounter) + ".sh");
foreach (string input in inputfiles)
{
if (!new Regex("^Apsim.*.bat$").IsMatch(input))
{
pbsInputs.Add(input);
}
}
PBSJobList.Append("Apsim." + Convert.ToString(jobCounter) + "|" + // Jobname
string.Join(",", pbsInputs) + "|" + // input files
"Apsim.singularity." + Convert.ToString(jobCounter) + ".sh\n"); //command
}
SubWriter.Close();
StreamWriter PBSWriter = new StreamWriter(Path.Combine(WorkingFolder, "Apsim.pbs"));
PBSWriter.NewLine = "\n";
PBSWriter.WriteLine("#!/bin/bash");
PBSWriter.WriteLine("# Construct a PBSPro job for each apsim job.");
PBSWriter.WriteLine("# Each job runs apsim on a bunch of simulations. They will execute in parallel under Apsim.exe.");
PBSWriter.WriteLine("# It should keep 1 node (of X CPUs) busy for a couple of hours.");
PBSWriter.WriteLine("srcdir=`dirname $(readlink -f $0)`");
if (jobCounter > 0)
{
PBSWriter.WriteLine("cat <<EOF | /opt/pbs/bin/qsub -J 0-" + Convert.ToString(jobCounter));
}
else
{
PBSWriter.WriteLine("cat <<EOF | qsub ");
}
PBSWriter.WriteLine("###### Select resources #####");
PBSWriter.WriteLine("#PBS -A UQ-QAAFI");
PBSWriter.WriteLine("#PBS -S /bin/bash");
PBSWriter.WriteLine("#PBS -N Apsim");
PBSWriter.WriteLine("#PBS -l select=1:ncpus=24:mem=120Gb\n");
PBSWriter.WriteLine("#PBS -l walltime=" + hoursPerJob + ":00:00");
PBSWriter.WriteLine("###### #####");
PBSWriter.WriteLine(" srcdir=$srcdir");
PBSWriter.WriteLine(" cd \\$TMPDIR");
PBSWriter.WriteLine("# extra environment settings");
PBSWriter.WriteLine(" module load singularity/2.3.1");
PBSWriter.WriteLine(" module load glibc/2.23");
PBSWriter.WriteLine(" export SINGULARITYENV_NUMBER_OF_PROCESSORS=24");
PBSWriter.WriteLine(" export SINGULARITYENV_R_LIBS_USER=$HOME/R");
PBSWriter.WriteLine(" mapfile -t joblist <<'XXXXXX'");
PBSWriter.WriteLine(PBSJobList.ToString() + "XXXXXX");
PBSWriter.WriteLine(" jobname=\\$(echo \\${joblist[\\$PBS_ARRAY_INDEX]} | cut -d\\| -f1)");
PBSWriter.WriteLine(" inputfiles=\\$(echo \\${joblist[\\$PBS_ARRAY_INDEX]} | cut -d\\| -f2)");
PBSWriter.WriteLine(" command=\\$(echo \\${joblist[\\$PBS_ARRAY_INDEX]} | cut -d\\| -f3)");
PBSWriter.WriteLine("# copy the job specific datafiles");
PBSWriter.WriteLine(" IFS=','");
PBSWriter.WriteLine(" for x in \\$inputfiles ; do cp \"\\$srcdir/\\$x\" ./ ; done");
PBSWriter.WriteLine(" unset IFS");
PBSWriter.WriteLine(" chmod +x \\$command; touch \\$command");
PBSWriter.WriteLine(" ./\\$command");
PBSWriter.WriteLine(" find . -maxdepth 1 -type f -newer \\$command -print | tar cfz \\$srcdir/\\$jobname.output.tar.gz --files-from -\n");
PBSWriter.WriteLine("EOF");
PBSWriter.Close();
#if false
// One day, we'll need a dependancy job to assemble output files. Not yet.
PBSWriter = new StreamWriter(Path.Combine(WorkingFolder, "Apsim.AssembleOutputs.pbs"));
PBSWriter.NewLine = "\n";
PBSWriter.WriteLine("#!/bin/bash");
PBSWriter.WriteLine("# Construct a PBS job to assemble the outputs of the first job.");
PBSWriter.WriteLine("# 1st argument is the id of the previous job");
PBSWriter.WriteLine("srcdir=`dirname $(readlink -f $0)`");
PBSWriter.WriteLine("cat <<EOF | qsub ");
PBSWriter.WriteLine("###### Select resources #####");
PBSWriter.WriteLine("#PBS -A UQ-QAAFI");
PBSWriter.WriteLine("#PBS -N Apsim");
PBSWriter.WriteLine("#PBS -l nodes=1:intel:ppn=1");
PBSWriter.WriteLine("#PBS -l mem=10Gb");
PBSWriter.WriteLine("#PBS -l vmem=10Gb");
PBSWriter.WriteLine("#PBS -l walltime=1:00:00");
if (jobCounter > 1)
{
PBSWriter.WriteLine("#PBS -W depend=afterokarray:$1");
}
else
{
PBSWriter.WriteLine("#PBS -W depend=afterok:$1");
}
PBSWriter.WriteLine("###### #####");
PBSWriter.WriteLine("echo > $srcdir/Apsim.finished");
PBSWriter.Close();
#endif
}
private int FillProjectWithAllFactorialJobs(ApsimFile AFile, string FileName, ref List <IJob> jobs)
{
int numFound = 0;
if (AFile.FactorComponent != null)
{
List <string> SimulationPaths = new List <string>();
ApsimFile.ExpandSimsToRun(AFile.RootComponent, ref SimulationPaths);
foreach (string simXmlPath in SimulationPaths)
{
FactorBuilder b = new FactorBuilder(AFile.FactorComponent);
Component Simulation = AFile.Find(simXmlPath);
List <string> allFactorials = Factor.CalcFactorialList(AFile, simXmlPath);
int totalCount = allFactorials.Count;
for (int instanceCount = 1; instanceCount <= totalCount; instanceCount++)
{
string rootName = Simulation.Name;
if (b.SaveExtraInfoInFilename)
{
rootName += ";" + allFactorials [instanceCount - 1];
}
else
{
//write a spacer to list sims in order eg: 01 or 001 or 0001 depending on count
string sPad = "";
double tot = Math.Floor(Math.Log10(totalCount) + 1);
double file = Math.Floor(Math.Log10(instanceCount) + 1);
for (int i = 0; i < (int)(tot - file); ++i)
{
sPad += "0";
}
rootName += "_" + sPad + instanceCount;
}
Job J = new Job();
J.WorkingDirectory = Path.GetDirectoryName(FileName);
string fullSimPath = simXmlPath + "@factorial=" + allFactorials [instanceCount - 1];
J.CommandLine += StringManip.DQuote(Path.Combine(Configuration.ApsimBinDirectory(), "ApsimToSim.exe")) +
" " + StringManip.DQuote(FileName) + " " + StringManip.DQuote("Simulation=" + fullSimPath);
J.CommandLineUnix = "mono " + J.CommandLine;
J.Name = "ApsimToSim " + FileName + " " + fullSimPath;
J.DependsOn = new List <DependsOn> ();
J.SendStdErrToConsole = false;
jobs.Add(J);
J = new Job();
J.WorkingDirectory = Path.GetDirectoryName(FileName);
J.CommandLine += StringManip.DQuote(Path.Combine(Configuration.ApsimBinDirectory(), "ApsimModel.exe")) +
" " + StringManip.DQuote(rootName + ".sim");
J.CommandLineUnix = J.CommandLine;
J.Name = "ApsimModel " + rootName + ".sim";
J.DependsOn = new List <DependsOn> ();
J.DependsOn.Add(new DependsOn("ApsimToSim " + FileName + " " + fullSimPath));
J.SendStdErrToConsole = false;
J.StdOutFilename = Path.Combine(Path.GetDirectoryName(FileName), rootName + ".sum");
jobs.Add(J);
J = CleanupJob(Path.Combine(Path.GetDirectoryName(FileName), rootName + ".sim"),
"ApsimModel " + rootName + ".sim");
jobs.Add(J);
numFound++;
}
}
}
return(numFound);
}
/// <summary>
/// The specified ApsimFile has a factorial in it. Create a series of jobs and add to ApsimJobs.
/// </summary>
private int FillProjectWithSpecifiedFactorialJobs(ApsimFile AFile, string FileName, string[] SimulationPaths, ref List <IJob> jobs)
{
int numFound = 0;
if (AFile.FactorComponent != null)
{
foreach (string SimulationPath in SimulationPaths)
{
string[] simPathParts = SimulationPath.Split(new string[] { "@factorial=" }, StringSplitOptions.None);
string simXmlPath = simPathParts [0];
string simPathFactorInstance = "";
if (simPathParts.Length < 2)
{
continue;
}
simPathFactorInstance = simPathParts [1].Trim(new char[] { '\'', ' ' });
List <string> allFactorials = Factor.CalcFactorialList(AFile, simXmlPath);
int totalCount = allFactorials.Count;
int instanceCount = 1 + allFactorials.IndexOf(simPathFactorInstance);
if (instanceCount < 1)
{
throw new Exception("Factor level " + simPathFactorInstance + "wasnt found");
}
FactorBuilder b = new FactorBuilder(AFile.FactorComponent);
Component Simulation = AFile.Find(simXmlPath);
string rootName = Simulation.Name;
if (b.SaveExtraInfoInFilename)
{
rootName += ";" + simPathFactorInstance;
}
else
{
//write a spacer to list sims in order eg: 01 or 001 or 0001 depending on count
string sPad = "";
double tot = Math.Floor(Math.Log10(totalCount) + 1);
double file = Math.Floor(Math.Log10(instanceCount) + 1);
for (int i = 0; i < (int)(tot - file); ++i)
{
sPad += "0";
}
rootName += "_" + sPad + instanceCount;
}
Job J = new Job();
J.WorkingDirectory = Path.GetDirectoryName(FileName);
J.CommandLine += StringManip.DQuote(Path.Combine(Configuration.ApsimBinDirectory(), "ApsimToSim.exe")) +
" " + StringManip.DQuote(FileName) + " " + StringManip.DQuote("Simulation=" + SimulationPath);
J.CommandLineUnix = "mono " + J.CommandLine;
J.Name = "ApsimToSim " + FileName + " " + SimulationPath;
J.DependsOn = new List <DependsOn> ();
J.SendStdErrToConsole = false;
jobs.Add(J);
J = new Job();
J.WorkingDirectory = Path.GetDirectoryName(FileName);
J.CommandLine += StringManip.DQuote(Path.Combine(Configuration.ApsimBinDirectory(), "ApsimModel.exe")) +
" " + StringManip.DQuote(rootName + ".sim");
J.CommandLineUnix = J.CommandLine;
J.Name = "ApsimModel " + rootName + ".sim";
J.DependsOn = new List <DependsOn> ();
J.DependsOn.Add(new DependsOn("ApsimToSim " + FileName + " " + SimulationPath));
J.SendStdErrToConsole = false;
J.StdOutFilename = Path.Combine(Path.GetDirectoryName(FileName), rootName + ".sum");
jobs.Add(J);
J = CleanupJob(Path.Combine(Path.GetDirectoryName(FileName), rootName + ".sim"),
"ApsimModel " + rootName + ".sim");
jobs.Add(J);
numFound++;
}
}
return(numFound);
}
/// <summary>
/// Create, and add to ApsimJobs, a series of jobs to run APSIM for each simulation.
/// </summary>
private void CreateJobsFromAPSIM(string FileName, string[] SimulationPaths, ref List <IJob> jobs, bool doAllFactors)
{
// Load all plugin (.xml) files.
if (Types.Instance.TypeNames.Length == 0)
{
PlugIns.LoadAll();
}
ApsimFile AFile = new ApsimFile(FileName);
// Look for factorials.
if (FillProjectWithSpecifiedFactorialJobs(AFile, FileName, SimulationPaths, ref jobs) > 0)
{
return;
}
else if (doAllFactors && FillProjectWithAllFactorialJobs(AFile, FileName, ref jobs) > 0)
{
return;
}
// No factorials.
if (SimulationPaths == null || SimulationPaths.Length == 0)
{
// No simulation specified. Run everything.
string ApsimToSimExe = Path.Combine(Configuration.ApsimBinDirectory(), "ApsimToSim.exe");
Process ApsimToSim = new Process();
ApsimToSim.StartInfo.FileName = ApsimToSimExe;
ApsimToSim.StartInfo.Arguments = StringManip.DQuote(Path.GetFileName(FileName));
ApsimToSim.StartInfo.WorkingDirectory = Directory.GetCurrentDirectory();
ApsimToSim.StartInfo.CreateNoWindow = true;
ApsimToSim.StartInfo.UseShellExecute = false;
ApsimToSim.Start();
ApsimToSim.WaitForExit();
List <String> AllPaths = new List <String>();
ApsimFile.ExpandSimsToRun(AFile.RootComponent, ref AllPaths);
for (int i = 0; i < AllPaths.Count; i++)
{
int lastSlash = AllPaths[i].LastIndexOf('/');
if (lastSlash != -1)
{
AllPaths[i] = Path.Combine(Directory.GetCurrentDirectory(), AllPaths[i].Substring(lastSlash + 1)) + ".sim";
}
}
// Create a series of jobs for each simulation in the .con file.
foreach (string SimFileName in AllPaths)
{
Job J = CreateJob(SimFileName, SimFileName.Replace(".sim", ".sum"));
jobs.Add(J);
J = CleanupJob(SimFileName, J.Name);
jobs.Add(J);
}
}
else if (SimulationPaths.Length == 1)
{
string SumFileName = SimulationPaths[0];
int PosLastSlash = SumFileName.LastIndexOf('/');
if (PosLastSlash != -1)
{
SumFileName = SumFileName.Substring(PosLastSlash + 1);
}
SumFileName += ".sum";
SumFileName = Path.Combine(Path.GetDirectoryName(FileName), SumFileName);
Job J = CreateJob(FileName, SumFileName, SimulationPaths[0]);
J.IgnoreErrors = false;
jobs.Add(J);
}
else if (SimulationPaths.Length > 1)
{
// For each path, create a simfile, and a job in our target.
foreach (string SimulationPath in SimulationPaths)
{
string simName = SimulationPath;
int PosLastSlash = simName.LastIndexOf('/');
if (PosLastSlash != -1)
{
simName = simName.Substring(PosLastSlash + 1);
}
string simFileName = Path.Combine(Path.GetDirectoryName(FileName), simName + ".sim");
try
{
Job J = CreateJob(FileName,
Path.Combine(Path.GetDirectoryName(FileName), simName + ".sum"),
SimulationPath);
J.IgnoreErrors = false;
jobs.Add(J);
J = CleanupJob(simFileName, J.Name);
jobs.Add(J);
}
catch (Exception err)
{
string sumFileName = Path.ChangeExtension(simFileName, ".sum");
WriteErrorToSummaryFile(sumFileName, err.Message);
}
}
}
}
protected override void Run()
{
Status = Status_t.Running;
//Console.WriteLine("Run: " + FileSpec);
// Scan a directory.
// 1. For each filename found, add a job for each simulation in each file.
List <string> FileNames = new List <string>();
Utility.FindFiles(Utility.ReplaceEnvironmentVariables(Path.GetDirectoryName(FileSpec)),
Path.GetFileName(FileSpec), ref FileNames, true, false);
Target myTarget = new Target();
myTarget.Name = TargetFolder;
myTarget.Project = Project;
foreach (string FileName in FileNames)
{
string Exe = "";
if (Path.GetExtension(FileName).ToLower() == ".con")
{
Exe = "%APSIM%/Model/ConToSim.exe";
}
else if (Path.GetExtension(FileName).ToLower() == ".apsim")
{
Exe = "%APSIM%/Model/ApsimToSim.exe";
}
Job convJob = new Job();
convJob.Name = "";
if (Exe != "")
{
convJob.Target = myTarget; convJob.Project = Project;
convJob.Name = "Convert " + FileName;
convJob.CommandLine = StringManip.DQuote(Exe) + " " + StringManip.DQuote(FileName);
convJob.WorkingDirectory = Path.GetDirectoryName(FileName);
myTarget.Jobs.Add(convJob);
}
foreach (string SimulationName in GetSimulationNamesFrom(FileName))
{
Job J = new Job();
J.Target = myTarget; J.Project = Project;
J.Name = FileName + ":" + SimulationName;
if (Path.GetExtension(FileName).ToLower() == ".apsim")
{
J.CommandLine = StringManip.DQuote("%APSIM%/Model/ApsimModel.exe") + " " +
StringManip.DQuote(SimulationName + ".sim");
J.DependsOn.Add(new DependsOn(convJob.Name));
J.StdOutFilename = Path.Combine(Path.GetDirectoryName(FileName), SimulationName + ".sum");
}
else if (Path.GetExtension(FileName).ToLower() == ".con")
{
string simfile = Path.ChangeExtension(FileName, "." + SimulationName + ".sim");
J.CommandLine = StringManip.DQuote("%APSIM%/Model/ApsimModel.exe") + " " +
StringManip.DQuote(simfile);
J.DependsOn.Add(new DependsOn(convJob.Name));
J.StdOutFilename = Path.Combine(Path.GetDirectoryName(FileName), Path.ChangeExtension(simfile, ".sum"));
}
else
{
// ???
}
J.IgnoreErrors = true;
J.WorkingDirectory = Path.GetDirectoryName(FileName);
myTarget.Jobs.Add(J);
}
}
Status = Status_t.Pass;
//FIXME StdOut += "Found " + FileNames.Count + " files\n";
Project.AddTarget(myTarget);
FinishTime = DateTime.Now;
}