public override void Run() { _IsRunning = true; // We need to run ConToSim first. When it has finished then we can // let the base class run APSIM. SimFileName = Path.Combine(Directory.GetCurrentDirectory(), Path.GetDirectoryName(_ConFileName), Path.GetFileNameWithoutExtension(_ConFileName) + "." + _SimulationName + ".sim"); _ConFileName = Path.Combine(Directory.GetCurrentDirectory(), _ConFileName); _ConToSimProcess = new ProcessCaller(Application.OpenForms[0]); if (Configuration.getArchitecture() == Configuration.architecture.unix) { _ConToSimProcess.FileName = "mono"; _ConToSimProcess.Arguments = StringManip.DQuote(Configuration.RemoveMacros(Path.Combine("%apsim%", "Model", "ConToSim.exe"))) + " " + StringManip.DQuote(_ConFileName) + " " + StringManip.DQuote(_SimulationName); } else { _ConToSimProcess.FileName = Configuration.RemoveMacros(Path.Combine("%apsim%", "Model", "ConToSim.exe")); _ConToSimProcess.Arguments = StringManip.DQuote(_ConFileName) + " " + StringManip.DQuote(_SimulationName); } _ConToSimProcess.AllFinished += OnConToSimExited; _ConToSimProcess.StdOutReceived += OnStdOut; _ConToSimProcess.StdErrReceived += OnStdError; _ConToSimProcess.Start(); }
/// <summary> /// Create a job for each simulation in the specified .con file. /// </summary> private void CreateJobsFromCON(string FileName, string[] SimulationPaths, ref List <IJob> jobs) { // Run ConToSim first. string ConToSimExe = Path.Combine(Configuration.ApsimBinDirectory(), "ConToSim.exe"); Process ConToSim = Utility.RunProcess(ConToSimExe, StringManip.DQuote(FileName), Path.GetDirectoryName(FileName)); Utility.CheckProcessExitedProperly(ConToSim); // If no paths were specified then get a list of all paths. if (SimulationPaths == null || SimulationPaths.Length == 0) { List <String> AllPaths = new List <String> (); AllPaths = ConFile.GetSimsInConFile(FileName); SimulationPaths = AllPaths.ToArray(); } // Create a series of jobs for each simulation in the .con file. foreach (string SimulationPath in SimulationPaths) { string SimFileName = Path.Combine(Path.GetDirectoryName(FileName), Path.GetFileNameWithoutExtension(FileName) + "." + SimulationPath + ".sim"); Job J = CreateJob(SimFileName, SimFileName.Replace(".sim", ".sum")); jobs.Add(J); J = CleanupJob(SimFileName, J.Name); jobs.Add(J); } }
/// <summary> /// User has clicked ok - create patch. /// </summary> private void OnOKButtonClick(object sender, EventArgs e) { if (SaveFileDialog.ShowDialog() == System.Windows.Forms.DialogResult.OK) { try { string DirectoryName = Directory.GetCurrentDirectory(); bool SomeAreOutOfDate = false; string RevisionsFileName = Path.Combine(Directory.GetCurrentDirectory(), "patch.revisions"); StreamWriter Revisions = new StreamWriter(RevisionsFileName); List <string> FileNames = new List <string>(); FileNames.Add(Path.GetFileName(RevisionsFileName)); foreach (ListViewItem Item in ListView.Items) { if (Item.Checked) { string Status = Item.SubItems[3].Text; if (Status != "Deleted") { FileNames.Add(Item.Text); } Revisions.WriteLine(StringManip.DQuote(Item.Text) + " " + Status + " " + Item.SubItems[4].Text); if (Item.SubItems[3].Text == "OutOfDate") { SomeAreOutOfDate = true; } } } Revisions.Close(); saveSelections(); if (SomeAreOutOfDate) { MessageBox.Show("Some of the selected files are out of date. You need to do an SVN update before submitting a patch to Bob.", "Error", MessageBoxButtons.OK, MessageBoxIcon.Error); } else { // Zip all files. Zip.ZipFilesWithDirectories(FileNames, SaveFileDialog.FileName, ""); if (MessageBox.Show("Patch file successfully created. Upload to Bob?", "Success", MessageBoxButtons.YesNo, MessageBoxIcon.Question) == System.Windows.Forms.DialogResult.Yes) { Process.Start("http://www.apsim.info/APSIM.Builds.Portal/Upload.aspx"); } Close(); } } catch (Exception err) { MessageBox.Show(err.Message, "Error", MessageBoxButtons.OK, MessageBoxIcon.Error); } } }
/// <summary> /// delete a file after a job has finished /// </summary> private Job CleanupJob(string FileName, string JobName) { // create job and return it. Job J = new Job(); J.CommandLine = "%ComSpec% /c del " + StringManip.DQuote(FileName); J.CommandLineUnix = "/bin/rm -f " + StringManip.DQuote(FileName); J.WorkingDirectory = Path.GetDirectoryName(FileName); J.Name = "Delete " + FileName; J.DependsOn = new List <DependsOn> (); J.DependsOn.Add(new DependsOn(JobName)); return(J); }
/// <summary> /// Create and return a job to run APSIM. /// </summary> private Job CreateJob(string FileName, string SumFileName, string SimulationPath = null) { NumApsimRuns++; string Executable = Path.Combine(Configuration.ApsimBinDirectory(), "ApsimModel.exe"); // Create arguments string Arguments = StringManip.DQuote(FileName); if (SimulationPath != null) { Arguments += " " + StringManip.DQuote("Simulation=" + SimulationPath); } if (MaxLinesInSummaryFile > 0) { Arguments += " MaxOutputLines=" + MaxLinesInSummaryFile.ToString(); } // create job and return it. Job J = new Job(); J.CommandLine = StringManip.DQuote(Executable) + " " + Arguments; J.WorkingDirectory = Path.GetDirectoryName(FileName); // A UNC Path cannot be used as the working directory, // so test for this and use the LocalApplicationData folder instead. // This folder is reasonably likely to have a non-UNC path, since it is // intended to be "local". Uri URI = null; bool isUNC = Uri.TryCreate(J.WorkingDirectory, UriKind.Absolute, out URI) && URI.IsUnc; if (isUNC) { J.WorkingDirectory = Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData); } J.Name = FileName + ":"; if (SimulationPath == null) { J.Name += Path.GetFileNameWithoutExtension(SumFileName); } else { J.Name += SimulationPath; } J.IgnoreErrors = true; J.maxLines = MaxLinesInSummaryFile; J.StdOutFilename = SumFileName; return(J); }
public RunEntireApsimFileJob(string ApsimFileName, JobRunner JobRunner) : base(Path.GetFileName(ApsimFileName), JobRunner) { _ApsimFileName = ApsimFileName; if (Configuration.getArchitecture() == Configuration.architecture.unix) { _Executable = "mono"; _Arguments = StringManip.DQuote(_Executable) + " " + StringManip.DQuote(_ApsimFileName); } else { _Executable = Configuration.RemoveMacros(Path.Combine("%apsim%", "Model", "ApsimToSim.exe")); _Arguments = StringManip.DQuote(_ApsimFileName); } }
public static void Plant2Documentation(BaseController Controller) { if ((!(BaseActions.FileSave(Controller)))) { return; } string XmlFileName = Controller.ApsimData.FileName; //Dim HtmlFileName As String = Path.GetTempPath() + Path.GetFileNameWithoutExtension(XmlFileName) + ".html" //removed from temp dir as Firefox can't handle abs dirs. JF string HtmlFileName = Path.Combine("..", "Documentation", "Plant2Docs", Path.GetFileNameWithoutExtension(XmlFileName) + ".html"); string Arguments = StringManip.DQuote(XmlFileName) + " " + StringManip.DQuote(HtmlFileName); //Dim P As Process = Process.Start(Configuration.ApsimBinDirectory + "\Plant2Documentation", Arguments) Process P = Utility.RunProcess(Path.Combine(Configuration.ApsimBinDirectory(), "Plant2Documentation.exe"), Arguments, Path.GetDirectoryName(XmlFileName)); Utility.CheckProcessExitedProperly(P); Process.Start(HtmlFileName); }
private static void Go() { string ApsimDirectory = Directory.GetCurrentDirectory(); int JobID = Convert.ToInt32(System.Environment.GetEnvironmentVariable("JobID")); string SVNFileName = CSGeneral.Utility.FindFileOnPath("svn.exe"); if (SVNFileName == "") { throw new Exception("Cannot find svn.exe on PATH"); } string url = "https://apsimdev.apsim.info/APSIM.Builds.Service/BuildsClassic.svc/GetDoCommit" + "?JobID=" + JobID; int doCommit = Utils.REST.CallService <int>(url); if (doCommit == 0) { Console.WriteLine("The commit option was unchecked on the upload patch form"); } else { url = "https://apsimdev.apsim.info/APSIM.Builds.Service/BuildsClassic.svc/GetBugID" + "?JobID=" + JobID; string BugID = Utils.REST.CallService <string>(url); url = "https://apsimdev.apsim.info/APSIM.Builds.Service/BuildsClassic.svc/GetDescription" + "?JobID=" + JobID; string Description = Utils.REST.CallService <string>(url); Description += "\r\nbugid: " + BugID; url = "https://apsimdev.apsim.info/APSIM.Builds.Service/BuildsClassic.svc/GetUserName" + "?JobID=" + JobID; string UserName = Utils.REST.CallService <string>(url); url = "https://apsimdev.apsim.info/APSIM.Builds.Service/BuildsClassic.svc/GetPassword" + "?JobID=" + JobID + "&DbConnectPassword="******"commit --username " + UserName + " --password " + Password + " -m " + StringManip.DQuote(Description); Process SVNP = Utility.RunProcess(SVNFileName, SVNArguments, ApsimDirectory); string StdOut = Utility.CheckProcessExitedProperly(SVNP); Console.WriteLine(StdOut); } }
public override void Run() { _IsRunning = true; // We need to run ApsimToSim first. When it has finished then we can // let the base class run APSIM. string SimulationName = _SimulationPath; int PosLastSlash = _SimulationPath.LastIndexOf('/'); if (PosLastSlash != -1) SimulationName = _SimulationPath.Substring(PosLastSlash + 1); SimFileName = Path.Combine(Path.GetDirectoryName(_ApsimFileName), SimulationName + ".sim"); _ApsimToSimProcess = new ProcessCaller(System.Windows.Forms.Application.OpenForms[0]); if (Configuration.getArchitecture() == Configuration.architecture.unix) _ApsimToSimProcess.FileName = "mono"; _ApsimToSimProcess.Arguments = StringManip.DQuote(Configuration.RemoveMacros(Path.Combine("%apsim%", "Model", "ApsimToSim.exe"))) _ + " " + StringManip.DQuote(_ApsimFileName) + " " + StringManip.DQuote(_SimulationPath); } else {
/// <summary> /// Call ConToSim to convert the .con file to a .sim file. /// </summary> private static void ConvertConToSims(string ConFileName) { Process PlugInProcess = new Process(); string Exe = Configuration.RemoveMacros(Path.Combine("%apsim%", "Model", "ConToSim.exe")); if (Configuration.getArchitecture() == Configuration.architecture.unix) { PlugInProcess.StartInfo.FileName = "mono"; PlugInProcess.StartInfo.Arguments = StringManip.DQuote(Exe) + " " + Path.GetFileName(ConFileName); } else { PlugInProcess.StartInfo.FileName = Exe; PlugInProcess.StartInfo.Arguments = Path.GetFileName(ConFileName); } // Determine whether or not the file is an executable; execute from the shell if it's not PlugInProcess.StartInfo.UseShellExecute = false; PlugInProcess.StartInfo.CreateNoWindow = true; PlugInProcess.StartInfo.WorkingDirectory = Path.GetDirectoryName(ConFileName); PlugInProcess.Start(); PlugInProcess.WaitForExit(); }
// Create a .sub file for condor public void CreateSubmitFile(XmlNode jobDoc) { StreamWriter SubWriter = new StreamWriter(Path.Combine(WorkingFolder, "Apsim.sub")); SubWriter.WriteLine("universe = vanilla"); SubWriter.WriteLine("should_transfer_files = YES"); SubWriter.WriteLine("when_to_transfer_output = ON_EXIT"); SubWriter.WriteLine("log = Apsim.condorlog"); SubWriter.WriteLine("skip_filechecks = true"); SubWriter.WriteLine("periodic_remove = (((CurrentTime - EnteredCurrentStatus) > 600) && JobStatus == 5)"); // Abort if held (missing files) SubWriter.WriteLine("nice_user = "******"TRUE" : "FALSE")); if (!useSingleCPU) { SubWriter.WriteLine("+RequiresWholeMachine = True"); } SubWriter.Write("requirements = "); if (arch == Configuration.architecture.unix) { SubWriter.Write(" OpSys == \"LINUX\""); } else if (arch == Configuration.architecture.win32) { SubWriter.Write(" Regexp(\"^WIN\", OpSys)"); } else if (arch == (Configuration.architecture.win32 | Configuration.architecture.unix)) { SubWriter.Write(" ((OpSys == \"LINUX\") || Regexp(\"^WIN\", OpSys))"); } else { throw new Exception("Please select at least one Operating System to run on"); } SubWriter.Write(" && ((Arch == \"INTEL\") || (Arch == \"X86_64\")) "); // see https://htcondor-wiki.cs.wisc.edu/index.cgi/wiki?p=WholeMachineSlots if (!useSingleCPU) { SubWriter.Write(" && (Target.CAN_RUN_WHOLE_MACHINE =?= True)"); } SubWriter.WriteLine(""); SubWriter.WriteLine("executable = Apsim.$$(OpSys).$$(Arch).bat"); if (useSingleCPU) { SubWriter.WriteLine("environment = \"NUMBER_OF_PROCESSORS=1\""); } // Create a top level batch file. StreamWriter ExeWriter; ExeWriter = new StreamWriter(Path.Combine(WorkingFolder, "Apsim.LINUX.INTEL.bat")); ExeWriter.NewLine = "\n"; ExeWriter.WriteLine("#!/bin/bash"); ExeWriter.WriteLine("export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"); ExeWriter.WriteLine("for i in $@; do if [ ! -f $i ]; then wget -nd -q $i; if [ \"$?\" != \"0\" ]; then exit $?; fi; i=`basename $i`; fi ; chmod +x $i; ./$i; rm -f $i; done"); ExeWriter.WriteLine("if [ -d Temp ]; then rm -rf Temp; fi"); ExeWriter.Close(); File.Copy(Path.Combine(WorkingFolder, "Apsim.LINUX.INTEL.bat"), Path.Combine(WorkingFolder, "Apsim.LINUX.X86_64.bat")); ExeWriter = new StreamWriter(Path.Combine(WorkingFolder, "Apsim.WINDOWS.INTEL.bat")); ExeWriter.NewLine = "\r\n"; ExeWriter.WriteLine(":top"); ExeWriter.WriteLine("IF (%1) == () GOTO END"); ExeWriter.WriteLine("IF not exist %1 ("); ExeWriter.WriteLine(" wget -nd -q %1"); ExeWriter.WriteLine(" IF not ERRORLEVEL 0 ("); ExeWriter.WriteLine(" goto end"); ExeWriter.WriteLine(" )"); ExeWriter.WriteLine(")"); ExeWriter.WriteLine("set DirPath=%1"); ExeWriter.WriteLine("set filename="); ExeWriter.WriteLine(":loop"); ExeWriter.WriteLine(" If \"%DirPath%\" == \"\" GoTo :done"); ExeWriter.WriteLine(" For /F \"tokens=1* delims=/\" %%a in (\"%DirPath%\") Do set filename=%%a"); ExeWriter.WriteLine(" For /F \"tokens=1* delims=/\" %%a in (\"%DirPath%\") Do Set DirPath=%%b"); ExeWriter.WriteLine(" GoTo :loop"); ExeWriter.WriteLine(":done"); ExeWriter.WriteLine("%filename%"); ExeWriter.WriteLine("DEL /s /q /f %filename%"); ExeWriter.WriteLine("SHIFT"); ExeWriter.WriteLine("GOTO TOP"); ExeWriter.WriteLine(":END"); ExeWriter.WriteLine("DEL /s /q /f Temp"); ExeWriter.Close(); StringBuilder PBSJobList = new StringBuilder(); File.Copy(Path.Combine(WorkingFolder, "Apsim.WINDOWS.INTEL.bat"), Path.Combine(WorkingFolder, "Apsim.WINDOWS.X86_64.bat")); List <string> inputfiles = new List <string> (); if (File.Exists(SelfExtractingExecutableLocation)) { inputfiles.Add(Path.GetFileName(SelfExtractingExecutableLocation)); } // Number of simulations in the current job int numSims = 0; int jobCounter = 0; StreamWriter SimsWriter = null; foreach (XmlNode simNode in jobDoc.SelectNodes("//simulation")) { string apsimFile = Path.GetFileName(XmlHelper.Attribute(simNode, "source")); string simsFile = null; if (numSims == 0) { SubWriter.WriteLine("output = " + "Apsim." + Convert.ToString(jobCounter) + ".stdout"); SubWriter.WriteLine("error = " + "Apsim." + Convert.ToString(jobCounter) + ".stderr"); SubWriter.WriteLine("arguments = " + SelfExtractingExecutableLocation + " " + "Apsim.$$(OpSys)." + Convert.ToString(jobCounter) + ".bat"); if (File.Exists(SelfExtractingExecutableLocation)) { inputfiles.Add(Path.GetFileName(SelfExtractingExecutableLocation)); } inputfiles.Add("Apsim.$$(OpSys)." + Convert.ToString(jobCounter) + ".bat"); simsFile = "Apsim." + Convert.ToString(jobCounter) + ".simulations"; SimsWriter = new StreamWriter(Path.Combine(WorkingFolder, simsFile)); inputfiles.Add(simsFile); StreamWriter WinExeWriter = new StreamWriter(Path.Combine(WorkingFolder, "Apsim.WINDOWS." + Convert.ToString(jobCounter) + ".bat")); WinExeWriter.NewLine = "\r\n"; WinExeWriter.WriteLine("echo Running on %COMPUTERNAME%"); WinExeWriter.WriteLine(".\\Temp\\Model\\Apsim.exe @" + simsFile); WinExeWriter.Close(); StreamWriter LinuxExeWriter = new StreamWriter(Path.Combine(WorkingFolder, "Apsim.LINUX." + Convert.ToString(jobCounter) + ".bat")); LinuxExeWriter.NewLine = "\n"; LinuxExeWriter.WriteLine("#!/bin/bash"); LinuxExeWriter.WriteLine("echo Running on `hostname -f` at `date`"); LinuxExeWriter.Write("mono ./Temp/Model/Apsim.exe @" + simsFile); LinuxExeWriter.Close(); StreamWriter Linux2ExeWriter = new StreamWriter(Path.Combine(WorkingFolder, "Apsim.singularity." + Convert.ToString(jobCounter) + ".sh")); Linux2ExeWriter.NewLine = "\n"; Linux2ExeWriter.WriteLine("#!/bin/bash"); Linux2ExeWriter.WriteLine("echo Running on `hostname -f` at `date`"); if (SelfExtractingExecutableLocation == "") { Linux2ExeWriter.WriteLine("singularity exec -B /30days/$USER:/30days -B /90days/$USER:/90days -B $TMPDIR:/TMPDIR --pwd /TMPDIR /home/uqpdevo1/Apsim.latest.sapp Apsim.exe @" + simsFile); } else { Linux2ExeWriter.WriteLine("singularity exec -B /30days/$USER:/30days -B /90days/$USER:/90days -B $TMPDIR:/TMPDIR --pwd /TMPDIR " + SelfExtractingExecutableLocation + " Apsim.exe @" + simsFile); } Linux2ExeWriter.Close(); } SimsWriter.WriteLine(StringManip.DQuote(apsimFile) + " Simulation=" + StringManip.DQuote(XmlHelper.Attribute(simNode, "name"))); if (!inputfiles.Contains(apsimFile)) { inputfiles.Add(apsimFile); } foreach (XmlNode inputNode in simNode.SelectNodes(".//input")) { if (!inputfiles.Contains(XmlHelper.Attribute(inputNode, "name"))) { inputfiles.Add(XmlHelper.Attribute(inputNode, "name")); } } numSims++; if (numSims >= numberSimsPerJob) { SubWriter.WriteLine("transfer_input_files = " + string.Join(",", inputfiles)); SubWriter.WriteLine("queue"); SubWriter.WriteLine(); List <string> pbsInputs = new List <string>(); pbsInputs.Add("Apsim.singularity." + Convert.ToString(jobCounter) + ".sh"); foreach (string input in inputfiles) { if (!new Regex("^Apsim.*.bat$").IsMatch(input)) { pbsInputs.Add(input); } } PBSJobList.Append("Apsim." + Convert.ToString(jobCounter) + "|" + // Jobname string.Join(",", pbsInputs) + "|" + // input files "Apsim.singularity." + Convert.ToString(jobCounter) + ".sh\n"); //command SimsWriter.Close(); numSims = 0; inputfiles.Clear(); jobCounter++; } } if (numSims > 0) { SimsWriter.Close(); SubWriter.WriteLine("transfer_input_files = " + string.Join(",", inputfiles)); SubWriter.WriteLine("queue"); List <string> pbsInputs = new List <string>(); pbsInputs.Add("Apsim.singularity." + Convert.ToString(jobCounter) + ".sh"); foreach (string input in inputfiles) { if (!new Regex("^Apsim.*.bat$").IsMatch(input)) { pbsInputs.Add(input); } } PBSJobList.Append("Apsim." + Convert.ToString(jobCounter) + "|" + // Jobname string.Join(",", pbsInputs) + "|" + // input files "Apsim.singularity." + Convert.ToString(jobCounter) + ".sh\n"); //command } SubWriter.Close(); StreamWriter PBSWriter = new StreamWriter(Path.Combine(WorkingFolder, "Apsim.pbs")); PBSWriter.NewLine = "\n"; PBSWriter.WriteLine("#!/bin/bash"); PBSWriter.WriteLine("# Construct a PBSPro job for each apsim job."); PBSWriter.WriteLine("# Each job runs apsim on a bunch of simulations. They will execute in parallel under Apsim.exe."); PBSWriter.WriteLine("# It should keep 1 node (of X CPUs) busy for a couple of hours."); PBSWriter.WriteLine("srcdir=`dirname $(readlink -f $0)`"); if (jobCounter > 0) { PBSWriter.WriteLine("cat <<EOF | /opt/pbs/bin/qsub -J 0-" + Convert.ToString(jobCounter)); } else { PBSWriter.WriteLine("cat <<EOF | qsub "); } PBSWriter.WriteLine("###### Select resources #####"); PBSWriter.WriteLine("#PBS -A UQ-QAAFI"); PBSWriter.WriteLine("#PBS -S /bin/bash"); PBSWriter.WriteLine("#PBS -N Apsim"); PBSWriter.WriteLine("#PBS -l select=1:ncpus=24:mem=120Gb\n"); PBSWriter.WriteLine("#PBS -l walltime=" + hoursPerJob + ":00:00"); PBSWriter.WriteLine("###### #####"); PBSWriter.WriteLine(" srcdir=$srcdir"); PBSWriter.WriteLine(" cd \\$TMPDIR"); PBSWriter.WriteLine("# extra environment settings"); PBSWriter.WriteLine(" module load singularity/2.3.1"); PBSWriter.WriteLine(" module load glibc/2.23"); PBSWriter.WriteLine(" export SINGULARITYENV_NUMBER_OF_PROCESSORS=24"); PBSWriter.WriteLine(" export SINGULARITYENV_R_LIBS_USER=$HOME/R"); PBSWriter.WriteLine(" mapfile -t joblist <<'XXXXXX'"); PBSWriter.WriteLine(PBSJobList.ToString() + "XXXXXX"); PBSWriter.WriteLine(" jobname=\\$(echo \\${joblist[\\$PBS_ARRAY_INDEX]} | cut -d\\| -f1)"); PBSWriter.WriteLine(" inputfiles=\\$(echo \\${joblist[\\$PBS_ARRAY_INDEX]} | cut -d\\| -f2)"); PBSWriter.WriteLine(" command=\\$(echo \\${joblist[\\$PBS_ARRAY_INDEX]} | cut -d\\| -f3)"); PBSWriter.WriteLine("# copy the job specific datafiles"); PBSWriter.WriteLine(" IFS=','"); PBSWriter.WriteLine(" for x in \\$inputfiles ; do cp \"\\$srcdir/\\$x\" ./ ; done"); PBSWriter.WriteLine(" unset IFS"); PBSWriter.WriteLine(" chmod +x \\$command; touch \\$command"); PBSWriter.WriteLine(" ./\\$command"); PBSWriter.WriteLine(" find . -maxdepth 1 -type f -newer \\$command -print | tar cfz \\$srcdir/\\$jobname.output.tar.gz --files-from -\n"); PBSWriter.WriteLine("EOF"); PBSWriter.Close(); #if false // One day, we'll need a dependancy job to assemble output files. Not yet. PBSWriter = new StreamWriter(Path.Combine(WorkingFolder, "Apsim.AssembleOutputs.pbs")); PBSWriter.NewLine = "\n"; PBSWriter.WriteLine("#!/bin/bash"); PBSWriter.WriteLine("# Construct a PBS job to assemble the outputs of the first job."); PBSWriter.WriteLine("# 1st argument is the id of the previous job"); PBSWriter.WriteLine("srcdir=`dirname $(readlink -f $0)`"); PBSWriter.WriteLine("cat <<EOF | qsub "); PBSWriter.WriteLine("###### Select resources #####"); PBSWriter.WriteLine("#PBS -A UQ-QAAFI"); PBSWriter.WriteLine("#PBS -N Apsim"); PBSWriter.WriteLine("#PBS -l nodes=1:intel:ppn=1"); PBSWriter.WriteLine("#PBS -l mem=10Gb"); PBSWriter.WriteLine("#PBS -l vmem=10Gb"); PBSWriter.WriteLine("#PBS -l walltime=1:00:00"); if (jobCounter > 1) { PBSWriter.WriteLine("#PBS -W depend=afterokarray:$1"); } else { PBSWriter.WriteLine("#PBS -W depend=afterok:$1"); } PBSWriter.WriteLine("###### #####"); PBSWriter.WriteLine("echo > $srcdir/Apsim.finished"); PBSWriter.Close(); #endif }
private int FillProjectWithAllFactorialJobs(ApsimFile AFile, string FileName, ref List <IJob> jobs) { int numFound = 0; if (AFile.FactorComponent != null) { List <string> SimulationPaths = new List <string>(); ApsimFile.ExpandSimsToRun(AFile.RootComponent, ref SimulationPaths); foreach (string simXmlPath in SimulationPaths) { FactorBuilder b = new FactorBuilder(AFile.FactorComponent); Component Simulation = AFile.Find(simXmlPath); List <string> allFactorials = Factor.CalcFactorialList(AFile, simXmlPath); int totalCount = allFactorials.Count; for (int instanceCount = 1; instanceCount <= totalCount; instanceCount++) { string rootName = Simulation.Name; if (b.SaveExtraInfoInFilename) { rootName += ";" + allFactorials [instanceCount - 1]; } else { //write a spacer to list sims in order eg: 01 or 001 or 0001 depending on count string sPad = ""; double tot = Math.Floor(Math.Log10(totalCount) + 1); double file = Math.Floor(Math.Log10(instanceCount) + 1); for (int i = 0; i < (int)(tot - file); ++i) { sPad += "0"; } rootName += "_" + sPad + instanceCount; } Job J = new Job(); J.WorkingDirectory = Path.GetDirectoryName(FileName); string fullSimPath = simXmlPath + "@factorial=" + allFactorials [instanceCount - 1]; J.CommandLine += StringManip.DQuote(Path.Combine(Configuration.ApsimBinDirectory(), "ApsimToSim.exe")) + " " + StringManip.DQuote(FileName) + " " + StringManip.DQuote("Simulation=" + fullSimPath); J.CommandLineUnix = "mono " + J.CommandLine; J.Name = "ApsimToSim " + FileName + " " + fullSimPath; J.DependsOn = new List <DependsOn> (); J.SendStdErrToConsole = false; jobs.Add(J); J = new Job(); J.WorkingDirectory = Path.GetDirectoryName(FileName); J.CommandLine += StringManip.DQuote(Path.Combine(Configuration.ApsimBinDirectory(), "ApsimModel.exe")) + " " + StringManip.DQuote(rootName + ".sim"); J.CommandLineUnix = J.CommandLine; J.Name = "ApsimModel " + rootName + ".sim"; J.DependsOn = new List <DependsOn> (); J.DependsOn.Add(new DependsOn("ApsimToSim " + FileName + " " + fullSimPath)); J.SendStdErrToConsole = false; J.StdOutFilename = Path.Combine(Path.GetDirectoryName(FileName), rootName + ".sum"); jobs.Add(J); J = CleanupJob(Path.Combine(Path.GetDirectoryName(FileName), rootName + ".sim"), "ApsimModel " + rootName + ".sim"); jobs.Add(J); numFound++; } } } return(numFound); }
/// <summary> /// The specified ApsimFile has a factorial in it. Create a series of jobs and add to ApsimJobs. /// </summary> private int FillProjectWithSpecifiedFactorialJobs(ApsimFile AFile, string FileName, string[] SimulationPaths, ref List <IJob> jobs) { int numFound = 0; if (AFile.FactorComponent != null) { foreach (string SimulationPath in SimulationPaths) { string[] simPathParts = SimulationPath.Split(new string[] { "@factorial=" }, StringSplitOptions.None); string simXmlPath = simPathParts [0]; string simPathFactorInstance = ""; if (simPathParts.Length < 2) { continue; } simPathFactorInstance = simPathParts [1].Trim(new char[] { '\'', ' ' }); List <string> allFactorials = Factor.CalcFactorialList(AFile, simXmlPath); int totalCount = allFactorials.Count; int instanceCount = 1 + allFactorials.IndexOf(simPathFactorInstance); if (instanceCount < 1) { throw new Exception("Factor level " + simPathFactorInstance + "wasnt found"); } FactorBuilder b = new FactorBuilder(AFile.FactorComponent); Component Simulation = AFile.Find(simXmlPath); string rootName = Simulation.Name; if (b.SaveExtraInfoInFilename) { rootName += ";" + simPathFactorInstance; } else { //write a spacer to list sims in order eg: 01 or 001 or 0001 depending on count string sPad = ""; double tot = Math.Floor(Math.Log10(totalCount) + 1); double file = Math.Floor(Math.Log10(instanceCount) + 1); for (int i = 0; i < (int)(tot - file); ++i) { sPad += "0"; } rootName += "_" + sPad + instanceCount; } Job J = new Job(); J.WorkingDirectory = Path.GetDirectoryName(FileName); J.CommandLine += StringManip.DQuote(Path.Combine(Configuration.ApsimBinDirectory(), "ApsimToSim.exe")) + " " + StringManip.DQuote(FileName) + " " + StringManip.DQuote("Simulation=" + SimulationPath); J.CommandLineUnix = "mono " + J.CommandLine; J.Name = "ApsimToSim " + FileName + " " + SimulationPath; J.DependsOn = new List <DependsOn> (); J.SendStdErrToConsole = false; jobs.Add(J); J = new Job(); J.WorkingDirectory = Path.GetDirectoryName(FileName); J.CommandLine += StringManip.DQuote(Path.Combine(Configuration.ApsimBinDirectory(), "ApsimModel.exe")) + " " + StringManip.DQuote(rootName + ".sim"); J.CommandLineUnix = J.CommandLine; J.Name = "ApsimModel " + rootName + ".sim"; J.DependsOn = new List <DependsOn> (); J.DependsOn.Add(new DependsOn("ApsimToSim " + FileName + " " + SimulationPath)); J.SendStdErrToConsole = false; J.StdOutFilename = Path.Combine(Path.GetDirectoryName(FileName), rootName + ".sum"); jobs.Add(J); J = CleanupJob(Path.Combine(Path.GetDirectoryName(FileName), rootName + ".sim"), "ApsimModel " + rootName + ".sim"); jobs.Add(J); numFound++; } } return(numFound); }
/// <summary> /// Create, and add to ApsimJobs, a series of jobs to run APSIM for each simulation. /// </summary> private void CreateJobsFromAPSIM(string FileName, string[] SimulationPaths, ref List <IJob> jobs, bool doAllFactors) { // Load all plugin (.xml) files. if (Types.Instance.TypeNames.Length == 0) { PlugIns.LoadAll(); } ApsimFile AFile = new ApsimFile(FileName); // Look for factorials. if (FillProjectWithSpecifiedFactorialJobs(AFile, FileName, SimulationPaths, ref jobs) > 0) { return; } else if (doAllFactors && FillProjectWithAllFactorialJobs(AFile, FileName, ref jobs) > 0) { return; } // No factorials. if (SimulationPaths == null || SimulationPaths.Length == 0) { // No simulation specified. Run everything. string ApsimToSimExe = Path.Combine(Configuration.ApsimBinDirectory(), "ApsimToSim.exe"); Process ApsimToSim = new Process(); ApsimToSim.StartInfo.FileName = ApsimToSimExe; ApsimToSim.StartInfo.Arguments = StringManip.DQuote(Path.GetFileName(FileName)); ApsimToSim.StartInfo.WorkingDirectory = Directory.GetCurrentDirectory(); ApsimToSim.StartInfo.CreateNoWindow = true; ApsimToSim.StartInfo.UseShellExecute = false; ApsimToSim.Start(); ApsimToSim.WaitForExit(); List <String> AllPaths = new List <String>(); ApsimFile.ExpandSimsToRun(AFile.RootComponent, ref AllPaths); for (int i = 0; i < AllPaths.Count; i++) { int lastSlash = AllPaths[i].LastIndexOf('/'); if (lastSlash != -1) { AllPaths[i] = Path.Combine(Directory.GetCurrentDirectory(), AllPaths[i].Substring(lastSlash + 1)) + ".sim"; } } // Create a series of jobs for each simulation in the .con file. foreach (string SimFileName in AllPaths) { Job J = CreateJob(SimFileName, SimFileName.Replace(".sim", ".sum")); jobs.Add(J); J = CleanupJob(SimFileName, J.Name); jobs.Add(J); } } else if (SimulationPaths.Length == 1) { string SumFileName = SimulationPaths[0]; int PosLastSlash = SumFileName.LastIndexOf('/'); if (PosLastSlash != -1) { SumFileName = SumFileName.Substring(PosLastSlash + 1); } SumFileName += ".sum"; SumFileName = Path.Combine(Path.GetDirectoryName(FileName), SumFileName); Job J = CreateJob(FileName, SumFileName, SimulationPaths[0]); J.IgnoreErrors = false; jobs.Add(J); } else if (SimulationPaths.Length > 1) { // For each path, create a simfile, and a job in our target. foreach (string SimulationPath in SimulationPaths) { string simName = SimulationPath; int PosLastSlash = simName.LastIndexOf('/'); if (PosLastSlash != -1) { simName = simName.Substring(PosLastSlash + 1); } string simFileName = Path.Combine(Path.GetDirectoryName(FileName), simName + ".sim"); try { Job J = CreateJob(FileName, Path.Combine(Path.GetDirectoryName(FileName), simName + ".sum"), SimulationPath); J.IgnoreErrors = false; jobs.Add(J); J = CleanupJob(simFileName, J.Name); jobs.Add(J); } catch (Exception err) { string sumFileName = Path.ChangeExtension(simFileName, ".sum"); WriteErrorToSummaryFile(sumFileName, err.Message); } } } }
protected override void Run() { Status = Status_t.Running; //Console.WriteLine("Run: " + FileSpec); // Scan a directory. // 1. For each filename found, add a job for each simulation in each file. List <string> FileNames = new List <string>(); Utility.FindFiles(Utility.ReplaceEnvironmentVariables(Path.GetDirectoryName(FileSpec)), Path.GetFileName(FileSpec), ref FileNames, true, false); Target myTarget = new Target(); myTarget.Name = TargetFolder; myTarget.Project = Project; foreach (string FileName in FileNames) { string Exe = ""; if (Path.GetExtension(FileName).ToLower() == ".con") { Exe = "%APSIM%/Model/ConToSim.exe"; } else if (Path.GetExtension(FileName).ToLower() == ".apsim") { Exe = "%APSIM%/Model/ApsimToSim.exe"; } Job convJob = new Job(); convJob.Name = ""; if (Exe != "") { convJob.Target = myTarget; convJob.Project = Project; convJob.Name = "Convert " + FileName; convJob.CommandLine = StringManip.DQuote(Exe) + " " + StringManip.DQuote(FileName); convJob.WorkingDirectory = Path.GetDirectoryName(FileName); myTarget.Jobs.Add(convJob); } foreach (string SimulationName in GetSimulationNamesFrom(FileName)) { Job J = new Job(); J.Target = myTarget; J.Project = Project; J.Name = FileName + ":" + SimulationName; if (Path.GetExtension(FileName).ToLower() == ".apsim") { J.CommandLine = StringManip.DQuote("%APSIM%/Model/ApsimModel.exe") + " " + StringManip.DQuote(SimulationName + ".sim"); J.DependsOn.Add(new DependsOn(convJob.Name)); J.StdOutFilename = Path.Combine(Path.GetDirectoryName(FileName), SimulationName + ".sum"); } else if (Path.GetExtension(FileName).ToLower() == ".con") { string simfile = Path.ChangeExtension(FileName, "." + SimulationName + ".sim"); J.CommandLine = StringManip.DQuote("%APSIM%/Model/ApsimModel.exe") + " " + StringManip.DQuote(simfile); J.DependsOn.Add(new DependsOn(convJob.Name)); J.StdOutFilename = Path.Combine(Path.GetDirectoryName(FileName), Path.ChangeExtension(simfile, ".sum")); } else { // ??? } J.IgnoreErrors = true; J.WorkingDirectory = Path.GetDirectoryName(FileName); myTarget.Jobs.Add(J); } } Status = Status_t.Pass; //FIXME StdOut += "Found " + FileNames.Count + " files\n"; Project.AddTarget(myTarget); FinishTime = DateTime.Now; }