/// <summary> /// Runs the solver described by the control object <paramref name="ctrl"/> on a batch system. /// The method returns immediately. /// </summary> /// <param name="ctrl"></param> /// <param name="BatchSys"></param> /// <returns></returns> public static Job RunBatch(this AppControl ctrl, BatchProcessorClient BatchSys) { ctrl.ProjectName = InteractiveShell.WorkflowMgm.CurrentProject; string JobName = ctrl.SessionName; int ctrl_idx = InteractiveShell.WorkflowMgm.RegisterControl(ctrl); if (JobName.IsEmptyOrWhite()) { JobName = "UnnamedJob_" + ctrl_idx; ctrl.SessionName = JobName; } Type solverClass = ctrl.GetSolverType(); Job job = new Job(JobName, solverClass); //job.ExecutionTime = executionTime; //job.NumberOfMPIProcs = NumberOfMPIProcs; //job.UseComputeNodesExclusive = UseComputeNodesExclusive; job.SetControlObject(ctrl); job.Activate(BatchSys); return(job); }
/// <summary> /// Adds an entry to <see cref="ExecutionQueues"/>. /// </summary> public static int AddExecutionQueue(BatchProcessorClient bpc) { if (executionQueues == null) { executionQueues = new List <BatchProcessorClient>(); } executionQueues.Add(bpc); return(executionQueues.Count - 1); }
/// <summary> /// Runs the solver described by the control object <paramref name="ctrl"/> on a batch system. /// The method returns immediately. /// </summary> /// <param name="ctrl"></param> /// <param name="BatchSys"></param> /// <returns></returns> public static Job RunBatch(this AppControl ctrl, BatchProcessorClient BatchSys) { ctrl.ProjectName = InteractiveShell.WorkflowMgm.CurrentProject; Type solverClass = ctrl.GetSolverType(); Job job = new Job(ctrl.SessionName, solverClass); //job.ExecutionTime = executionTime; //job.NumberOfMPIProcs = NumberOfMPIProcs; //job.UseComputeNodesExclusive = UseComputeNodesExclusive; job.SetControlObject(ctrl); job.Activate(BatchSys); return(job); }
/// <summary> /// Activates the job; can only be called once in this objects lifetime. /// </summary> /// <remarks> /// The job is submitted to the batch processor if no /// By using a unique name, see <see cref="Name"/>, the job becomes persistent: e.g., assume /// that this method is called within a worksheet. Later, the worksheet is restarted, or closed and reopened, /// thus this method is called a second time. By identifying the job in the batch processor with a unique name and /// by tagging sessions with the job name, it is ensured that the job is _not_ submitted a second time, /// resp. every time the worksheet is restarted. /// </remarks> public void Activate(BatchProcessorClient bpc) { // ============================================ // ensure that this method is only called once. // ============================================ if (this.AssignedBatchProc != null) { throw new NotSupportedException("Job can only be activated once."); } AssignedBatchProc = bpc; if (this.Name == "J2560_k2_exp_softpcg_mg") { Console.WriteLine(); } // ================ // check job status // ================ string DeployDir; int SubmitCount; var status = GetStatus(out SubmitCount, out DeployDir); this.DeploymentDirectory = DeployDir; // ============== // check Database // ============== var RR = this.AllSessions; // ================= // decide what to do // ================= switch (status) { case JobStatus.PreActivation: DeployDir = null; this.DeploymentDirectory = null; Console.WriteLine("Job not submitted yet, or no result session is known - starting submission."); break; case JobStatus.Failed: if (RR.Length <= 0) { DeployDir = null; this.DeploymentDirectory = null; Console.WriteLine("Job is marked as failed by job manager, no database entry is found; performing new deployment and submission."); break; } if (RetryCount <= SubmitCount) { Console.WriteLine("Job is failed {0} times, maximum number of tries reached, no further action.", SubmitCount); return; } else { DeployDir = null; this.DeploymentDirectory = null; Console.WriteLine("Job is failed, retrying (Submitted {0} times so far); performing new deployment and submission.", SubmitCount); break; } case JobStatus.PendingInExecutionQueue: Console.WriteLine("Job submitted, waiting for launch - no further action."); return; case JobStatus.FinishedSuccessful: if (RR.Length <= 0) { DeployDir = null; this.DeploymentDirectory = null; Console.WriteLine("Job is marked as success by job manager, but no session info in database is found; performing new deployment and submission."); break; } ISessionInfo LatestSession = RR.OrderBy(sinf => sinf.CreationTime).Last(); Console.WriteLine("Job was successful (according to job manager), latest session related to job is:"); Console.WriteLine(LatestSession.ToString()); Console.WriteLine("No further action."); return; case JobStatus.InProgress: Console.Write("Job has been started (according to job manager), "); if (RR.Length > 0) { Console.WriteLine("latest known session is:"); ISessionInfo LatestSession2 = RR.OrderBy(sinf => sinf.CreationTime).Last(); Console.WriteLine(LatestSession2.ToString()); } else { Console.WriteLine("no session information available at this point."); } Console.WriteLine("No further action."); return; default: throw new NotImplementedException(); } // ======================================================================== // finally, it might be necessary to submit the job to the batch processor. // ======================================================================== // deploy executables: bool RequiresDeploy = false; if (DeployDir == null) { if (!Directory.Exists(this.DeploymentDirectory) || !File.Exists(Path.Combine(DeployDir, Path.GetFileName(this.EntryAssembly.Location)))) { RequiresDeploy = true; } } else { RequiresDeploy = false; } if (RequiresDeploy) { this.DeploymentDirectory = bpc.GetNewDeploymentDir(this); bpc.DeployExecuteables(this, AdditionalDeploymentFiles); } // submit job this.BatchProcessorIdentifierToken = bpc.Submit(this); }
private void FiddleControlFile(BatchProcessorClient bpc) { if (m_ctrl == null) { return; } // check the database // ================== IDatabaseInfo ctrl_db = m_ctrl.GetDatabase(); if (bpc.AllowedDatabases != null && bpc.AllowedDatabases.Count > 0) { IDatabaseInfo newDb = null; if (ctrl_db == null) { newDb = bpc.AllowedDatabases[0]; } else { bool ok = false; foreach (var allow_dba in bpc.AllowedDatabases) { if (allow_dba.Equals(ctrl_db)) { ok = true; break; } } if (!ok) { newDb = bpc.AllowedDatabases[0]; } } if (newDb != null) { Console.WriteLine("Resetting database for control object to " + newDb.ToString()); //newDb.AlternateDbPaths m_ctrl.SetDatabase(newDb); ctrl_db = newDb; } Console.WriteLine("Submitting job with the following database info: "); Console.WriteLine("Primary: " + m_ctrl.DbPath); if (ctrl_db.AlternateDbPaths != null && ctrl_db.AlternateDbPaths.Length > 0) { int cnt = 0; foreach (var t in ctrl_db.AlternateDbPaths) { Console.WriteLine($" Alternative[{cnt}]: {t.DbPath}, MachineFilter: '{t.MachineFilter}'"); cnt++; } } else { Console.WriteLine("No alternative paths specified."); } } else { Console.WriteLine(""); } // check grid & restart info // ========================= if (ctrl_db != null) { if (!m_ctrl.GridGuid.Equals(Guid.Empty)) { var GridIn_ctrl_db = ctrl_db.Grids.FirstOrDefault(GrdInf => GrdInf.ID.Equals(m_ctrl.GridGuid)); if (GridIn_ctrl_db == null) { Console.WriteLine($"Grid {m_ctrl.GridGuid} is not present in database - copy to target system..."); var grid2copy = InteractiveShell.AllGrids.FirstOrDefault(dbGrid => dbGrid.ID.Equals(m_ctrl.GridGuid)); if (grid2copy == null) { // maybe replace exception with a warning, if job should be tried anyway throw new IOException($"Unable to find grid '{m_ctrl.GridGuid}' in any database - job will most likely crash."); } else { grid2copy.Copy(ctrl_db); } Console.WriteLine("done."); } } else { Console.Error.WriteLine($"Warning: no grid seems to be specified for the job to submit."); } if (m_ctrl.RestartInfo != null) { Guid Rstsess_guid = m_ctrl.RestartInfo.Item1; var Rstsess_ctrl_db = ctrl_db.Sessions.FirstOrDefault(sinf => sinf.ID.Equals(Rstsess_guid)); if (Rstsess_ctrl_db == null) { Console.WriteLine($"Session {m_ctrl.GridGuid} to restart from is not present in database - copy to target system..."); var sess_2copy = InteractiveShell.AllSessions.FirstOrDefault(sinf => sinf.ID.Equals(Rstsess_guid)); if (sess_2copy == null) { // maybe replace exception with a warning, if job should be tried anyway throw new IOException($"Unable to find session '{sess_2copy}' in any database - job will most likely crash."); } else { sess_2copy.Copy(ctrl_db); } Console.WriteLine("done."); } } } else { Console.Error.WriteLine($"Warning: no database is set for the job to submit; nothing ma be saved."); } // finally, serialize the object // ============================= { string text; m_ctrl_index = -1; if (m_ctrl.GeneratedFromCode) { text = m_ctrl.ControlFileText; ControlName = "control.cs"; m_ctrl_index = m_ctrl.ControlFileText_Index; } else { text = m_ctrl.Serialize(); ControlName = "control.obj"; } byte[] buffer = Encoding.UTF8.GetBytes(text); int remIdx = AdditionalDeploymentFiles.IndexWhere(tt => tt.Item2 == ControlName); if (remIdx >= 0) { AdditionalDeploymentFiles.RemoveAt(remIdx); } AdditionalDeploymentFiles.Add(new Tuple <byte[], string>(buffer, ControlName)); } }