/// <summary>
/// Runs the solver described by the control object <paramref name="ctrl"/> on a batch system.
/// The method returns immediately.
/// </summary>
/// <param name="ctrl"></param>
/// <param name="BatchSys"></param>
/// <returns></returns>
public static Job RunBatch(this AppControl ctrl, BatchProcessorClient BatchSys)
{
ctrl.ProjectName = InteractiveShell.WorkflowMgm.CurrentProject;
string JobName = ctrl.SessionName;
int ctrl_idx = InteractiveShell.WorkflowMgm.RegisterControl(ctrl);
if (JobName.IsEmptyOrWhite())
{
JobName = "UnnamedJob_" + ctrl_idx;
ctrl.SessionName = JobName;
}
Type solverClass = ctrl.GetSolverType();
Job job = new Job(JobName, solverClass);
//job.ExecutionTime = executionTime;
//job.NumberOfMPIProcs = NumberOfMPIProcs;
//job.UseComputeNodesExclusive = UseComputeNodesExclusive;
job.SetControlObject(ctrl);
job.Activate(BatchSys);
return(job);
}
/// <summary>
/// Adds an entry to <see cref="ExecutionQueues"/>.
/// </summary>
public static int AddExecutionQueue(BatchProcessorClient bpc)
{
if (executionQueues == null)
{
executionQueues = new List <BatchProcessorClient>();
}
executionQueues.Add(bpc);
return(executionQueues.Count - 1);
}
/// <summary>
/// Runs the solver described by the control object <paramref name="ctrl"/> on a batch system.
/// The method returns immediately.
/// </summary>
/// <param name="ctrl"></param>
/// <param name="BatchSys"></param>
/// <returns></returns>
public static Job RunBatch(this AppControl ctrl, BatchProcessorClient BatchSys)
{
ctrl.ProjectName = InteractiveShell.WorkflowMgm.CurrentProject;
Type solverClass = ctrl.GetSolverType();
Job job = new Job(ctrl.SessionName, solverClass);
//job.ExecutionTime = executionTime;
//job.NumberOfMPIProcs = NumberOfMPIProcs;
//job.UseComputeNodesExclusive = UseComputeNodesExclusive;
job.SetControlObject(ctrl);
job.Activate(BatchSys);
return(job);
}
/// <summary>
/// Activates the job; can only be called once in this objects lifetime.
/// </summary>
/// <remarks>
/// The job is submitted to the batch processor if no
/// By using a unique name, see <see cref="Name"/>, the job becomes persistent: e.g., assume
/// that this method is called within a worksheet. Later, the worksheet is restarted, or closed and reopened,
/// thus this method is called a second time. By identifying the job in the batch processor with a unique name and
/// by tagging sessions with the job name, it is ensured that the job is _not_ submitted a second time,
/// resp. every time the worksheet is restarted.
/// </remarks>
public void Activate(BatchProcessorClient bpc)
{
// ============================================
// ensure that this method is only called once.
// ============================================
if (this.AssignedBatchProc != null)
{
throw new NotSupportedException("Job can only be activated once.");
}
AssignedBatchProc = bpc;
if (this.Name == "J2560_k2_exp_softpcg_mg")
{
Console.WriteLine();
}
// ================
// check job status
// ================
string DeployDir;
int SubmitCount;
var status = GetStatus(out SubmitCount, out DeployDir);
this.DeploymentDirectory = DeployDir;
// ==============
// check Database
// ==============
var RR = this.AllSessions;
// =================
// decide what to do
// =================
switch (status)
{
case JobStatus.PreActivation:
DeployDir = null;
this.DeploymentDirectory = null;
Console.WriteLine("Job not submitted yet, or no result session is known - starting submission.");
break;
case JobStatus.Failed:
if (RR.Length <= 0)
{
DeployDir = null;
this.DeploymentDirectory = null;
Console.WriteLine("Job is marked as failed by job manager, no database entry is found; performing new deployment and submission.");
break;
}
if (RetryCount <= SubmitCount)
{
Console.WriteLine("Job is failed {0} times, maximum number of tries reached, no further action.", SubmitCount);
return;
}
else
{
DeployDir = null;
this.DeploymentDirectory = null;
Console.WriteLine("Job is failed, retrying (Submitted {0} times so far); performing new deployment and submission.", SubmitCount);
break;
}
case JobStatus.PendingInExecutionQueue:
Console.WriteLine("Job submitted, waiting for launch - no further action.");
return;
case JobStatus.FinishedSuccessful:
if (RR.Length <= 0)
{
DeployDir = null;
this.DeploymentDirectory = null;
Console.WriteLine("Job is marked as success by job manager, but no session info in database is found; performing new deployment and submission.");
break;
}
ISessionInfo LatestSession = RR.OrderBy(sinf => sinf.CreationTime).Last();
Console.WriteLine("Job was successful (according to job manager), latest session related to job is:");
Console.WriteLine(LatestSession.ToString());
Console.WriteLine("No further action.");
return;
case JobStatus.InProgress:
Console.Write("Job has been started (according to job manager), ");
if (RR.Length > 0)
{
Console.WriteLine("latest known session is:");
ISessionInfo LatestSession2 = RR.OrderBy(sinf => sinf.CreationTime).Last();
Console.WriteLine(LatestSession2.ToString());
}
else
{
Console.WriteLine("no session information available at this point.");
}
Console.WriteLine("No further action.");
return;
default:
throw new NotImplementedException();
}
// ========================================================================
// finally, it might be necessary to submit the job to the batch processor.
// ========================================================================
// deploy executables:
bool RequiresDeploy = false;
if (DeployDir == null)
{
if (!Directory.Exists(this.DeploymentDirectory) ||
!File.Exists(Path.Combine(DeployDir, Path.GetFileName(this.EntryAssembly.Location))))
{
RequiresDeploy = true;
}
}
else
{
RequiresDeploy = false;
}
if (RequiresDeploy)
{
this.DeploymentDirectory = bpc.GetNewDeploymentDir(this);
bpc.DeployExecuteables(this, AdditionalDeploymentFiles);
}
// submit job
this.BatchProcessorIdentifierToken = bpc.Submit(this);
}
private void FiddleControlFile(BatchProcessorClient bpc)
{
if (m_ctrl == null)
{
return;
}
// check the database
// ==================
IDatabaseInfo ctrl_db = m_ctrl.GetDatabase();
if (bpc.AllowedDatabases != null && bpc.AllowedDatabases.Count > 0)
{
IDatabaseInfo newDb = null;
if (ctrl_db == null)
{
newDb = bpc.AllowedDatabases[0];
}
else
{
bool ok = false;
foreach (var allow_dba in bpc.AllowedDatabases)
{
if (allow_dba.Equals(ctrl_db))
{
ok = true;
break;
}
}
if (!ok)
{
newDb = bpc.AllowedDatabases[0];
}
}
if (newDb != null)
{
Console.WriteLine("Resetting database for control object to " + newDb.ToString());
//newDb.AlternateDbPaths
m_ctrl.SetDatabase(newDb);
ctrl_db = newDb;
}
Console.WriteLine("Submitting job with the following database info: ");
Console.WriteLine("Primary: " + m_ctrl.DbPath);
if (ctrl_db.AlternateDbPaths != null && ctrl_db.AlternateDbPaths.Length > 0)
{
int cnt = 0;
foreach (var t in ctrl_db.AlternateDbPaths)
{
Console.WriteLine($" Alternative[{cnt}]: {t.DbPath}, MachineFilter: '{t.MachineFilter}'");
cnt++;
}
}
else
{
Console.WriteLine("No alternative paths specified.");
}
}
else
{
Console.WriteLine("");
}
// check grid & restart info
// =========================
if (ctrl_db != null)
{
if (!m_ctrl.GridGuid.Equals(Guid.Empty))
{
var GridIn_ctrl_db = ctrl_db.Grids.FirstOrDefault(GrdInf => GrdInf.ID.Equals(m_ctrl.GridGuid));
if (GridIn_ctrl_db == null)
{
Console.WriteLine($"Grid {m_ctrl.GridGuid} is not present in database - copy to target system...");
var grid2copy = InteractiveShell.AllGrids.FirstOrDefault(dbGrid => dbGrid.ID.Equals(m_ctrl.GridGuid));
if (grid2copy == null)
{
// maybe replace exception with a warning, if job should be tried anyway
throw new IOException($"Unable to find grid '{m_ctrl.GridGuid}' in any database - job will most likely crash.");
}
else
{
grid2copy.Copy(ctrl_db);
}
Console.WriteLine("done.");
}
}
else
{
Console.Error.WriteLine($"Warning: no grid seems to be specified for the job to submit.");
}
if (m_ctrl.RestartInfo != null)
{
Guid Rstsess_guid = m_ctrl.RestartInfo.Item1;
var Rstsess_ctrl_db = ctrl_db.Sessions.FirstOrDefault(sinf => sinf.ID.Equals(Rstsess_guid));
if (Rstsess_ctrl_db == null)
{
Console.WriteLine($"Session {m_ctrl.GridGuid} to restart from is not present in database - copy to target system...");
var sess_2copy = InteractiveShell.AllSessions.FirstOrDefault(sinf => sinf.ID.Equals(Rstsess_guid));
if (sess_2copy == null)
{
// maybe replace exception with a warning, if job should be tried anyway
throw new IOException($"Unable to find session '{sess_2copy}' in any database - job will most likely crash.");
}
else
{
sess_2copy.Copy(ctrl_db);
}
Console.WriteLine("done.");
}
}
}
else
{
Console.Error.WriteLine($"Warning: no database is set for the job to submit; nothing ma be saved.");
}
// finally, serialize the object
// =============================
{
string text;
m_ctrl_index = -1;
if (m_ctrl.GeneratedFromCode)
{
text = m_ctrl.ControlFileText;
ControlName = "control.cs";
m_ctrl_index = m_ctrl.ControlFileText_Index;
}
else
{
text = m_ctrl.Serialize();
ControlName = "control.obj";
}
byte[] buffer = Encoding.UTF8.GetBytes(text);
int remIdx = AdditionalDeploymentFiles.IndexWhere(tt => tt.Item2 == ControlName);
if (remIdx >= 0)
{
AdditionalDeploymentFiles.RemoveAt(remIdx);
}
AdditionalDeploymentFiles.Add(new Tuple <byte[], string>(buffer, ControlName));
}
}
