/// <summary>
/// a parametric sweep is a parallel computing job that consists of running multiple iterations of same command using different input values and output files
/// each occurrence of an asterisk found on the command line is replaced with the value
/// </summary>
/// <param name="jobName">the name of job</param>
/// <param name="commandLine">a parametric command for the task</param>
/// <param name="startValue">the starting instance value. the value must be > 0, inclusive</param>
/// <param name="endValue">the ending value. >= startValue. inclusive</param>
/// <param name="incrementValue">the increment value</param>
/// <param name="username">the name of the runas user, in the form domain\username</param>
/// <param name="password">the password for the runas user</param>
public void CreateParametricJob(string jobName, string commandLine, int startValue, int endValue, int incrementValue, string username, string password)
{
ISchedulerJob job = null;
ISchedulerTask task = null;
manualReset.Reset();
//create a job
job = _scheduler.CreateJob();
job.OnJobState += OnJobStateCallback;
job.OnTaskState += OnTaskStateCallback;
job.Name = jobName;
task = job.CreateTask();
//each occurrence of an asterisk found on the command line is replaced with the value
task.CommandLine = commandLine;
task.Type = TaskType.ParametricSweep;
task.StartValue = startValue;
task.EndValue = endValue;
task.IncrementValue = incrementValue;
job.AddTask(task);
_scheduler.SubmitJob(job, username, password);
manualReset.WaitOne();
}
/// <summary>
/// Submit the job to the Microsoft HPC server.
/// </summary>
public override object Submit(Job myJob)
{
string PrjName = InteractiveShell.WorkflowMgm.CurrentProject;
ISchedulerJob job = null;
ISchedulerTask task = null;
// Create a job and add a task to the job.
job = m_scheduler.CreateJob();
job.Name = myJob.Name;
job.Project = PrjName;
job.MaximumNumberOfCores = myJob.NumberOfMPIProcs;
job.MinimumNumberOfCores = myJob.NumberOfMPIProcs;
job.UserName = m_Username;
task = job.CreateTask();
task.MaximumNumberOfCores = myJob.NumberOfMPIProcs;
task.MinimumNumberOfCores = myJob.NumberOfMPIProcs;
task.WorkDirectory = myJob.DeploymentDirectory;
using (var str = new StringWriter()) {
str.Write("mpiexec ");
str.Write(Path.GetFileName(myJob.EntryAssembly.Location));
foreach (string arg in myJob.CommandLineArguments)
{
str.Write(" ");
str.Write(arg);
}
task.CommandLine = str.ToString();
}
foreach (var kv in myJob.EnvironmentVars)
{
string name = kv.Key;
string valu = kv.Value;
task.SetEnvironmentVariable(name, valu);
}
task.StdOutFilePath = Path.Combine(myJob.DeploymentDirectory, "stdout.txt");
task.StdErrFilePath = Path.Combine(myJob.DeploymentDirectory, "stderr.txt");
if (m_ComputeNodes != null)
{
foreach (string node in m_ComputeNodes)
{
job.RequestedNodes.Add(node);
}
}
job.AddTask(task);
// Start the job.
m_scheduler.SubmitJob(job, m_Password != null ? m_Username : null, m_Password);
return(job.Id);
}
static void Main(string[] args)
{
Console.Write("hpc: ");
string serverName = Console.ReadLine();
Console.Write("user: "******"job template (<Enter> - 'default'): ");
string jobTemplate = Console.ReadLine();
if (string.IsNullOrWhiteSpace(jobTemplate))
{
jobTemplate = "Default";
}
IScheduler scheduler = new Scheduler();
scheduler.Connect(serverName);
ISchedulerJob job = scheduler.CreateJob();
job.SetJobTemplate(jobTemplate);
ISchedulerTask task = job.CreateTask();
task.CommandLine = "dir";
job.AddTask(task);
// Specify the events that you want to receive.
job.OnJobState += JobStateCallback;
job.OnTaskState += TaskStateCallback;
// Start the job.
scheduler.SubmitJob(job, user, null);
manualEvent.WaitOne();
}
/// <summary>
/// The job template that you specify for the job defines the initial default values and constraints for many of the job properties.
/// Any changes that you make to properties must be made within the constraints of the template.
/// </summary>
/// <param name="jobName">the name of job</param>
/// <param name="commandLine">the command line for the task</param>
/// <param name="username">the name of the runas user, in the form domain\username</param>
/// <param name="password">the password for the runas user</param>
public void CreateJob(string jobName, string commandLine, string username, string password)
{
ISchedulerJob job = null;
ISchedulerTask task = null;
manualReset.Reset();
//create job
job = _scheduler.CreateJob();
job.Name = jobName;
//create task
task = job.CreateTask();
task.CommandLine = commandLine;
//add task to job
job.AddTask(task);
//specify the events that you want to receive
job.OnJobState += OnJobStateCallback;
job.OnTaskState += OnTaskStateCallback;
//start the job
_scheduler.SubmitJob(job, username, password);
//block so the events get delivered
manualReset.WaitOne();
}
private static bool submit_job(IScheduler scheduler, string taskName, string workDir, List <string> requestedNodes,
string userName, string password, bool waitFlag, string taskSuffix = "")
{
try
{
ISchedulerJob job = null;
ISchedulerTask task = null;
Console.WriteLine(" working dir: " + workDir);
Console.WriteLine(" task: " + taskName);
foreach (string node in requestedNodes)
{
job = scheduler.CreateJob();
job.UnitType = JobUnitType.Node;
job.IsExclusive = true;
StringCollection nodes = new StringCollection()
{
node
};
Console.WriteLine(" node: " + node);
job.RequestedNodes = nodes;
job.Name = node + taskSuffix;
task = job.CreateTask();
task.CommandLine = taskName;
task.WorkDirectory = workDir;
task.StdOutFilePath = workDir + "\\output.txt";
task.StdErrFilePath = workDir + "\\error.txt";
task.Name = node;
job.AddTask(task);
scheduler.SubmitJob(job, userName, password);
}
//if (waitFlag)
//{
// Console.WriteLine("Waiting for job to finish...");
// while (job.State != JobState.Finished)
// {
// System.Threading.Thread.Sleep(5000);
// }
//}
if (waitFlag)
{
Console.WriteLine("hit any key to continue");
Console.ReadKey();
}
//manualEvent.WaitOne();
return(true);
}
catch (Exception e)
{
Console.WriteLine("unable to start job successfully: " + taskName);
Console.WriteLine(e);
return(false);
}
}
public void ExecuteWait(IEnumerable <string> tasksCmdLines,
int jobMinimumNumberOfCores = 1, int jobMaximumNumberOfCores = 1,
int taskMinimumNumberOfCores = 1, int taskMaximumNumberOfCores = 1,
string outFolder = "")
{
ISchedulerJob job = null;
ISchedulerTask task = null;
using (IScheduler scheduler = new Scheduler( ))
{
scheduler.Connect(headNodeName);
job = scheduler.CreateJob( );
int i = 0;
foreach (var taskDef in tasksCmdLines)
{
i++;
task = job.CreateTask( );
task.CommandLine = taskDef;
task.MinimumNumberOfCores = taskMinimumNumberOfCores;
task.MaximumNumberOfCores = taskMaximumNumberOfCores;
task.StdOutFilePath = createOutFileName(outFolder, "stdout", i);
task.StdErrFilePath = createOutFileName(outFolder, "stderr", i);
job.AddTask(task);
}
try
{
job.AutoCalculateMin = false;
job.AutoCalculateMax = false;
job.MinimumNumberOfCores = jobMinimumNumberOfCores;
job.MaximumNumberOfCores = jobMaximumNumberOfCores;
job.OnJobState += new EventHandler <JobStateEventArg>(jobStateCallback);
job.OnTaskState += new EventHandler <TaskStateEventArg>(taskStateCallback);
// Start the job.
scheduler.SubmitJob(job, string.Empty, null);
// Blocks so the events get delivered. One of your event
// handlers need to set this event.
manualEvent.WaitOne( );
}
finally
{
job.OnJobState -= jobStateCallback;
job.OnTaskState -= taskStateCallback;
}
}
}
static void Main(string[] args)
{
IScheduler scheduler = new Scheduler();
string clustername = null;
string username = null;
if (args.Length != 2)
{
Console.Error.WriteLine("Usage: Finish clustername username ");
return;
}
clustername = args[0];
username = args[1];
scheduler.Connect(clustername);
ISchedulerJob job = scheduler.CreateJob();
job.UnitType = JobUnitType.Core;
job.MinimumNumberOfCores = 1;
job.MaximumNumberOfCores = 1;
scheduler.AddJob(job);
ISchedulerTask task = job.CreateTask();
task.CommandLine = @"ping -t localhost";
job.AddTask(task);
scheduler.SubmitJob(job, username, null);
Console.WriteLine("job {0} Submitted ", job.Id);
Thread.Sleep(12 * 1000);
job.Refresh();
Console.WriteLine("Job {0} State {1}", job.Id, job.State);
((ISchedulerJobV3)job).Finish();
Thread.Sleep(10000);
job.Refresh();
task.Refresh();
Console.WriteLine("After finish Job {0} State {1} message {2}", job.Id, job.State, task.Output);
}
private static bool submit_job(IScheduler scheduler, string taskName, string workDir, List <string> requestedNodes,
string userName, string password, bool waitFlag)
{
try
{
ISchedulerJob job = null;
ISchedulerTask task = null;
Console.WriteLine(" working dir: " + workDir);
Console.WriteLine(" task: " + taskName);
foreach (string node in requestedNodes)
{
job = scheduler.CreateJob();
job.UnitType = JobUnitType.Node;
job.IsExclusive = true;
StringCollection nodes = new StringCollection()
{
node
};
Console.WriteLine(" node: " + node);
job.RequestedNodes = nodes;
job.Name = node;
task = job.CreateTask();
task.CommandLine = taskName;
task.WorkDirectory = workDir;
task.StdOutFilePath = workDir + "\\output.txt";
task.StdErrFilePath = workDir + "\\error.txt";
task.Name = node;
job.AddTask(task);
scheduler.SubmitJob(job, userName, password);
}
//manualEvent.WaitOne();
return(true);
}
catch (Exception e)
{
Console.WriteLine("unable to start job successfully: " + taskName);
Console.WriteLine(e);
return(false);
}
}
static bool submitJobs()
{
//wait for a maximum of one minute for scheduler connect before exiting
if (connected.WaitOne(1 * 1000)) //timesout in one second
{
//create a job equivalent to "job submit echo Hello World"
ISchedulerJob job = scheduler.CreateJob();
ISchedulerTask task = job.CreateTask();
task.CommandLine = "echo Hello World";
job.AddTask(task);
scheduler.SubmitJob(job, null, null);
job.Refresh();
Console.WriteLine("Job {0} was submitted", job.Id);
Thread.Sleep(2 * 1000); //pause for 2 seconds
return(true);
}
return(false);
}
private static ISchedulerTask CreateCleanupTask(ISchedulerJob job, string internalRemoteDirectoryName, string stdErrDirName, string stdOutDirName, string name, bool isFinalCleanup)
{
ISchedulerTask cleanupTask = job.CreateTask();
cleanupTask.WorkDirectory = internalRemoteDirectoryName;
cleanupTask.Name = name;
cleanupTask.IsExclusive = false;
cleanupTask.StdErrFilePath = string.Format(@"{0}\{1}.txt", stdErrDirName, name);
cleanupTask.StdOutFilePath = string.Format(@"{0}\{1}.txt", stdOutDirName, name);
if (isFinalCleanup)
{
}
else
{
cleanupTask.CommandLine = "ECHO The cleanup task is running.";
}
return(cleanupTask);
}
static void Main(string[] args)
{
if (args.Length != 2)
{
Console.WriteLine("Usage: WaitForJob <headnode> <test executable>");
Console.WriteLine("e.g: WaitForJob myscheduler \"sleep 60\"");
Environment.ExitCode = 1;
return;
}
string headNode = args[0];
IScheduler scheduler = new Scheduler();
scheduler.Connect(headNode);
ISchedulerJob job = scheduler.CreateJob();
ISchedulerTask task = job.CreateTask();
task.CommandLine = args[1];
job.AddTask(task);
scheduler.SubmitJob(job, null, null);
WaitForJob(scheduler, job);
}
public static bool CreateJob(int endValue)
{
string headnode = ConfigurationManager.AppSettings["HeadNodeName"];
string targetNodes = ConfigurationManager.AppSettings["NodeGroup"];
bool retVal = false;
if (!string.IsNullOrEmpty(headnode))
{
try
{
Scheduler scheduler = new Scheduler();
scheduler.Connect(headnode);
// Define job settings
ISchedulerJob job = scheduler.CreateJob();
job.Name = "Aqsis on Azure";
job.MinimumNumberOfCores = 1;
job.MaximumNumberOfCores = 1;
job.UnitType = JobUnitType.Core;
// Let the scheduler calculate the required resources for the job
job.AutoCalculateMax = true;
job.NodeGroups.Add(targetNodes);
// Create a parametric sweep task
ISchedulerTask task = job.CreateTask();
task.Type = TaskType.ParametricSweep;
task.StartValue = 0;
task.EndValue = endValue;
task.IncrementValue = 1;
// Run the aqsis command to render the images
// The (*) wildcard is used as a placeholder for the current index value
task.CommandLine = @"%CCP_PACKAGE_ROOT%\Aqsis\bin\run.cmd frame-*";
task.WorkDirectory = "%CCP_PACKAGE_ROOT%";
Console.WriteLine("Running job");
job.AddTask(task);
scheduler.SubmitJob(job, username: null, password: null);
job.Refresh();
while (job.State != JobState.Finished &&
job.State != JobState.Canceled &&
job.State != JobState.Failed)
{
// Wait for the job to complete
Thread.Sleep(5000);
job.Refresh();
}
switch (job.State)
{
case JobState.Canceled:
Console.WriteLine("Job canceled");
break;
case JobState.Finished:
Console.WriteLine("Job finished");
retVal = true;
break;
case JobState.Failed:
Console.WriteLine("Job failed");
break;
}
}
catch (Exception ex)
{
Utility.Logger("CreateJob Failed. Exception Message: " + ex.Message);
}
}
return(retVal);
}
static void Main(string[] args)
{
string clusterName = Environment.GetEnvironmentVariable("CCP_SCHEDULER");
//create a scheduler object used to connect to the scheduler
using (IScheduler scheduler = new Scheduler())
{
//connect to the scheduler
Console.WriteLine("Connecting to cluster {0}", clusterName);
scheduler.Connect(clusterName);
//create a job equilvalent to the cmdline string: job submit /parametric:1-500 "echo *"
Console.WriteLine("Creating parametric sweep job");
//first create a SchedulerJob object
ISchedulerJob job = scheduler.CreateJob();
//and a task object
ISchedulerTask task = job.CreateTask();
//set the command line to "echo *"
task.CommandLine = "echo *";
//and we set the parametric task settings
task.Type = TaskType.ParametricSweep;
task.StartValue = 1;
task.IncrementValue = 1;
task.EndValue = 500;
//add the task to the job
job.AddTask(task);
//Create an event handler so that we know when the job starts running
job.OnJobState += new EventHandler <JobStateEventArg>(job_OnJobState);
//and submit
//you will be prompted for your credentials if they aren't already cached
Console.WriteLine("Submitting job...");
scheduler.SubmitJob(job, null, null);
Console.WriteLine("Job submitted");
//Wait for the job to start running
jobStatus.WaitOne();
jobStatus.Reset();
//you can get realtime updates on the job through the api
//we'll keep checking every second for 5 seconds
for (int i = 0; i < 5; i++)
{
//refresh the job object with updates from the cluster
job.Refresh();
Console.Write("Current job progress: " + job.Progress);
Console.SetCursorPosition(0, Console.CursorTop);
//we want to check again after a second
Thread.Sleep(1 * 1000);
}
//this field isn't read-only. You can specify your own progress value depending on your needs
Console.WriteLine();
Console.WriteLine("Manually changing job progress");
job.Progress = 0;
//commit the changes to the server
job.Commit();
Console.WriteLine("Current job progress: " + job.Progress);
//you can also set progress messages, which will also be viewable in the Job Management UI
Console.WriteLine("Setting job progress message");
job.ProgressMessage = "Job is still running";
//commit the changes to the server
job.Commit();
Console.WriteLine("Progress message: " + job.ProgressMessage);
//Wait for the job to finish
Console.WriteLine("Waiting for the job to finish...");
jobStatus.WaitOne();
//job.Progress will no longer increment automatically
//the job will finish regardless of the value of job.Progress
Console.WriteLine("Finished job progress: " + job.Progress);
//close the scheduler connection
scheduler.Close();
}
}
public void SubmitHPCRecoveryJob(string db, int eid, string cluster, int priority, int nworkers, string executor, string jobTemplate, int jobTimeout, int taskTimeout)
{
SqlConnection sql = Connect(db);
SqlCommand cmd = new SqlCommand("SELECT * FROM Experiments WHERE ID=" + eid, sql);
SqlDataReader r = cmd.ExecuteReader();
if (!r.Read())
{
throw new Exception("Could not read from database.");
}
string nodegroup = (string)r["Nodegroup"];
string locality = (string)r["Locality"];
string sharedDir = (string)r["SharedDir"];
string eFullpath = System.IO.Path.GetFullPath(executor);
string eFilename = System.IO.Path.GetFileName(executor);
string sExecutor = eid.ToString() + "_" + eFilename;
if (!File.Exists(sharedDir + "\\" + sExecutor) ||
File.GetLastWriteTime(sharedDir + "\\" + sExecutor) < File.GetLastWriteTime(eFullpath))
{
retry:
try
{
File.Copy(eFullpath, sharedDir + "\\" + sExecutor, true);
}
catch (UnauthorizedAccessException)
{
sExecutor = sExecutor.Substring(0, sExecutor.Length - 4) + "-.exe";
goto retry;
}
catch (IOException)
{
sExecutor = sExecutor.Substring(0, sExecutor.Length - 4) + "-.exe";
goto retry;
}
}
scheduler.Connect(cluster);
ISchedulerJob hpcJob = scheduler.CreateJob();
if (jobTemplate != null)
{
hpcJob.SetJobTemplate(jobTemplate);
}
if (jobTimeout != 0)
{
hpcJob.Runtime = jobTimeout;
}
try
{
if (nodegroup != "<Any>")
{
hpcJob.NodeGroups.Add(nodegroup);
}
hpcJob.Name = "Z3 Performance Test Recovery (" + eid + ")";
hpcJob.IsExclusive = true;
hpcJob.CanPreempt = true;
SetPriority(hpcJob, priority);
hpcJob.Project = "Z3";
if (locality == "Socket")
{
hpcJob.UnitType = JobUnitType.Socket;
}
else if (locality == "Core")
{
hpcJob.UnitType = JobUnitType.Core;
}
else if (locality == "Node")
{
hpcJob.UnitType = JobUnitType.Node;
}
else
{
throw new Exception("Unknown locality.");
}
int max = nworkers == 0 ? 1 : nworkers;
int progressTotal = max + 3;
// Add population task.
ReportProgress(Convert.ToInt32(100.0 * 1 / (double)max));
ISchedulerTask populateTask = hpcJob.CreateTask();
SetResources(populateTask, locality);
populateTask.IsRerunnable = false;
populateTask.IsExclusive = false;
populateTask.WorkDirectory = sharedDir;
populateTask.CommandLine = sExecutor + " " + eid + " * \"" + db + "\""; // * means recovery
populateTask.Name = "Populate";
if (taskTimeout != 0)
{
populateTask.Runtime = taskTimeout;
}
populateTask.FailJobOnFailure = true;
hpcJob.AddTask(populateTask);
for (int i = 0; i < max; i++)
{
// Add worker task.
ReportProgress(Convert.ToInt32(100.0 * (i + 1) / (double)max));
ISchedulerTask task = hpcJob.CreateTask();
SetResources(task, locality);
task.WorkDirectory = sharedDir;
task.CommandLine = sExecutor + " " + eid + " \"" + db + "\"";
task.IsExclusive = false;
task.IsRerunnable = true;
task.DependsOn.Add("Populate");
task.Name = "Worker";
task.FailJobOnFailure = false;
if (taskTimeout != 0)
{
task.Runtime = taskTimeout;
}
hpcJob.AddTask(task);
}
// No additional recovery task.
// Add deletion task.
ReportProgress(Convert.ToInt32(100.0 * (progressTotal) / (double)max));
ISchedulerTask dTask = hpcJob.CreateTask();
SetResources(dTask, locality);
dTask.IsRerunnable = true;
dTask.IsExclusive = false;
dTask.WorkDirectory = sharedDir;
dTask.CommandLine = "del " + sharedDir + "\\" + executor;
dTask.Name = "Delete worker";
dTask.DependsOn.Add("Worker");
if (taskTimeout != 0)
{
dTask.Runtime = taskTimeout;
}
dTask.FailJobOnFailure = false;
hpcJob.AddTask(dTask);
scheduler.AddJob(hpcJob);
scheduler.SubmitJob(hpcJob, null, null);
}
catch (Exception ex)
{
MessageBox.Show("Error submitting job: " + ex.Message, "Error", MessageBoxButton.OK, MessageBoxImage.Error);
}
}
static int Main(string[] args)
{
bool debug = false;
string clusterName = Environment.GetEnvironmentVariable("CCP_SCHEDULER");
string userName = null;
for (int i = 0; i < args.Length; i++)
{
switch (args[i])
{
case "-d":
debug = true;
break;
case "-u":
if (++i == args.Length)
{
ShowHelp();
return(1);
}
userName = args[i];
break;
case "-c":
if (++i == args.Length)
{
ShowHelp();
return(1);
}
clusterName = args[i];
break;
default:
ShowHelp();
return(1);
}
}
if (debug)
{
Console.Write("Press any key to start...");
Console.Read();
}
// Create a scheduler object to be used to
// establish a connection to the scheduler on the headnode
using (IScheduler scheduler = new Scheduler())
{
try
{
if (userName != null)
{
// Connect to the scheduler as another user
Console.WriteLine("Connecting to {0} as {1}...", clusterName, userName);
scheduler.ConnectServiceAsClient(clusterName, () => userName);
}
else
{
// Connect to the scheduler
Console.WriteLine("Connecting to {0}...", clusterName);
scheduler.Connect(clusterName);
}
}
catch (Exception e)
{
Console.Error.WriteLine("Could not connect to the scheduler: {0}", e.Message);
return(1); //abort if no connection could be made
}
//Create a job to submit to the scheduler
//the job will be equivalent to the CLI command: job submit /numcores:1-1 "echo hello world"
job = scheduler.CreateJob();
//Some of the optional job parameters to specify. If omitted, defaults are:
// Name = {blank}
// UnitType = Core
// Min/Max Resources = Autocalculated
// etc...
job.Name = "HPCSchedulerBasics Job";
Console.WriteLine("Creating job name {0}...", job.Name);
job.UnitType = JobUnitType.Core;
job.AutoCalculateMin = false;
job.AutoCalculateMax = false;
job.MinimumNumberOfCores = 1;
job.MaximumNumberOfCores = 1;
//Create a task to submit to the job
task = job.CreateTask();
task.Name = "Hello World";
Console.WriteLine("Creating a {0} task...", task.Name);
//The commandline parameter tells the scheduler what the task should do
//CommandLine is the only mandatory parameter you must set for every task
task.CommandLine = "echo Hello World";
//Don't forget to add the task to the job!
job.AddTask(task);
//Use callback to check if a job is finished
job.OnJobState += new EventHandler <JobStateEventArg>(job_OnJobState);
//And to submit the job.
//You can specify your username and password in the parameters, or set them to null and you will be prompted for your credentials
Console.WriteLine("Submitting job to the cluster...");
Console.WriteLine();
scheduler.SubmitJob(job, null, null);
//wait for job to finish
jobFinishedEvent.WaitOne();
//Close the connection
scheduler.Close();
return(0);
} //Call scheduler.Dispose() to free the object when finished
}
private static ISchedulerTask CreateTask(int?taskNumber, ClusterSubmitterArgs clusterArgs, ISchedulerJob job, IDistributable distributableObj, IStringCollection nodesToUse)
{
Locally local = new Locally()
{
Cleanup = false,
TaskCount = clusterArgs.TaskCount,
Tasks = taskNumber.HasValue ? new RangeCollection(taskNumber.Value) : null,
};
ISchedulerTask task = job.CreateTask();
if (nodesToUse != null)
{
task.RequiredNodes = nodesToUse;
}
if (clusterArgs.NumCoresPerTask != null)
{
task.MinimumNumberOfCores = clusterArgs.NumCoresPerTask.Value;
task.MaximumNumberOfCores = clusterArgs.NumCoresPerTask.Value;
task.MaximumNumberOfNodes = 1;
local.ParallelOptions.MaxDegreeOfParallelism = clusterArgs.NumCoresPerTask.Value;
}
if (!clusterArgs.IsExclusive)
{
local.ParallelOptions = new ParallelOptions()
{
MaxDegreeOfParallelism = 1
};
}
task.WorkDirectory = clusterArgs.ExternalRemoteDirectoryName;
DistributeApp.Distribute distributeExe = new DistributeApp.Distribute()
{
Distributable = distributableObj,
Distributor = local
};
string taskArgString = CreateTaskString(distributeExe, clusterArgs.MinimalCommandLine);
string exeName = distributeExe.Distributable is DistributableWrapper ? clusterArgs.ExeName : distributeExe.GetType().Assembly.GetName().Name;
string taskCommandLine = null;
if (clusterArgs.UseMPI)
{
taskCommandLine = string.Format("mpiexec -n {0} {1}\\{2} {3}", clusterArgs.NumCoresPerTask, clusterArgs.ExeRelativeDirectoryName, exeName, taskArgString);
}
else
{
taskCommandLine = string.Format("{0}\\{1} {2}", clusterArgs.ExeRelativeDirectoryName, exeName, taskArgString);
}
task.CommandLine = taskCommandLine;
string taskNumberAsString = taskNumber.HasValue ? taskNumber.Value.ToString() : "*";
task.Name = Helper.CreateDelimitedString(" ", distributableObj.JobName, taskNumberAsString);
Console.WriteLine(Resource.StdOutRelativeDirName + clusterArgs.StdOutRelativeDirName);
task.StdErrFilePath = string.Format(@"{0}\{1}.txt", clusterArgs.StdErrRelativeDirName, taskNumberAsString);
task.StdOutFilePath = string.Format(@"{0}\{1}.txt", clusterArgs.StdOutRelativeDirName, taskNumberAsString);
Console.WriteLine(Resource.CreateTask, task.CommandLine.Length, task.CommandLine);
if (task.StdErrFilePath.Length >= 160)
{
Console.WriteLine(Resource.Caution, task.StdErrFilePath.Length);
}
return(task);
}
public void SubmitCatchall(string db, string cluster, string locality, int priority, string nodegroup, string executor, string min, string max, string jobTemplate, int jobTimeout, int taskTimeout)
{
scheduler.Connect(cluster);
ISchedulerJob hpcJob = scheduler.CreateJob();
if (jobTemplate != null)
{
hpcJob.SetJobTemplate(jobTemplate);
}
if (jobTimeout != 0)
{
hpcJob.Runtime = jobTimeout;
}
try
{
if (nodegroup != "<Any>")
{
hpcJob.NodeGroups.Add(nodegroup);
}
hpcJob.Name = "Z3 Performance Test (catchall)";
hpcJob.IsExclusive = true;
hpcJob.CanPreempt = true;
SetPriority(hpcJob, priority);
hpcJob.Project = "Z3";
uint fmin = 0;
uint fmax = 0;
string limitsMinTrimmed = min.Trim();
string limitsMaxTrimmed = max.Trim();
if (limitsMinTrimmed != "")
{
try { fmin = Convert.ToUInt32(limitsMinTrimmed); }
catch { fmin = 0; }
}
if (limitsMaxTrimmed != "")
{
try { fmax = Convert.ToUInt32(limitsMaxTrimmed); }
catch { fmax = 0; }
}
ISchedulerCounters ctrs = scheduler.GetCounters();
if (locality == "Socket")
{
hpcJob.UnitType = JobUnitType.Socket;
if (fmin > 0)
{
hpcJob.MinimumNumberOfSockets = (int)fmin;
}
fmax = ((fmax == 0) ? (uint)ctrs.TotalSockets : fmax);
hpcJob.MaximumNumberOfSockets = (int)fmax;
}
else if (locality == "Core")
{
hpcJob.UnitType = JobUnitType.Core;
if (fmin > 0)
{
hpcJob.MinimumNumberOfCores = (int)fmin;
}
fmax = ((fmax == 0) ? (uint)ctrs.TotalCores : fmax);
hpcJob.MaximumNumberOfCores = (int)fmax;
}
else if (locality == "Node")
{
hpcJob.UnitType = JobUnitType.Node;
if (fmin > 0)
{
hpcJob.MinimumNumberOfNodes = (int)fmin;
}
fmax = ((fmax == 0) ? (uint)ctrs.TotalNodes : fmax);
hpcJob.MaximumNumberOfNodes = (int)fmax;
}
for (int i = 0; i < fmax; i++)
{
// Add worker task.
if (WorkerReportsProgress)
{
ReportProgress(Convert.ToInt32(100.0 * (i + 1) / (double)fmax));
}
ISchedulerTask task = hpcJob.CreateTask();
SetResources(task, locality);
task.WorkDirectory = Path.GetDirectoryName(Path.GetFullPath(executor));
task.CommandLine = "pushd " + Path.GetDirectoryName(Path.GetFullPath(executor)) + " & " + Path.GetFileName(executor) + " \"" + db + "\"";
// task.CommandLine = Path.GetFileName(executor) + " \"" + db + "\"";
task.IsExclusive = false;
task.IsRerunnable = true;
task.Name = "Worker";
task.Runtime = taskTimeout;
hpcJob.AddTask(task);
}
scheduler.AddJob(hpcJob);
scheduler.SubmitJob(hpcJob, null, null);
}
catch (Exception ex)
{
scheduler.CancelJob(hpcJob.Id, "Aborted.");
throw ex;
}
}
//for best results, run this sample code in queued scheduling mode
static void Main(string[] args)
{
string clusterName = Environment.GetEnvironmentVariable("CCP_SCHEDULER");
using (IScheduler scheduler = new Scheduler())
{
Console.WriteLine("Connecting to {0}", clusterName);
scheduler.Connect(clusterName);
//assume you have two nodegroups, NodeGroup1 and NodeGroup2
IStringCollection nodeGroup1 = scheduler.GetNodesInNodeGroup("NodeGroup1");
IStringCollection nodeGroup2 = scheduler.GetNodesInNodeGroup("NodeGroup2");
if (nodeGroup1.Count == 0 || nodeGroup2.Count == 0)
{
Console.WriteLine("Node groups are not set up correctly");
return;
}
//and nodes in NodeGroup2 are not in NodeGroup1, and vise versa.
string nodeToMove = "";
foreach (string node in nodeGroup2)
{
if (!nodeGroup1.Contains(node))
{
nodeToMove = node;
break;
}
}
if (string.IsNullOrEmpty(nodeToMove))
{
Console.WriteLine("No eligible nodes to move");
return;
}
//create a job to run on NodeGroup1
ISchedulerJob job = scheduler.CreateJob();
job.NodeGroups.Add("NodeGroup1");
//Set unit type to node, but let it autocalculate resources
job.UnitType = JobUnitType.Node;
ISchedulerTask task = job.CreateTask();
task.CommandLine = "ver";
task.Type = TaskType.Service;
job.AddTask(task);
job.OnTaskState += new EventHandler <TaskStateEventArg>(job_OnTaskState);
Console.WriteLine("Submitting job on NodeGroup1");
scheduler.SubmitJob(job, null, null);
Console.WriteLine("Job {0} Submitted", job.Id);
//wait for the job to start running
running.WaitOne();
job.Refresh();
int allocationCount = job.AllocatedNodes.Count;
Console.WriteLine("Number of allocated nodes: {0}", allocationCount);
//Check the status of NodeGroup1 nodes
int idleCores = 0;
foreach (string nodename in nodeGroup1)
{
ISchedulerNode node = scheduler.OpenNodeByName(nodename);
idleCores += node.GetCounters().IdleCoreCount;
}
//There are no more idle cores remaining in this node group
//So we'll place one of the nodes from NodeGroup2 allow the job to grow
if (idleCores == 0)
{
running.Reset();
//Changing nodegroups is available through the UI or PowerShell
string powershellScript = String.Format("add-pssnapin microsoft.hpc; " +
"add-hpcgroup -scheduler {0} -name {1} -nodename {2}",
clusterName, "NodeGroup1", nodeToMove);
using (PowerShell ps = PowerShell.Create())
{
ps.AddScript(powershellScript, true);
ps.Invoke();
}
running.WaitOne();
Console.WriteLine("(Waiting 5 seconds for job to update the scheduler)");
Thread.Sleep(5 * 1000);
job.Refresh();
int newAllocationCount = job.AllocatedNodes.Count;
//verify that job has grown
if (newAllocationCount > allocationCount)
{
Console.WriteLine("Job has grown to {0} nodes", newAllocationCount);
}
}
else
{
Console.WriteLine("There are still idle cores in the nodegroup");
}
}
}
public void SubmitHPCJob(string db, bool isNew, int newID, string cluster, string nodegroup, int priority,
string locality, string limitsMin, string limitsMax, string sharedDir,
string executor,
string jobTemplate,
int jobTimeout, int taskTimeout,
int nworkers = 0)
{
string limitsMinTrimmed = limitsMin.Trim();
string limitsMaxTrimmed = limitsMax.Trim();
SqlConnection sql = Connect(db);
SqlCommand cmd = null;
scheduler.Connect(cluster);
ISchedulerJob hpcJob = scheduler.CreateJob();
if (jobTemplate != null)
{
hpcJob.SetJobTemplate(jobTemplate);
}
if (jobTimeout != 0)
{
hpcJob.Runtime = jobTimeout;
}
hpcJob.FailOnTaskFailure = false;
try
{
if (nodegroup != "<Any>")
{
hpcJob.NodeGroups.Add(nodegroup);
}
hpcJob.Name = "Z3 Performance Test (" + newID + ")";
hpcJob.IsExclusive = true;
hpcJob.CanPreempt = true;
SetPriority(hpcJob, priority);
hpcJob.Project = "Z3";
if (locality == "Socket")
{
hpcJob.UnitType = JobUnitType.Socket;
}
else if (locality == "Core")
{
hpcJob.UnitType = JobUnitType.Core;
}
else if (locality == "Node")
{
hpcJob.UnitType = JobUnitType.Node;
}
else
{
throw new Exception("Unknown locality.");
}
uint min = 0;
uint max = 0;
if (limitsMinTrimmed != "")
{
try { min = Convert.ToUInt32(limitsMinTrimmed); }
catch { min = 0; }
}
if (limitsMax != "")
{
try { max = Convert.ToUInt32(limitsMaxTrimmed); }
catch { max = 0; }
}
ISchedulerCounters ctrs = scheduler.GetCounters();
if (locality == "Socket")
{
if (min > 0)
{
hpcJob.MinimumNumberOfSockets = (int)min;
}
max = ((max == 0) ? (uint)ctrs.TotalSockets: max);
hpcJob.MaximumNumberOfSockets = (int)max;
}
else if (locality == "Core")
{
if (min > 0)
{
hpcJob.MinimumNumberOfCores = (int)min;
}
max = ((max == 0) ? (uint)ctrs.TotalCores : max);
hpcJob.MaximumNumberOfCores = (int)max;
}
else if (locality == "Node")
{
if (min > 0)
{
hpcJob.MinimumNumberOfNodes = (int)min;
}
max = ((max == 0) ? (uint)ctrs.TotalNodes: max);
hpcJob.MaximumNumberOfNodes = (int)max;
}
uint progressTotal = max + 3;
// Add population task.
if (WorkerReportsProgress)
{
ReportProgress(Convert.ToInt32(100.0 * 1 / (double)max));
}
ISchedulerTask populateTask = hpcJob.CreateTask();
SetResources(populateTask, locality);
populateTask.IsRerunnable = false;
populateTask.IsExclusive = false;
// populateTask.WorkDirectory = sharedDir;
//populateTask.CommandLine = executor + " " + newID + " ? \"" + db + "\"";
populateTask.CommandLine = "pushd " + sharedDir + " & " + Path.GetFileName(executor) + " " + newID + " ? \"" + db + "\"";
populateTask.Name = "Populate";
if (taskTimeout != 0)
{
populateTask.Runtime = taskTimeout;
}
populateTask.FailJobOnFailure = true;
hpcJob.AddTask(populateTask);
for (int i = 0; i < max; i++)
{
// Add worker task.
if (WorkerReportsProgress)
{
ReportProgress(Convert.ToInt32(100.0 * (i + 1) / (double)max));
}
ISchedulerTask task = hpcJob.CreateTask();
SetResources(task, locality);
// task.WorkDirectory = sharedDir;
// task.CommandLine = executor + " " + newID + " \"" + db + "\"";
task.CommandLine = "pushd " + sharedDir + " & " + Path.GetFileName(executor) + " " + newID + " \"" + db + "\"";
task.IsExclusive = false;
task.IsRerunnable = true;
task.DependsOn.Add("Populate");
task.Name = "Worker";
if (taskTimeout != 0)
{
task.Runtime = taskTimeout;
}
populateTask.FailJobOnFailure = false;
hpcJob.AddTask(task);
}
// Add recovery task.
if (WorkerReportsProgress)
{
ReportProgress(Convert.ToInt32(100.0 * (progressTotal - 1) / (double)max));
}
ISchedulerTask rTask = hpcJob.CreateTask();
SetResources(rTask, locality);
rTask.IsRerunnable = true;
rTask.IsExclusive = false;
// rTask.WorkDirectory = sharedDir;
// rTask.CommandLine = executor + " " + newID + " ! \"" + db + "\"";
rTask.CommandLine = "pushd " + sharedDir + " & " + Path.GetFileName(executor) + " " + newID + " ! \"" + db + "\"";
rTask.DependsOn.Add("Worker");
rTask.Name = "Recovery";
if (taskTimeout != 0)
{
rTask.Runtime = taskTimeout;
}
rTask.FailJobOnFailure = true;
hpcJob.AddTask(rTask);
// Add deletion task.
if (WorkerReportsProgress)
{
ReportProgress(Convert.ToInt32(100.0 * (progressTotal) / (double)max));
}
ISchedulerTask dTask = hpcJob.CreateTask();
SetResources(dTask, locality);
dTask.IsRerunnable = true;
dTask.IsExclusive = false;
// dTask.WorkDirectory = sharedDir;
// dTask.CommandLine = "del " + sharedDir + "\\" + executor;
dTask.CommandLine = "pushd " + sharedDir + " & del " + Path.GetFileName(executor);
dTask.Name = "Delete worker";
dTask.DependsOn.Add("Recovery");
if (taskTimeout != 0)
{
dTask.Runtime = taskTimeout;
}
dTask.FailJobOnFailure = false;
hpcJob.AddTask(dTask);
scheduler.AddJob(hpcJob);
scheduler.SubmitJob(hpcJob, null, null);
if (isNew)
{
cmd = new SqlCommand("UPDATE Experiments SET ClusterJobID=" + hpcJob.Id.ToString() + " WHERE ID=" + newID.ToString() + "; ", sql);
cmd.CommandTimeout = 0;
cmd.ExecuteNonQuery();
}
}
catch (Exception ex)
{
cmd = new SqlCommand("DELETE FROM JobQueue WHERE ExperimentID=" + newID + "; DELETE FROM Experiments WHERE ID=" + newID, sql);
cmd.CommandTimeout = 0;
cmd.ExecuteNonQuery();
if (hpcJob.State == JobState.Configuring ||
hpcJob.State == JobState.ExternalValidation ||
hpcJob.State == JobState.Queued ||
hpcJob.State == JobState.Running ||
hpcJob.State == JobState.Submitted ||
hpcJob.State == JobState.Validating)
{
try { scheduler.CancelJob(hpcJob.Id, "Aborted."); }
catch (Exception) { }
}
throw ex;
}
//return totalJobs;
}