private static Timer HeartbeatTimer(int jobID, string clusterName, JobWaitingParams jobWaitingParams, int heartBeatPeriod)
{
Timer timer = new Timer(state =>
{
try
{
jobWaitingParams.Job.Refresh();
//Console.WriteLine("Job is still connected. Status is " + jobWaitingParams.JobState);
}
catch
{
Console.WriteLine("Lost connection to job. Attempting reconnect.");
v2008R2.IScheduler scheduler = null;
for (int iTry = 0; iTry < 10 && scheduler == null; iTry++)
{
HpcLib.TryConnect(clusterName, out scheduler);
}
if (scheduler == null)
{
Console.WriteLine("Unable to reconnect to cluster. Going back to sleep.");
}
else
{
jobWaitingParams.Job = scheduler.OpenJob(jobID);
jobWaitingParams.Job.Refresh();
SetupJobEventHandler(jobWaitingParams);
Console.WriteLine("Reconnect succeeded.");
}
}
jobWaitingParams.JobState = jobWaitingParams.Job.State;
if (JobIsFinished(jobWaitingParams.Job.State))
{
jobWaitingParams.ManualResetEvent.Set();
}
}, null, heartBeatPeriod, heartBeatPeriod);
return(timer);
}
private static v2008R2.ISchedulerTask AddCleanupTaskToJob(ClusterSubmitterArgs clusterArgs, v2008R2.IScheduler scheduler, v2008R2.ISchedulerJob job, IDistributable distributableJob)
{
v2008R2.ISchedulerCollection taskList = job.GetTaskList(scheduler.CreateFilterCollection(), scheduler.CreateSortCollection(), true);
v2008R2.IStringCollection dependencyTasks = scheduler.CreateStringCollection();
if (!clusterArgs.OnlyDoCleanup)
{
dependencyTasks.Add(((v2008R2.ISchedulerTask)taskList[0]).Name);
}
v2008R2.ISchedulerTask cleanupTask = CreateCleanupTask(job, clusterArgs.ExternalRemoteDirectoryName, clusterArgs.StdErrDirName, clusterArgs.StdOutDirName, "cleanup", true);
Distribute.Locally local = new Distribute.Locally()
{
Cleanup = true,
TaskCount = clusterArgs.TaskCount,
Tasks = new RangeCollection(),
ParallelOptions = new ParallelOptions()
{
MaxDegreeOfParallelism = 1
}
};
Distribute.Distribute distributeExe = new Distribute.Distribute()
{
Distributor = local,
Distributable = distributableJob
};
string exeName = distributableJob is DistributableWrapper ? clusterArgs.ExeName : distributeExe.GetType().Assembly.GetName().Name;
//args.AddOptionalFlag("cleanup");
//args.AddOptional("tasks", "empty");
string taskCommandLine = string.Format("{0}\\{1} {2}", clusterArgs.ExeRelativeDirectoryName, exeName, CreateTaskString(distributeExe, clusterArgs.MinimalCommandLine));
cleanupTask.CommandLine = taskCommandLine;
if (!clusterArgs.OnlyDoCleanup)
{
cleanupTask.DependsOn = dependencyTasks;
}
job.AddTask(cleanupTask);
return(cleanupTask);
}
private static v2008R2.IStringCollection GetNodesToUse(ClusterSubmitterArgs clusterArgs, v2008R2.IScheduler scheduler, v2008R2.ISchedulerJob job)
{
job.AutoCalculateMax = false;
job.AutoCalculateMin = false;
var availableNodes = scheduler.GetNodeList(null, null);
v2008R2.IStringCollection nodesToUse = scheduler.CreateStringCollection();
List <string> nodesFound = new List <string>();
foreach (var node in availableNodes)
{
string nodeName = ((Microsoft.Hpc.Scheduler.SchedulerNode)node).Name;
if (!clusterArgs.NodeExclusionList.Contains(nodeName))
{
nodesToUse.Add(nodeName);
}
else
{
nodesFound.Add(nodeName);
}
}
Helper.CheckCondition(nodesFound.Count != clusterArgs.NodeExclusionList.Count, "not all nodes in exclusion list found: check for typo " + clusterArgs.NodeExclusionList);
return(nodesToUse);
}
