public Dictionary <int, ISchedulerJob> GetJobs(JobState state)
{
IFilterCollection jobFilters = hpcScheduler.CreateFilterCollection();
jobFilters.Add(FilterOperator.Equal, PropId.Job_State, state);
ISchedulerCollection jobs = hpcScheduler.GetJobList(jobFilters, null);
Dictionary <int, ISchedulerJob> jobDict = new Dictionary <int, ISchedulerJob>();
foreach (ISchedulerJob job in jobs)
{
updateMinimum(job);
if (!jobDict.ContainsKey(job.Id))
{
jobDict[job.Id] = job;
}
}
if (state == JobState.Running) //|| state == JobState.Finished || state == JobState.Finishing )
{
lastMinStartTime = tempMinStartTime;
}
return(jobDict);
}
static void generateTaskLogLines(ISchedulerCollection tasks)
{
StringBuilder buffer = new StringBuilder();
foreach (ISchedulerTask task in tasks)
{
tempMaxEndDate = task.EndTime > tempMaxEndDate ? task.EndTime : tempMaxEndDate;
if (tracker.ContainsKey(task.TaskId))
{
continue;
}
if (task.EndTime > taskMaxEndDate)
{
tracker[task.TaskId] = 1;
buffer.Append(generateTaskLogLine(task));
//System.Console.WriteLine(generateTaskLogLine(task));
lineCounter++;
}
// System.Console.WriteLine(generateTaskLogLine(task));
}
System.Console.WriteLine(buffer.ToString());
}
static void generateTaskLogLines(DateTime dt, ISchedulerJob job, ISchedulerCollection tasks)
{
StringBuilder buffer = new StringBuilder();
foreach (ISchedulerTask task in tasks)
{
tempMaxEndDate = task.EndTime > tempMaxEndDate ? task.EndTime : tempMaxEndDate;
ISchedulerCore core;
ISchedulerNode node;
if (tracker.ContainsKey(task.TaskId))
{
continue;
}
if (task.EndTime > taskMaxEndDate)
{
tracker[task.TaskId] = 1;
buffer.Append(generateTaskLogLine(dt, job, task));
buffer.Append("\n");
//System.Console.WriteLine(generateTaskLogLine(task));
lineCounter++;
}
}
if (buffer.ToString().Length != 0)
{
System.Console.WriteLine(buffer.ToString().Trim());
}
}
public TaskListViewModel(ISchedulerJob job, IEnumerable <VM> activeVMList, Scheduler hpcSched,
CompositeCommand _CancelTaskListCommand, CompositeCommand _RequeueTaskListCommand)
: base()
{
TaskList = new ObservableCollection <TaskViewModel>();
SelectAllCommand = new CompositeCommand();
UnselectAllCommand = new CompositeCommand();
CancelTaskListCommand = _CancelTaskListCommand;
this.RequeueTaskListCommand = _RequeueTaskListCommand;
//IEnumerable<VM> activeVMList = VMModel.GetInstance().ActiveVMList;
try
{
//scheduler.Connect(JobListViewModel.MainNodeName);
job.Refresh();
ISchedulerCollection tasks = job.GetTaskList(null, null, true);
var query = from ISchedulerTask task
in tasks
where activeVMList.Any(j => task.AllocatedNodes.Contains(j.GetDomainName()))
select task;
foreach (ISchedulerTask task in query)
{
TaskList.Add(new TaskViewModel(task, job, this));
}
}
catch (Exception ex)
{
}
//foreach (ISchedulerTask task in job.GetTaskList(null, null, true))
//{
// TaskList.Add(new TaskViewModel(task, job));
//}
}
public Dictionary <int, ISchedulerJob> GetJobs(DateTime lastDate)
{
IFilterCollection jobFilters = hpcScheduler.CreateFilterCollection();
jobFilters.Add(FilterOperator.GreaterThanOrEqual, PropId.Job_StartTime, lastDate);
ISchedulerCollection jobs = hpcScheduler.GetJobList(jobFilters, null);
Dictionary <int, ISchedulerJob> jobDict = new Dictionary <int, ISchedulerJob>();
foreach (ISchedulerJob job in jobs)
{
if (job.State == JobState.Running)
{
updateMinimum(job);
}
else
{
continue;
}
if (!jobDict.ContainsKey(job.Id))
{
jobDict[job.Id] = job;
}
}
lastMinStartTime = tempMinStartTime;
lastGetJobsRun = jobs;
return(jobDict);
}
/// <summary>
/// Converts a ISchedulerCollection of ISchedulerJob into corresponding wrappers
/// </summary>
/// <param name="collection">Collection to be converted</param>
/// <returns>A new collection with wrappers</returns>
private static ISchedulerCollection ConvertJobList(ISchedulerCollection collection)
{
SchedulerCollection <ISchedulerJob> newCollection = new SchedulerCollection <ISchedulerJob>();
foreach (ISchedulerJob job in collection)
{
newCollection.Add(job);
}
return(newCollection);
}
/// <summary>
/// get the broker nodes from the scheduler.
/// </summary>
private List <BrokerNodeItem> GetBrokerNodesFromScheduler()
{
List <BrokerNodeItem> brokerNodesInCluster = new List <BrokerNodeItem>();
// brokerNodesFilterField is immutable so accessing brokerNodesFilterField is thread safe
if (brokerNodesFilterField == null)
{
FilterCollection brokerNodesFilters = new FilterCollection();
brokerNodesFilters.Add(new FilterProperty(FilterOperator.HasBitSet, new StoreProperty(NodePropertyIds.JobType, JobType.Broker)));
// TODO: Why arent these used?
// brokerNodesFilters.Add(new FilterProperty(FilterOperator.Equal, new StoreProperty(NodePropertyIds.Reachable, true)));
// brokerNodesFilters.Add(new FilterProperty(FilterOperator.Equal, new StoreProperty(NodePropertyIds.State, NodeState.Online)));
brokerNodesFilterField = brokerNodesFilters;
}
ISchedulerCollection nodeList = null;
try
{
nodeList = this.schedulerField.GetNodeList(brokerNodesFilterField, new SortCollection());
}
catch (SchedulerException e)
{
TraceHelper.TraceEvent(TraceEventType.Error,
"Failed to get the node list from the scheduler, Exception:{0}, {1}",
e.Message, e);
throw;
}
TraceHelper.TraceEvent(
TraceEventType.Information,
"Get {0} broker nodes from the scheduler.",
nodeList.Count);
foreach (ISchedulerNode brokerNode in nodeList)
{
if (brokerNode != null && ((brokerNode.JobType & JobType.Broker) != 0))
{
brokerNodesInCluster.Add(
new BrokerNodeItem(brokerNode.Name, brokerNode.DnsSuffix, brokerNode.State, brokerNode.Reachable));
TraceHelper.TraceEvent(
TraceEventType.Information,
"Add broker node {0} to the list, domain name is {1}.",
brokerNode.Name,
brokerNode.DnsSuffix);
}
}
return(brokerNodesInCluster);
}
static void createLogData()
{
IFilterCollection jobFilters = scheduler.CreateFilterCollection();
jobFilters.Add(FilterOperator.GreaterThanOrEqual, PropId.Job_StartTime, jobMinStartDate);
ISchedulerCollection jobs = scheduler.GetJobList(jobFilters, null); //getJobs (jobFilters);
DateTime now = DateTime.Now;
foreach (ISchedulerJob job in jobs)
{
generateTaskLogLines(now, job, getTasks(job, TaskState.Failed));
generateTaskLogLines(now, job, getTasks(job, TaskState.Finished));
}
}
static void getMinimumStartDate()
{
taskMaxEndDate = new DateTime(1970, 1, 1);
tempMaxEndDate = new DateTime(1970, 1, 1);
jobMinStartDate = new DateTime(3000, 1, 1);
tempMinStartDate = new DateTime(3000, 1, 1);
ISchedulerCollection jobs;
while (true)
{
IFilterCollection jobFilter = scheduler.CreateFilterCollection();
jobFilter.Add(FilterOperator.Equal, PropId.Job_State, JobState.Running);
jobs = scheduler.GetJobList(jobFilter, null);
foreach (ISchedulerJob job in jobs)
{
tempMinStartDate = job.StartTime < tempMinStartDate ? job.StartTime : tempMinStartDate;
ISchedulerCollection tasks = getTasks(job, TaskState.Finished);
foreach (ISchedulerTask task in tasks)
{
tempMaxEndDate = task.EndTime > tempMaxEndDate ? task.EndTime : tempMaxEndDate;
}
}
if (jobs.Count > 0)
{
info("There are now running jobs. Sleeping (" + runningJobPollDuration + ")");
return;
}
else
{
warn("There are no running jobs. Sleeping (" + runningJobPollDuration + ")");
}
Thread.Sleep(runningJobPollDuration * 1000);
}
taskMaxEndDate = tempMaxEndDate;
jobMinStartDate = tempMinStartDate;
}
/// <summary>
/// Check the caller if it is a node in the cluster.
/// </summary>
/// <param name="identity">indicating caller's identity</param>
/// <param name="nodes">collection of cluster node</param>
public static bool IsClusterNode(WindowsIdentity identity, ISchedulerCollection nodes)
{
if (identity.IsSystem)
{
return(true);
}
string nodeName = ExtractMachineName(identity.Name);
foreach (ISchedulerNode node in nodes)
{
if (string.Equals(nodeName, node.Name, StringComparison.InvariantCultureIgnoreCase))
{
return(true);
}
}
return(false);
}
private static ISchedulerTask AddCleanupTaskToJob(ClusterSubmitterArgs clusterArgs, IScheduler scheduler, ISchedulerJob job, IDistributable distributableJob)
{
ISchedulerCollection taskList = job.GetTaskList(scheduler.CreateFilterCollection(), scheduler.CreateSortCollection(), true);
IStringCollection dependencyTasks = scheduler.CreateStringCollection();
if (!clusterArgs.OnlyDoCleanup)
{
dependencyTasks.Add(((ISchedulerTask)taskList[0]).Name);
}
ISchedulerTask cleanupTask = CreateCleanupTask(job, clusterArgs.ExternalRemoteDirectoryName, clusterArgs.StdErrRelativeDirName, clusterArgs.StdOutRelativeDirName, "cleanup", isFinalCleanup: true);
Locally local = new Locally()
{
Cleanup = true,
TaskCount = clusterArgs.TaskCount,
Tasks = new RangeCollection(),
ParallelOptions = new ParallelOptions()
{
MaxDegreeOfParallelism = 1
}
};
DistributeApp.Distribute distributeExe = new DistributeApp.Distribute()
{
Distributor = local,
Distributable = distributableJob
};
string exeName = distributableJob is DistributableWrapper ? clusterArgs.ExeName : distributeExe.GetType().Assembly.GetName().Name;
string taskCommandLine = string.Format("{0}\\{1} {2}", clusterArgs.ExeRelativeDirectoryName, exeName, CreateTaskString(distributeExe, clusterArgs.MinimalCommandLine));
cleanupTask.CommandLine = taskCommandLine;
if (!clusterArgs.OnlyDoCleanup)
{
cleanupTask.DependsOn = dependencyTasks;
}
job.AddTask(cleanupTask);
return(cleanupTask);
}
/// <summary>
/// Job status.
/// </summary>
public override void EvaluateStatus(string idToken, object optInfo, string DeployDir, out bool isRunning, out bool isTerminated, out int ExitCode)
{
using (var tr = new FuncTrace()) {
int id = int.Parse(idToken);
ISchedulerJob JD;
//if (optInfo != null && optInfo is ISchedulerJob _JD) {
// JD = _JD;
//} else {
using (new BlockTrace("Scheduler.OpenJob", tr)) {
JD = Scheduler.OpenJob(id);
}
//}
/*
* the following seems really slow:
*
*
* List<SchedulerJob> allFoundJobs = new List<SchedulerJob>();
* ISchedulerCollection allJobs;
* using (new BlockTrace("Scheduler.GetJobList", tr)) {
* allJobs = Scheduler.Get
* }
* int cc = allJobs.Count;
* Console.WriteLine("MsHpcClient: " + cc + " jobs.");
* tr.Logger.Info("list of " + cc + " jobs.");
* using (new BlockTrace("ID_FILTERING", tr)) {
* foreach (SchedulerJob sJob in allJobs) {
* if (sJob.Id != id)
* continue;
* allFoundJobs.Add(sJob);
* }
*
* if (allFoundJobs.Count <= 0) {
* // some weird state
* isRunning = false;
* isTerminated = false;
* ExitCode = int.MinValue;
* return;
* }
* }
*
* SchedulerJob JD;
* using (new BlockTrace("SORTING", tr)) {
* JD = allFoundJobs.ElementAtMax(MsHpcJob => MsHpcJob.SubmitTime);
* }
*/
using (new BlockTrace("TASK_FILTERING", tr)) {
ISchedulerCollection tasks = JD.GetTaskList(null, null, false);
ExitCode = int.MinValue;
foreach (ISchedulerTask t in tasks)
{
DeployDir = t.WorkDirectory;
ExitCode = t.ExitCode;
}
}
using (new BlockTrace("STATE_EVAL", tr)) {
switch (JD.State)
{
case JobState.Configuring:
case JobState.Submitted:
case JobState.Validating:
case JobState.ExternalValidation:
case JobState.Queued:
isRunning = false;
isTerminated = false;
break;
case JobState.Running:
case JobState.Finishing:
isRunning = true;
isTerminated = false;
break;
case JobState.Finished:
isRunning = false;
isTerminated = true;
break;
case JobState.Failed:
case JobState.Canceled:
case JobState.Canceling:
isRunning = false;
isTerminated = true;
break;
default:
throw new NotImplementedException("Unknown job state: " + JD.State);
}
}
}
}
static void Main(string[] args)
{
//change the headnode name here
const string headnode = "[headnode]";
const string serviceName = "EchoService";
const int numRequests = 8;
SessionStartInfo info = new SessionStartInfo(headnode, serviceName);
//the sample code needs at least 2 cores in the cluster
info.SessionResourceUnitType = SessionUnitType.Core;
info.MaximumUnits = 2;
info.MinimumUnits = 2;
Console.Write("Creating a session for EchoService...");
using (DurableSession session = DurableSession.CreateSession(info))
{
Console.WriteLine("done session id = {0}", session.Id);
NetTcpBinding binding = new NetTcpBinding(SecurityMode.Transport);
using (BrokerClient <IService1> client = new BrokerClient <IService1>(session, binding))
{
Console.Write("Sending {0} requests...", numRequests);
for (int i = 0; i < numRequests; i++)
{
EchoOnExitRequest request = new EchoOnExitRequest(new TimeSpan(0, 0, 5));
client.SendRequest <EchoOnExitRequest>(request, i);
}
client.EndRequests();
Console.WriteLine("done");
// cancel half of the service tasks when processing the requests
ThreadPool.QueueUserWorkItem(delegate
{
//wait 5 seconds to try cancel service tasks.
Thread.Sleep(3 * 1000);
try
{
Scheduler scheduler = new Scheduler();
try
{
scheduler.Connect(headnode);
}
catch (Exception e)
{
Console.WriteLine("Error connecting store.{0}", e.ToString());
return;
}
int jobId = session.GetProperty <int>("HPC_ServiceJobId");
ISchedulerJob job = scheduler.OpenJob(jobId);
job.Refresh();
ISchedulerCollection taskList = job.GetTaskList(null, null, true);
int onFlag = 0;
foreach (ISchedulerTask task in taskList)
{
// cancel half of the service tasks
if (onFlag++ % 2 == 0)
{
try
{
if (task.State == TaskState.Running)
{
Console.WriteLine("Try to cancel task {0}", task.TaskId);
job.CancelTask(task.TaskId);
job.Commit();
}
}
catch (Exception ex)
{
Console.WriteLine("Got exception when trying to cancel task {0}:{1}", task.TaskId, ex.Message);
}
}
}
}
catch (Exception ex)
{
Console.WriteLine("Exception when trying to cancel the service tasks. {0}", ex.Message);
}
});
Console.WriteLine("Retrieving responses...");
try
{
int count = 0;
foreach (var response in client.GetResponses <EchoOnExitResponse>())
{
try
{
string reply = response.Result.EchoOnExitResult;
Console.WriteLine("\tReceived response for request {0}: {1}", response.GetUserData <int>(), reply);
count++;
}
catch (Exception ex)
{
Console.WriteLine("Error occured while processing {0}-th request: {1}", response.GetUserData <int>(), ex.Message);
}
}
Console.WriteLine("Done retrieving responses.{0}/{1} responses retrieved ", count, numRequests);
}
catch (SessionException ex)
{
Console.WriteLine("SessionException while getting responses: {0}", ex.Message);
}
catch (Exception ex)
{
Console.WriteLine("Exception while getting responses: {0}", ex.Message);
}
}
// Close connections and delete messages stored in the system
session.Close();
Console.WriteLine("Press any key to exit.");
Console.ReadKey();
}
}
static int iRoundToSecondsMinimum = 10; // If a TimeSpan takes more than 10 seconds, round off the milliseconds
/// <summary>
/// Command Line entrypoint
/// </summary>
/// <param name="args">parsed parameter string</param>
/// <returns></returns>
static int Main(string[] args)
{
// In case it has been renamed, find out what the current name in use is
// Filename for Help or Error messages
Location l = new Location();
sExeFilename = Path.GetFileNameWithoutExtension(Path.GetFileName(l.locationFullPath));
if (ParseParameters(args) == false)
{
return(-1);
}
// If no job id or user is defined, give help message
if ((jobId == 0) && (userName == null))
{
help(sExeFilename);
return(-1);
}
try {
using (IScheduler scheduler = new Scheduler()) {
ISchedulerCollection jobs = null;
IFilterCollection filter = null;
ISortCollection sort = null;
scheduler.Connect(clusterName);
// Filter the jobs requested by the parameters, either a single job or a range, and/or a user
filter = scheduler.CreateFilterCollection();
if (jobIdRange != 0)
{
filter.Add(FilterOperator.GreaterThanOrEqual, JobPropertyIds.Id, jobId);
filter.Add(FilterOperator.LessThanOrEqual, JobPropertyIds.Id, jobIdRange);
}
else
{
filter.Add(FilterOperator.Equal, JobPropertyIds.Id, jobId);
}
if (userName != null)
{
filter.Add(FilterOperator.Equal, JobPropertyIds.Owner, userName);
}
// Sort the jobs relative to when they were created
sort = scheduler.CreateSortCollection();
sort.Add(SortProperty.SortOrder.Ascending, PropId.Job_CreateTime);
jobs = scheduler.GetJobList(filter, sort);
// Be sure the job(s) can be found
if (jobs.Count == 0)
{
Console.Error.WriteLine(sExeFilename + NOJOBSFOUND);
return(-1);
}
else
{
foreach (ISchedulerJob job in jobs)
{
if (bVerbose)
{
Console.WriteLine("job {0} project: {1}", job.Id, job.Project);
}
// Is the job part of the project?
if ((projectName != null) && (job.Project != projectName))
{
continue;
}
else if ((projectNameIgnore != null) && (job.Project.Equals(projectNameIgnore, StringComparison.InvariantCultureIgnoreCase) == false))
{
continue;
}
else // Exact match overrides StartsWith and Contains, only test them if previous tests are not attempted
{
if ((projectNameBegins != null) && (job.Project.StartsWith(projectNameBegins, StringComparison.InvariantCultureIgnoreCase) == false))
{
continue;
}
if ((projectNameContains != null) && (job.Project.Contains(projectNameContains) == false))
{
continue;
}
}
iFilteredJobs++;
Console.WriteLine("Job {0} - {1}", job.Id, job.State);
Console.WriteLine(job.Name);
collectAllocatedInfo(job);
}
}
// Jobs were found within the range, but none had the appropriate project criteria
if (iFilteredJobs <= 0)
{
Console.Error.WriteLine(sExeFilename + NOJOBSFOUND);
return(-1);
}
// Round up/down to seconds if greater than minimum
if (tsAllJobUsage.TotalSeconds >= iRoundToSecondsMinimum)
{
if (tsAllJobUsage.Milliseconds >= 500)
{
tsAllJobUsage = tsAllJobUsage.Add(TimeSpan.FromMilliseconds(1000 - tsAllJobUsage.Milliseconds));
}
else
{
tsAllJobUsage = tsAllJobUsage.Subtract(TimeSpan.FromMilliseconds(tsAllJobUsage.Milliseconds));
}
}
// If a range of jobs was given, or other criteria created selected multiple jobs, show summary
if ((jobIdRange > 0) || (iFilteredJobs > 1))
{
Console.WriteLine("Number of jobs: {0}", iFilteredJobs);
Console.WriteLine("Number of {0}: {1}", bNodesOnly ? "nodes" : "cores", iAllJobThreads);
Console.WriteLine("{0} usage across all jobs: {1}", bNodesOnly ? "Node" : "Core", tsAllJobUsage);
}
}
} catch (Exception e) {
Console.Error.WriteLine("Exception!");
Console.Error.WriteLine(e.Message);
}
return(iFilteredJobs);
}
public JobStatus GetStatus()
{
if (this.m_status == JobStatus.Success ||
this.m_status == JobStatus.Failure)
{
return(this.m_status);
}
if (this.m_job == null)
{
return(JobStatus.NotSubmitted);
}
switch (this.m_job.State)
{
case JobState.ExternalValidation:
case JobState.Queued:
case JobState.Submitted:
case JobState.Validating:
{
this.m_status = JobStatus.Waiting;
break;
}
case JobState.Configuring:
case JobState.Running:
case JobState.Canceling:
case JobState.Finishing:
{
this.m_status = JobStatus.Running;
break;
}
case JobState.Failed:
// a job only fails if the job manager fails.
{
ISchedulerCollection tasks = this.m_job.Job.GetTaskList(null, null, false);
if (tasks.Count < 1)
{
this.ErrorMsg = this.m_job.ErrorMessage;
this.m_status = JobStatus.Failure;
}
else
{
ISchedulerTask jm = tasks[0] as ISchedulerTask;
switch (jm.State)
{
case TaskState.Finished:
this.m_status = JobStatus.Success;
break;
default:
this.m_status = JobStatus.Failure;
this.ErrorMsg = "JM error: " + jm.ErrorMessage;
break;
}
}
break;
}
case JobState.Canceled:
{
this.ErrorMsg = this.m_job.ErrorMessage;
this.m_status = JobStatus.Failure;
break;
}
case JobState.Finished:
{
this.m_status = JobStatus.Success;
break;
}
}
return(this.m_status);
}
static int Main(string[] args)
{
Dictionary <string, string> arguments = getArgs(args);
string userName = string.Empty;
string password = string.Empty;
int secs = 60;
string clusterName = "localhost";
if (arguments.ContainsKey("scheduler"))
{
clusterName = arguments["scheduler"];
}
try
{
if (arguments.ContainsKey("wait"))
{
secs = int.Parse(arguments["wait"]);
}
}
catch (Exception e)
{
warn("" + DateTime.Now.ToString() + "\t Invalid sleep argument ! Switching ti default (" + secs + ")");
warn(e.StackTrace);
}
Dictionary <int, ISchedulerJob> prevRunningJobs = new Dictionary <int, ISchedulerJob>();
Dictionary <int, ISchedulerJob> runningJobs = new Dictionary <int, ISchedulerJob>();
HashSet <int> prev = new HashSet <int>();
HashSet <int> curr = new HashSet <int>();
HashSet <int> diff = new HashSet <int>();
IScheduler scheduler = new Scheduler();
try
{
scheduler.Connect(clusterName);
}
catch (Exception e)
{
System.Console.Error.WriteLine("" + DateTime.Now.ToString() + "\t Could not connect to " + clusterName + " ! Exiting !!!");
System.Console.Error.WriteLine(e.StackTrace);
return(-1);
}
try
{
if (arguments.ContainsKey("outputPath"))
{
outputPath = arguments["outputPath"];
if (!File.Exists(outputPath))
{
outputPath = ".";
}
}
else
{
outputPath = "."; outputPath = ".";
}
}
catch (Exception e)
{
debug("\t Could not use path");
debug(e.StackTrace);
}
string timeStampFile = string.Format("{0}\\jobTimeStampFile.txt", outputPath);
debug("\t timestampfile: path=" + timeStampFile + "");
if (File.Exists(timeStampFile))
{
DateTime dt = getMinimumStartDateFromFile(timeStampFile);
debug("\t Using date from timestampfile: path=" + timeStampFile + "\n" + "\t Time Used : " + dt.ToString());
if ((DateTime.Now - dt).Hours < 24)
{
Grid tempGrid = new Grid(scheduler);
Dictionary <int, ISchedulerJob> jobs = tempGrid.GetJobs(dt);
ISchedulerCollection jobValues = tempGrid.lastGetJobsRun;
// foreach (ISchedulerJob job in jobs.Values)
foreach (ISchedulerJob job in jobValues)
{
System.Console.WriteLine(getLogLine(job).Trim());
}
}
else
{
debug("\t Serice stop time is more than 24hours. [" + dt + "]");
}
}
Grid grid = new Grid(scheduler);
try {
bool pollingGridFlag = true;
while (pollingGridFlag)
{
runningJobs = grid.GetJobs(JobState.Running);
if (runningJobs.Count > 0)
{
pollingGridFlag = false;
info("There are now running jobs. Sleeping (" + runningJobPollDuration + ")");
continue;
}
else
{
warn("There are no running jobs. Sleeping (" + runningJobPollDuration + ")");
}
Thread.Sleep(runningJobPollDuration * 1000);
}
while (true)
{
Dictionary <int, ISchedulerJob> allJobs = new Dictionary <int, ISchedulerJob>();
List <int> jobIds = new List <int>();
prevRunningJobs = runningJobs;
runningJobs = grid.GetJobsByStartTime();
prev = curr;
curr = new HashSet <int>(runningJobs.Keys);
// diff = new HashSet<int>(prev.Except(curr));
diff = grid.getSetDifference(prev, curr);
foreach (int jobId in diff)
{
// print log line
ISchedulerJob job = prevRunningJobs [jobId];
job.Refresh();
//System.Console.WriteLine(getLogLine(job).Trim());
allJobs[job.Id] = job;
jobIds.Add(job.Id);
}
foreach (ISchedulerJob job in runningJobs.Values)
{
//System.Console.WriteLine(getLogLine(job).Trim());
allJobs[job.Id] = job;
jobIds.Add(job.Id);
}
List <ISchedulerJob> jobs = new List <ISchedulerJob> (grid.GetJobs(JobState.Queued).Values.ToArray());
foreach (ISchedulerJob job in jobs)
{
//System.Console.WriteLine(getLogLine(job).Trim().Trim());
allJobs[job.Id] = job;
jobIds.Add(job.Id);
}
jobIds.Sort(new JobComparer(allJobs));
foreach (int jobId in jobIds)
{
ISchedulerJob job = allJobs[jobId];
System.Console.WriteLine(getLogLine(job).Trim());
}
File.WriteAllText(timeStampFile, grid.MinJobStartTime.ToString());
Thread.Sleep(secs * 1000);
}
}catch (Exception e)
{
debug("\t + ");
}
return(0);
}
/// <summary>
/// Job status.
/// </summary>
public override void EvaluateStatus(Job myJob, out int SubmitCount, out bool isRunning, out bool wasSuccessful, out bool isFailed, out string DeployDir)
{
string PrjName = InteractiveShell.WorkflowMgm.CurrentProject;
DeployDir = null;
List <SchedulerJob> allFoundJobs = new List <SchedulerJob>();
ISchedulerCollection allJobs = m_scheduler.GetJobList(null, null);
foreach (SchedulerJob sJob in allJobs)
{
if (!sJob.Project.Equals(PrjName))
{
continue;
}
if (!sJob.Name.Equals(myJob.Name))
{
continue;
}
if (!sJob.UserName.Equals(m_Username, StringComparison.OrdinalIgnoreCase) && !sJob.Owner.Equals(m_Username, StringComparison.OrdinalIgnoreCase))
{
continue;
}
if (!sJob.UserName.Equals(sJob.Owner, StringComparison.OrdinalIgnoreCase))
{
// ignore weird stuff
continue;
}
allFoundJobs.Add(sJob);
}
SubmitCount = allFoundJobs.Count;
if (allFoundJobs.Count <= 0)
{
isRunning = false;
wasSuccessful = false;
isFailed = false;
return;
}
//if (allFoundJobs.Count > 1) {
// throw new ApplicationException(string.Format("Found {0} Microsoft-HPC-jobs with matching criteria (project is '{1}', name is '{2}', user name is '{3}'). Unable to determine which one correlates to given meta-schedule-job.",allFoundJobs.Count, PrjName, myJob.Name, m_Username));
//}
//allFoundJobs[0].SubmitTime
SchedulerJob JD = allFoundJobs.ElementAtMax(MsHpcJob => MsHpcJob.SubmitTime);
ISchedulerCollection tasks = JD.GetTaskList(null, null, false);
foreach (ISchedulerTask t in tasks)
{
DeployDir = t.WorkDirectory;
}
switch (JD.State)
{
case JobState.Configuring:
case JobState.Submitted:
case JobState.Validating:
case JobState.ExternalValidation:
case JobState.Queued:
isRunning = false;
wasSuccessful = false;
isFailed = false;
break;
case JobState.Running:
case JobState.Finishing:
isRunning = true;
wasSuccessful = false;
isFailed = false;
break;
case JobState.Finished:
isRunning = false;
wasSuccessful = true;
isFailed = false;
break;
case JobState.Failed:
case JobState.Canceled:
case JobState.Canceling:
wasSuccessful = false;
isFailed = true;
isRunning = false;
break;
default:
throw new NotImplementedException("Unknown job state: " + JD.State);
}
}
