/// <summary>
/// Initializes a new instance of the JobInfo struct by job counters
/// </summary>
/// <param name="id">indicating the job id</param>
/// <param name="counters">indicating the job counters</param>
public JobInfo(int id, ISchedulerJobCounters counters)
{
this.id = id;
this.runningTasks = counters.RunningTaskCount;
this.failedTasks = counters.FailedTaskCount;
this.canceledTasks = counters.CanceledTaskCount;
this.totalTasks = counters.TaskCount;
this.finishedTasks = counters.FinishedTaskCount;
}
/// <summary>
/// Refreshes job status
/// </summary>
public void RefreshJobStatus()
{
if (_jobListener != null)
{
LogEntry oldVersion = (LogEntry)this.MemberwiseClone();
JobState = _jobListener.JobState;
ISchedulerJobCounters counters = _jobListener.JobCounters; //_job.GetCounters();
string stateStr = string.Format("{0}/{1}/{2}/{3}/{4}", counters.QueuedTaskCount, counters.RunningTaskCount, counters.FailedTaskCount, counters.CanceledTaskCount, counters.FinishedTaskCount);
FailedTaskCount = counters.FailedTaskCount;
TaskStatus = stateStr;
if (FailedTaskCount > 0)
{
IEnumerable <ISchedulerTask> tasklist = ClusterSubmitter.GetFailedAndCanceledTasks(_jobListener.Scheduler, _jobListener.Job);
string failedTaskRangeAsString = tasklist.Select(task => task.TaskId.JobTaskId).StringJoin(",");
if ("" != failedTaskRangeAsString)
{
this.FailedTasks = RangeCollection.Parse(failedTaskRangeAsString).ToString();
}
else
{
FailedTasks = "";
}
}
else
{
FailedTasks = "";
}
if (_jobListener.JobState == JobState.Finished)
{
if (WallTime.Ticks == 0)
{
DateTime startTime = _job.SubmitTime;
DateTime endTime = _job.EndTime;
WallTime = endTime - startTime;
}
if (CpuTime.Ticks == 0)
{
var tasklist = _job.GetTaskList(null, null, true).Cast <ISchedulerTask>();
var totalTicks = tasklist.Select(task => (task.EndTime - task.StartTime).Ticks).Sum();
CpuTime = new TimeSpan(totalTicks);
}
}
}
}
/// <summary>
/// Refreshes both the job state and the task status
/// </summary>
/// <returns>Flags specifying whether the JobStatus and/or TaskState changed.</returns>
public UpdateResult RefreshJobStatus_old()
{
UpdateResult result = UpdateResult.NoChange;
if ((string.IsNullOrEmpty(TaskStatus) ||
JobState != JobState.Finished && JobState != JobState.Canceled && JobState != JobState.Failed) &&
_job != null)
//CCSLibSettings.ActiveClusters.Contains(Cluster, StringComparer.CurrentCultureIgnoreCase) &&
//Connect())
{
try
{
_job.Refresh();
//ISchedulerJob job = GetJob();
//if (job == null) return result;
if (JobState != _job.State)
{
JobState = _job.State;
result |= UpdateResult.JobStateChanged;
}
ISchedulerJobCounters counters = _job.GetCounters();
string stateStr = string.Format("{0}/{1}/{2}/{3}", counters.QueuedTaskCount, counters.RunningTaskCount, counters.FailedTaskCount, counters.FinishedTaskCount);
if (FailedTaskCount != counters.FailedTaskCount || (string.IsNullOrEmpty(FailedTasks) != (FailedTaskCount == 0)))
{
UpdateFailedTaskString(_job);
result |= UpdateResult.TaskStatusChanged;
}
FailedTaskCount = counters.FailedTaskCount;
if (stateStr != TaskStatus)
{
TaskStatus = stateStr;
result |= UpdateResult.TaskStatusChanged;
}
}
catch
{
// probably a problem with the scheduler.
_scheduler = null;
return(result);
}
}
return(result);
}
/// <summary>
/// Refreshes both the job state and the task status
/// </summary>
/// <returns>Flags specifying whether the JobStatus and/or TaskState changed.</returns>
public UpdateResult RefreshJobStatus_old()
{
UpdateResult result = UpdateResult.NoChange;
if ((string.IsNullOrEmpty(TaskStatus) ||
JobState != JobState.Finished && JobState != JobState.Canceled && JobState != JobState.Failed) &&
_job != null)
{
try
{
_job.Refresh();
if (JobState != _job.State)
{
JobState = _job.State;
result |= UpdateResult.JobStateChanged;
}
ISchedulerJobCounters counters = _job.GetCounters();
string stateStr = string.Format("{0}/{1}/{2}/{3}/{4}", counters.QueuedTaskCount, counters.RunningTaskCount, counters.FailedTaskCount, counters.CanceledTaskCount, counters.FinishedTaskCount);
if (FailedTaskCount != counters.FailedTaskCount || (string.IsNullOrEmpty(FailedTasks) != (FailedTaskCount == 0)))
{
UpdateFailedTaskString(_job);
result |= UpdateResult.TaskStatusChanged;
}
FailedTaskCount = counters.FailedTaskCount;
if (stateStr != TaskStatus)
{
TaskStatus = stateStr;
result |= UpdateResult.TaskStatusChanged;
}
}
catch
{
// probably a problem with the scheduler.
return(result);
}
}
return(result);
}
/// <summary>
/// Query task info
/// </summary>
private void QueryTaskInfo()
{
TraceHelper.TraceEvent(this.sessionid, TraceEventType.Verbose, "[JobMonitorEntry] Enters QueryTaskInfo method.");
if (Interlocked.Increment(ref this.registerdPullTask) != 1)
{
// register count doesn't change from 0 to 1 means somebody is pulling task, quit
return;
}
bool shouldExit = false;
List <TaskInfo> taskInfoList = null;
this.pullTaskGap = PullTaskMinGap;
while (true)
{
try
{
TraceHelper.TraceEvent(this.sessionid, TraceEventType.Verbose, "[JobMonitorEntry] Starting get job counters.");
ISchedulerJobCounters counters = this.schedulerJob.GetCounters();
JobInfo jobInfo = new JobInfo(this.sessionid, counters);
JobState state = this.schedulerJob.State;
TraceHelper.TraceEvent(this.sessionid, TraceEventType.Information, "[JobMonitorEntry] Starting query task info: JobState = {0}\nJobInfo: {1}", state, jobInfo);
if (state != this.previousState)
{
this.previousState = state;
if (this.context != null)
{
// Bug 7144: dispose JobMonitorEntry instance and unsubscribe events if job state changed to Canceled/Finished/Failed
shouldExit = (0 == isRequeuingJob) && (state == JobState.Canceled || state == JobState.Finished || state == JobState.Failed);
try
{
// ignore JobState change that happened during job requeue operation.
// Note: requeue job takes 3 steps: cancel job, configure job, submit job. Job state transitions during job requeue, i.e.,
// (running) -> cancelling -> cancelled -> configuring -> submitted -> validating ->(queued), will all be ignored.
if (0 == this.isRequeuingJob)
{
ISchedulerNotify proxy = this.context;
proxy.JobStateChanged(JobStateConverter.FromHpcJobState(state)).ContinueWith(this.OnEndJobStateChanged);
TraceHelper.TraceEvent(this.sessionid, TraceEventType.Information, "[JobMonitorEntry] Job state change event triggered, new state: {0}", state);
}
}
catch (CommunicationException e)
{
// Channel is aborted, set the context to null
TraceHelper.TraceEvent(this.sessionid, TraceEventType.Error, "[JobMonitorEntry] Callback channel is aborted: {0}", e);
this.context = null;
}
catch (Exception e)
{
TraceHelper.TraceEvent(this.sessionid, TraceEventType.Error, "[JobMonitorEntry] Failed to trigger job state change event: {0}", e);
}
}
}
if (this.context != null && (taskInfoList == null || !jobInfo.Equals(this.previousJobInfo)))
{
try
{
taskInfoList = this.GetTaskInfo();
if (taskInfoList != null)
{
ISchedulerNotify proxy = this.context;
proxy.TaskStateChanged(taskInfoList).ContinueWith(this.OnEndTaskStateChanged, jobInfo);
TraceHelper.TraceEvent(this.sessionid, TraceEventType.Information, "[JobMonitorEntry] Task state change event triggered.");
}
}
catch (CommunicationException e)
{
// Channel is aborted, set the context to null
TraceHelper.TraceEvent(this.sessionid, TraceEventType.Error, "[JobMonitorEntry] Callback channel is aborted: {0}", e);
this.context = null;
}
catch (Exception e)
{
TraceHelper.TraceEvent(this.sessionid, TraceEventType.Error, "[JobMonitorEntry] Failed to trigger task state change event: {0}", e);
}
}
}
catch (Exception e)
{
TraceHelper.TraceEvent(this.sessionid, TraceEventType.Warning, "[JobMonitorEntry] Exception thrown when querying task info: {0}", e);
}
// pull task is not registered, quit
if (Interlocked.Decrement(ref this.registerdPullTask) == 0)
{
break;
}
TraceHelper.TraceEvent(this.sessionid, TraceEventType.Information, "[JobMonitorEntry] Waiting {0} miliseconds and start another round of getting task info.", this.pullTaskGap);
// Sleep and pull task again, clear the register pull task flag
Thread.Sleep(this.pullTaskGap);
if (this.pullTaskGap < PullTaskMaxGap)
{
this.pullTaskGap *= 2;
if (this.pullTaskGap > PullTaskMaxGap)
{
this.pullTaskGap = PullTaskMaxGap;
}
}
this.registerdPullTask = 1;
}
if (shouldExit)
{
if (this.Exit != null)
{
this.Exit(this, EventArgs.Empty);
}
}
}
public static string getLogLine(ISchedulerJob job)
{
/* #date time JobId Queed Running Finished Failed Total Progress User
* Tue Jan 24 15:13:05 GMT 2012 7016 1 12 30 40 Running AJG\gomoz
*
* Tue Jan 24 15:13:05 GMT 2012 7017 1 8 23 34.7826086956522 Running AJG\gomoz
*
* Tue Jan 24 15:13:05 GMT 2012 7018 11 8 28 28.5714285714286 Running AJG\gomoz
*
* Tue Jan 24 15:13:05 GMT 2012 7019 111 4 15 26.6666666666667 Running AJG\gomoz
*
* Tue Jan 24 15:13:29 GMT 2012 7016 12 14 30 46.6666666666667 Running AJG\gomoz */
// System.Console.WriteLine("{0}\t{1}\t{2}\t{3}\T{4}", dateStamp, job.Id, job.);
DateTime dateStamp = DateTime.Now;
if (job.State == JobState.Finished)
{
dateStamp = job.EndTime;
}
if (job.State == JobState.Canceled)
{
dateStamp = job.EndTime;
}
ISchedulerJobCounters counters = job.GetCounters();
List <string> groups = new List <string>(job.NodeGroups);
float progress = 100 * ((float)counters.FinishedTaskCount / (float)counters.TaskCount);
string nodeGroups = "[" + string.Join(",", groups.ToArray()) + "]";
string line = string.Empty;
if (job.State == JobState.Running)
{
if (job.EndTime < new DateTime(1970, 1, 1))
{
line = string.Format("{0}\t{1}\t{2}\t{3}\t{4}\t{5}\t{6}\t{7}\t{8}\t{9}\t{10}\t{11}\t{12}\t{13}\t{14}", dateStamp.ToString(hpcDateTimeFormat), job.Id, counters.QueuedTaskCount, counters.RunningTaskCount, counters.FinishedTaskCount, counters.FailedTaskCount, counters.TaskCount, progress, job.State, job.Owner, nodeGroups, job.StartTime, "NULL", (DateTime.Now - job.StartTime).Minutes, job.Name);
}
else
{
line = string.Format("{0}\t{1}\t{2}\t{3}\t{4}\t{5}\t{6}\t{7}\t{8}\t{9}\t{10}\t{11}\t{12}\t{13}\t{14}", dateStamp.ToString(hpcDateTimeFormat), job.Id, counters.QueuedTaskCount, counters.RunningTaskCount, counters.FinishedTaskCount, counters.FailedTaskCount, counters.TaskCount, progress, job.State, job.Owner, nodeGroups, job.StartTime, job.EndTime, (DateTime.Now - job.StartTime).Minutes, job.Name);
}
}
else if (job.State == JobState.Finished)
{
line = string.Format("{0}\t{1}\t{2}\t{3}\t{4}\t{5}\t{6}\t{7}\t{8}\t{9}\t{10}\t{11}\t{12}\t{13}\t{14}", dateStamp.ToString(hpcDateTimeFormat), job.Id, counters.QueuedTaskCount, counters.RunningTaskCount, counters.FinishedTaskCount, counters.FailedTaskCount, counters.TaskCount, progress, job.State, job.Owner, nodeGroups, job.StartTime, job.EndTime, (job.EndTime - job.StartTime).Minutes, job.Name);
}
else if (job.State == JobState.Failed)
{
line = string.Format("{0}\t{1}\t{2}\t{3}\t{4}\t{5}\t{6}\t{7}\t{8}\t{9}\t{10}\t{11}\t{12}\t{13}\t{14}", dateStamp.ToString(hpcDateTimeFormat), job.Id, counters.QueuedTaskCount, counters.RunningTaskCount, counters.FinishedTaskCount, counters.FailedTaskCount, counters.TaskCount, progress, job.State, job.Owner, nodeGroups, job.StartTime, job.EndTime, (job.EndTime - job.StartTime).Minutes, job.Name);
}
else
{
if (job.EndTime < new DateTime(1970, 1, 1))
{
line = string.Format("{0}\t{1}\t{2}\t{3}\t{4}\t{5}\t{6}\t{7}\t{8}\t{9}\t{10}\t{11}\t{12}\t{13}\t{14}", dateStamp.ToString(hpcDateTimeFormat), job.Id, counters.QueuedTaskCount, counters.RunningTaskCount, counters.FinishedTaskCount, counters.FailedTaskCount, counters.TaskCount, progress, job.State, job.Owner, nodeGroups, job.StartTime, "NULL", 0, job.Name);
}
else
{
line = string.Format("{0}\t{1}\t{2}\t{3}\t{4}\t{5}\t{6}\t{7}\t{8}\t{9}\t{10}\t{11}\t{12}\t{13}\t{14}", dateStamp.ToString(hpcDateTimeFormat), job.Id, counters.QueuedTaskCount, counters.RunningTaskCount, counters.FinishedTaskCount, counters.FailedTaskCount, counters.TaskCount, progress, job.State, job.Owner, nodeGroups, job.StartTime, job.EndTime, 0, job.Name);
}
}
//System.Console.WriteLine( line.Trim() );
return(line.Trim());
}
public void RefreshJobStatus()
{
if (_job != null)
{
LogEntry oldVersion = (LogEntry)this.MemberwiseClone();
_job.Refresh();
JobState = _job.State;
ISchedulerJobCounters counters = _job.GetCounters();
string stateStr = string.Format("{0}/{1}/{2}/{3}", counters.QueuedTaskCount, counters.RunningTaskCount, counters.FailedTaskCount, counters.FinishedTaskCount);
FailedTaskCount = counters.FailedTaskCount;
TaskStatus = stateStr;
if (FailedTaskCount > 0)
{
IEnumerable <ISchedulerTask> tasklist = GetFailedTasks(_job);
string failedTaskRangeAsString = tasklist.Select(task => task.TaskId.JobTaskId).StringJoin(",");
if ("" != failedTaskRangeAsString)
{
this.FailedTasks = RangeCollection.Parse(failedTaskRangeAsString).ToString();
}
else
{
FailedTasks = "";
}
}
else
{
FailedTasks = "";
}
if (JobState == JobState.Finished)
{
if (WallTime.Ticks == 0)
{
DateTime startTime = _job.SubmitTime;
DateTime endTime = _job.EndTime;
WallTime = endTime - startTime;
}
if (CpuTime.Ticks == 0)
{
var tasklist = _job.GetTaskList(null, null, true).Cast <ISchedulerTask>();
var totalTicks = tasklist.Select(task => (task.EndTime - task.StartTime).Ticks).Sum();
CpuTime = new TimeSpan(totalTicks);
}
}
bool taskStateChanged = FailedTasks != oldVersion.FailedTasks || TaskStatus != oldVersion.TaskStatus;
bool jobStateChanged = JobState != oldVersion.JobState ||
(FailedTaskCount == 0) != (oldVersion.FailedTaskCount == 0) ||
string.IsNullOrEmpty(FailedTasks) != string.IsNullOrEmpty(oldVersion.FailedTasks) ||
CpuTime != oldVersion.CpuTime || WallTime != oldVersion.WallTime;
//if (_taskStateChangedSinceLastEvent != taskStateChanged || _jobStateChangedSinceLastEvent != jobStateChanged)
// Console.WriteLine("bad");
if (taskStateChanged)
{
RaiseTaskStateChangedEvent();
}
if (jobStateChanged)
{
RaiseJobStateChangedEvent();
}
}
}