/// <summary> /// Initializes a new instance of the JobInfo struct by job counters /// </summary> /// <param name="id">indicating the job id</param> /// <param name="counters">indicating the job counters</param> public JobInfo(int id, ISchedulerJobCounters counters) { this.id = id; this.runningTasks = counters.RunningTaskCount; this.failedTasks = counters.FailedTaskCount; this.canceledTasks = counters.CanceledTaskCount; this.totalTasks = counters.TaskCount; this.finishedTasks = counters.FinishedTaskCount; }
/// <summary> /// Refreshes job status /// </summary> public void RefreshJobStatus() { if (_jobListener != null) { LogEntry oldVersion = (LogEntry)this.MemberwiseClone(); JobState = _jobListener.JobState; ISchedulerJobCounters counters = _jobListener.JobCounters; //_job.GetCounters(); string stateStr = string.Format("{0}/{1}/{2}/{3}/{4}", counters.QueuedTaskCount, counters.RunningTaskCount, counters.FailedTaskCount, counters.CanceledTaskCount, counters.FinishedTaskCount); FailedTaskCount = counters.FailedTaskCount; TaskStatus = stateStr; if (FailedTaskCount > 0) { IEnumerable <ISchedulerTask> tasklist = ClusterSubmitter.GetFailedAndCanceledTasks(_jobListener.Scheduler, _jobListener.Job); string failedTaskRangeAsString = tasklist.Select(task => task.TaskId.JobTaskId).StringJoin(","); if ("" != failedTaskRangeAsString) { this.FailedTasks = RangeCollection.Parse(failedTaskRangeAsString).ToString(); } else { FailedTasks = ""; } } else { FailedTasks = ""; } if (_jobListener.JobState == JobState.Finished) { if (WallTime.Ticks == 0) { DateTime startTime = _job.SubmitTime; DateTime endTime = _job.EndTime; WallTime = endTime - startTime; } if (CpuTime.Ticks == 0) { var tasklist = _job.GetTaskList(null, null, true).Cast <ISchedulerTask>(); var totalTicks = tasklist.Select(task => (task.EndTime - task.StartTime).Ticks).Sum(); CpuTime = new TimeSpan(totalTicks); } } } }
/// <summary> /// Refreshes both the job state and the task status /// </summary> /// <returns>Flags specifying whether the JobStatus and/or TaskState changed.</returns> public UpdateResult RefreshJobStatus_old() { UpdateResult result = UpdateResult.NoChange; if ((string.IsNullOrEmpty(TaskStatus) || JobState != JobState.Finished && JobState != JobState.Canceled && JobState != JobState.Failed) && _job != null) //CCSLibSettings.ActiveClusters.Contains(Cluster, StringComparer.CurrentCultureIgnoreCase) && //Connect()) { try { _job.Refresh(); //ISchedulerJob job = GetJob(); //if (job == null) return result; if (JobState != _job.State) { JobState = _job.State; result |= UpdateResult.JobStateChanged; } ISchedulerJobCounters counters = _job.GetCounters(); string stateStr = string.Format("{0}/{1}/{2}/{3}", counters.QueuedTaskCount, counters.RunningTaskCount, counters.FailedTaskCount, counters.FinishedTaskCount); if (FailedTaskCount != counters.FailedTaskCount || (string.IsNullOrEmpty(FailedTasks) != (FailedTaskCount == 0))) { UpdateFailedTaskString(_job); result |= UpdateResult.TaskStatusChanged; } FailedTaskCount = counters.FailedTaskCount; if (stateStr != TaskStatus) { TaskStatus = stateStr; result |= UpdateResult.TaskStatusChanged; } } catch { // probably a problem with the scheduler. _scheduler = null; return(result); } } return(result); }
/// <summary> /// Refreshes both the job state and the task status /// </summary> /// <returns>Flags specifying whether the JobStatus and/or TaskState changed.</returns> public UpdateResult RefreshJobStatus_old() { UpdateResult result = UpdateResult.NoChange; if ((string.IsNullOrEmpty(TaskStatus) || JobState != JobState.Finished && JobState != JobState.Canceled && JobState != JobState.Failed) && _job != null) { try { _job.Refresh(); if (JobState != _job.State) { JobState = _job.State; result |= UpdateResult.JobStateChanged; } ISchedulerJobCounters counters = _job.GetCounters(); string stateStr = string.Format("{0}/{1}/{2}/{3}/{4}", counters.QueuedTaskCount, counters.RunningTaskCount, counters.FailedTaskCount, counters.CanceledTaskCount, counters.FinishedTaskCount); if (FailedTaskCount != counters.FailedTaskCount || (string.IsNullOrEmpty(FailedTasks) != (FailedTaskCount == 0))) { UpdateFailedTaskString(_job); result |= UpdateResult.TaskStatusChanged; } FailedTaskCount = counters.FailedTaskCount; if (stateStr != TaskStatus) { TaskStatus = stateStr; result |= UpdateResult.TaskStatusChanged; } } catch { // probably a problem with the scheduler. return(result); } } return(result); }
/// <summary> /// Query task info /// </summary> private void QueryTaskInfo() { TraceHelper.TraceEvent(this.sessionid, TraceEventType.Verbose, "[JobMonitorEntry] Enters QueryTaskInfo method."); if (Interlocked.Increment(ref this.registerdPullTask) != 1) { // register count doesn't change from 0 to 1 means somebody is pulling task, quit return; } bool shouldExit = false; List <TaskInfo> taskInfoList = null; this.pullTaskGap = PullTaskMinGap; while (true) { try { TraceHelper.TraceEvent(this.sessionid, TraceEventType.Verbose, "[JobMonitorEntry] Starting get job counters."); ISchedulerJobCounters counters = this.schedulerJob.GetCounters(); JobInfo jobInfo = new JobInfo(this.sessionid, counters); JobState state = this.schedulerJob.State; TraceHelper.TraceEvent(this.sessionid, TraceEventType.Information, "[JobMonitorEntry] Starting query task info: JobState = {0}\nJobInfo: {1}", state, jobInfo); if (state != this.previousState) { this.previousState = state; if (this.context != null) { // Bug 7144: dispose JobMonitorEntry instance and unsubscribe events if job state changed to Canceled/Finished/Failed shouldExit = (0 == isRequeuingJob) && (state == JobState.Canceled || state == JobState.Finished || state == JobState.Failed); try { // ignore JobState change that happened during job requeue operation. // Note: requeue job takes 3 steps: cancel job, configure job, submit job. Job state transitions during job requeue, i.e., // (running) -> cancelling -> cancelled -> configuring -> submitted -> validating ->(queued), will all be ignored. if (0 == this.isRequeuingJob) { ISchedulerNotify proxy = this.context; proxy.JobStateChanged(JobStateConverter.FromHpcJobState(state)).ContinueWith(this.OnEndJobStateChanged); TraceHelper.TraceEvent(this.sessionid, TraceEventType.Information, "[JobMonitorEntry] Job state change event triggered, new state: {0}", state); } } catch (CommunicationException e) { // Channel is aborted, set the context to null TraceHelper.TraceEvent(this.sessionid, TraceEventType.Error, "[JobMonitorEntry] Callback channel is aborted: {0}", e); this.context = null; } catch (Exception e) { TraceHelper.TraceEvent(this.sessionid, TraceEventType.Error, "[JobMonitorEntry] Failed to trigger job state change event: {0}", e); } } } if (this.context != null && (taskInfoList == null || !jobInfo.Equals(this.previousJobInfo))) { try { taskInfoList = this.GetTaskInfo(); if (taskInfoList != null) { ISchedulerNotify proxy = this.context; proxy.TaskStateChanged(taskInfoList).ContinueWith(this.OnEndTaskStateChanged, jobInfo); TraceHelper.TraceEvent(this.sessionid, TraceEventType.Information, "[JobMonitorEntry] Task state change event triggered."); } } catch (CommunicationException e) { // Channel is aborted, set the context to null TraceHelper.TraceEvent(this.sessionid, TraceEventType.Error, "[JobMonitorEntry] Callback channel is aborted: {0}", e); this.context = null; } catch (Exception e) { TraceHelper.TraceEvent(this.sessionid, TraceEventType.Error, "[JobMonitorEntry] Failed to trigger task state change event: {0}", e); } } } catch (Exception e) { TraceHelper.TraceEvent(this.sessionid, TraceEventType.Warning, "[JobMonitorEntry] Exception thrown when querying task info: {0}", e); } // pull task is not registered, quit if (Interlocked.Decrement(ref this.registerdPullTask) == 0) { break; } TraceHelper.TraceEvent(this.sessionid, TraceEventType.Information, "[JobMonitorEntry] Waiting {0} miliseconds and start another round of getting task info.", this.pullTaskGap); // Sleep and pull task again, clear the register pull task flag Thread.Sleep(this.pullTaskGap); if (this.pullTaskGap < PullTaskMaxGap) { this.pullTaskGap *= 2; if (this.pullTaskGap > PullTaskMaxGap) { this.pullTaskGap = PullTaskMaxGap; } } this.registerdPullTask = 1; } if (shouldExit) { if (this.Exit != null) { this.Exit(this, EventArgs.Empty); } } }
public static string getLogLine(ISchedulerJob job) { /* #date time JobId Queed Running Finished Failed Total Progress User * Tue Jan 24 15:13:05 GMT 2012 7016 1 12 30 40 Running AJG\gomoz * * Tue Jan 24 15:13:05 GMT 2012 7017 1 8 23 34.7826086956522 Running AJG\gomoz * * Tue Jan 24 15:13:05 GMT 2012 7018 11 8 28 28.5714285714286 Running AJG\gomoz * * Tue Jan 24 15:13:05 GMT 2012 7019 111 4 15 26.6666666666667 Running AJG\gomoz * * Tue Jan 24 15:13:29 GMT 2012 7016 12 14 30 46.6666666666667 Running AJG\gomoz */ // System.Console.WriteLine("{0}\t{1}\t{2}\t{3}\T{4}", dateStamp, job.Id, job.); DateTime dateStamp = DateTime.Now; if (job.State == JobState.Finished) { dateStamp = job.EndTime; } if (job.State == JobState.Canceled) { dateStamp = job.EndTime; } ISchedulerJobCounters counters = job.GetCounters(); List <string> groups = new List <string>(job.NodeGroups); float progress = 100 * ((float)counters.FinishedTaskCount / (float)counters.TaskCount); string nodeGroups = "[" + string.Join(",", groups.ToArray()) + "]"; string line = string.Empty; if (job.State == JobState.Running) { if (job.EndTime < new DateTime(1970, 1, 1)) { line = string.Format("{0}\t{1}\t{2}\t{3}\t{4}\t{5}\t{6}\t{7}\t{8}\t{9}\t{10}\t{11}\t{12}\t{13}\t{14}", dateStamp.ToString(hpcDateTimeFormat), job.Id, counters.QueuedTaskCount, counters.RunningTaskCount, counters.FinishedTaskCount, counters.FailedTaskCount, counters.TaskCount, progress, job.State, job.Owner, nodeGroups, job.StartTime, "NULL", (DateTime.Now - job.StartTime).Minutes, job.Name); } else { line = string.Format("{0}\t{1}\t{2}\t{3}\t{4}\t{5}\t{6}\t{7}\t{8}\t{9}\t{10}\t{11}\t{12}\t{13}\t{14}", dateStamp.ToString(hpcDateTimeFormat), job.Id, counters.QueuedTaskCount, counters.RunningTaskCount, counters.FinishedTaskCount, counters.FailedTaskCount, counters.TaskCount, progress, job.State, job.Owner, nodeGroups, job.StartTime, job.EndTime, (DateTime.Now - job.StartTime).Minutes, job.Name); } } else if (job.State == JobState.Finished) { line = string.Format("{0}\t{1}\t{2}\t{3}\t{4}\t{5}\t{6}\t{7}\t{8}\t{9}\t{10}\t{11}\t{12}\t{13}\t{14}", dateStamp.ToString(hpcDateTimeFormat), job.Id, counters.QueuedTaskCount, counters.RunningTaskCount, counters.FinishedTaskCount, counters.FailedTaskCount, counters.TaskCount, progress, job.State, job.Owner, nodeGroups, job.StartTime, job.EndTime, (job.EndTime - job.StartTime).Minutes, job.Name); } else if (job.State == JobState.Failed) { line = string.Format("{0}\t{1}\t{2}\t{3}\t{4}\t{5}\t{6}\t{7}\t{8}\t{9}\t{10}\t{11}\t{12}\t{13}\t{14}", dateStamp.ToString(hpcDateTimeFormat), job.Id, counters.QueuedTaskCount, counters.RunningTaskCount, counters.FinishedTaskCount, counters.FailedTaskCount, counters.TaskCount, progress, job.State, job.Owner, nodeGroups, job.StartTime, job.EndTime, (job.EndTime - job.StartTime).Minutes, job.Name); } else { if (job.EndTime < new DateTime(1970, 1, 1)) { line = string.Format("{0}\t{1}\t{2}\t{3}\t{4}\t{5}\t{6}\t{7}\t{8}\t{9}\t{10}\t{11}\t{12}\t{13}\t{14}", dateStamp.ToString(hpcDateTimeFormat), job.Id, counters.QueuedTaskCount, counters.RunningTaskCount, counters.FinishedTaskCount, counters.FailedTaskCount, counters.TaskCount, progress, job.State, job.Owner, nodeGroups, job.StartTime, "NULL", 0, job.Name); } else { line = string.Format("{0}\t{1}\t{2}\t{3}\t{4}\t{5}\t{6}\t{7}\t{8}\t{9}\t{10}\t{11}\t{12}\t{13}\t{14}", dateStamp.ToString(hpcDateTimeFormat), job.Id, counters.QueuedTaskCount, counters.RunningTaskCount, counters.FinishedTaskCount, counters.FailedTaskCount, counters.TaskCount, progress, job.State, job.Owner, nodeGroups, job.StartTime, job.EndTime, 0, job.Name); } } //System.Console.WriteLine( line.Trim() ); return(line.Trim()); }
public void RefreshJobStatus() { if (_job != null) { LogEntry oldVersion = (LogEntry)this.MemberwiseClone(); _job.Refresh(); JobState = _job.State; ISchedulerJobCounters counters = _job.GetCounters(); string stateStr = string.Format("{0}/{1}/{2}/{3}", counters.QueuedTaskCount, counters.RunningTaskCount, counters.FailedTaskCount, counters.FinishedTaskCount); FailedTaskCount = counters.FailedTaskCount; TaskStatus = stateStr; if (FailedTaskCount > 0) { IEnumerable <ISchedulerTask> tasklist = GetFailedTasks(_job); string failedTaskRangeAsString = tasklist.Select(task => task.TaskId.JobTaskId).StringJoin(","); if ("" != failedTaskRangeAsString) { this.FailedTasks = RangeCollection.Parse(failedTaskRangeAsString).ToString(); } else { FailedTasks = ""; } } else { FailedTasks = ""; } if (JobState == JobState.Finished) { if (WallTime.Ticks == 0) { DateTime startTime = _job.SubmitTime; DateTime endTime = _job.EndTime; WallTime = endTime - startTime; } if (CpuTime.Ticks == 0) { var tasklist = _job.GetTaskList(null, null, true).Cast <ISchedulerTask>(); var totalTicks = tasklist.Select(task => (task.EndTime - task.StartTime).Ticks).Sum(); CpuTime = new TimeSpan(totalTicks); } } bool taskStateChanged = FailedTasks != oldVersion.FailedTasks || TaskStatus != oldVersion.TaskStatus; bool jobStateChanged = JobState != oldVersion.JobState || (FailedTaskCount == 0) != (oldVersion.FailedTaskCount == 0) || string.IsNullOrEmpty(FailedTasks) != string.IsNullOrEmpty(oldVersion.FailedTasks) || CpuTime != oldVersion.CpuTime || WallTime != oldVersion.WallTime; //if (_taskStateChangedSinceLastEvent != taskStateChanged || _jobStateChangedSinceLastEvent != jobStateChanged) // Console.WriteLine("bad"); if (taskStateChanged) { RaiseTaskStateChangedEvent(); } if (jobStateChanged) { RaiseJobStateChangedEvent(); } } }