/// <summary>Apply the filter (status) on the parsed job and generate summary</summary> public FilteredJob(JobHistoryParser.JobInfo job, string status) { filter = status; IDictionary <TaskID, JobHistoryParser.TaskInfo> tasks = job.GetAllTasks(); foreach (JobHistoryParser.TaskInfo task in tasks.Values) { IDictionary <TaskAttemptID, JobHistoryParser.TaskAttemptInfo> attempts = task.GetAllTaskAttempts (); foreach (JobHistoryParser.TaskAttemptInfo attempt in attempts.Values) { if (attempt.GetTaskStatus().Equals(status)) { string hostname = attempt.GetHostname(); TaskID id = attempt.GetAttemptId().GetTaskID(); ICollection <TaskID> set = badNodesToFilteredTasks[hostname]; if (set == null) { set = new TreeSet <TaskID>(); set.AddItem(id); badNodesToFilteredTasks[hostname] = set; } else { set.AddItem(id); } } } } }
/// <summary>Generate analysis information for the parsed job</summary> public AnalyzedJob(JobHistoryParser.JobInfo job) { IDictionary <TaskID, JobHistoryParser.TaskInfo> tasks = job.GetAllTasks(); int finishedMaps = (int)job.GetFinishedMaps(); int finishedReduces = (int)job.GetFinishedReduces(); mapTasks = new JobHistoryParser.TaskAttemptInfo[finishedMaps]; reduceTasks = new JobHistoryParser.TaskAttemptInfo[finishedReduces]; int mapIndex = 0; int reduceIndex = 0; avgMapTime = 0; avgReduceTime = 0; avgShuffleTime = 0; foreach (JobHistoryParser.TaskInfo task in tasks.Values) { IDictionary <TaskAttemptID, JobHistoryParser.TaskAttemptInfo> attempts = task.GetAllTaskAttempts (); foreach (JobHistoryParser.TaskAttemptInfo attempt in attempts.Values) { if (attempt.GetTaskStatus().Equals(TaskStatus.State.Succeeded.ToString())) { long avgFinishTime = (attempt.GetFinishTime() - attempt.GetStartTime()); if (attempt.GetTaskType().Equals(TaskType.Map)) { mapTasks[mapIndex++] = attempt; avgMapTime += avgFinishTime; } else { if (attempt.GetTaskType().Equals(TaskType.Reduce)) { reduceTasks[reduceIndex++] = attempt; avgShuffleTime += (attempt.GetShuffleFinishTime() - attempt.GetStartTime()); avgReduceTime += (attempt.GetFinishTime() - attempt.GetShuffleFinishTime()); } } break; } } } if (finishedMaps > 0) { avgMapTime /= finishedMaps; } if (finishedReduces > 0) { avgReduceTime /= finishedReduces; avgShuffleTime /= finishedReduces; } }
/// <summary>Create summary information for the parsed job</summary> public SummarizedJob(JobHistoryParser.JobInfo job) { tasks = job.GetAllTasks(); foreach (JobHistoryParser.TaskInfo task in tasks.Values) { IDictionary <TaskAttemptID, JobHistoryParser.TaskAttemptInfo> attempts = task.GetAllTaskAttempts (); //allHosts.put(task.getHo(Keys.HOSTNAME), ""); foreach (JobHistoryParser.TaskAttemptInfo attempt in attempts.Values) { long startTime = attempt.GetStartTime(); long finishTime = attempt.GetFinishTime(); if (attempt.GetTaskType().Equals(TaskType.Map)) { if (mapStarted == 0 || mapStarted > startTime) { mapStarted = startTime; } if (mapFinished < finishTime) { mapFinished = finishTime; } totalMaps++; if (attempt.GetTaskStatus().Equals(TaskStatus.State.Failed.ToString())) { numFailedMaps++; } else { if (attempt.GetTaskStatus().Equals(TaskStatus.State.Killed.ToString())) { numKilledMaps++; } } } else { if (attempt.GetTaskType().Equals(TaskType.Reduce)) { if (reduceStarted == 0 || reduceStarted > startTime) { reduceStarted = startTime; } if (reduceFinished < finishTime) { reduceFinished = finishTime; } totalReduces++; if (attempt.GetTaskStatus().Equals(TaskStatus.State.Failed.ToString())) { numFailedReduces++; } else { if (attempt.GetTaskStatus().Equals(TaskStatus.State.Killed.ToString())) { numKilledReduces++; } } } else { if (attempt.GetTaskType().Equals(TaskType.JobCleanup)) { if (cleanupStarted == 0 || cleanupStarted > startTime) { cleanupStarted = startTime; } if (cleanupFinished < finishTime) { cleanupFinished = finishTime; } totalCleanups++; if (attempt.GetTaskStatus().Equals(TaskStatus.State.Succeeded.ToString())) { numFinishedCleanups++; } else { if (attempt.GetTaskStatus().Equals(TaskStatus.State.Failed.ToString())) { numFailedCleanups++; } else { if (attempt.GetTaskStatus().Equals(TaskStatus.State.Killed.ToString())) { numKilledCleanups++; } } } } else { if (attempt.GetTaskType().Equals(TaskType.JobSetup)) { if (setupStarted == 0 || setupStarted > startTime) { setupStarted = startTime; } if (setupFinished < finishTime) { setupFinished = finishTime; } totalSetups++; if (attempt.GetTaskStatus().Equals(TaskStatus.State.Succeeded.ToString())) { numFinishedSetups++; } else { if (attempt.GetTaskStatus().Equals(TaskStatus.State.Failed.ToString())) { numFailedSetups++; } else { if (attempt.GetTaskStatus().Equals(TaskStatus.State.Killed.ToString())) { numKilledSetups++; } } } } } } } } } }