/// <summary>Apply the filter (status) on the parsed job and generate summary</summary> public FilteredJob(JobHistoryParser.JobInfo job, string status) { filter = status; IDictionary <TaskID, JobHistoryParser.TaskInfo> tasks = job.GetAllTasks(); foreach (JobHistoryParser.TaskInfo task in tasks.Values) { IDictionary <TaskAttemptID, JobHistoryParser.TaskAttemptInfo> attempts = task.GetAllTaskAttempts (); foreach (JobHistoryParser.TaskAttemptInfo attempt in attempts.Values) { if (attempt.GetTaskStatus().Equals(status)) { string hostname = attempt.GetHostname(); TaskID id = attempt.GetAttemptId().GetTaskID(); ICollection <TaskID> set = badNodesToFilteredTasks[hostname]; if (set == null) { set = new TreeSet <TaskID>(); set.AddItem(id); badNodesToFilteredTasks[hostname] = set; } else { set.AddItem(id); } } } } }
/// <summary>Constructs the HistoryViewer object</summary> /// <param name="historyFile">The fully qualified Path of the History File</param> /// <param name="conf">The Configuration file</param> /// <param name="printAll">Toggle to print all status to only killed/failed status</param> /// <exception cref="System.IO.IOException"/> public HistoryViewer(string historyFile, Configuration conf, bool printAll) { this.printAll = printAll; string errorMsg = "Unable to initialize History Viewer"; try { Path jobFile = new Path(historyFile); fs = jobFile.GetFileSystem(conf); string[] jobDetails = jobFile.GetName().Split("_"); if (jobDetails.Length < 2) { // NOT a valid name System.Console.Error.WriteLine("Ignore unrecognized file: " + jobFile.GetName()); throw new IOException(errorMsg); } JobHistoryParser parser = new JobHistoryParser(fs, jobFile); job = parser.Parse(); jobId = job.GetJobId().ToString(); } catch (Exception e) { throw new IOException(errorMsg, e); } }
public virtual JobHistoryParser.JobInfo Parse(EventReader reader) { lock (this) { if (info != null) { return(info); } info = new JobHistoryParser.JobInfo(); Parse(reader, this); return(info); } }
/// <summary>Generate analysis information for the parsed job</summary> public AnalyzedJob(JobHistoryParser.JobInfo job) { IDictionary <TaskID, JobHistoryParser.TaskInfo> tasks = job.GetAllTasks(); int finishedMaps = (int)job.GetFinishedMaps(); int finishedReduces = (int)job.GetFinishedReduces(); mapTasks = new JobHistoryParser.TaskAttemptInfo[finishedMaps]; reduceTasks = new JobHistoryParser.TaskAttemptInfo[finishedReduces]; int mapIndex = 0; int reduceIndex = 0; avgMapTime = 0; avgReduceTime = 0; avgShuffleTime = 0; foreach (JobHistoryParser.TaskInfo task in tasks.Values) { IDictionary <TaskAttemptID, JobHistoryParser.TaskAttemptInfo> attempts = task.GetAllTaskAttempts (); foreach (JobHistoryParser.TaskAttemptInfo attempt in attempts.Values) { if (attempt.GetTaskStatus().Equals(TaskStatus.State.Succeeded.ToString())) { long avgFinishTime = (attempt.GetFinishTime() - attempt.GetStartTime()); if (attempt.GetTaskType().Equals(TaskType.Map)) { mapTasks[mapIndex++] = attempt; avgMapTime += avgFinishTime; } else { if (attempt.GetTaskType().Equals(TaskType.Reduce)) { reduceTasks[reduceIndex++] = attempt; avgShuffleTime += (attempt.GetShuffleFinishTime() - attempt.GetStartTime()); avgReduceTime += (attempt.GetFinishTime() - attempt.GetShuffleFinishTime()); } } break; } } } if (finishedMaps > 0) { avgMapTime /= finishedMaps; } if (finishedReduces > 0) { avgReduceTime /= finishedReduces; avgShuffleTime /= finishedReduces; } }
/// <summary>Create summary information for the parsed job</summary> public SummarizedJob(JobHistoryParser.JobInfo job) { tasks = job.GetAllTasks(); foreach (JobHistoryParser.TaskInfo task in tasks.Values) { IDictionary <TaskAttemptID, JobHistoryParser.TaskAttemptInfo> attempts = task.GetAllTaskAttempts (); //allHosts.put(task.getHo(Keys.HOSTNAME), ""); foreach (JobHistoryParser.TaskAttemptInfo attempt in attempts.Values) { long startTime = attempt.GetStartTime(); long finishTime = attempt.GetFinishTime(); if (attempt.GetTaskType().Equals(TaskType.Map)) { if (mapStarted == 0 || mapStarted > startTime) { mapStarted = startTime; } if (mapFinished < finishTime) { mapFinished = finishTime; } totalMaps++; if (attempt.GetTaskStatus().Equals(TaskStatus.State.Failed.ToString())) { numFailedMaps++; } else { if (attempt.GetTaskStatus().Equals(TaskStatus.State.Killed.ToString())) { numKilledMaps++; } } } else { if (attempt.GetTaskType().Equals(TaskType.Reduce)) { if (reduceStarted == 0 || reduceStarted > startTime) { reduceStarted = startTime; } if (reduceFinished < finishTime) { reduceFinished = finishTime; } totalReduces++; if (attempt.GetTaskStatus().Equals(TaskStatus.State.Failed.ToString())) { numFailedReduces++; } else { if (attempt.GetTaskStatus().Equals(TaskStatus.State.Killed.ToString())) { numKilledReduces++; } } } else { if (attempt.GetTaskType().Equals(TaskType.JobCleanup)) { if (cleanupStarted == 0 || cleanupStarted > startTime) { cleanupStarted = startTime; } if (cleanupFinished < finishTime) { cleanupFinished = finishTime; } totalCleanups++; if (attempt.GetTaskStatus().Equals(TaskStatus.State.Succeeded.ToString())) { numFinishedCleanups++; } else { if (attempt.GetTaskStatus().Equals(TaskStatus.State.Failed.ToString())) { numFailedCleanups++; } else { if (attempt.GetTaskStatus().Equals(TaskStatus.State.Killed.ToString())) { numKilledCleanups++; } } } } else { if (attempt.GetTaskType().Equals(TaskType.JobSetup)) { if (setupStarted == 0 || setupStarted > startTime) { setupStarted = startTime; } if (setupFinished < finishTime) { setupFinished = finishTime; } totalSetups++; if (attempt.GetTaskStatus().Equals(TaskStatus.State.Succeeded.ToString())) { numFinishedSetups++; } else { if (attempt.GetTaskStatus().Equals(TaskStatus.State.Failed.ToString())) { numFailedSetups++; } else { if (attempt.GetTaskStatus().Equals(TaskStatus.State.Killed.ToString())) { numKilledSetups++; } } } } } } } } } }