/// <summary>Apply the filter (status) on the parsed job and generate summary</summary>
public FilteredJob(JobHistoryParser.JobInfo job, string status)
{
filter = status;
IDictionary <TaskID, JobHistoryParser.TaskInfo> tasks = job.GetAllTasks();
foreach (JobHistoryParser.TaskInfo task in tasks.Values)
{
IDictionary <TaskAttemptID, JobHistoryParser.TaskAttemptInfo> attempts = task.GetAllTaskAttempts
();
foreach (JobHistoryParser.TaskAttemptInfo attempt in attempts.Values)
{
if (attempt.GetTaskStatus().Equals(status))
{
string hostname = attempt.GetHostname();
TaskID id = attempt.GetAttemptId().GetTaskID();
ICollection <TaskID> set = badNodesToFilteredTasks[hostname];
if (set == null)
{
set = new TreeSet <TaskID>();
set.AddItem(id);
badNodesToFilteredTasks[hostname] = set;
}
else
{
set.AddItem(id);
}
}
}
}
}
/// <summary>Constructs the HistoryViewer object</summary>
/// <param name="historyFile">The fully qualified Path of the History File</param>
/// <param name="conf">The Configuration file</param>
/// <param name="printAll">Toggle to print all status to only killed/failed status</param>
/// <exception cref="System.IO.IOException"/>
public HistoryViewer(string historyFile, Configuration conf, bool printAll)
{
this.printAll = printAll;
string errorMsg = "Unable to initialize History Viewer";
try
{
Path jobFile = new Path(historyFile);
fs = jobFile.GetFileSystem(conf);
string[] jobDetails = jobFile.GetName().Split("_");
if (jobDetails.Length < 2)
{
// NOT a valid name
System.Console.Error.WriteLine("Ignore unrecognized file: " + jobFile.GetName());
throw new IOException(errorMsg);
}
JobHistoryParser parser = new JobHistoryParser(fs, jobFile);
job = parser.Parse();
jobId = job.GetJobId().ToString();
}
catch (Exception e)
{
throw new IOException(errorMsg, e);
}
}
public virtual JobHistoryParser.JobInfo Parse(EventReader reader)
{
lock (this)
{
if (info != null)
{
return(info);
}
info = new JobHistoryParser.JobInfo();
Parse(reader, this);
return(info);
}
}
/// <summary>Generate analysis information for the parsed job</summary>
public AnalyzedJob(JobHistoryParser.JobInfo job)
{
IDictionary <TaskID, JobHistoryParser.TaskInfo> tasks = job.GetAllTasks();
int finishedMaps = (int)job.GetFinishedMaps();
int finishedReduces = (int)job.GetFinishedReduces();
mapTasks = new JobHistoryParser.TaskAttemptInfo[finishedMaps];
reduceTasks = new JobHistoryParser.TaskAttemptInfo[finishedReduces];
int mapIndex = 0;
int reduceIndex = 0;
avgMapTime = 0;
avgReduceTime = 0;
avgShuffleTime = 0;
foreach (JobHistoryParser.TaskInfo task in tasks.Values)
{
IDictionary <TaskAttemptID, JobHistoryParser.TaskAttemptInfo> attempts = task.GetAllTaskAttempts
();
foreach (JobHistoryParser.TaskAttemptInfo attempt in attempts.Values)
{
if (attempt.GetTaskStatus().Equals(TaskStatus.State.Succeeded.ToString()))
{
long avgFinishTime = (attempt.GetFinishTime() - attempt.GetStartTime());
if (attempt.GetTaskType().Equals(TaskType.Map))
{
mapTasks[mapIndex++] = attempt;
avgMapTime += avgFinishTime;
}
else
{
if (attempt.GetTaskType().Equals(TaskType.Reduce))
{
reduceTasks[reduceIndex++] = attempt;
avgShuffleTime += (attempt.GetShuffleFinishTime() - attempt.GetStartTime());
avgReduceTime += (attempt.GetFinishTime() - attempt.GetShuffleFinishTime());
}
}
break;
}
}
}
if (finishedMaps > 0)
{
avgMapTime /= finishedMaps;
}
if (finishedReduces > 0)
{
avgReduceTime /= finishedReduces;
avgShuffleTime /= finishedReduces;
}
}
/// <summary>Create summary information for the parsed job</summary>
public SummarizedJob(JobHistoryParser.JobInfo job)
{
tasks = job.GetAllTasks();
foreach (JobHistoryParser.TaskInfo task in tasks.Values)
{
IDictionary <TaskAttemptID, JobHistoryParser.TaskAttemptInfo> attempts = task.GetAllTaskAttempts
();
//allHosts.put(task.getHo(Keys.HOSTNAME), "");
foreach (JobHistoryParser.TaskAttemptInfo attempt in attempts.Values)
{
long startTime = attempt.GetStartTime();
long finishTime = attempt.GetFinishTime();
if (attempt.GetTaskType().Equals(TaskType.Map))
{
if (mapStarted == 0 || mapStarted > startTime)
{
mapStarted = startTime;
}
if (mapFinished < finishTime)
{
mapFinished = finishTime;
}
totalMaps++;
if (attempt.GetTaskStatus().Equals(TaskStatus.State.Failed.ToString()))
{
numFailedMaps++;
}
else
{
if (attempt.GetTaskStatus().Equals(TaskStatus.State.Killed.ToString()))
{
numKilledMaps++;
}
}
}
else
{
if (attempt.GetTaskType().Equals(TaskType.Reduce))
{
if (reduceStarted == 0 || reduceStarted > startTime)
{
reduceStarted = startTime;
}
if (reduceFinished < finishTime)
{
reduceFinished = finishTime;
}
totalReduces++;
if (attempt.GetTaskStatus().Equals(TaskStatus.State.Failed.ToString()))
{
numFailedReduces++;
}
else
{
if (attempt.GetTaskStatus().Equals(TaskStatus.State.Killed.ToString()))
{
numKilledReduces++;
}
}
}
else
{
if (attempt.GetTaskType().Equals(TaskType.JobCleanup))
{
if (cleanupStarted == 0 || cleanupStarted > startTime)
{
cleanupStarted = startTime;
}
if (cleanupFinished < finishTime)
{
cleanupFinished = finishTime;
}
totalCleanups++;
if (attempt.GetTaskStatus().Equals(TaskStatus.State.Succeeded.ToString()))
{
numFinishedCleanups++;
}
else
{
if (attempt.GetTaskStatus().Equals(TaskStatus.State.Failed.ToString()))
{
numFailedCleanups++;
}
else
{
if (attempt.GetTaskStatus().Equals(TaskStatus.State.Killed.ToString()))
{
numKilledCleanups++;
}
}
}
}
else
{
if (attempt.GetTaskType().Equals(TaskType.JobSetup))
{
if (setupStarted == 0 || setupStarted > startTime)
{
setupStarted = startTime;
}
if (setupFinished < finishTime)
{
setupFinished = finishTime;
}
totalSetups++;
if (attempt.GetTaskStatus().Equals(TaskStatus.State.Succeeded.ToString()))
{
numFinishedSetups++;
}
else
{
if (attempt.GetTaskStatus().Equals(TaskStatus.State.Failed.ToString()))
{
numFailedSetups++;
}
else
{
if (attempt.GetTaskStatus().Equals(TaskStatus.State.Killed.ToString()))
{
numKilledSetups++;
}
}
}
}
}
}
}
}
}
}
