private void LoadAllTasks()
{
if (tasksLoaded.Get())
{
return;
}
tasksLock.Lock();
try
{
if (tasksLoaded.Get())
{
return;
}
foreach (KeyValuePair <TaskID, JobHistoryParser.TaskInfo> entry in jobInfo.GetAllTasks
())
{
TaskId yarnTaskID = TypeConverter.ToYarn(entry.Key);
JobHistoryParser.TaskInfo taskInfo = entry.Value;
Task task = new CompletedTask(yarnTaskID, taskInfo);
tasks[yarnTaskID] = task;
if (task.GetType() == TaskType.Map)
{
mapTasks[task.GetID()] = task;
}
else
{
if (task.GetType() == TaskType.Reduce)
{
reduceTasks[task.GetID()] = task;
}
}
}
tasksLoaded.Set(true);
}
finally
{
tasksLock.Unlock();
}
}
/// <exception cref="System.Exception"/>
public virtual void TestCountersForFailedTask()
{
Log.Info("STARTING testCountersForFailedTask");
try
{
Configuration conf = new Configuration();
conf.SetClass(CommonConfigurationKeysPublic.NetTopologyNodeSwitchMappingImplKey,
typeof(TestJobHistoryParsing.MyResolver), typeof(DNSToSwitchMapping));
RackResolver.Init(conf);
MRApp app = new TestJobHistoryParsing.MRAppWithHistoryWithFailedTask(2, 1, true,
this.GetType().FullName, true);
app.Submit(conf);
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.GetContext().GetAllJobs().Values
.GetEnumerator().Next();
JobId jobId = job.GetID();
app.WaitForState(job, JobState.Failed);
// make sure all events are flushed
app.WaitForState(Service.STATE.Stopped);
JobHistory jobHistory = new JobHistory();
jobHistory.Init(conf);
HistoryFileManager.HistoryFileInfo fileInfo = jobHistory.GetJobFileInfo(jobId);
JobHistoryParser parser;
JobHistoryParser.JobInfo jobInfo;
lock (fileInfo)
{
Path historyFilePath = fileInfo.GetHistoryFile();
FSDataInputStream @in = null;
FileContext fc = null;
try
{
fc = FileContext.GetFileContext(conf);
@in = fc.Open(fc.MakeQualified(historyFilePath));
}
catch (IOException ioe)
{
Log.Info("Can not open history file: " + historyFilePath, ioe);
throw (new Exception("Can not open History File"));
}
parser = new JobHistoryParser(@in);
jobInfo = parser.Parse();
}
Exception parseException = parser.GetParseException();
NUnit.Framework.Assert.IsNull("Caught an expected exception " + parseException, parseException
);
foreach (KeyValuePair <TaskID, JobHistoryParser.TaskInfo> entry in jobInfo.GetAllTasks
())
{
TaskId yarnTaskID = TypeConverter.ToYarn(entry.Key);
CompletedTask ct = new CompletedTask(yarnTaskID, entry.Value);
NUnit.Framework.Assert.IsNotNull("completed task report has null counters", ct.GetReport
().GetCounters());
}
IList <string> originalDiagnostics = job.GetDiagnostics();
string historyError = jobInfo.GetErrorInfo();
NUnit.Framework.Assert.IsTrue("No original diagnostics for a failed job", originalDiagnostics
!= null && !originalDiagnostics.IsEmpty());
NUnit.Framework.Assert.IsNotNull("No history error info for a failed job ", historyError
);
foreach (string diagString in originalDiagnostics)
{
NUnit.Framework.Assert.IsTrue(historyError.Contains(diagString));
}
}
finally
{
Log.Info("FINISHED testCountersForFailedTask");
}
}
