private void LoadAllTasks() { if (tasksLoaded.Get()) { return; } tasksLock.Lock(); try { if (tasksLoaded.Get()) { return; } foreach (KeyValuePair <TaskID, JobHistoryParser.TaskInfo> entry in jobInfo.GetAllTasks ()) { TaskId yarnTaskID = TypeConverter.ToYarn(entry.Key); JobHistoryParser.TaskInfo taskInfo = entry.Value; Task task = new CompletedTask(yarnTaskID, taskInfo); tasks[yarnTaskID] = task; if (task.GetType() == TaskType.Map) { mapTasks[task.GetID()] = task; } else { if (task.GetType() == TaskType.Reduce) { reduceTasks[task.GetID()] = task; } } } tasksLoaded.Set(true); } finally { tasksLock.Unlock(); } }
/// <exception cref="System.Exception"/> public virtual void TestCountersForFailedTask() { Log.Info("STARTING testCountersForFailedTask"); try { Configuration conf = new Configuration(); conf.SetClass(CommonConfigurationKeysPublic.NetTopologyNodeSwitchMappingImplKey, typeof(TestJobHistoryParsing.MyResolver), typeof(DNSToSwitchMapping)); RackResolver.Init(conf); MRApp app = new TestJobHistoryParsing.MRAppWithHistoryWithFailedTask(2, 1, true, this.GetType().FullName, true); app.Submit(conf); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.GetContext().GetAllJobs().Values .GetEnumerator().Next(); JobId jobId = job.GetID(); app.WaitForState(job, JobState.Failed); // make sure all events are flushed app.WaitForState(Service.STATE.Stopped); JobHistory jobHistory = new JobHistory(); jobHistory.Init(conf); HistoryFileManager.HistoryFileInfo fileInfo = jobHistory.GetJobFileInfo(jobId); JobHistoryParser parser; JobHistoryParser.JobInfo jobInfo; lock (fileInfo) { Path historyFilePath = fileInfo.GetHistoryFile(); FSDataInputStream @in = null; FileContext fc = null; try { fc = FileContext.GetFileContext(conf); @in = fc.Open(fc.MakeQualified(historyFilePath)); } catch (IOException ioe) { Log.Info("Can not open history file: " + historyFilePath, ioe); throw (new Exception("Can not open History File")); } parser = new JobHistoryParser(@in); jobInfo = parser.Parse(); } Exception parseException = parser.GetParseException(); NUnit.Framework.Assert.IsNull("Caught an expected exception " + parseException, parseException ); foreach (KeyValuePair <TaskID, JobHistoryParser.TaskInfo> entry in jobInfo.GetAllTasks ()) { TaskId yarnTaskID = TypeConverter.ToYarn(entry.Key); CompletedTask ct = new CompletedTask(yarnTaskID, entry.Value); NUnit.Framework.Assert.IsNotNull("completed task report has null counters", ct.GetReport ().GetCounters()); } IList <string> originalDiagnostics = job.GetDiagnostics(); string historyError = jobInfo.GetErrorInfo(); NUnit.Framework.Assert.IsTrue("No original diagnostics for a failed job", originalDiagnostics != null && !originalDiagnostics.IsEmpty()); NUnit.Framework.Assert.IsNotNull("No history error info for a failed job ", historyError ); foreach (string diagString in originalDiagnostics) { NUnit.Framework.Assert.IsTrue(historyError.Contains(diagString)); } } finally { Log.Info("FINISHED testCountersForFailedTask"); } }