/// <exception cref="System.IO.IOException"/> public virtual GetTaskAttemptReportResponse GetTaskAttemptReport(GetTaskAttemptReportRequest request) { TaskAttemptId taskAttemptId = request.GetTaskAttemptId(); GetTaskAttemptReportResponse response = this.recordFactory.NewRecordInstance <GetTaskAttemptReportResponse >(); response.SetTaskAttemptReport(this.VerifyAndGetAttempt(taskAttemptId, JobACL.ViewJob ).GetReport()); return(response); }
/// <exception cref="System.IO.IOException"/> public virtual GetTaskAttemptReportResponse GetTaskAttemptReport(GetTaskAttemptReportRequest request) { TaskAttemptId taskAttemptId = request.GetTaskAttemptId(); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = this.VerifyAndGetJob(taskAttemptId .GetTaskId().GetJobId(), true); GetTaskAttemptReportResponse response = this.recordFactory.NewRecordInstance <GetTaskAttemptReportResponse >(); response.SetTaskAttemptReport(job.GetTask(taskAttemptId.GetTaskId()).GetAttempt(taskAttemptId ).GetReport()); return(response); }
/// <exception cref="System.IO.IOException"/> public virtual GetTaskAttemptReportResponse GetTaskAttemptReport(GetTaskAttemptReportRequest request) { MRServiceProtos.GetTaskAttemptReportRequestProto requestProto = ((GetTaskAttemptReportRequestPBImpl )request).GetProto(); try { return(new GetTaskAttemptReportResponsePBImpl(proxy.GetTaskAttemptReport(null, requestProto ))); } catch (ServiceException e) { throw UnwrapAndThrowException(e); } }
/// <exception cref="System.IO.IOException"/> public virtual LogParams GetLogFilePath(JobID oldJobID, TaskAttemptID oldTaskAttemptID ) { JobId jobId = TypeConverter.ToYarn(oldJobID); GetJobReportRequest request = recordFactory.NewRecordInstance <GetJobReportRequest >(); request.SetJobId(jobId); JobReport report = ((GetJobReportResponse)Invoke("getJobReport", typeof(GetJobReportRequest ), request)).GetJobReport(); if (EnumSet.Of(JobState.Succeeded, JobState.Failed, JobState.Killed, JobState.Error ).Contains(report.GetJobState())) { if (oldTaskAttemptID != null) { GetTaskAttemptReportRequest taRequest = recordFactory.NewRecordInstance <GetTaskAttemptReportRequest >(); taRequest.SetTaskAttemptId(TypeConverter.ToYarn(oldTaskAttemptID)); TaskAttemptReport taReport = ((GetTaskAttemptReportResponse)Invoke("getTaskAttemptReport" , typeof(GetTaskAttemptReportRequest), taRequest)).GetTaskAttemptReport(); if (taReport.GetContainerId() == null || taReport.GetNodeManagerHost() == null) { throw new IOException("Unable to get log information for task: " + oldTaskAttemptID ); } return(new LogParams(taReport.GetContainerId().ToString(), taReport.GetContainerId ().GetApplicationAttemptId().GetApplicationId().ToString(), NodeId.NewInstance(taReport .GetNodeManagerHost(), taReport.GetNodeManagerPort()).ToString(), report.GetUser ())); } else { if (report.GetAMInfos() == null || report.GetAMInfos().Count == 0) { throw new IOException("Unable to get log information for job: " + oldJobID); } AMInfo amInfo = report.GetAMInfos()[report.GetAMInfos().Count - 1]; return(new LogParams(amInfo.GetContainerId().ToString(), amInfo.GetAppAttemptId() .GetApplicationId().ToString(), NodeId.NewInstance(amInfo.GetNodeManagerHost(), amInfo.GetNodeManagerPort()).ToString(), report.GetUser())); } } else { throw new IOException("Cannot get log path for a in-progress job"); } }
public virtual void Test() { TestMRClientService.MRAppWithClientService app = new TestMRClientService.MRAppWithClientService (this, 1, 0, false); Configuration conf = new Configuration(); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(conf); app.WaitForState(job, JobState.Running); NUnit.Framework.Assert.AreEqual("Num tasks not correct", 1, job.GetTasks().Count); IEnumerator <Task> it = job.GetTasks().Values.GetEnumerator(); Task task = it.Next(); app.WaitForState(task, TaskState.Running); TaskAttempt attempt = task.GetAttempts().Values.GetEnumerator().Next(); app.WaitForState(attempt, TaskAttemptState.Running); // send the diagnostic string diagnostic1 = "Diagnostic1"; string diagnostic2 = "Diagnostic2"; app.GetContext().GetEventHandler().Handle(new TaskAttemptDiagnosticsUpdateEvent(attempt .GetID(), diagnostic1)); // send the status update TaskAttemptStatusUpdateEvent.TaskAttemptStatus taskAttemptStatus = new TaskAttemptStatusUpdateEvent.TaskAttemptStatus (); taskAttemptStatus.id = attempt.GetID(); taskAttemptStatus.progress = 0.5f; taskAttemptStatus.stateString = "RUNNING"; taskAttemptStatus.taskState = TaskAttemptState.Running; taskAttemptStatus.phase = Phase.Map; // send the status update app.GetContext().GetEventHandler().Handle(new TaskAttemptStatusUpdateEvent(attempt .GetID(), taskAttemptStatus)); //verify that all object are fully populated by invoking RPCs. YarnRPC rpc = YarnRPC.Create(conf); MRClientProtocol proxy = (MRClientProtocol)rpc.GetProxy(typeof(MRClientProtocol), app.clientService.GetBindAddress(), conf); GetCountersRequest gcRequest = recordFactory.NewRecordInstance <GetCountersRequest >(); gcRequest.SetJobId(job.GetID()); NUnit.Framework.Assert.IsNotNull("Counters is null", proxy.GetCounters(gcRequest) .GetCounters()); GetJobReportRequest gjrRequest = recordFactory.NewRecordInstance <GetJobReportRequest >(); gjrRequest.SetJobId(job.GetID()); JobReport jr = proxy.GetJobReport(gjrRequest).GetJobReport(); VerifyJobReport(jr); GetTaskAttemptCompletionEventsRequest gtaceRequest = recordFactory.NewRecordInstance <GetTaskAttemptCompletionEventsRequest>(); gtaceRequest.SetJobId(job.GetID()); gtaceRequest.SetFromEventId(0); gtaceRequest.SetMaxEvents(10); NUnit.Framework.Assert.IsNotNull("TaskCompletionEvents is null", proxy.GetTaskAttemptCompletionEvents (gtaceRequest).GetCompletionEventList()); GetDiagnosticsRequest gdRequest = recordFactory.NewRecordInstance <GetDiagnosticsRequest >(); gdRequest.SetTaskAttemptId(attempt.GetID()); NUnit.Framework.Assert.IsNotNull("Diagnostics is null", proxy.GetDiagnostics(gdRequest ).GetDiagnosticsList()); GetTaskAttemptReportRequest gtarRequest = recordFactory.NewRecordInstance <GetTaskAttemptReportRequest >(); gtarRequest.SetTaskAttemptId(attempt.GetID()); TaskAttemptReport tar = proxy.GetTaskAttemptReport(gtarRequest).GetTaskAttemptReport (); VerifyTaskAttemptReport(tar); GetTaskReportRequest gtrRequest = recordFactory.NewRecordInstance <GetTaskReportRequest >(); gtrRequest.SetTaskId(task.GetID()); NUnit.Framework.Assert.IsNotNull("TaskReport is null", proxy.GetTaskReport(gtrRequest ).GetTaskReport()); GetTaskReportsRequest gtreportsRequest = recordFactory.NewRecordInstance <GetTaskReportsRequest >(); gtreportsRequest.SetJobId(job.GetID()); gtreportsRequest.SetTaskType(TaskType.Map); NUnit.Framework.Assert.IsNotNull("TaskReports for map is null", proxy.GetTaskReports (gtreportsRequest).GetTaskReportList()); gtreportsRequest = recordFactory.NewRecordInstance <GetTaskReportsRequest>(); gtreportsRequest.SetJobId(job.GetID()); gtreportsRequest.SetTaskType(TaskType.Reduce); NUnit.Framework.Assert.IsNotNull("TaskReports for reduce is null", proxy.GetTaskReports (gtreportsRequest).GetTaskReportList()); IList <string> diag = proxy.GetDiagnostics(gdRequest).GetDiagnosticsList(); NUnit.Framework.Assert.AreEqual("Num diagnostics not correct", 1, diag.Count); NUnit.Framework.Assert.AreEqual("Diag 1 not correct", diagnostic1, diag[0].ToString ()); TaskReport taskReport = proxy.GetTaskReport(gtrRequest).GetTaskReport(); NUnit.Framework.Assert.AreEqual("Num diagnostics not correct", 1, taskReport.GetDiagnosticsCount ()); //send the done signal to the task app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(task.GetAttempts() .Values.GetEnumerator().Next().GetID(), TaskAttemptEventType.TaDone)); app.WaitForState(job, JobState.Succeeded); // For invalid jobid, throw IOException gtreportsRequest = recordFactory.NewRecordInstance <GetTaskReportsRequest>(); gtreportsRequest.SetJobId(TypeConverter.ToYarn(JobID.ForName("job_1415730144495_0001" ))); gtreportsRequest.SetTaskType(TaskType.Reduce); try { proxy.GetTaskReports(gtreportsRequest); NUnit.Framework.Assert.Fail("IOException not thrown for invalid job id"); } catch (IOException) { } }
//Test reports of JobHistoryServer. History server should get log files from MRApp and read them /// <exception cref="System.Exception"/> public virtual void TestReports() { Configuration config = new Configuration(); config.SetClass(CommonConfigurationKeysPublic.NetTopologyNodeSwitchMappingImplKey , typeof(TestJobHistoryParsing.MyResolver), typeof(DNSToSwitchMapping)); RackResolver.Init(config); MRApp app = new TestJobHistoryEvents.MRAppWithHistory(1, 1, true, this.GetType(). FullName, true); app.Submit(config); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.GetContext().GetAllJobs().Values .GetEnumerator().Next(); app.WaitForState(job, JobState.Succeeded); historyServer = new JobHistoryServer(); historyServer.Init(config); historyServer.Start(); // search JobHistory service JobHistory jobHistory = null; foreach (Org.Apache.Hadoop.Service.Service service in historyServer.GetServices()) { if (service is JobHistory) { jobHistory = (JobHistory)service; } } IDictionary <JobId, Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job> jobs = jobHistory. GetAllJobs(); NUnit.Framework.Assert.AreEqual(1, jobs.Count); NUnit.Framework.Assert.AreEqual("job_0_0000", jobs.Keys.GetEnumerator().Next().ToString ()); Task task = job.GetTasks().Values.GetEnumerator().Next(); TaskAttempt attempt = task.GetAttempts().Values.GetEnumerator().Next(); HistoryClientService historyService = historyServer.GetClientService(); MRClientProtocol protocol = historyService.GetClientHandler(); GetTaskAttemptReportRequest gtarRequest = recordFactory.NewRecordInstance <GetTaskAttemptReportRequest >(); // test getTaskAttemptReport TaskAttemptId taId = attempt.GetID(); taId.SetTaskId(task.GetID()); taId.GetTaskId().SetJobId(job.GetID()); gtarRequest.SetTaskAttemptId(taId); GetTaskAttemptReportResponse response = protocol.GetTaskAttemptReport(gtarRequest ); NUnit.Framework.Assert.AreEqual("container_0_0000_01_000000", response.GetTaskAttemptReport ().GetContainerId().ToString()); NUnit.Framework.Assert.IsTrue(response.GetTaskAttemptReport().GetDiagnosticInfo() .IsEmpty()); // counters NUnit.Framework.Assert.IsNotNull(response.GetTaskAttemptReport().GetCounters().GetCounter (TaskCounter.PhysicalMemoryBytes)); NUnit.Framework.Assert.AreEqual(taId.ToString(), response.GetTaskAttemptReport(). GetTaskAttemptId().ToString()); // test getTaskReport GetTaskReportRequest request = recordFactory.NewRecordInstance <GetTaskReportRequest >(); TaskId taskId = task.GetID(); taskId.SetJobId(job.GetID()); request.SetTaskId(taskId); GetTaskReportResponse reportResponse = protocol.GetTaskReport(request); NUnit.Framework.Assert.AreEqual(string.Empty, reportResponse.GetTaskReport().GetDiagnosticsList ().GetEnumerator().Next()); // progress NUnit.Framework.Assert.AreEqual(1.0f, reportResponse.GetTaskReport().GetProgress( ), 0.01); // report has corrected taskId NUnit.Framework.Assert.AreEqual(taskId.ToString(), reportResponse.GetTaskReport() .GetTaskId().ToString()); // Task state should be SUCCEEDED NUnit.Framework.Assert.AreEqual(TaskState.Succeeded, reportResponse.GetTaskReport ().GetTaskState()); // For invalid jobid, throw IOException GetTaskReportsRequest gtreportsRequest = recordFactory.NewRecordInstance <GetTaskReportsRequest >(); gtreportsRequest.SetJobId(TypeConverter.ToYarn(JobID.ForName("job_1415730144495_0001" ))); gtreportsRequest.SetTaskType(TaskType.Reduce); try { protocol.GetTaskReports(gtreportsRequest); NUnit.Framework.Assert.Fail("IOException not thrown for invalid job id"); } catch (IOException) { } // Expected // test getTaskAttemptCompletionEvents GetTaskAttemptCompletionEventsRequest taskAttemptRequest = recordFactory.NewRecordInstance <GetTaskAttemptCompletionEventsRequest>(); taskAttemptRequest.SetJobId(job.GetID()); GetTaskAttemptCompletionEventsResponse taskAttemptCompletionEventsResponse = protocol .GetTaskAttemptCompletionEvents(taskAttemptRequest); NUnit.Framework.Assert.AreEqual(0, taskAttemptCompletionEventsResponse.GetCompletionEventCount ()); // test getDiagnostics GetDiagnosticsRequest diagnosticRequest = recordFactory.NewRecordInstance <GetDiagnosticsRequest >(); diagnosticRequest.SetTaskAttemptId(taId); GetDiagnosticsResponse diagnosticResponse = protocol.GetDiagnostics(diagnosticRequest ); // it is strange : why one empty string ? NUnit.Framework.Assert.AreEqual(1, diagnosticResponse.GetDiagnosticsCount()); NUnit.Framework.Assert.AreEqual(string.Empty, diagnosticResponse.GetDiagnostics(0 )); }
/// <exception cref="System.IO.IOException"/> public virtual GetTaskAttemptReportResponse GetTaskAttemptReport(GetTaskAttemptReportRequest request) { //not invoked by anybody throw new NotImplementedException(); }
/// <exception cref="System.IO.IOException"/> public virtual GetTaskAttemptReportResponse GetTaskAttemptReport(GetTaskAttemptReportRequest request) { return(null); }