Exemplo n.º 1
0
            /// <exception cref="System.IO.IOException"/>
            public virtual GetTaskAttemptReportResponse GetTaskAttemptReport(GetTaskAttemptReportRequest
                                                                             request)
            {
                TaskAttemptId taskAttemptId           = request.GetTaskAttemptId();
                GetTaskAttemptReportResponse response = this.recordFactory.NewRecordInstance <GetTaskAttemptReportResponse
                                                                                              >();

                response.SetTaskAttemptReport(this.VerifyAndGetAttempt(taskAttemptId, JobACL.ViewJob
                                                                       ).GetReport());
                return(response);
            }
Exemplo n.º 2
0
            /// <exception cref="System.IO.IOException"/>
            public virtual GetTaskAttemptReportResponse GetTaskAttemptReport(GetTaskAttemptReportRequest
                                                                             request)
            {
                TaskAttemptId taskAttemptId = request.GetTaskAttemptId();

                Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = this.VerifyAndGetJob(taskAttemptId
                                                                                      .GetTaskId().GetJobId(), true);
                GetTaskAttemptReportResponse response = this.recordFactory.NewRecordInstance <GetTaskAttemptReportResponse
                                                                                              >();

                response.SetTaskAttemptReport(job.GetTask(taskAttemptId.GetTaskId()).GetAttempt(taskAttemptId
                                                                                                ).GetReport());
                return(response);
            }
Exemplo n.º 3
0
 /// <exception cref="System.IO.IOException"/>
 public virtual GetTaskAttemptReportResponse GetTaskAttemptReport(GetTaskAttemptReportRequest
                                                                  request)
 {
     MRServiceProtos.GetTaskAttemptReportRequestProto requestProto = ((GetTaskAttemptReportRequestPBImpl
                                                                       )request).GetProto();
     try
     {
         return(new GetTaskAttemptReportResponsePBImpl(proxy.GetTaskAttemptReport(null, requestProto
                                                                                  )));
     }
     catch (ServiceException e)
     {
         throw UnwrapAndThrowException(e);
     }
 }
Exemplo n.º 4
0
        /// <exception cref="System.IO.IOException"/>
        public virtual LogParams GetLogFilePath(JobID oldJobID, TaskAttemptID oldTaskAttemptID
                                                )
        {
            JobId jobId = TypeConverter.ToYarn(oldJobID);
            GetJobReportRequest request = recordFactory.NewRecordInstance <GetJobReportRequest
                                                                           >();

            request.SetJobId(jobId);
            JobReport report = ((GetJobReportResponse)Invoke("getJobReport", typeof(GetJobReportRequest
                                                                                    ), request)).GetJobReport();

            if (EnumSet.Of(JobState.Succeeded, JobState.Failed, JobState.Killed, JobState.Error
                           ).Contains(report.GetJobState()))
            {
                if (oldTaskAttemptID != null)
                {
                    GetTaskAttemptReportRequest taRequest = recordFactory.NewRecordInstance <GetTaskAttemptReportRequest
                                                                                             >();
                    taRequest.SetTaskAttemptId(TypeConverter.ToYarn(oldTaskAttemptID));
                    TaskAttemptReport taReport = ((GetTaskAttemptReportResponse)Invoke("getTaskAttemptReport"
                                                                                       , typeof(GetTaskAttemptReportRequest), taRequest)).GetTaskAttemptReport();
                    if (taReport.GetContainerId() == null || taReport.GetNodeManagerHost() == null)
                    {
                        throw new IOException("Unable to get log information for task: " + oldTaskAttemptID
                                              );
                    }
                    return(new LogParams(taReport.GetContainerId().ToString(), taReport.GetContainerId
                                             ().GetApplicationAttemptId().GetApplicationId().ToString(), NodeId.NewInstance(taReport
                                                                                                                            .GetNodeManagerHost(), taReport.GetNodeManagerPort()).ToString(), report.GetUser
                                             ()));
                }
                else
                {
                    if (report.GetAMInfos() == null || report.GetAMInfos().Count == 0)
                    {
                        throw new IOException("Unable to get log information for job: " + oldJobID);
                    }
                    AMInfo amInfo = report.GetAMInfos()[report.GetAMInfos().Count - 1];
                    return(new LogParams(amInfo.GetContainerId().ToString(), amInfo.GetAppAttemptId()
                                         .GetApplicationId().ToString(), NodeId.NewInstance(amInfo.GetNodeManagerHost(),
                                                                                            amInfo.GetNodeManagerPort()).ToString(), report.GetUser()));
                }
            }
            else
            {
                throw new IOException("Cannot get log path for a in-progress job");
            }
        }
Exemplo n.º 5
0
        public virtual void Test()
        {
            TestMRClientService.MRAppWithClientService app = new TestMRClientService.MRAppWithClientService
                                                                 (this, 1, 0, false);
            Configuration conf = new Configuration();

            Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(conf);
            app.WaitForState(job, JobState.Running);
            NUnit.Framework.Assert.AreEqual("Num tasks not correct", 1, job.GetTasks().Count);
            IEnumerator <Task> it = job.GetTasks().Values.GetEnumerator();
            Task task             = it.Next();

            app.WaitForState(task, TaskState.Running);
            TaskAttempt attempt = task.GetAttempts().Values.GetEnumerator().Next();

            app.WaitForState(attempt, TaskAttemptState.Running);
            // send the diagnostic
            string diagnostic1 = "Diagnostic1";
            string diagnostic2 = "Diagnostic2";

            app.GetContext().GetEventHandler().Handle(new TaskAttemptDiagnosticsUpdateEvent(attempt
                                                                                            .GetID(), diagnostic1));
            // send the status update
            TaskAttemptStatusUpdateEvent.TaskAttemptStatus taskAttemptStatus = new TaskAttemptStatusUpdateEvent.TaskAttemptStatus
                                                                                   ();
            taskAttemptStatus.id          = attempt.GetID();
            taskAttemptStatus.progress    = 0.5f;
            taskAttemptStatus.stateString = "RUNNING";
            taskAttemptStatus.taskState   = TaskAttemptState.Running;
            taskAttemptStatus.phase       = Phase.Map;
            // send the status update
            app.GetContext().GetEventHandler().Handle(new TaskAttemptStatusUpdateEvent(attempt
                                                                                       .GetID(), taskAttemptStatus));
            //verify that all object are fully populated by invoking RPCs.
            YarnRPC          rpc   = YarnRPC.Create(conf);
            MRClientProtocol proxy = (MRClientProtocol)rpc.GetProxy(typeof(MRClientProtocol),
                                                                    app.clientService.GetBindAddress(), conf);
            GetCountersRequest gcRequest = recordFactory.NewRecordInstance <GetCountersRequest
                                                                            >();

            gcRequest.SetJobId(job.GetID());
            NUnit.Framework.Assert.IsNotNull("Counters is null", proxy.GetCounters(gcRequest)
                                             .GetCounters());
            GetJobReportRequest gjrRequest = recordFactory.NewRecordInstance <GetJobReportRequest
                                                                              >();

            gjrRequest.SetJobId(job.GetID());
            JobReport jr = proxy.GetJobReport(gjrRequest).GetJobReport();

            VerifyJobReport(jr);
            GetTaskAttemptCompletionEventsRequest gtaceRequest = recordFactory.NewRecordInstance
                                                                 <GetTaskAttemptCompletionEventsRequest>();

            gtaceRequest.SetJobId(job.GetID());
            gtaceRequest.SetFromEventId(0);
            gtaceRequest.SetMaxEvents(10);
            NUnit.Framework.Assert.IsNotNull("TaskCompletionEvents is null", proxy.GetTaskAttemptCompletionEvents
                                                 (gtaceRequest).GetCompletionEventList());
            GetDiagnosticsRequest gdRequest = recordFactory.NewRecordInstance <GetDiagnosticsRequest
                                                                               >();

            gdRequest.SetTaskAttemptId(attempt.GetID());
            NUnit.Framework.Assert.IsNotNull("Diagnostics is null", proxy.GetDiagnostics(gdRequest
                                                                                         ).GetDiagnosticsList());
            GetTaskAttemptReportRequest gtarRequest = recordFactory.NewRecordInstance <GetTaskAttemptReportRequest
                                                                                       >();

            gtarRequest.SetTaskAttemptId(attempt.GetID());
            TaskAttemptReport tar = proxy.GetTaskAttemptReport(gtarRequest).GetTaskAttemptReport
                                        ();

            VerifyTaskAttemptReport(tar);
            GetTaskReportRequest gtrRequest = recordFactory.NewRecordInstance <GetTaskReportRequest
                                                                               >();

            gtrRequest.SetTaskId(task.GetID());
            NUnit.Framework.Assert.IsNotNull("TaskReport is null", proxy.GetTaskReport(gtrRequest
                                                                                       ).GetTaskReport());
            GetTaskReportsRequest gtreportsRequest = recordFactory.NewRecordInstance <GetTaskReportsRequest
                                                                                      >();

            gtreportsRequest.SetJobId(job.GetID());
            gtreportsRequest.SetTaskType(TaskType.Map);
            NUnit.Framework.Assert.IsNotNull("TaskReports for map is null", proxy.GetTaskReports
                                                 (gtreportsRequest).GetTaskReportList());
            gtreportsRequest = recordFactory.NewRecordInstance <GetTaskReportsRequest>();
            gtreportsRequest.SetJobId(job.GetID());
            gtreportsRequest.SetTaskType(TaskType.Reduce);
            NUnit.Framework.Assert.IsNotNull("TaskReports for reduce is null", proxy.GetTaskReports
                                                 (gtreportsRequest).GetTaskReportList());
            IList <string> diag = proxy.GetDiagnostics(gdRequest).GetDiagnosticsList();

            NUnit.Framework.Assert.AreEqual("Num diagnostics not correct", 1, diag.Count);
            NUnit.Framework.Assert.AreEqual("Diag 1 not correct", diagnostic1, diag[0].ToString
                                                ());
            TaskReport taskReport = proxy.GetTaskReport(gtrRequest).GetTaskReport();

            NUnit.Framework.Assert.AreEqual("Num diagnostics not correct", 1, taskReport.GetDiagnosticsCount
                                                ());
            //send the done signal to the task
            app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(task.GetAttempts()
                                                                           .Values.GetEnumerator().Next().GetID(), TaskAttemptEventType.TaDone));
            app.WaitForState(job, JobState.Succeeded);
            // For invalid jobid, throw IOException
            gtreportsRequest = recordFactory.NewRecordInstance <GetTaskReportsRequest>();
            gtreportsRequest.SetJobId(TypeConverter.ToYarn(JobID.ForName("job_1415730144495_0001"
                                                                         )));
            gtreportsRequest.SetTaskType(TaskType.Reduce);
            try
            {
                proxy.GetTaskReports(gtreportsRequest);
                NUnit.Framework.Assert.Fail("IOException not thrown for invalid job id");
            }
            catch (IOException)
            {
            }
        }
Exemplo n.º 6
0
        //Test reports of  JobHistoryServer. History server should get log files from  MRApp and read them
        /// <exception cref="System.Exception"/>
        public virtual void TestReports()
        {
            Configuration config = new Configuration();

            config.SetClass(CommonConfigurationKeysPublic.NetTopologyNodeSwitchMappingImplKey
                            , typeof(TestJobHistoryParsing.MyResolver), typeof(DNSToSwitchMapping));
            RackResolver.Init(config);
            MRApp app = new TestJobHistoryEvents.MRAppWithHistory(1, 1, true, this.GetType().
                                                                  FullName, true);

            app.Submit(config);
            Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.GetContext().GetAllJobs().Values
                                                             .GetEnumerator().Next();
            app.WaitForState(job, JobState.Succeeded);
            historyServer = new JobHistoryServer();
            historyServer.Init(config);
            historyServer.Start();
            // search JobHistory  service
            JobHistory jobHistory = null;

            foreach (Org.Apache.Hadoop.Service.Service service in historyServer.GetServices())
            {
                if (service is JobHistory)
                {
                    jobHistory = (JobHistory)service;
                }
            }
            IDictionary <JobId, Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job> jobs = jobHistory.
                                                                                   GetAllJobs();

            NUnit.Framework.Assert.AreEqual(1, jobs.Count);
            NUnit.Framework.Assert.AreEqual("job_0_0000", jobs.Keys.GetEnumerator().Next().ToString
                                                ());
            Task                        task           = job.GetTasks().Values.GetEnumerator().Next();
            TaskAttempt                 attempt        = task.GetAttempts().Values.GetEnumerator().Next();
            HistoryClientService        historyService = historyServer.GetClientService();
            MRClientProtocol            protocol       = historyService.GetClientHandler();
            GetTaskAttemptReportRequest gtarRequest    = recordFactory.NewRecordInstance <GetTaskAttemptReportRequest
                                                                                          >();
            // test getTaskAttemptReport
            TaskAttemptId taId = attempt.GetID();

            taId.SetTaskId(task.GetID());
            taId.GetTaskId().SetJobId(job.GetID());
            gtarRequest.SetTaskAttemptId(taId);
            GetTaskAttemptReportResponse response = protocol.GetTaskAttemptReport(gtarRequest
                                                                                  );

            NUnit.Framework.Assert.AreEqual("container_0_0000_01_000000", response.GetTaskAttemptReport
                                                ().GetContainerId().ToString());
            NUnit.Framework.Assert.IsTrue(response.GetTaskAttemptReport().GetDiagnosticInfo()
                                          .IsEmpty());
            // counters
            NUnit.Framework.Assert.IsNotNull(response.GetTaskAttemptReport().GetCounters().GetCounter
                                                 (TaskCounter.PhysicalMemoryBytes));
            NUnit.Framework.Assert.AreEqual(taId.ToString(), response.GetTaskAttemptReport().
                                            GetTaskAttemptId().ToString());
            // test getTaskReport
            GetTaskReportRequest request = recordFactory.NewRecordInstance <GetTaskReportRequest
                                                                            >();
            TaskId taskId = task.GetID();

            taskId.SetJobId(job.GetID());
            request.SetTaskId(taskId);
            GetTaskReportResponse reportResponse = protocol.GetTaskReport(request);

            NUnit.Framework.Assert.AreEqual(string.Empty, reportResponse.GetTaskReport().GetDiagnosticsList
                                                ().GetEnumerator().Next());
            // progress
            NUnit.Framework.Assert.AreEqual(1.0f, reportResponse.GetTaskReport().GetProgress(
                                                ), 0.01);
            // report has corrected taskId
            NUnit.Framework.Assert.AreEqual(taskId.ToString(), reportResponse.GetTaskReport()
                                            .GetTaskId().ToString());
            // Task state should be SUCCEEDED
            NUnit.Framework.Assert.AreEqual(TaskState.Succeeded, reportResponse.GetTaskReport
                                                ().GetTaskState());
            // For invalid jobid, throw IOException
            GetTaskReportsRequest gtreportsRequest = recordFactory.NewRecordInstance <GetTaskReportsRequest
                                                                                      >();

            gtreportsRequest.SetJobId(TypeConverter.ToYarn(JobID.ForName("job_1415730144495_0001"
                                                                         )));
            gtreportsRequest.SetTaskType(TaskType.Reduce);
            try
            {
                protocol.GetTaskReports(gtreportsRequest);
                NUnit.Framework.Assert.Fail("IOException not thrown for invalid job id");
            }
            catch (IOException)
            {
            }
            // Expected
            // test getTaskAttemptCompletionEvents
            GetTaskAttemptCompletionEventsRequest taskAttemptRequest = recordFactory.NewRecordInstance
                                                                       <GetTaskAttemptCompletionEventsRequest>();

            taskAttemptRequest.SetJobId(job.GetID());
            GetTaskAttemptCompletionEventsResponse taskAttemptCompletionEventsResponse = protocol
                                                                                         .GetTaskAttemptCompletionEvents(taskAttemptRequest);

            NUnit.Framework.Assert.AreEqual(0, taskAttemptCompletionEventsResponse.GetCompletionEventCount
                                                ());
            // test getDiagnostics
            GetDiagnosticsRequest diagnosticRequest = recordFactory.NewRecordInstance <GetDiagnosticsRequest
                                                                                       >();

            diagnosticRequest.SetTaskAttemptId(taId);
            GetDiagnosticsResponse diagnosticResponse = protocol.GetDiagnostics(diagnosticRequest
                                                                                );

            // it is strange : why one empty string ?
            NUnit.Framework.Assert.AreEqual(1, diagnosticResponse.GetDiagnosticsCount());
            NUnit.Framework.Assert.AreEqual(string.Empty, diagnosticResponse.GetDiagnostics(0
                                                                                            ));
        }
Exemplo n.º 7
0
 /// <exception cref="System.IO.IOException"/>
 public virtual GetTaskAttemptReportResponse GetTaskAttemptReport(GetTaskAttemptReportRequest
                                                                  request)
 {
     //not invoked by anybody
     throw new NotImplementedException();
 }
Exemplo n.º 8
0
 /// <exception cref="System.IO.IOException"/>
 public virtual GetTaskAttemptReportResponse GetTaskAttemptReport(GetTaskAttemptReportRequest
                                                                  request)
 {
     return(null);
 }