private void ConstructTaskAttemptReport() { report = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <TaskAttemptReport>(); report.SetTaskAttemptId(attemptId); report.SetTaskAttemptState(state); report.SetProgress(GetProgress()); report.SetStartTime(attemptInfo.GetStartTime()); report.SetFinishTime(attemptInfo.GetFinishTime()); report.SetShuffleFinishTime(attemptInfo.GetShuffleFinishTime()); report.SetSortFinishTime(attemptInfo.GetSortFinishTime()); if (localDiagMessage != null) { report.SetDiagnosticInfo(attemptInfo.GetError() + ", " + localDiagMessage); } else { report.SetDiagnosticInfo(attemptInfo.GetError()); } // report.setPhase(attemptInfo.get); //TODO report.SetStateString(attemptInfo.GetState()); report.SetCounters(TypeConverter.ToYarn(GetCounters())); report.SetContainerId(attemptInfo.GetContainerId()); if (attemptInfo.GetHostname() == null) { report.SetNodeManagerHost("UNKNOWN"); } else { report.SetNodeManagerHost(attemptInfo.GetHostname()); report.SetNodeManagerPort(attemptInfo.GetPort()); } report.SetNodeManagerHttpPort(attemptInfo.GetHttpPort()); }
public static TaskAttemptReport NewTaskAttemptReport(TaskAttemptId id) { ApplicationAttemptId appAttemptId = ApplicationAttemptId.NewInstance(id.GetTaskId ().GetJobId().GetAppId(), 0); ContainerId containerId = ContainerId.NewContainerId(appAttemptId, 0); TaskAttemptReport report = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <TaskAttemptReport >(); report.SetTaskAttemptId(id); report.SetStartTime(Runtime.CurrentTimeMillis() - (int)(Math.Random() * Dt)); report.SetFinishTime(Runtime.CurrentTimeMillis() + (int)(Math.Random() * Dt) + 1); if (id.GetTaskId().GetTaskType() == TaskType.Reduce) { report.SetShuffleFinishTime((report.GetFinishTime() + report.GetStartTime()) / 2); report.SetSortFinishTime((report.GetFinishTime() + report.GetShuffleFinishTime()) / 2); } report.SetPhase(Phases.Next()); report.SetTaskAttemptState(TaskAttemptStates.Next()); report.SetProgress((float)Math.Random()); report.SetCounters(TypeConverter.ToYarn(NewCounters())); report.SetContainerId(containerId); report.SetDiagnosticInfo(Diags.Next()); report.SetStateString("Moving average " + Math.Random()); return(report); }
public virtual void VerifyCompleted() { foreach (Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job in GetContext().GetAllJobs ().Values) { JobReport jobReport = job.GetReport(); System.Console.Out.WriteLine("Job start time :" + jobReport.GetStartTime()); System.Console.Out.WriteLine("Job finish time :" + jobReport.GetFinishTime()); NUnit.Framework.Assert.IsTrue("Job start time is not less than finish time", jobReport .GetStartTime() <= jobReport.GetFinishTime()); NUnit.Framework.Assert.IsTrue("Job finish time is in future", jobReport.GetFinishTime () <= Runtime.CurrentTimeMillis()); foreach (Task task in job.GetTasks().Values) { TaskReport taskReport = task.GetReport(); System.Console.Out.WriteLine("Task start time : " + taskReport.GetStartTime()); System.Console.Out.WriteLine("Task finish time : " + taskReport.GetFinishTime()); NUnit.Framework.Assert.IsTrue("Task start time is not less than finish time", taskReport .GetStartTime() <= taskReport.GetFinishTime()); foreach (TaskAttempt attempt in task.GetAttempts().Values) { TaskAttemptReport attemptReport = attempt.GetReport(); NUnit.Framework.Assert.IsTrue("Attempt start time is not less than finish time", attemptReport.GetStartTime() <= attemptReport.GetFinishTime()); } } } }
public virtual void SetTaskAttemptReport(TaskAttemptReport taskAttemptReport) { MaybeInitBuilder(); if (taskAttemptReport == null) { builder.ClearTaskAttemptReport(); } this.taskAttemptReport = taskAttemptReport; }
private void VerifyTaskAttemptReport(TaskAttemptReport tar) { NUnit.Framework.Assert.AreEqual(TaskAttemptState.Running, tar.GetTaskAttemptState ()); NUnit.Framework.Assert.IsNotNull("TaskAttemptReport is null", tar); NUnit.Framework.Assert.AreEqual(MRApp.NmHost, tar.GetNodeManagerHost()); NUnit.Framework.Assert.AreEqual(MRApp.NmPort, tar.GetNodeManagerPort()); NUnit.Framework.Assert.AreEqual(MRApp.NmHttpPort, tar.GetNodeManagerHttpPort()); NUnit.Framework.Assert.AreEqual(1, tar.GetContainerId().GetApplicationAttemptId() .GetAttemptId()); }
public static TaskAttempt NewTaskAttempt(TaskId tid, int i) { TaskAttemptId taid = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <TaskAttemptId> (); taid.SetTaskId(tid); taid.SetId(i); TaskAttemptReport report = NewTaskAttemptReport(taid); return(new _TaskAttempt_248(taid, report)); }
public virtual TaskAttemptReport GetTaskAttemptReport() { MRServiceProtos.GetTaskAttemptReportResponseProtoOrBuilder p = viaProto ? proto : builder; if (this.taskAttemptReport != null) { return(this.taskAttemptReport); } if (!p.HasTaskAttemptReport()) { return(null); } this.taskAttemptReport = ConvertFromProtoFormat(p.GetTaskAttemptReport()); return(this.taskAttemptReport); }
/// <exception cref="System.IO.IOException"/> public virtual LogParams GetLogFilePath(JobID oldJobID, TaskAttemptID oldTaskAttemptID ) { JobId jobId = TypeConverter.ToYarn(oldJobID); GetJobReportRequest request = recordFactory.NewRecordInstance <GetJobReportRequest >(); request.SetJobId(jobId); JobReport report = ((GetJobReportResponse)Invoke("getJobReport", typeof(GetJobReportRequest ), request)).GetJobReport(); if (EnumSet.Of(JobState.Succeeded, JobState.Failed, JobState.Killed, JobState.Error ).Contains(report.GetJobState())) { if (oldTaskAttemptID != null) { GetTaskAttemptReportRequest taRequest = recordFactory.NewRecordInstance <GetTaskAttemptReportRequest >(); taRequest.SetTaskAttemptId(TypeConverter.ToYarn(oldTaskAttemptID)); TaskAttemptReport taReport = ((GetTaskAttemptReportResponse)Invoke("getTaskAttemptReport" , typeof(GetTaskAttemptReportRequest), taRequest)).GetTaskAttemptReport(); if (taReport.GetContainerId() == null || taReport.GetNodeManagerHost() == null) { throw new IOException("Unable to get log information for task: " + oldTaskAttemptID ); } return(new LogParams(taReport.GetContainerId().ToString(), taReport.GetContainerId ().GetApplicationAttemptId().GetApplicationId().ToString(), NodeId.NewInstance(taReport .GetNodeManagerHost(), taReport.GetNodeManagerPort()).ToString(), report.GetUser ())); } else { if (report.GetAMInfos() == null || report.GetAMInfos().Count == 0) { throw new IOException("Unable to get log information for job: " + oldJobID); } AMInfo amInfo = report.GetAMInfos()[report.GetAMInfos().Count - 1]; return(new LogParams(amInfo.GetContainerId().ToString(), amInfo.GetAppAttemptId() .GetApplicationId().ToString(), NodeId.NewInstance(amInfo.GetNodeManagerHost(), amInfo.GetNodeManagerPort()).ToString(), report.GetUser())); } } else { throw new IOException("Cannot get log path for a in-progress job"); } }
/// <exception cref="System.Exception"/> public virtual void WaitForState(TaskAttempt attempt, TaskAttemptState finalState ) { int timeoutSecs = 0; TaskAttemptReport report = attempt.GetReport(); while (!finalState.Equals(report.GetTaskAttemptState()) && timeoutSecs++ < 20) { System.Console.Out.WriteLine("TaskAttempt State is : " + report.GetTaskAttemptState () + " Waiting for state : " + finalState + " progress : " + report.GetProgress ()); report = attempt.GetReport(); Sharpen.Thread.Sleep(500); } System.Console.Out.WriteLine("TaskAttempt State is : " + report.GetTaskAttemptState ()); NUnit.Framework.Assert.AreEqual("TaskAttempt state is not correct (timedout)", finalState , report.GetTaskAttemptState()); }
/// <exception cref="System.Exception"/> public virtual void TestCompletedTaskAttempt() { HistoryFileManager.HistoryFileInfo info = Org.Mockito.Mockito.Mock <HistoryFileManager.HistoryFileInfo >(); Org.Mockito.Mockito.When(info.GetConfFile()).ThenReturn(fullConfPath); completedJob = new CompletedJob(conf, jobId, fullHistoryPath, loadTasks, "user", info, jobAclsManager); TaskId mt1Id = MRBuilderUtils.NewTaskId(jobId, 0, TaskType.Map); TaskId rt1Id = MRBuilderUtils.NewTaskId(jobId, 0, TaskType.Reduce); TaskAttemptId mta1Id = MRBuilderUtils.NewTaskAttemptId(mt1Id, 0); TaskAttemptId rta1Id = MRBuilderUtils.NewTaskAttemptId(rt1Id, 0); Task mt1 = completedJob.GetTask(mt1Id); Task rt1 = completedJob.GetTask(rt1Id); TaskAttempt mta1 = mt1.GetAttempt(mta1Id); NUnit.Framework.Assert.AreEqual(TaskAttemptState.Succeeded, mta1.GetState()); NUnit.Framework.Assert.AreEqual("localhost:45454", mta1.GetAssignedContainerMgrAddress ()); NUnit.Framework.Assert.AreEqual("localhost:9999", mta1.GetNodeHttpAddress()); TaskAttemptReport mta1Report = mta1.GetReport(); NUnit.Framework.Assert.AreEqual(TaskAttemptState.Succeeded, mta1Report.GetTaskAttemptState ()); NUnit.Framework.Assert.AreEqual("localhost", mta1Report.GetNodeManagerHost()); NUnit.Framework.Assert.AreEqual(45454, mta1Report.GetNodeManagerPort()); NUnit.Framework.Assert.AreEqual(9999, mta1Report.GetNodeManagerHttpPort()); TaskAttempt rta1 = rt1.GetAttempt(rta1Id); NUnit.Framework.Assert.AreEqual(TaskAttemptState.Succeeded, rta1.GetState()); NUnit.Framework.Assert.AreEqual("localhost:45454", rta1.GetAssignedContainerMgrAddress ()); NUnit.Framework.Assert.AreEqual("localhost:9999", rta1.GetNodeHttpAddress()); TaskAttemptReport rta1Report = rta1.GetReport(); NUnit.Framework.Assert.AreEqual(TaskAttemptState.Succeeded, rta1Report.GetTaskAttemptState ()); NUnit.Framework.Assert.AreEqual("localhost", rta1Report.GetNodeManagerHost()); NUnit.Framework.Assert.AreEqual(45454, rta1Report.GetNodeManagerPort()); NUnit.Framework.Assert.AreEqual(9999, rta1Report.GetNodeManagerHttpPort()); }
public TaskAttemptInfo(TaskAttempt ta, TaskType type, bool isRunning) { TaskAttemptReport report = ta.GetReport(); this.type = type.ToString(); this.id = MRApps.ToString(ta.GetID()); this.nodeHttpAddress = ta.GetNodeHttpAddress(); this.startTime = report.GetStartTime(); this.finishTime = report.GetFinishTime(); this.assignedContainerId = ConverterUtils.ToString(report.GetContainerId()); this.assignedContainer = report.GetContainerId(); this.progress = report.GetProgress() * 100; this.status = report.GetStateString(); this.state = report.GetTaskAttemptState(); this.elapsedTime = Times.Elapsed(this.startTime, this.finishTime, isRunning); if (this.elapsedTime == -1) { this.elapsedTime = 0; } this.diagnostics = report.GetDiagnosticInfo(); this.rack = ta.GetNodeRackName(); }
private MRProtos.TaskAttemptReportProto ConvertToProtoFormat(TaskAttemptReport t) { return(((TaskAttemptReportPBImpl)t).GetProto()); }
public virtual void Test() { TestMRClientService.MRAppWithClientService app = new TestMRClientService.MRAppWithClientService (this, 1, 0, false); Configuration conf = new Configuration(); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(conf); app.WaitForState(job, JobState.Running); NUnit.Framework.Assert.AreEqual("Num tasks not correct", 1, job.GetTasks().Count); IEnumerator <Task> it = job.GetTasks().Values.GetEnumerator(); Task task = it.Next(); app.WaitForState(task, TaskState.Running); TaskAttempt attempt = task.GetAttempts().Values.GetEnumerator().Next(); app.WaitForState(attempt, TaskAttemptState.Running); // send the diagnostic string diagnostic1 = "Diagnostic1"; string diagnostic2 = "Diagnostic2"; app.GetContext().GetEventHandler().Handle(new TaskAttemptDiagnosticsUpdateEvent(attempt .GetID(), diagnostic1)); // send the status update TaskAttemptStatusUpdateEvent.TaskAttemptStatus taskAttemptStatus = new TaskAttemptStatusUpdateEvent.TaskAttemptStatus (); taskAttemptStatus.id = attempt.GetID(); taskAttemptStatus.progress = 0.5f; taskAttemptStatus.stateString = "RUNNING"; taskAttemptStatus.taskState = TaskAttemptState.Running; taskAttemptStatus.phase = Phase.Map; // send the status update app.GetContext().GetEventHandler().Handle(new TaskAttemptStatusUpdateEvent(attempt .GetID(), taskAttemptStatus)); //verify that all object are fully populated by invoking RPCs. YarnRPC rpc = YarnRPC.Create(conf); MRClientProtocol proxy = (MRClientProtocol)rpc.GetProxy(typeof(MRClientProtocol), app.clientService.GetBindAddress(), conf); GetCountersRequest gcRequest = recordFactory.NewRecordInstance <GetCountersRequest >(); gcRequest.SetJobId(job.GetID()); NUnit.Framework.Assert.IsNotNull("Counters is null", proxy.GetCounters(gcRequest) .GetCounters()); GetJobReportRequest gjrRequest = recordFactory.NewRecordInstance <GetJobReportRequest >(); gjrRequest.SetJobId(job.GetID()); JobReport jr = proxy.GetJobReport(gjrRequest).GetJobReport(); VerifyJobReport(jr); GetTaskAttemptCompletionEventsRequest gtaceRequest = recordFactory.NewRecordInstance <GetTaskAttemptCompletionEventsRequest>(); gtaceRequest.SetJobId(job.GetID()); gtaceRequest.SetFromEventId(0); gtaceRequest.SetMaxEvents(10); NUnit.Framework.Assert.IsNotNull("TaskCompletionEvents is null", proxy.GetTaskAttemptCompletionEvents (gtaceRequest).GetCompletionEventList()); GetDiagnosticsRequest gdRequest = recordFactory.NewRecordInstance <GetDiagnosticsRequest >(); gdRequest.SetTaskAttemptId(attempt.GetID()); NUnit.Framework.Assert.IsNotNull("Diagnostics is null", proxy.GetDiagnostics(gdRequest ).GetDiagnosticsList()); GetTaskAttemptReportRequest gtarRequest = recordFactory.NewRecordInstance <GetTaskAttemptReportRequest >(); gtarRequest.SetTaskAttemptId(attempt.GetID()); TaskAttemptReport tar = proxy.GetTaskAttemptReport(gtarRequest).GetTaskAttemptReport (); VerifyTaskAttemptReport(tar); GetTaskReportRequest gtrRequest = recordFactory.NewRecordInstance <GetTaskReportRequest >(); gtrRequest.SetTaskId(task.GetID()); NUnit.Framework.Assert.IsNotNull("TaskReport is null", proxy.GetTaskReport(gtrRequest ).GetTaskReport()); GetTaskReportsRequest gtreportsRequest = recordFactory.NewRecordInstance <GetTaskReportsRequest >(); gtreportsRequest.SetJobId(job.GetID()); gtreportsRequest.SetTaskType(TaskType.Map); NUnit.Framework.Assert.IsNotNull("TaskReports for map is null", proxy.GetTaskReports (gtreportsRequest).GetTaskReportList()); gtreportsRequest = recordFactory.NewRecordInstance <GetTaskReportsRequest>(); gtreportsRequest.SetJobId(job.GetID()); gtreportsRequest.SetTaskType(TaskType.Reduce); NUnit.Framework.Assert.IsNotNull("TaskReports for reduce is null", proxy.GetTaskReports (gtreportsRequest).GetTaskReportList()); IList <string> diag = proxy.GetDiagnostics(gdRequest).GetDiagnosticsList(); NUnit.Framework.Assert.AreEqual("Num diagnostics not correct", 1, diag.Count); NUnit.Framework.Assert.AreEqual("Diag 1 not correct", diagnostic1, diag[0].ToString ()); TaskReport taskReport = proxy.GetTaskReport(gtrRequest).GetTaskReport(); NUnit.Framework.Assert.AreEqual("Num diagnostics not correct", 1, taskReport.GetDiagnosticsCount ()); //send the done signal to the task app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(task.GetAttempts() .Values.GetEnumerator().Next().GetID(), TaskAttemptEventType.TaDone)); app.WaitForState(job, JobState.Succeeded); // For invalid jobid, throw IOException gtreportsRequest = recordFactory.NewRecordInstance <GetTaskReportsRequest>(); gtreportsRequest.SetJobId(TypeConverter.ToYarn(JobID.ForName("job_1415730144495_0001" ))); gtreportsRequest.SetTaskType(TaskType.Reduce); try { proxy.GetTaskReports(gtreportsRequest); NUnit.Framework.Assert.Fail("IOException not thrown for invalid job id"); } catch (IOException) { } }
public _TaskAttempt_248(TaskAttemptId taid, TaskAttemptReport report) { this.taid = taid; this.report = report; }
public virtual void TestAttemptsBlock() { AppContext ctx = Org.Mockito.Mockito.Mock <AppContext>(); AppForTest app = new AppForTest(ctx); Task task = GetTask(0); IDictionary <TaskAttemptId, TaskAttempt> attempts = new Dictionary <TaskAttemptId, TaskAttempt>(); TaskAttempt attempt = Org.Mockito.Mockito.Mock <TaskAttempt>(); TaskAttemptId taId = new TaskAttemptIdPBImpl(); taId.SetId(0); taId.SetTaskId(task.GetID()); Org.Mockito.Mockito.When(attempt.GetID()).ThenReturn(taId); Org.Mockito.Mockito.When(attempt.GetNodeHttpAddress()).ThenReturn("Node address"); ApplicationId appId = ApplicationIdPBImpl.NewInstance(0, 5); ApplicationAttemptId appAttemptId = ApplicationAttemptIdPBImpl.NewInstance(appId, 1); ContainerId containerId = ContainerIdPBImpl.NewContainerId(appAttemptId, 1); Org.Mockito.Mockito.When(attempt.GetAssignedContainerID()).ThenReturn(containerId ); Org.Mockito.Mockito.When(attempt.GetAssignedContainerMgrAddress()).ThenReturn("assignedContainerMgrAddress" ); Org.Mockito.Mockito.When(attempt.GetNodeRackName()).ThenReturn("nodeRackName"); long taStartTime = 100002L; long taFinishTime = 100012L; long taShuffleFinishTime = 100010L; long taSortFinishTime = 100011L; TaskAttemptState taState = TaskAttemptState.Succeeded; Org.Mockito.Mockito.When(attempt.GetLaunchTime()).ThenReturn(taStartTime); Org.Mockito.Mockito.When(attempt.GetFinishTime()).ThenReturn(taFinishTime); Org.Mockito.Mockito.When(attempt.GetShuffleFinishTime()).ThenReturn(taShuffleFinishTime ); Org.Mockito.Mockito.When(attempt.GetSortFinishTime()).ThenReturn(taSortFinishTime ); Org.Mockito.Mockito.When(attempt.GetState()).ThenReturn(taState); TaskAttemptReport taReport = Org.Mockito.Mockito.Mock <TaskAttemptReport>(); Org.Mockito.Mockito.When(taReport.GetStartTime()).ThenReturn(taStartTime); Org.Mockito.Mockito.When(taReport.GetFinishTime()).ThenReturn(taFinishTime); Org.Mockito.Mockito.When(taReport.GetShuffleFinishTime()).ThenReturn(taShuffleFinishTime ); Org.Mockito.Mockito.When(taReport.GetSortFinishTime()).ThenReturn(taSortFinishTime ); Org.Mockito.Mockito.When(taReport.GetContainerId()).ThenReturn(containerId); Org.Mockito.Mockito.When(taReport.GetProgress()).ThenReturn(1.0f); Org.Mockito.Mockito.When(taReport.GetStateString()).ThenReturn("Processed 128/128 records <p> \n" ); Org.Mockito.Mockito.When(taReport.GetTaskAttemptState()).ThenReturn(taState); Org.Mockito.Mockito.When(taReport.GetDiagnosticInfo()).ThenReturn(string.Empty); Org.Mockito.Mockito.When(attempt.GetReport()).ThenReturn(taReport); attempts[taId] = attempt; Org.Mockito.Mockito.When(task.GetAttempts()).ThenReturn(attempts); app.SetTask(task); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job >(); Org.Mockito.Mockito.When(job.GetUserName()).ThenReturn("User"); app.SetJob(job); TestBlocks.AttemptsBlockForTest block = new TestBlocks.AttemptsBlockForTest(this, app); block.AddParameter(AMParams.TaskType, "r"); PrintWriter pWriter = new PrintWriter(data); HtmlBlock.Block html = new BlockForTest(new TestBlocks.HtmlBlockForTest(this), pWriter , 0, false); block.Render(html); pWriter.Flush(); // should be printed information about attempts NUnit.Framework.Assert.IsTrue(data.ToString().Contains("0 attempt_0_0001_r_000000_0" )); NUnit.Framework.Assert.IsTrue(data.ToString().Contains("SUCCEEDED")); NUnit.Framework.Assert.IsFalse(data.ToString().Contains("Processed 128/128 records <p> \n" )); NUnit.Framework.Assert.IsTrue(data.ToString().Contains("Processed 128\\/128 records <p> \\n" )); NUnit.Framework.Assert.IsTrue(data.ToString().Contains("_0005_01_000001:attempt_0_0001_r_000000_0:User:"******"100002")); NUnit.Framework.Assert.IsTrue(data.ToString().Contains("100010")); NUnit.Framework.Assert.IsTrue(data.ToString().Contains("100011")); NUnit.Framework.Assert.IsTrue(data.ToString().Contains("100012")); }