Пример #1
0
 public virtual void VerifyReduceTaskAttemptGeneric(TaskAttempt ta, long shuffleFinishTime
                                                    , long mergeFinishTime, long elapsedShuffleTime, long elapsedMergeTime, long elapsedReduceTime
                                                    )
 {
     NUnit.Framework.Assert.AreEqual("shuffleFinishTime wrong", ta.GetShuffleFinishTime
                                         (), shuffleFinishTime);
     NUnit.Framework.Assert.AreEqual("mergeFinishTime wrong", ta.GetSortFinishTime(),
                                     mergeFinishTime);
     NUnit.Framework.Assert.AreEqual("elapsedShuffleTime wrong", ta.GetShuffleFinishTime
                                         () - ta.GetLaunchTime(), elapsedShuffleTime);
     NUnit.Framework.Assert.AreEqual("elapsedMergeTime wrong", ta.GetSortFinishTime()
                                     - ta.GetShuffleFinishTime(), elapsedMergeTime);
     NUnit.Framework.Assert.AreEqual("elapsedReduceTime wrong", ta.GetFinishTime() - ta
                                     .GetSortFinishTime(), elapsedReduceTime);
 }
Пример #2
0
        public virtual void TestAverageReduceTime()
        {
            Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = Org.Mockito.Mockito.Mock <CompletedJob
                                                                                       >();
            Task          task1          = Org.Mockito.Mockito.Mock <Task>();
            Task          task2          = Org.Mockito.Mockito.Mock <Task>();
            JobId         jobId          = MRBuilderUtils.NewJobId(1L, 1, 1);
            TaskId        taskId1        = MRBuilderUtils.NewTaskId(jobId, 1, TaskType.Reduce);
            TaskId        taskId2        = MRBuilderUtils.NewTaskId(jobId, 2, TaskType.Reduce);
            TaskAttemptId taskAttemptId1 = MRBuilderUtils.NewTaskAttemptId(taskId1, 1);
            TaskAttemptId taskAttemptId2 = MRBuilderUtils.NewTaskAttemptId(taskId2, 2);
            TaskAttempt   taskAttempt1   = Org.Mockito.Mockito.Mock <TaskAttempt>();
            TaskAttempt   taskAttempt2   = Org.Mockito.Mockito.Mock <TaskAttempt>();
            JobReport     jobReport      = Org.Mockito.Mockito.Mock <JobReport>();

            Org.Mockito.Mockito.When(taskAttempt1.GetState()).ThenReturn(TaskAttemptState.Succeeded
                                                                         );
            Org.Mockito.Mockito.When(taskAttempt1.GetLaunchTime()).ThenReturn(0L);
            Org.Mockito.Mockito.When(taskAttempt1.GetShuffleFinishTime()).ThenReturn(4L);
            Org.Mockito.Mockito.When(taskAttempt1.GetSortFinishTime()).ThenReturn(6L);
            Org.Mockito.Mockito.When(taskAttempt1.GetFinishTime()).ThenReturn(8L);
            Org.Mockito.Mockito.When(taskAttempt2.GetState()).ThenReturn(TaskAttemptState.Succeeded
                                                                         );
            Org.Mockito.Mockito.When(taskAttempt2.GetLaunchTime()).ThenReturn(5L);
            Org.Mockito.Mockito.When(taskAttempt2.GetShuffleFinishTime()).ThenReturn(10L);
            Org.Mockito.Mockito.When(taskAttempt2.GetSortFinishTime()).ThenReturn(22L);
            Org.Mockito.Mockito.When(taskAttempt2.GetFinishTime()).ThenReturn(42L);
            Org.Mockito.Mockito.When(task1.GetType()).ThenReturn(TaskType.Reduce);
            Org.Mockito.Mockito.When(task2.GetType()).ThenReturn(TaskType.Reduce);
            Org.Mockito.Mockito.When(task1.GetAttempts()).ThenReturn(new _Dictionary_120(taskAttemptId1
                                                                                         , taskAttempt1));
            Org.Mockito.Mockito.When(task2.GetAttempts()).ThenReturn(new _Dictionary_123(taskAttemptId2
                                                                                         , taskAttempt2));
            Org.Mockito.Mockito.When(job.GetTasks()).ThenReturn(new _Dictionary_127(taskId1,
                                                                                    task1, taskId2, task2));
            Org.Mockito.Mockito.When(job.GetID()).ThenReturn(jobId);
            Org.Mockito.Mockito.When(job.GetReport()).ThenReturn(jobReport);
            Org.Mockito.Mockito.When(job.GetName()).ThenReturn("TestJobInfo");
            Org.Mockito.Mockito.When(job.GetState()).ThenReturn(JobState.Succeeded);
            JobInfo jobInfo = new JobInfo(job);

            NUnit.Framework.Assert.AreEqual(11L, jobInfo.GetAvgReduceTime());
        }
Пример #3
0
        public virtual void VerifyTaskAttemptGeneric(TaskAttempt ta, TaskType ttype, string
                                                     id, string state, string type, string rack, string nodeHttpAddress, string diagnostics
                                                     , string assignedContainerId, long startTime, long finishTime, long elapsedTime,
                                                     float progress)
        {
            TaskAttemptId attid     = ta.GetID();
            string        attemptId = MRApps.ToString(attid);

            WebServicesTestUtils.CheckStringMatch("id", attemptId, id);
            WebServicesTestUtils.CheckStringMatch("type", ttype.ToString(), type);
            WebServicesTestUtils.CheckStringMatch("state", ta.GetState().ToString(), state);
            WebServicesTestUtils.CheckStringMatch("rack", ta.GetNodeRackName(), rack);
            WebServicesTestUtils.CheckStringMatch("nodeHttpAddress", ta.GetNodeHttpAddress(),
                                                  nodeHttpAddress);
            string         expectDiag      = string.Empty;
            IList <string> diagnosticsList = ta.GetDiagnostics();

            if (diagnosticsList != null && !diagnostics.IsEmpty())
            {
                StringBuilder b = new StringBuilder();
                foreach (string diag in diagnosticsList)
                {
                    b.Append(diag);
                }
                expectDiag = b.ToString();
            }
            WebServicesTestUtils.CheckStringMatch("diagnostics", expectDiag, diagnostics);
            WebServicesTestUtils.CheckStringMatch("assignedContainerId", ConverterUtils.ToString
                                                      (ta.GetAssignedContainerID()), assignedContainerId);
            NUnit.Framework.Assert.AreEqual("startTime wrong", ta.GetLaunchTime(), startTime);
            NUnit.Framework.Assert.AreEqual("finishTime wrong", ta.GetFinishTime(), finishTime
                                            );
            NUnit.Framework.Assert.AreEqual("elapsedTime wrong", finishTime - startTime, elapsedTime
                                            );
            NUnit.Framework.Assert.AreEqual("progress wrong", ta.GetProgress() * 100, progress
                                            , 1e-3f);
        }
Пример #4
0
        public virtual void TestAttemptsBlock()
        {
            AppContext ctx  = Org.Mockito.Mockito.Mock <AppContext>();
            AppForTest app  = new AppForTest(ctx);
            Task       task = GetTask(0);
            IDictionary <TaskAttemptId, TaskAttempt> attempts = new Dictionary <TaskAttemptId,
                                                                                TaskAttempt>();
            TaskAttempt   attempt = Org.Mockito.Mockito.Mock <TaskAttempt>();
            TaskAttemptId taId    = new TaskAttemptIdPBImpl();

            taId.SetId(0);
            taId.SetTaskId(task.GetID());
            Org.Mockito.Mockito.When(attempt.GetID()).ThenReturn(taId);
            Org.Mockito.Mockito.When(attempt.GetNodeHttpAddress()).ThenReturn("Node address");
            ApplicationId        appId        = ApplicationIdPBImpl.NewInstance(0, 5);
            ApplicationAttemptId appAttemptId = ApplicationAttemptIdPBImpl.NewInstance(appId,
                                                                                       1);
            ContainerId containerId = ContainerIdPBImpl.NewContainerId(appAttemptId, 1);

            Org.Mockito.Mockito.When(attempt.GetAssignedContainerID()).ThenReturn(containerId
                                                                                  );
            Org.Mockito.Mockito.When(attempt.GetAssignedContainerMgrAddress()).ThenReturn("assignedContainerMgrAddress"
                                                                                          );
            Org.Mockito.Mockito.When(attempt.GetNodeRackName()).ThenReturn("nodeRackName");
            long             taStartTime         = 100002L;
            long             taFinishTime        = 100012L;
            long             taShuffleFinishTime = 100010L;
            long             taSortFinishTime    = 100011L;
            TaskAttemptState taState             = TaskAttemptState.Succeeded;

            Org.Mockito.Mockito.When(attempt.GetLaunchTime()).ThenReturn(taStartTime);
            Org.Mockito.Mockito.When(attempt.GetFinishTime()).ThenReturn(taFinishTime);
            Org.Mockito.Mockito.When(attempt.GetShuffleFinishTime()).ThenReturn(taShuffleFinishTime
                                                                                );
            Org.Mockito.Mockito.When(attempt.GetSortFinishTime()).ThenReturn(taSortFinishTime
                                                                             );
            Org.Mockito.Mockito.When(attempt.GetState()).ThenReturn(taState);
            TaskAttemptReport taReport = Org.Mockito.Mockito.Mock <TaskAttemptReport>();

            Org.Mockito.Mockito.When(taReport.GetStartTime()).ThenReturn(taStartTime);
            Org.Mockito.Mockito.When(taReport.GetFinishTime()).ThenReturn(taFinishTime);
            Org.Mockito.Mockito.When(taReport.GetShuffleFinishTime()).ThenReturn(taShuffleFinishTime
                                                                                 );
            Org.Mockito.Mockito.When(taReport.GetSortFinishTime()).ThenReturn(taSortFinishTime
                                                                              );
            Org.Mockito.Mockito.When(taReport.GetContainerId()).ThenReturn(containerId);
            Org.Mockito.Mockito.When(taReport.GetProgress()).ThenReturn(1.0f);
            Org.Mockito.Mockito.When(taReport.GetStateString()).ThenReturn("Processed 128/128 records <p> \n"
                                                                           );
            Org.Mockito.Mockito.When(taReport.GetTaskAttemptState()).ThenReturn(taState);
            Org.Mockito.Mockito.When(taReport.GetDiagnosticInfo()).ThenReturn(string.Empty);
            Org.Mockito.Mockito.When(attempt.GetReport()).ThenReturn(taReport);
            attempts[taId] = attempt;
            Org.Mockito.Mockito.When(task.GetAttempts()).ThenReturn(attempts);
            app.SetTask(task);
            Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job
                                                                                       >();
            Org.Mockito.Mockito.When(job.GetUserName()).ThenReturn("User");
            app.SetJob(job);
            TestBlocks.AttemptsBlockForTest block = new TestBlocks.AttemptsBlockForTest(this,
                                                                                        app);
            block.AddParameter(AMParams.TaskType, "r");
            PrintWriter pWriter = new PrintWriter(data);

            HtmlBlock.Block html = new BlockForTest(new TestBlocks.HtmlBlockForTest(this), pWriter
                                                    , 0, false);
            block.Render(html);
            pWriter.Flush();
            // should be printed information about attempts
            NUnit.Framework.Assert.IsTrue(data.ToString().Contains("0 attempt_0_0001_r_000000_0"
                                                                   ));
            NUnit.Framework.Assert.IsTrue(data.ToString().Contains("SUCCEEDED"));
            NUnit.Framework.Assert.IsFalse(data.ToString().Contains("Processed 128/128 records <p> \n"
                                                                    ));
            NUnit.Framework.Assert.IsTrue(data.ToString().Contains("Processed 128\\/128 records &lt;p&gt; \\n"
                                                                   ));
            NUnit.Framework.Assert.IsTrue(data.ToString().Contains("_0005_01_000001:attempt_0_0001_r_000000_0:User:"******"100002"));
            NUnit.Framework.Assert.IsTrue(data.ToString().Contains("100010"));
            NUnit.Framework.Assert.IsTrue(data.ToString().Contains("100011"));
            NUnit.Framework.Assert.IsTrue(data.ToString().Contains("100012"));
        }