Esempio n. 1
0
        /// <summary>test JobConf</summary>
        /// <exception cref="System.Exception"/>
        public virtual void TestNetworkedJob()
        {
            // mock creation
            MiniMRClientCluster mr      = null;
            FileSystem          fileSys = null;

            try
            {
                mr = CreateMiniClusterWithCapacityScheduler();
                JobConf job = new JobConf(mr.GetConfig());
                fileSys = FileSystem.Get(job);
                fileSys.Delete(testDir, true);
                FSDataOutputStream @out = fileSys.Create(inFile, true);
                @out.WriteBytes("This is a test file");
                @out.Close();
                FileInputFormat.SetInputPaths(job, inFile);
                FileOutputFormat.SetOutputPath(job, outDir);
                job.SetInputFormat(typeof(TextInputFormat));
                job.SetOutputFormat(typeof(TextOutputFormat));
                job.SetMapperClass(typeof(IdentityMapper));
                job.SetReducerClass(typeof(IdentityReducer));
                job.SetNumReduceTasks(0);
                JobClient              client     = new JobClient(mr.GetConfig());
                RunningJob             rj         = client.SubmitJob(job);
                JobID                  jobId      = rj.GetID();
                JobClient.NetworkedJob runningJob = (JobClient.NetworkedJob)client.GetJob(jobId);
                runningJob.SetJobPriority(JobPriority.High.ToString());
                // test getters
                NUnit.Framework.Assert.IsTrue(runningJob.GetConfiguration().ToString().EndsWith("0001/job.xml"
                                                                                                ));
                NUnit.Framework.Assert.AreEqual(runningJob.GetID(), jobId);
                NUnit.Framework.Assert.AreEqual(runningJob.GetJobID(), jobId.ToString());
                NUnit.Framework.Assert.AreEqual(runningJob.GetJobName(), "N/A");
                NUnit.Framework.Assert.IsTrue(runningJob.GetJobFile().EndsWith(".staging/" + runningJob
                                                                               .GetJobID() + "/job.xml"));
                NUnit.Framework.Assert.IsTrue(runningJob.GetTrackingURL().Length > 0);
                NUnit.Framework.Assert.IsTrue(runningJob.MapProgress() == 0.0f);
                NUnit.Framework.Assert.IsTrue(runningJob.ReduceProgress() == 0.0f);
                NUnit.Framework.Assert.IsTrue(runningJob.CleanupProgress() == 0.0f);
                NUnit.Framework.Assert.IsTrue(runningJob.SetupProgress() == 0.0f);
                TaskCompletionEvent[] tce = runningJob.GetTaskCompletionEvents(0);
                NUnit.Framework.Assert.AreEqual(tce.Length, 0);
                NUnit.Framework.Assert.AreEqual(runningJob.GetHistoryUrl(), string.Empty);
                NUnit.Framework.Assert.IsFalse(runningJob.IsRetired());
                NUnit.Framework.Assert.AreEqual(runningJob.GetFailureInfo(), string.Empty);
                NUnit.Framework.Assert.AreEqual(runningJob.GetJobStatus().GetJobName(), "N/A");
                NUnit.Framework.Assert.AreEqual(client.GetMapTaskReports(jobId).Length, 0);
                try
                {
                    client.GetSetupTaskReports(jobId);
                }
                catch (YarnRuntimeException e)
                {
                    NUnit.Framework.Assert.AreEqual(e.Message, "Unrecognized task type: JOB_SETUP");
                }
                try
                {
                    client.GetCleanupTaskReports(jobId);
                }
                catch (YarnRuntimeException e)
                {
                    NUnit.Framework.Assert.AreEqual(e.Message, "Unrecognized task type: JOB_CLEANUP");
                }
                NUnit.Framework.Assert.AreEqual(client.GetReduceTaskReports(jobId).Length, 0);
                // test ClusterStatus
                ClusterStatus status = client.GetClusterStatus(true);
                NUnit.Framework.Assert.AreEqual(status.GetActiveTrackerNames().Count, 2);
                // it method does not implemented and always return empty array or null;
                NUnit.Framework.Assert.AreEqual(status.GetBlacklistedTrackers(), 0);
                NUnit.Framework.Assert.AreEqual(status.GetBlacklistedTrackerNames().Count, 0);
                NUnit.Framework.Assert.AreEqual(status.GetBlackListedTrackersInfo().Count, 0);
                NUnit.Framework.Assert.AreEqual(status.GetJobTrackerStatus(), Cluster.JobTrackerStatus
                                                .Running);
                NUnit.Framework.Assert.AreEqual(status.GetMapTasks(), 1);
                NUnit.Framework.Assert.AreEqual(status.GetMaxMapTasks(), 20);
                NUnit.Framework.Assert.AreEqual(status.GetMaxReduceTasks(), 4);
                NUnit.Framework.Assert.AreEqual(status.GetNumExcludedNodes(), 0);
                NUnit.Framework.Assert.AreEqual(status.GetReduceTasks(), 1);
                NUnit.Framework.Assert.AreEqual(status.GetTaskTrackers(), 2);
                NUnit.Framework.Assert.AreEqual(status.GetTTExpiryInterval(), 0);
                NUnit.Framework.Assert.AreEqual(status.GetJobTrackerStatus(), Cluster.JobTrackerStatus
                                                .Running);
                NUnit.Framework.Assert.AreEqual(status.GetGraylistedTrackers(), 0);
                // test read and write
                ByteArrayOutputStream dataOut = new ByteArrayOutputStream();
                status.Write(new DataOutputStream(dataOut));
                ClusterStatus status2 = new ClusterStatus();
                status2.ReadFields(new DataInputStream(new ByteArrayInputStream(dataOut.ToByteArray
                                                                                    ())));
                NUnit.Framework.Assert.AreEqual(status.GetActiveTrackerNames(), status2.GetActiveTrackerNames
                                                    ());
                NUnit.Framework.Assert.AreEqual(status.GetBlackListedTrackersInfo(), status2.GetBlackListedTrackersInfo
                                                    ());
                NUnit.Framework.Assert.AreEqual(status.GetMapTasks(), status2.GetMapTasks());
                try
                {
                }
                catch (RuntimeException e)
                {
                    NUnit.Framework.Assert.IsTrue(e.Message.EndsWith("not found on CLASSPATH"));
                }
                // test taskStatusfilter
                JobClient.SetTaskOutputFilter(job, JobClient.TaskStatusFilter.All);
                NUnit.Framework.Assert.AreEqual(JobClient.GetTaskOutputFilter(job), JobClient.TaskStatusFilter
                                                .All);
                // runningJob.setJobPriority(JobPriority.HIGH.name());
                // test default map
                NUnit.Framework.Assert.AreEqual(client.GetDefaultMaps(), 20);
                NUnit.Framework.Assert.AreEqual(client.GetDefaultReduces(), 4);
                NUnit.Framework.Assert.AreEqual(client.GetSystemDir().GetName(), "jobSubmitDir");
                // test queue information
                JobQueueInfo[] rootQueueInfo = client.GetRootQueues();
                NUnit.Framework.Assert.AreEqual(rootQueueInfo.Length, 1);
                NUnit.Framework.Assert.AreEqual(rootQueueInfo[0].GetQueueName(), "default");
                JobQueueInfo[] qinfo = client.GetQueues();
                NUnit.Framework.Assert.AreEqual(qinfo.Length, 1);
                NUnit.Framework.Assert.AreEqual(qinfo[0].GetQueueName(), "default");
                NUnit.Framework.Assert.AreEqual(client.GetChildQueues("default").Length, 0);
                NUnit.Framework.Assert.AreEqual(client.GetJobsFromQueue("default").Length, 1);
                NUnit.Framework.Assert.IsTrue(client.GetJobsFromQueue("default")[0].GetJobFile().
                                              EndsWith("/job.xml"));
                JobQueueInfo qi = client.GetQueueInfo("default");
                NUnit.Framework.Assert.AreEqual(qi.GetQueueName(), "default");
                NUnit.Framework.Assert.AreEqual(qi.GetQueueState(), "running");
                QueueAclsInfo[] aai = client.GetQueueAclsForCurrentUser();
                NUnit.Framework.Assert.AreEqual(aai.Length, 2);
                NUnit.Framework.Assert.AreEqual(aai[0].GetQueueName(), "root");
                NUnit.Framework.Assert.AreEqual(aai[1].GetQueueName(), "default");
                // test token
                Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier> token = client.
                                                                                           GetDelegationToken(new Text(UserGroupInformation.GetCurrentUser().GetShortUserName
                                                                                                                           ()));
                NUnit.Framework.Assert.AreEqual(token.GetKind().ToString(), "RM_DELEGATION_TOKEN"
                                                );
                // test JobClient
                // The following asserts read JobStatus twice and ensure the returned
                // JobStatus objects correspond to the same Job.
                NUnit.Framework.Assert.AreEqual("Expected matching JobIDs", jobId, ((JobID)client
                                                                                    .GetJob(jobId).GetJobStatus().GetJobID()));
                NUnit.Framework.Assert.AreEqual("Expected matching startTimes", rj.GetJobStatus()
                                                .GetStartTime(), client.GetJob(jobId).GetJobStatus().GetStartTime());
            }
            finally
            {
                if (fileSys != null)
                {
                    fileSys.Delete(testDir, true);
                }
                if (mr != null)
                {
                    mr.Stop();
                }
            }
        }