public virtual void TestTaskIdXML()
        {
            WebResource r = Resource();
            IDictionary <JobId, Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job> jobsMap = appContext
                                                                                      .GetAllJobs();

            foreach (JobId id in jobsMap.Keys)
            {
                string jobId = MRApps.ToString(id);
                foreach (Task task in jobsMap[id].GetTasks().Values)
                {
                    string         tid      = MRApps.ToString(task.GetID());
                    ClientResponse response = r.Path("ws").Path("v1").Path("mapreduce").Path("jobs").
                                              Path(jobId).Path("tasks").Path(tid).Accept(MediaType.ApplicationXml).Get <ClientResponse
                                                                                                                        >();
                    NUnit.Framework.Assert.AreEqual(MediaType.ApplicationXmlType, response.GetType());
                    string xml = response.GetEntity <string>();
                    DocumentBuilderFactory dbf = DocumentBuilderFactory.NewInstance();
                    DocumentBuilder        db  = dbf.NewDocumentBuilder();
                    InputSource            @is = new InputSource();
                    @is.SetCharacterStream(new StringReader(xml));
                    Document dom   = db.Parse(@is);
                    NodeList nodes = dom.GetElementsByTagName("task");
                    for (int i = 0; i < nodes.GetLength(); i++)
                    {
                        Element element = (Element)nodes.Item(i);
                        VerifyAMSingleTaskXML(element, task);
                    }
                }
            }
        }
Esempio n. 2
0
        public TaskInfo(Task task)
        {
            TaskType ttype = task.GetType();

            this.type = ttype.ToString();
            TaskReport report = task.GetReport();

            this.startTime   = report.GetStartTime();
            this.finishTime  = report.GetFinishTime();
            this.state       = report.GetTaskState();
            this.elapsedTime = Times.Elapsed(this.startTime, this.finishTime, this.state == TaskState
                                             .Running);
            if (this.elapsedTime == -1)
            {
                this.elapsedTime = 0;
            }
            this.progress   = report.GetProgress() * 100;
            this.status     = report.GetStatus();
            this.id         = MRApps.ToString(task.GetID());
            this.taskNum    = task.GetID().GetId();
            this.successful = GetSuccessfulAttempt(task);
            if (successful != null)
            {
                this.successfulAttempt = MRApps.ToString(successful.GetID());
            }
            else
            {
                this.successfulAttempt = string.Empty;
            }
        }
        public virtual void TestTasksXML()
        {
            WebResource r = Resource();
            IDictionary <JobId, Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job> jobsMap = appContext
                                                                                      .GetAllJobs();

            foreach (JobId id in jobsMap.Keys)
            {
                string         jobId    = MRApps.ToString(id);
                ClientResponse response = r.Path("ws").Path("v1").Path("mapreduce").Path("jobs").
                                          Path(jobId).Path("tasks").Accept(MediaType.ApplicationXml).Get <ClientResponse>();
                NUnit.Framework.Assert.AreEqual(MediaType.ApplicationXmlType, response.GetType());
                string xml = response.GetEntity <string>();
                DocumentBuilderFactory dbf = DocumentBuilderFactory.NewInstance();
                DocumentBuilder        db  = dbf.NewDocumentBuilder();
                InputSource            @is = new InputSource();
                @is.SetCharacterStream(new StringReader(xml));
                Document dom   = db.Parse(@is);
                NodeList tasks = dom.GetElementsByTagName("tasks");
                NUnit.Framework.Assert.AreEqual("incorrect number of elements", 1, tasks.GetLength
                                                    ());
                NodeList task = dom.GetElementsByTagName("task");
                VerifyAMTaskXML(task, jobsMap[id]);
            }
        }
Esempio n. 4
0
        public virtual void SetUp()
        {
            AppContext context = Org.Mockito.Mockito.Mock <AppContext>();

            Org.Mockito.Mockito.When(context.GetApplicationID()).ThenReturn(ApplicationId.NewInstance
                                                                                (0, 0));
            Org.Mockito.Mockito.When(context.GetApplicationName()).ThenReturn("AppName");
            Org.Mockito.Mockito.When(context.GetUser()).ThenReturn("User");
            Org.Mockito.Mockito.When(context.GetStartTime()).ThenReturn(Runtime.CurrentTimeMillis
                                                                            ());
            job = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job>();
            Task task = Org.Mockito.Mockito.Mock <Task>();

            Org.Mockito.Mockito.When(job.GetTask(Any <TaskId>())).ThenReturn(task);
            JobId jobID = MRApps.ToJobID("job_01_01");

            Org.Mockito.Mockito.When(context.GetJob(jobID)).ThenReturn(job);
            Org.Mockito.Mockito.When(job.CheckAccess(Any <UserGroupInformation>(), Any <JobACL>
                                                         ())).ThenReturn(true);
            Org.Apache.Hadoop.Mapreduce.V2.App.Webapp.App app = new Org.Apache.Hadoop.Mapreduce.V2.App.Webapp.App
                                                                    (context);
            Configuration configuration = new Configuration();

            ctx           = Org.Mockito.Mockito.Mock <Controller.RequestContext>();
            appController = new AppControllerForTest(app, configuration, ctx);
            appController.GetProperty()[AMParams.JobId]  = "job_01_01";
            appController.GetProperty()[AMParams.TaskId] = "task_01_01_m01_01";
        }
Esempio n. 5
0
		/// <exception cref="System.IO.IOException"/>
		/// <exception cref="System.Exception"/>
		public virtual void TestMRAppMasterMaxAppAttempts()
		{
			// No matter what's the maxAppAttempt or attempt id, the isLastRetry always
			// equals to false
			bool[] expectedBools = new bool[] { false, false, false };
			string applicationAttemptIdStr = "appattempt_1317529182569_0004_000002";
			string containerIdStr = "container_1317529182569_0004_000002_1";
			string userName = "******";
			ApplicationAttemptId applicationAttemptId = ConverterUtils.ToApplicationAttemptId
				(applicationAttemptIdStr);
			ContainerId containerId = ConverterUtils.ToContainerId(containerIdStr);
			JobConf conf = new JobConf();
			conf.Set(MRJobConfig.MrAmStagingDir, stagingDir);
			FilePath stagingDir = new FilePath(MRApps.GetStagingAreaDir(conf, userName).ToString
				());
			stagingDir.Mkdirs();
			for (int i = 0; i < expectedBools.Length; ++i)
			{
				MRAppMasterTest appMaster = new MRAppMasterTest(applicationAttemptId, containerId
					, "host", -1, -1, Runtime.CurrentTimeMillis(), false, true);
				MRAppMaster.InitAndStartAppMaster(appMaster, conf, userName);
				NUnit.Framework.Assert.AreEqual("isLastAMRetry is correctly computed.", expectedBools
					[i], appMaster.IsLastAMRetry());
			}
		}
Esempio n. 6
0
        /// <exception cref="Org.Codehaus.Jettison.Json.JSONException"/>
        public virtual void VerifyHsJobTaskAttemptCounters(JSONObject info, TaskAttempt att
                                                           )
        {
            NUnit.Framework.Assert.AreEqual("incorrect number of elements", 2, info.Length());
            WebServicesTestUtils.CheckStringMatch("id", MRApps.ToString(att.GetID()), info.GetString
                                                      ("id"));
            // just do simple verification of fields - not data is correct
            // in the fields
            JSONArray counterGroups = info.GetJSONArray("taskAttemptCounterGroup");

            for (int i = 0; i < counterGroups.Length(); i++)
            {
                JSONObject counterGroup = counterGroups.GetJSONObject(i);
                string     name         = counterGroup.GetString("counterGroupName");
                NUnit.Framework.Assert.IsTrue("name not set", (name != null && !name.IsEmpty()));
                JSONArray counters = counterGroup.GetJSONArray("counter");
                for (int j = 0; j < counters.Length(); j++)
                {
                    JSONObject counter     = counters.GetJSONObject(j);
                    string     counterName = counter.GetString("name");
                    NUnit.Framework.Assert.IsTrue("name not set", (counterName != null && !counterName
                                                                   .IsEmpty()));
                    long value = counter.GetLong("value");
                    NUnit.Framework.Assert.IsTrue("value  >= 0", value >= 0);
                }
            }
        }
Esempio n. 7
0
        public virtual void TestTaskAttemptIdCounters()
        {
            WebResource r = Resource();
            IDictionary <JobId, Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job> jobsMap = appContext
                                                                                      .GetAllJobs();

            foreach (JobId id in jobsMap.Keys)
            {
                string jobId = MRApps.ToString(id);
                foreach (Task task in jobsMap[id].GetTasks().Values)
                {
                    string tid = MRApps.ToString(task.GetID());
                    foreach (TaskAttempt att in task.GetAttempts().Values)
                    {
                        TaskAttemptId  attemptid = att.GetID();
                        string         attid     = MRApps.ToString(attemptid);
                        ClientResponse response  = r.Path("ws").Path("v1").Path("history").Path("mapreduce"
                                                                                                ).Path("jobs").Path(jobId).Path("tasks").Path(tid).Path("attempts").Path(attid).
                                                   Path("counters").Accept(MediaType.ApplicationJson).Get <ClientResponse>();
                        NUnit.Framework.Assert.AreEqual(MediaType.ApplicationJsonType, response.GetType()
                                                        );
                        JSONObject json = response.GetEntity <JSONObject>();
                        NUnit.Framework.Assert.AreEqual("incorrect number of elements", 1, json.Length());
                        JSONObject info = json.GetJSONObject("jobTaskAttemptCounters");
                        VerifyHsJobTaskAttemptCounters(info, att);
                    }
                }
            }
        }
Esempio n. 8
0
        public virtual void TestTaskAttemptIdXMLCounters()
        {
            WebResource r = Resource();
            IDictionary <JobId, Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job> jobsMap = appContext
                                                                                      .GetAllJobs();

            foreach (JobId id in jobsMap.Keys)
            {
                string jobId = MRApps.ToString(id);
                foreach (Task task in jobsMap[id].GetTasks().Values)
                {
                    string tid = MRApps.ToString(task.GetID());
                    foreach (TaskAttempt att in task.GetAttempts().Values)
                    {
                        TaskAttemptId  attemptid = att.GetID();
                        string         attid     = MRApps.ToString(attemptid);
                        ClientResponse response  = r.Path("ws").Path("v1").Path("history").Path("mapreduce"
                                                                                                ).Path("jobs").Path(jobId).Path("tasks").Path(tid).Path("attempts").Path(attid).
                                                   Path("counters").Accept(MediaType.ApplicationXml).Get <ClientResponse>();
                        NUnit.Framework.Assert.AreEqual(MediaType.ApplicationXmlType, response.GetType());
                        string xml = response.GetEntity <string>();
                        DocumentBuilderFactory dbf = DocumentBuilderFactory.NewInstance();
                        DocumentBuilder        db  = dbf.NewDocumentBuilder();
                        InputSource            @is = new InputSource();
                        @is.SetCharacterStream(new StringReader(xml));
                        Document dom   = db.Parse(@is);
                        NodeList nodes = dom.GetElementsByTagName("jobTaskAttemptCounters");
                        VerifyHsTaskCountersXML(nodes, att);
                    }
                }
            }
        }
Esempio n. 9
0
        /// <exception cref="Org.Codehaus.Jettison.Json.JSONException"/>
        public virtual void VerifyHsTaskAttempts(JSONObject json, Task task)
        {
            NUnit.Framework.Assert.AreEqual("incorrect number of elements", 1, json.Length());
            JSONObject attempts = json.GetJSONObject("taskAttempts");

            NUnit.Framework.Assert.AreEqual("incorrect number of elements", 1, json.Length());
            JSONArray arr = attempts.GetJSONArray("taskAttempt");

            foreach (TaskAttempt att in task.GetAttempts().Values)
            {
                TaskAttemptId id    = att.GetID();
                string        attid = MRApps.ToString(id);
                bool          found = false;
                for (int i = 0; i < arr.Length(); i++)
                {
                    JSONObject info = arr.GetJSONObject(i);
                    if (attid.Matches(info.GetString("id")))
                    {
                        found = true;
                        VerifyHsTaskAttempt(info, att, task.GetType());
                    }
                }
                NUnit.Framework.Assert.IsTrue("task attempt with id: " + attid + " not in web service output"
                                              , found);
            }
        }
Esempio n. 10
0
        /// <summary>convert a job id string to an actual job and handle all the error checking.
        ///     </summary>
        /// <exception cref="Org.Apache.Hadoop.Yarn.Webapp.NotFoundException"/>
        public static Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job GetJobFromJobIdString(string
                                                                                       jid, AppContext appCtx)
        {
            JobId jobId;

            Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job;
            try
            {
                jobId = MRApps.ToJobID(jid);
            }
            catch (YarnRuntimeException e)
            {
                // TODO: after MAPREDUCE-2793 YarnRuntimeException is probably not expected here
                // anymore but keeping it for now just in case other stuff starts failing.
                // Also, the webservice should ideally return BadRequest (HTTP:400) when
                // the id is malformed instead of NotFound (HTTP:404). The webserver on
                // top of which AMWebServices is built seems to automatically do that for
                // unhandled exceptions
                throw new NotFoundException(e.Message);
            }
            catch (ArgumentException e)
            {
                throw new NotFoundException(e.Message);
            }
            if (jobId == null)
            {
                throw new NotFoundException("job, " + jid + ", is not found");
            }
            job = appCtx.GetJob(jobId);
            if (job == null)
            {
                throw new NotFoundException("job, " + jid + ", is not found");
            }
            return(job);
        }
Esempio n. 11
0
        /// <exception cref="System.Exception"/>
        public virtual Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job Submit(Configuration conf
                                                                         , bool mapSpeculative, bool reduceSpeculative)
        {
            string user = conf.Get(MRJobConfig.UserName, UserGroupInformation.GetCurrentUser(
                                       ).GetShortUserName());

            conf.Set(MRJobConfig.UserName, user);
            conf.Set(MRJobConfig.MrAmStagingDir, testAbsPath.ToString());
            conf.SetBoolean(MRJobConfig.MrAmCreateJhIntermediateBaseDir, true);
            // TODO: fix the bug where the speculator gets events with
            // not-fully-constructed objects. For now, disable speculative exec
            conf.SetBoolean(MRJobConfig.MapSpeculative, mapSpeculative);
            conf.SetBoolean(MRJobConfig.ReduceSpeculative, reduceSpeculative);
            Init(conf);
            Start();
            DefaultMetricsSystem.Shutdown();
            Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = GetContext().GetAllJobs().Values
                                                             .GetEnumerator().Next();
            if (assignedQueue != null)
            {
                job.SetQueueName(assignedQueue);
            }
            // Write job.xml
            string jobFile = MRApps.GetJobFile(conf, user, TypeConverter.FromYarn(job.GetID()
                                                                                  ));

            Log.Info("Writing job conf to " + jobFile);
            new FilePath(jobFile).GetParentFile().Mkdirs();
            conf.WriteXml(new FileOutputStream(jobFile));
            return(job);
        }
Esempio n. 12
0
 public virtual void VerifyHsTaskCountersXML(NodeList nodes, TaskAttempt att)
 {
     for (int i = 0; i < nodes.GetLength(); i++)
     {
         Element element = (Element)nodes.Item(i);
         WebServicesTestUtils.CheckStringMatch("id", MRApps.ToString(att.GetID()), WebServicesTestUtils
                                               .GetXmlString(element, "id"));
         // just do simple verification of fields - not data is correct
         // in the fields
         NodeList groups = element.GetElementsByTagName("taskAttemptCounterGroup");
         for (int j = 0; j < groups.GetLength(); j++)
         {
             Element counters = (Element)groups.Item(j);
             NUnit.Framework.Assert.IsNotNull("should have counters in the web service info",
                                              counters);
             string name = WebServicesTestUtils.GetXmlString(counters, "counterGroupName");
             NUnit.Framework.Assert.IsTrue("name not set", (name != null && !name.IsEmpty()));
             NodeList counterArr = counters.GetElementsByTagName("counter");
             for (int z = 0; z < counterArr.GetLength(); z++)
             {
                 Element counter     = (Element)counterArr.Item(z);
                 string  counterName = WebServicesTestUtils.GetXmlString(counter, "name");
                 NUnit.Framework.Assert.IsTrue("counter name not set", (counterName != null && !counterName
                                                                        .IsEmpty()));
                 long value = WebServicesTestUtils.GetXmlLong(counter, "value");
                 NUnit.Framework.Assert.IsTrue("value not >= 0", value >= 0);
             }
         }
     }
 }
Esempio n. 13
0
        // acls not being checked since
        // we are using mock job instead of CompletedJob
        public static void VerifyHsJobGeneric(Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job
                                              job, string id, string user, string name, string state, string queue, long startTime
                                              , long finishTime, int mapsTotal, int mapsCompleted, int reducesTotal, int reducesCompleted
                                              )
        {
            JobReport report = job.GetReport();

            WebServicesTestUtils.CheckStringMatch("id", MRApps.ToString(job.GetID()), id);
            WebServicesTestUtils.CheckStringMatch("user", job.GetUserName().ToString(), user);
            WebServicesTestUtils.CheckStringMatch("name", job.GetName(), name);
            WebServicesTestUtils.CheckStringMatch("state", job.GetState().ToString(), state);
            WebServicesTestUtils.CheckStringMatch("queue", job.GetQueueName(), queue);
            NUnit.Framework.Assert.AreEqual("startTime incorrect", report.GetStartTime(), startTime
                                            );
            NUnit.Framework.Assert.AreEqual("finishTime incorrect", report.GetFinishTime(), finishTime
                                            );
            NUnit.Framework.Assert.AreEqual("mapsTotal incorrect", job.GetTotalMaps(), mapsTotal
                                            );
            NUnit.Framework.Assert.AreEqual("mapsCompleted incorrect", job.GetCompletedMaps()
                                            , mapsCompleted);
            NUnit.Framework.Assert.AreEqual("reducesTotal incorrect", job.GetTotalReduces(),
                                            reducesTotal);
            NUnit.Framework.Assert.AreEqual("reducesCompleted incorrect", job.GetCompletedReduces
                                                (), reducesCompleted);
        }
Esempio n. 14
0
        /// <exception cref="System.Exception"/>
        public virtual void TestCommandLine()
        {
            TestMapReduceChildJVM.MyMRApp app = new TestMapReduceChildJVM.MyMRApp(1, 0, true,
                                                                                  this.GetType().FullName, true);
            Configuration conf = new Configuration();

            conf.SetBoolean(MRConfig.MapreduceAppSubmissionCrossPlatform, true);
            Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(conf);
            app.WaitForState(job, JobState.Succeeded);
            app.VerifyCompleted();
            NUnit.Framework.Assert.AreEqual("[" + MRApps.CrossPlatformify("JAVA_HOME") + "/bin/java"
                                            + " -Djava.net.preferIPv4Stack=true" + " -Dhadoop.metrics.log.level=WARN" + "  -Xmx200m -Djava.io.tmpdir="
                                            + MRApps.CrossPlatformify("PWD") + "/tmp" + " -Dlog4j.configuration=container-log4j.properties"
                                            + " -Dyarn.app.container.log.dir=<LOG_DIR>" + " -Dyarn.app.container.log.filesize=0"
                                            + " -Dhadoop.root.logger=INFO,CLA -Dhadoop.root.logfile=syslog" + " org.apache.hadoop.mapred.YarnChild 127.0.0.1"
                                            + " 54321" + " attempt_0_0000_m_000000_0" + " 0" + " 1><LOG_DIR>/stdout" + " 2><LOG_DIR>/stderr ]"
                                            , app.myCommandLine);
            NUnit.Framework.Assert.IsTrue("HADOOP_ROOT_LOGGER not set for job", app.cmdEnvironment
                                          .Contains("HADOOP_ROOT_LOGGER"));
            NUnit.Framework.Assert.AreEqual("INFO,console", app.cmdEnvironment["HADOOP_ROOT_LOGGER"
                                            ]);
            NUnit.Framework.Assert.IsTrue("HADOOP_CLIENT_OPTS not set for job", app.cmdEnvironment
                                          .Contains("HADOOP_CLIENT_OPTS"));
            NUnit.Framework.Assert.AreEqual(string.Empty, app.cmdEnvironment["HADOOP_CLIENT_OPTS"
                                            ]);
        }
Esempio n. 15
0
        public virtual void TestAMStandardEnv()
        {
            string  AdminLibPath = "foo";
            string  UserLibPath  = "bar";
            string  UserShell    = "shell";
            JobConf jobConf      = new JobConf();

            jobConf.Set(MRJobConfig.MrAmAdminUserEnv, "LD_LIBRARY_PATH=" + AdminLibPath);
            jobConf.Set(MRJobConfig.MrAmEnv, "LD_LIBRARY_PATH=" + UserLibPath);
            jobConf.Set(MRJobConfig.MapredAdminUserShell, UserShell);
            YARNRunner yarnRunner = new YARNRunner(jobConf);
            ApplicationSubmissionContext appSubCtx = BuildSubmitContext(yarnRunner, jobConf);
            // make sure PWD is first in the lib path
            ContainerLaunchContext       clc = appSubCtx.GetAMContainerSpec();
            IDictionary <string, string> env = clc.GetEnvironment();
            string libPath = env[ApplicationConstants.Environment.LdLibraryPath.ToString()];

            NUnit.Framework.Assert.IsNotNull("LD_LIBRARY_PATH not set", libPath);
            string cps = jobConf.GetBoolean(MRConfig.MapreduceAppSubmissionCrossPlatform, MRConfig
                                            .DefaultMapreduceAppSubmissionCrossPlatform) ? ApplicationConstants.ClassPathSeparator
                                 : FilePath.pathSeparator;

            NUnit.Framework.Assert.AreEqual("Bad AM LD_LIBRARY_PATH setting", MRApps.CrossPlatformifyMREnv
                                                (conf, ApplicationConstants.Environment.Pwd) + cps + AdminLibPath + cps + UserLibPath
                                            , libPath);
            // make sure SHELL is set
            string shell = env[ApplicationConstants.Environment.Shell.ToString()];

            NUnit.Framework.Assert.IsNotNull("SHELL not set", shell);
            NUnit.Framework.Assert.AreEqual("Bad SHELL setting", UserShell, shell);
        }
Esempio n. 16
0
        /// <summary>Ensure that a TASK_ID was passed into the page.</summary>
        public virtual void RequireTask()
        {
            if ($(TaskId).IsEmpty())
            {
                BadRequest("missing task ID");
                throw new RuntimeException("missing task ID");
            }
            TaskId taskID = MRApps.ToTaskID($(TaskId));

            Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.context.GetJob(taskID.GetJobId
                                                                                    ());
            app.SetJob(job);
            if (app.GetJob() == null)
            {
                NotFound(MRApps.ToString(taskID.GetJobId()));
                throw new RuntimeException("Not Found: " + $(JobId));
            }
            else
            {
                app.SetTask(app.GetJob().GetTask(taskID));
                if (app.GetTask() == null)
                {
                    NotFound($(TaskId));
                    throw new RuntimeException("Not Found: " + $(TaskId));
                }
            }
            if (!CheckAccess(job))
            {
                AccessDenied("User " + Request().GetRemoteUser() + " does not have " + " permission to view job "
                             + $(JobId));
                throw new RuntimeException("Access denied: User " + Request().GetRemoteUser() + " does not have permission to view job "
                                           + $(JobId));
            }
        }
Esempio n. 17
0
        public virtual TasksInfo GetJobTasks(HttpServletRequest hsr, string jid, string type
                                             )
        {
            Init();
            Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = AMWebServices.GetJobFromJobIdString
                                                                 (jid, ctx);
            CheckAccess(job, hsr);
            TasksInfo allTasks = new TasksInfo();

            foreach (Task task in job.GetTasks().Values)
            {
                TaskType ttype = null;
                if (type != null && !type.IsEmpty())
                {
                    try
                    {
                        ttype = MRApps.TaskType(type);
                    }
                    catch (YarnRuntimeException)
                    {
                        throw new BadRequestException("tasktype must be either m or r");
                    }
                }
                if (ttype != null && task.GetType() != ttype)
                {
                    continue;
                }
                allTasks.Add(new TaskInfo(task));
            }
            return(allTasks);
        }
Esempio n. 18
0
        protected override void Render(HtmlBlock.Block html)
        {
            string rmweb = $(AMParams.RmWeb);

            Hamlet.DIV <Org.Apache.Hadoop.Yarn.Webapp.Hamlet.Hamlet> nav = html.Div("#nav").H3
                                                                               ("Cluster").Ul().Li().A(Url(rmweb, "cluster", "cluster"), "About").().Li().A(Url
                                                                                                                                                                (rmweb, "cluster", "apps"), "Applications").().Li().A(Url(rmweb, "cluster", "scheduler"
                                                                                                                                                                                                                          ), "Scheduler").().().H3("Application").Ul().Li().A(Url("app/info"), "About").()
                                                                           .Li().A(Url("app"), "Jobs").().();
            if (app.GetJob() != null)
            {
                string         jobid           = MRApps.ToString(app.GetJob().GetID());
                IList <AMInfo> amInfos         = app.GetJob().GetAMInfos();
                AMInfo         thisAmInfo      = amInfos[amInfos.Count - 1];
                string         nodeHttpAddress = thisAmInfo.GetNodeManagerHost() + ":" + thisAmInfo.GetNodeManagerHttpPort
                                                     ();
                nav.H3("Job").Ul().Li().A(Url("job", jobid), "Overview").().Li().A(Url("jobcounters"
                                                                                       , jobid), "Counters").().Li().A(Url("conf", jobid), "Configuration").().Li().A(Url
                                                                                                                                                                          ("tasks", jobid, "m"), "Map tasks").().Li().A(Url("tasks", jobid, "r"), "Reduce tasks"
                                                                                                                                                                                                                        ).().Li().A(".logslink", Url(MRWebAppUtil.GetYARNWebappScheme(), nodeHttpAddress
                                                                                                                                                                                                                                                     , "node", "containerlogs", thisAmInfo.GetContainerId().ToString(), app.GetJob().
                                                                                                                                                                                                                                                     GetUserName()), "AM Logs").().();
                if (app.GetTask() != null)
                {
                    string taskid = MRApps.ToString(app.GetTask().GetID());
                    nav.H3("Task").Ul().Li().A(Url("task", taskid), "Task Overview").().Li().A(Url("taskcounters"
                                                                                                   , taskid), "Counters").().();
                }
            }
            nav.H3("Tools").Ul().Li().A("/conf", "Configuration").().Li().A("/logs", "Local logs"
                                                                            ).().Li().A("/stacks", "Server stacks").().Li().A("/jmx?qry=Hadoop:*", "Server metrics"
                                                                                                                              ).().().();
        }
Esempio n. 19
0
        /// <summary>Ensure that a JOB_ID was passed into the page.</summary>
        public virtual void RequireJob()
        {
            if ($(JobId).IsEmpty())
            {
                BadRequest("missing job ID");
                throw new RuntimeException("Bad Request: Missing job ID");
            }
            JobId jobID = MRApps.ToJobID($(JobId));

            app.SetJob(app.context.GetJob(jobID));
            if (app.GetJob() == null)
            {
                NotFound($(JobId));
                throw new RuntimeException("Not Found: " + $(JobId));
            }
            /* check for acl access */
            Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.context.GetJob(jobID);
            if (!CheckAccess(job))
            {
                AccessDenied("User " + Request().GetRemoteUser() + " does not have " + " permission to view job "
                             + $(JobId));
                throw new RuntimeException("Access denied: User " + Request().GetRemoteUser() + " does not have permission to view job "
                                           + $(JobId));
            }
        }
Esempio n. 20
0
        public virtual void TestDeletionofStaging()
        {
            conf.Set(MRJobConfig.MapreduceJobDir, stagingJobDir);
            fs = Org.Mockito.Mockito.Mock <FileSystem>();
            Org.Mockito.Mockito.When(fs.Delete(Matchers.Any <Path>(), Matchers.AnyBoolean())).
            ThenReturn(true);
            //Staging Dir exists
            string user       = UserGroupInformation.GetCurrentUser().GetShortUserName();
            Path   stagingDir = MRApps.GetStagingAreaDir(conf, user);

            Org.Mockito.Mockito.When(fs.Exists(stagingDir)).ThenReturn(true);
            ApplicationId        appId     = ApplicationId.NewInstance(Runtime.CurrentTimeMillis(), 0);
            ApplicationAttemptId attemptId = ApplicationAttemptId.NewInstance(appId, 1);
            JobId jobid = recordFactory.NewRecordInstance <JobId>();

            jobid.SetAppId(appId);
            ContainerAllocator mockAlloc = Org.Mockito.Mockito.Mock <ContainerAllocator>();

            NUnit.Framework.Assert.IsTrue(MRJobConfig.DefaultMrAmMaxAttempts > 1);
            MRAppMaster appMaster = new TestStagingCleanup.TestMRApp(this, attemptId, mockAlloc
                                                                     , JobStateInternal.Running, MRJobConfig.DefaultMrAmMaxAttempts);

            appMaster.Init(conf);
            appMaster.Start();
            appMaster.ShutDownJob();
            //test whether notifyIsLastAMRetry called
            NUnit.Framework.Assert.AreEqual(true, ((TestStagingCleanup.TestMRApp)appMaster).GetTestIsLastAMRetry
                                                ());
            Org.Mockito.Mockito.Verify(fs).Delete(stagingJobPath, true);
        }
Esempio n. 21
0
        /// <returns>
        /// The end of the JS map that is the jquery datatable config for the
        /// attempts table.
        /// </returns>
        private string AttemptsTableInit()
        {
            TaskType type   = null;
            string   symbol = $(AMParams.TaskType);

            if (!symbol.IsEmpty())
            {
                type = MRApps.TaskType(symbol);
            }
            else
            {
                TaskId taskID = MRApps.ToTaskID($(AMParams.TaskId));
                type = taskID.GetTaskType();
            }
            StringBuilder b = JQueryUI.TableInit().Append(", 'aaData': attemptsTableData").Append
                                  (", bDeferRender: true").Append(", bProcessing: true").Append("\n,aoColumnDefs:[\n"
                                                                                                ).Append("\n{'aTargets': [ 4 ]").Append(", 'bSearchable': false }").Append("\n, {'sType':'numeric', 'aTargets': [ 0 ]"
                                                                                                                                                                           ).Append(", 'mRender': parseHadoopAttemptID }").Append("\n, {'sType':'numeric', 'aTargets': [ 5, 6"
                                                                                                                                                                                                                                  ).Append(type == TaskType.Reduce ? ", 7, 8" : string.Empty).Append(" ], 'mRender': renderHadoopDate }"
                                                                                                                                                                                                                                                                                                     ).Append("\n, {'sType':'numeric', 'aTargets': [").Append(type == TaskType.Reduce
                                 ? "9, 10, 11, 12" : "7").Append(" ], 'mRender': renderHadoopElapsedTime }]").Append
                                  ("\n, aaSorting: [[0, 'asc']]").Append("}");

            //logs column should not filterable (it includes container ID which may pollute searches)
            //Column numbers are different for maps and reduces
            // Sort by id upon page load
            return(b.ToString());
        }
Esempio n. 22
0
        // FIXME:
        // Disabled this test because currently, when shutdown hook triggered at
        // lastRetry in RM view, cleanup will not do. This should be supported after
        // YARN-2261 completed
        //   @Test (timeout = 30000)
        /// <exception cref="System.IO.IOException"/>
        public virtual void TestDeletionofStagingOnKillLastTry()
        {
            conf.Set(MRJobConfig.MapreduceJobDir, stagingJobDir);
            fs = Org.Mockito.Mockito.Mock <FileSystem>();
            Org.Mockito.Mockito.When(fs.Delete(Matchers.Any <Path>(), Matchers.AnyBoolean())).
            ThenReturn(true);
            //Staging Dir exists
            string user       = UserGroupInformation.GetCurrentUser().GetShortUserName();
            Path   stagingDir = MRApps.GetStagingAreaDir(conf, user);

            Org.Mockito.Mockito.When(fs.Exists(stagingDir)).ThenReturn(true);
            ApplicationId        appId     = ApplicationId.NewInstance(Runtime.CurrentTimeMillis(), 0);
            ApplicationAttemptId attemptId = ApplicationAttemptId.NewInstance(appId, 1);
            JobId jobid = recordFactory.NewRecordInstance <JobId>();

            jobid.SetAppId(appId);
            ContainerAllocator mockAlloc = Org.Mockito.Mockito.Mock <ContainerAllocator>();
            MRAppMaster        appMaster = new TestStagingCleanup.TestMRApp(this, attemptId, mockAlloc
                                                                            );

            //no retry
            appMaster.Init(conf);
            NUnit.Framework.Assert.IsTrue("appMaster.isLastAMRetry() is false", appMaster.IsLastAMRetry
                                              ());
            //simulate the process being killed
            MRAppMaster.MRAppMasterShutdownHook hook = new MRAppMaster.MRAppMasterShutdownHook
                                                           (appMaster);
            hook.Run();
            NUnit.Framework.Assert.IsTrue("MRAppMaster isn't stopped", appMaster.IsInState(Service.STATE
                                                                                           .Stopped));
            Org.Mockito.Mockito.Verify(fs).Delete(stagingJobPath, true);
        }
Esempio n. 23
0
 /// <exception cref="System.Exception"/>
 protected override void ServiceInit(Configuration conf)
 {
     try
     {
         //Create the staging directory if it does not exist
         string     user       = UserGroupInformation.GetCurrentUser().GetShortUserName();
         Path       stagingDir = MRApps.GetStagingAreaDir(conf, user);
         FileSystem fs         = GetFileSystem(conf);
         fs.Mkdirs(stagingDir);
     }
     catch (Exception e)
     {
         throw new YarnRuntimeException("Error creating staging dir", e);
     }
     base.ServiceInit(conf);
     if (this.clusterInfo != null)
     {
         GetContext().GetClusterInfo().SetMaxContainerCapability(this.clusterInfo.GetMaxContainerCapability
                                                                     ());
     }
     else
     {
         GetContext().GetClusterInfo().SetMaxContainerCapability(Resource.NewInstance(10240
                                                                                      , 1));
     }
 }
Esempio n. 24
0
        /// <exception cref="System.Exception"/>
        private void TestReduceCommandLine(Configuration conf)
        {
            TestMapReduceChildJVM.MyMRApp app = new TestMapReduceChildJVM.MyMRApp(0, 1, true,
                                                                                  this.GetType().FullName, true);
            conf.SetBoolean(MRConfig.MapreduceAppSubmissionCrossPlatform, true);
            Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(conf);
            app.WaitForState(job, JobState.Succeeded);
            app.VerifyCompleted();
            long   shuffleLogSize = conf.GetLong(MRJobConfig.ShuffleLogKb, 0L) * 1024L;
            int    shuffleBackups = conf.GetInt(MRJobConfig.ShuffleLogBackups, 0);
            string appenderName   = shuffleLogSize > 0L && shuffleBackups > 0 ? "shuffleCRLA" :
                                    "shuffleCLA";

            NUnit.Framework.Assert.AreEqual("[" + MRApps.CrossPlatformify("JAVA_HOME") + "/bin/java"
                                            + " -Djava.net.preferIPv4Stack=true" + " -Dhadoop.metrics.log.level=WARN" + "  -Xmx200m -Djava.io.tmpdir="
                                            + MRApps.CrossPlatformify("PWD") + "/tmp" + " -Dlog4j.configuration=container-log4j.properties"
                                            + " -Dyarn.app.container.log.dir=<LOG_DIR>" + " -Dyarn.app.container.log.filesize=0"
                                            + " -Dhadoop.root.logger=INFO,CLA -Dhadoop.root.logfile=syslog" + " -Dyarn.app.mapreduce.shuffle.logger=INFO,"
                                            + appenderName + " -Dyarn.app.mapreduce.shuffle.logfile=syslog.shuffle" + " -Dyarn.app.mapreduce.shuffle.log.filesize="
                                            + shuffleLogSize + " -Dyarn.app.mapreduce.shuffle.log.backups=" + shuffleBackups
                                            + " org.apache.hadoop.mapred.YarnChild 127.0.0.1" + " 54321" + " attempt_0_0000_r_000000_0"
                                            + " 0" + " 1><LOG_DIR>/stdout" + " 2><LOG_DIR>/stderr ]", app.myCommandLine);
            NUnit.Framework.Assert.IsTrue("HADOOP_ROOT_LOGGER not set for job", app.cmdEnvironment
                                          .Contains("HADOOP_ROOT_LOGGER"));
            NUnit.Framework.Assert.AreEqual("INFO,console", app.cmdEnvironment["HADOOP_ROOT_LOGGER"
                                            ]);
            NUnit.Framework.Assert.IsTrue("HADOOP_CLIENT_OPTS not set for job", app.cmdEnvironment
                                          .Contains("HADOOP_CLIENT_OPTS"));
            NUnit.Framework.Assert.AreEqual(string.Empty, app.cmdEnvironment["HADOOP_CLIENT_OPTS"
                                            ]);
        }
Esempio n. 25
0
        public static IDictionary <string, string> GetJobParams(AppContext appContext)
        {
            JobId jobId = appContext.GetAllJobs().GetEnumerator().Next().Key;
            IDictionary <string, string> @params = new Dictionary <string, string>();

            @params[AMParams.JobId] = MRApps.ToString(jobId);
            return(@params);
        }
Esempio n. 26
0
        /*
         * (non-Javadoc)
         * @see org.apache.hadoop.yarn.webapp.view.HtmlBlock#render(org.apache.hadoop.yarn.webapp.view.HtmlBlock.Block)
         */
        protected override void Render(HtmlBlock.Block html)
        {
            string jid = $(AMParams.JobId);

            if (jid.IsEmpty())
            {
                html.P().("Sorry, can't do anything without a JobID.").();
                return;
            }
            JobId jobID = MRApps.ToJobID(jid);

            Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = appContext.GetJob(jobID);
            if (job == null)
            {
                html.P().("Sorry, ", jid, " not found.").();
                return;
            }
            Path confPath = job.GetConfFile();

            try
            {
                ConfInfo info = new ConfInfo(job);
                html.Div().H3(confPath.ToString()).();
                Hamlet.TBODY <Hamlet.TABLE <Org.Apache.Hadoop.Yarn.Webapp.Hamlet.Hamlet> > tbody = html
                                                                                                   .Table("#conf").Thead().Tr().Th(JQueryUI.Th, "key").Th(JQueryUI.Th, "value").Th(
                    JQueryUI.Th, "source chain").().().Tbody();
                // Tasks table
                foreach (ConfEntryInfo entry in info.GetProperties())
                {
                    StringBuilder buffer  = new StringBuilder();
                    string[]      sources = entry.GetSource();
                    //Skip the last entry, because it is always the same HDFS file, and
                    // output them in reverse order so most recent is output first
                    bool first = true;
                    for (int i = (sources.Length - 2); i >= 0; i--)
                    {
                        if (!first)
                        {
                            // \u2B05 is an arrow <--
                            buffer.Append(" \u2B05 ");
                        }
                        first = false;
                        buffer.Append(sources[i]);
                    }
                    tbody.Tr().Td(entry.GetName()).Td(entry.GetValue()).Td(buffer.ToString()).();
                }
                tbody.().Tfoot().Tr().Th().Input("search_init").$type(HamletSpec.InputType.text).
                $name("key").$value("key").().().Th().Input("search_init").$type(HamletSpec.InputType
                                                                                 .text).$name("value").$value("value").().().Th().Input("search_init").$type(HamletSpec.InputType
                                                                                                                                                             .text).$name("source chain").$value("source chain").().().().().();
            }
            catch (IOException e)
            {
                Log.Error("Error while reading " + confPath, e);
                html.P().("Sorry got an error while reading conf file. ", confPath);
            }
        }
Esempio n. 27
0
        /// <exception cref="System.IO.IOException"/>
        /// <exception cref="System.Exception"/>
        public virtual string GetStagingAreaDir()
        {
            //    Path path = new Path(MRJobConstants.JOB_SUBMIT_DIR);
            string user = UserGroupInformation.GetCurrentUser().GetShortUserName();
            Path   path = MRApps.GetStagingAreaDir(conf, user);

            Log.Debug("getStagingAreaDir: dir=" + path);
            return(path.ToString());
        }
Esempio n. 28
0
        /// <summary>Gets the configured directory prefix for In Progress history files.</summary>
        /// <param name="conf">the configuration for hte job</param>
        /// <param name="jobId">the id of the job the history file is for.</param>
        /// <returns>A string representation of the prefix.</returns>
        /// <exception cref="System.IO.IOException"/>
        public static string GetConfiguredHistoryStagingDirPrefix(Configuration conf, string
                                                                  jobId)
        {
            string user        = UserGroupInformation.GetCurrentUser().GetShortUserName();
            Path   stagingPath = MRApps.GetStagingAreaDir(conf, user);
            Path   path        = new Path(stagingPath, jobId);
            string logDir      = path.ToString();

            return(EnsurePathInDefaultFileSystem(logDir, conf));
        }
Esempio n. 29
0
        public JobInfo(Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job)
        {
            this.id = MRApps.ToString(job.GetID());
            JobReport report = job.GetReport();

            this.mapsTotal        = job.GetTotalMaps();
            this.mapsCompleted    = job.GetCompletedMaps();
            this.reducesTotal     = job.GetTotalReduces();
            this.reducesCompleted = job.GetCompletedReduces();
            this.submitTime       = report.GetSubmitTime();
            this.startTime        = report.GetStartTime();
            this.finishTime       = report.GetFinishTime();
            this.name             = job.GetName().ToString();
            this.queue            = job.GetQueueName();
            this.user             = job.GetUserName();
            this.state            = job.GetState().ToString();
            this.acls             = new AList <ConfEntryInfo>();
            if (job is CompletedJob)
            {
                avgMapTime               = 0l;
                avgReduceTime            = 0l;
                avgShuffleTime           = 0l;
                avgMergeTime             = 0l;
                failedReduceAttempts     = 0;
                killedReduceAttempts     = 0;
                successfulReduceAttempts = 0;
                failedMapAttempts        = 0;
                killedMapAttempts        = 0;
                successfulMapAttempts    = 0;
                CountTasksAndAttempts(job);
                this.uberized    = job.IsUber();
                this.diagnostics = string.Empty;
                IList <string> diagnostics = job.GetDiagnostics();
                if (diagnostics != null && !diagnostics.IsEmpty())
                {
                    StringBuilder b = new StringBuilder();
                    foreach (string diag in diagnostics)
                    {
                        b.Append(diag);
                    }
                    this.diagnostics = b.ToString();
                }
                IDictionary <JobACL, AccessControlList> allacls = job.GetJobACLs();
                if (allacls != null)
                {
                    foreach (KeyValuePair <JobACL, AccessControlList> entry in allacls)
                    {
                        this.acls.AddItem(new ConfEntryInfo(entry.Key.GetAclName(), entry.Value.GetAclString
                                                                ()));
                    }
                }
            }
        }
Esempio n. 30
0
        public JobInfo(Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job, bool hasAccess)
        {
            // ok for any user to see
            // these should only be seen if acls allow
            this.id = MRApps.ToString(job.GetID());
            JobReport report = job.GetReport();

            this.startTime   = report.GetStartTime();
            this.finishTime  = report.GetFinishTime();
            this.elapsedTime = Times.Elapsed(this.startTime, this.finishTime);
            if (this.elapsedTime == -1)
            {
                this.elapsedTime = 0;
            }
            this.name                  = job.GetName().ToString();
            this.user                  = job.GetUserName();
            this.state                 = job.GetState();
            this.mapsTotal             = job.GetTotalMaps();
            this.mapsCompleted         = job.GetCompletedMaps();
            this.mapProgress           = report.GetMapProgress() * 100;
            this.mapProgressPercent    = StringHelper.Percent(report.GetMapProgress());
            this.reducesTotal          = job.GetTotalReduces();
            this.reducesCompleted      = job.GetCompletedReduces();
            this.reduceProgress        = report.GetReduceProgress() * 100;
            this.reduceProgressPercent = StringHelper.Percent(report.GetReduceProgress());
            this.acls                  = new AList <ConfEntryInfo>();
            if (hasAccess)
            {
                this.diagnostics = string.Empty;
                CountTasksAndAttempts(job);
                this.uberized = job.IsUber();
                IList <string> diagnostics = job.GetDiagnostics();
                if (diagnostics != null && !diagnostics.IsEmpty())
                {
                    StringBuilder b = new StringBuilder();
                    foreach (string diag in diagnostics)
                    {
                        b.Append(diag);
                    }
                    this.diagnostics = b.ToString();
                }
                IDictionary <JobACL, AccessControlList> allacls = job.GetJobACLs();
                if (allacls != null)
                {
                    foreach (KeyValuePair <JobACL, AccessControlList> entry in allacls)
                    {
                        this.acls.AddItem(new ConfEntryInfo(entry.Key.GetAclName(), entry.Value.GetAclString
                                                                ()));
                    }
                }
            }
        }