// acls not being checked since // we are using mock job instead of CompletedJob public static void VerifyHsJobGeneric(Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job, string id, string user, string name, string state, string queue, long startTime , long finishTime, int mapsTotal, int mapsCompleted, int reducesTotal, int reducesCompleted ) { JobReport report = job.GetReport(); WebServicesTestUtils.CheckStringMatch("id", MRApps.ToString(job.GetID()), id); WebServicesTestUtils.CheckStringMatch("user", job.GetUserName().ToString(), user); WebServicesTestUtils.CheckStringMatch("name", job.GetName(), name); WebServicesTestUtils.CheckStringMatch("state", job.GetState().ToString(), state); WebServicesTestUtils.CheckStringMatch("queue", job.GetQueueName(), queue); NUnit.Framework.Assert.AreEqual("startTime incorrect", report.GetStartTime(), startTime ); NUnit.Framework.Assert.AreEqual("finishTime incorrect", report.GetFinishTime(), finishTime ); NUnit.Framework.Assert.AreEqual("mapsTotal incorrect", job.GetTotalMaps(), mapsTotal ); NUnit.Framework.Assert.AreEqual("mapsCompleted incorrect", job.GetCompletedMaps() , mapsCompleted); NUnit.Framework.Assert.AreEqual("reducesTotal incorrect", job.GetTotalReduces(), reducesTotal); NUnit.Framework.Assert.AreEqual("reducesCompleted incorrect", job.GetCompletedReduces (), reducesCompleted); }
public virtual void TestJobsQueryState() { WebResource r = Resource(); // we only create 3 jobs and it cycles through states so we should have 3 unique states IDictionary <JobId, Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job> jobsMap = appContext .GetAllJobs(); string queryState = "BOGUS"; JobId jid = null; foreach (KeyValuePair <JobId, Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job> entry in jobsMap) { jid = entry.Value.GetID(); queryState = entry.Value.GetState().ToString(); break; } ClientResponse response = r.Path("ws").Path("v1").Path("history").Path("mapreduce" ).Path("jobs").QueryParam("state", queryState).Accept(MediaType.ApplicationJson) .Get <ClientResponse>(); NUnit.Framework.Assert.AreEqual(MediaType.ApplicationJsonType, response.GetType() ); JSONObject json = response.GetEntity <JSONObject>(); NUnit.Framework.Assert.AreEqual("incorrect number of elements", 1, json.Length()); JSONObject jobs = json.GetJSONObject("jobs"); JSONArray arr = jobs.GetJSONArray("job"); NUnit.Framework.Assert.AreEqual("incorrect number of elements", 1, arr.Length()); JSONObject info = arr.GetJSONObject(0); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = appContext.GetPartialJob(jid); VerifyJobsUtils.VerifyHsJobPartial(info, job); }
public virtual void VerifyHsJobConfXML(NodeList nodes, Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job) { NUnit.Framework.Assert.AreEqual("incorrect number of elements", 1, nodes.GetLength ()); for (int i = 0; i < nodes.GetLength(); i++) { Element element = (Element)nodes.Item(i); WebServicesTestUtils.CheckStringMatch("path", job.GetConfFile().ToString(), WebServicesTestUtils .GetXmlString(element, "path")); // just do simple verification of fields - not data is correct // in the fields NodeList properties = element.GetElementsByTagName("property"); for (int j = 0; j < properties.GetLength(); j++) { Element property = (Element)properties.Item(j); NUnit.Framework.Assert.IsNotNull("should have counters in the web service info", property); string name = WebServicesTestUtils.GetXmlString(property, "name"); string value = WebServicesTestUtils.GetXmlString(property, "value"); NUnit.Framework.Assert.IsTrue("name not set", (name != null && !name.IsEmpty())); NUnit.Framework.Assert.IsTrue("name not set", (value != null && !value.IsEmpty()) ); } } }
/// <summary>Ensure that a JOB_ID was passed into the page.</summary> public virtual void RequireJob() { if ($(JobId).IsEmpty()) { BadRequest("missing job ID"); throw new RuntimeException("Bad Request: Missing job ID"); } JobId jobID = MRApps.ToJobID($(JobId)); app.SetJob(app.context.GetJob(jobID)); if (app.GetJob() == null) { NotFound($(JobId)); throw new RuntimeException("Not Found: " + $(JobId)); } /* check for acl access */ Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.context.GetJob(jobID); if (!CheckAccess(job)) { AccessDenied("User " + Request().GetRemoteUser() + " does not have " + " permission to view job " + $(JobId)); throw new RuntimeException("Access denied: User " + Request().GetRemoteUser() + " does not have permission to view job " + $(JobId)); } }
/// <summary>Ensure that a TASK_ID was passed into the page.</summary> public virtual void RequireTask() { if ($(TaskId).IsEmpty()) { BadRequest("missing task ID"); throw new RuntimeException("missing task ID"); } TaskId taskID = MRApps.ToTaskID($(TaskId)); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.context.GetJob(taskID.GetJobId ()); app.SetJob(job); if (app.GetJob() == null) { NotFound(MRApps.ToString(taskID.GetJobId())); throw new RuntimeException("Not Found: " + $(JobId)); } else { app.SetTask(app.GetJob().GetTask(taskID)); if (app.GetTask() == null) { NotFound($(TaskId)); throw new RuntimeException("Not Found: " + $(TaskId)); } } if (!CheckAccess(job)) { AccessDenied("User " + Request().GetRemoteUser() + " does not have " + " permission to view job " + $(JobId)); throw new RuntimeException("Access denied: User " + Request().GetRemoteUser() + " does not have permission to view job " + $(JobId)); } }
/// <exception cref="System.Exception"/> private void TestReduceCommandLine(Configuration conf) { TestMapReduceChildJVM.MyMRApp app = new TestMapReduceChildJVM.MyMRApp(0, 1, true, this.GetType().FullName, true); conf.SetBoolean(MRConfig.MapreduceAppSubmissionCrossPlatform, true); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(conf); app.WaitForState(job, JobState.Succeeded); app.VerifyCompleted(); long shuffleLogSize = conf.GetLong(MRJobConfig.ShuffleLogKb, 0L) * 1024L; int shuffleBackups = conf.GetInt(MRJobConfig.ShuffleLogBackups, 0); string appenderName = shuffleLogSize > 0L && shuffleBackups > 0 ? "shuffleCRLA" : "shuffleCLA"; NUnit.Framework.Assert.AreEqual("[" + MRApps.CrossPlatformify("JAVA_HOME") + "/bin/java" + " -Djava.net.preferIPv4Stack=true" + " -Dhadoop.metrics.log.level=WARN" + " -Xmx200m -Djava.io.tmpdir=" + MRApps.CrossPlatformify("PWD") + "/tmp" + " -Dlog4j.configuration=container-log4j.properties" + " -Dyarn.app.container.log.dir=<LOG_DIR>" + " -Dyarn.app.container.log.filesize=0" + " -Dhadoop.root.logger=INFO,CLA -Dhadoop.root.logfile=syslog" + " -Dyarn.app.mapreduce.shuffle.logger=INFO," + appenderName + " -Dyarn.app.mapreduce.shuffle.logfile=syslog.shuffle" + " -Dyarn.app.mapreduce.shuffle.log.filesize=" + shuffleLogSize + " -Dyarn.app.mapreduce.shuffle.log.backups=" + shuffleBackups + " org.apache.hadoop.mapred.YarnChild 127.0.0.1" + " 54321" + " attempt_0_0000_r_000000_0" + " 0" + " 1><LOG_DIR>/stdout" + " 2><LOG_DIR>/stderr ]", app.myCommandLine); NUnit.Framework.Assert.IsTrue("HADOOP_ROOT_LOGGER not set for job", app.cmdEnvironment .Contains("HADOOP_ROOT_LOGGER")); NUnit.Framework.Assert.AreEqual("INFO,console", app.cmdEnvironment["HADOOP_ROOT_LOGGER" ]); NUnit.Framework.Assert.IsTrue("HADOOP_CLIENT_OPTS not set for job", app.cmdEnvironment .Contains("HADOOP_CLIENT_OPTS")); NUnit.Framework.Assert.AreEqual(string.Empty, app.cmdEnvironment["HADOOP_CLIENT_OPTS" ]); }
/// <exception cref="System.Exception"/> public virtual void TestCommandLine() { TestMapReduceChildJVM.MyMRApp app = new TestMapReduceChildJVM.MyMRApp(1, 0, true, this.GetType().FullName, true); Configuration conf = new Configuration(); conf.SetBoolean(MRConfig.MapreduceAppSubmissionCrossPlatform, true); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(conf); app.WaitForState(job, JobState.Succeeded); app.VerifyCompleted(); NUnit.Framework.Assert.AreEqual("[" + MRApps.CrossPlatformify("JAVA_HOME") + "/bin/java" + " -Djava.net.preferIPv4Stack=true" + " -Dhadoop.metrics.log.level=WARN" + " -Xmx200m -Djava.io.tmpdir=" + MRApps.CrossPlatformify("PWD") + "/tmp" + " -Dlog4j.configuration=container-log4j.properties" + " -Dyarn.app.container.log.dir=<LOG_DIR>" + " -Dyarn.app.container.log.filesize=0" + " -Dhadoop.root.logger=INFO,CLA -Dhadoop.root.logfile=syslog" + " org.apache.hadoop.mapred.YarnChild 127.0.0.1" + " 54321" + " attempt_0_0000_m_000000_0" + " 0" + " 1><LOG_DIR>/stdout" + " 2><LOG_DIR>/stderr ]" , app.myCommandLine); NUnit.Framework.Assert.IsTrue("HADOOP_ROOT_LOGGER not set for job", app.cmdEnvironment .Contains("HADOOP_ROOT_LOGGER")); NUnit.Framework.Assert.AreEqual("INFO,console", app.cmdEnvironment["HADOOP_ROOT_LOGGER" ]); NUnit.Framework.Assert.IsTrue("HADOOP_CLIENT_OPTS not set for job", app.cmdEnvironment .Contains("HADOOP_CLIENT_OPTS")); NUnit.Framework.Assert.AreEqual(string.Empty, app.cmdEnvironment["HADOOP_CLIENT_OPTS" ]); }
public virtual void TestEnvironmentVariables() { TestMapReduceChildJVM.MyMRApp app = new TestMapReduceChildJVM.MyMRApp(1, 0, true, this.GetType().FullName, true); Configuration conf = new Configuration(); conf.Set(JobConf.MapredMapTaskEnv, "HADOOP_CLIENT_OPTS=test"); conf.SetStrings(MRJobConfig.MapLogLevel, "WARN"); conf.SetBoolean(MRConfig.MapreduceAppSubmissionCrossPlatform, false); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(conf); app.WaitForState(job, JobState.Succeeded); app.VerifyCompleted(); NUnit.Framework.Assert.IsTrue("HADOOP_ROOT_LOGGER not set for job", app.cmdEnvironment .Contains("HADOOP_ROOT_LOGGER")); NUnit.Framework.Assert.AreEqual("WARN,console", app.cmdEnvironment["HADOOP_ROOT_LOGGER" ]); NUnit.Framework.Assert.IsTrue("HADOOP_CLIENT_OPTS not set for job", app.cmdEnvironment .Contains("HADOOP_CLIENT_OPTS")); NUnit.Framework.Assert.AreEqual("test", app.cmdEnvironment["HADOOP_CLIENT_OPTS"]); // Try one more. app = new TestMapReduceChildJVM.MyMRApp(1, 0, true, this.GetType().FullName, true ); conf = new Configuration(); conf.Set(JobConf.MapredMapTaskEnv, "HADOOP_ROOT_LOGGER=trace"); job = app.Submit(conf); app.WaitForState(job, JobState.Succeeded); app.VerifyCompleted(); NUnit.Framework.Assert.IsTrue("HADOOP_ROOT_LOGGER not set for job", app.cmdEnvironment .Contains("HADOOP_ROOT_LOGGER")); NUnit.Framework.Assert.AreEqual("trace", app.cmdEnvironment["HADOOP_ROOT_LOGGER"] ); }
/* * (non-Javadoc) * @see org.apache.hadoop.yarn.webapp.view.HtmlBlock#render(org.apache.hadoop.yarn.webapp.view.HtmlBlock.Block) */ protected override void Render(HtmlBlock.Block html) { string jid = $(AMParams.JobId); if (jid.IsEmpty()) { html.P().("Sorry, can't do anything without a JobID.").(); return; } JobId jobID = MRApps.ToJobID(jid); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = appContext.GetJob(jobID); if (job == null) { html.P().("Sorry, ", jid, " not found.").(); return; } Path confPath = job.GetConfFile(); try { ConfInfo info = new ConfInfo(job); html.Div().H3(confPath.ToString()).(); Hamlet.TBODY <Hamlet.TABLE <Org.Apache.Hadoop.Yarn.Webapp.Hamlet.Hamlet> > tbody = html .Table("#conf").Thead().Tr().Th(JQueryUI.Th, "key").Th(JQueryUI.Th, "value").Th( JQueryUI.Th, "source chain").().().Tbody(); // Tasks table foreach (ConfEntryInfo entry in info.GetProperties()) { StringBuilder buffer = new StringBuilder(); string[] sources = entry.GetSource(); //Skip the last entry, because it is always the same HDFS file, and // output them in reverse order so most recent is output first bool first = true; for (int i = (sources.Length - 2); i >= 0; i--) { if (!first) { // \u2B05 is an arrow <-- buffer.Append(" \u2B05 "); } first = false; buffer.Append(sources[i]); } tbody.Tr().Td(entry.GetName()).Td(entry.GetValue()).Td(buffer.ToString()).(); } tbody.().Tfoot().Tr().Th().Input("search_init").$type(HamletSpec.InputType.text). $name("key").$value("key").().().Th().Input("search_init").$type(HamletSpec.InputType .text).$name("value").$value("value").().().Th().Input("search_init").$type(HamletSpec.InputType .text).$name("source chain").$value("source chain").().().().().(); } catch (IOException e) { Log.Error("Error while reading " + confPath, e); html.P().("Sorry got an error while reading conf file. ", confPath); } }
/// <exception cref="Org.Codehaus.Jettison.Json.JSONException"/> public static void VerifyHsJobPartial(JSONObject info, Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job) { NUnit.Framework.Assert.AreEqual("incorrect number of elements", 12, info.Length() ); // everyone access fields VerifyHsJobGeneric(job, info.GetString("id"), info.GetString("user"), info.GetString ("name"), info.GetString("state"), info.GetString("queue"), info.GetLong("startTime" ), info.GetLong("finishTime"), info.GetInt("mapsTotal"), info.GetInt("mapsCompleted" ), info.GetInt("reducesTotal"), info.GetInt("reducesCompleted")); }
/// <exception cref="System.Exception"/> private void PrintStat(Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job, long startTime ) { long currentTime = Runtime.CurrentTimeMillis(); Runtime.GetRuntime().Gc(); long mem = Runtime.GetRuntime().TotalMemory() - Runtime.GetRuntime().FreeMemory(); System.Console.Out.WriteLine("JobState:" + job.GetState() + " CompletedMaps:" + job .GetCompletedMaps() + " CompletedReduces:" + job.GetCompletedReduces() + " Memory(total-free)(KB):" + mem / 1024 + " ElapsedTime(ms):" + (currentTime - startTime)); }
/// <exception cref="System.IO.IOException"/> public ConfInfo(Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job) { this.property = new AList <ConfEntryInfo>(); Configuration jobConf = job.LoadConfFile(); this.path = job.GetConfFile().ToString(); foreach (KeyValuePair <string, string> entry in jobConf) { this.property.AddItem(new ConfEntryInfo(entry.Key, entry.Value, jobConf.GetPropertySources (entry.Key))); } }
public MockAppContext(int appid, int numTasks, int numAttempts, Path confPath) { appID = MockJobs.NewAppID(appid); appAttemptID = ApplicationAttemptId.NewInstance(appID, 0); IDictionary <JobId, Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job> map = Maps.NewHashMap (); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = MockJobs.NewJob(appID, 0, numTasks , numAttempts, confPath); map[job.GetID()] = job; jobs = map; }
/// <summary>Runs memory and time benchmark with Mock MRApp.</summary> /// <exception cref="System.Exception"/> public virtual void Run(MRApp app) { Logger rootLogger = LogManager.GetRootLogger(); rootLogger.SetLevel(Level.Warn); long startTime = Runtime.CurrentTimeMillis(); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(new Configuration()); while (!job.GetReport().GetJobState().Equals(JobState.Succeeded)) { PrintStat(job, startTime); Sharpen.Thread.Sleep(2000); } PrintStat(job, startTime); }
/// <summary>check for job access.</summary> /// <param name="job">the job that is being accessed</param> /// <returns>True if the requesting user has permission to view the job</returns> internal virtual bool CheckAccess(Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job) { string remoteUser = Request().GetRemoteUser(); UserGroupInformation callerUGI = null; if (remoteUser != null) { callerUGI = UserGroupInformation.CreateRemoteUser(remoteUser); } if (callerUGI != null && !job.CheckAccess(callerUGI, JobACL.ViewJob)) { return(false); } return(true); }
private static AppContext CreateAppContext() { ApplicationId appId = ApplicationId.NewInstance(1, 1); ApplicationAttemptId attemptId = ApplicationAttemptId.NewInstance(appId, 1); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job >(); EventHandler eventHandler = Org.Mockito.Mockito.Mock <EventHandler>(); AppContext ctx = Org.Mockito.Mockito.Mock <AppContext>(); Org.Mockito.Mockito.When(ctx.GetApplicationID()).ThenReturn(appId); Org.Mockito.Mockito.When(ctx.GetApplicationAttemptId()).ThenReturn(attemptId); Org.Mockito.Mockito.When(ctx.GetJob(Matchers.IsA <JobId>())).ThenReturn(job); Org.Mockito.Mockito.When(ctx.GetClusterInfo()).ThenReturn(new ClusterInfo(Resource .NewInstance(10240, 1))); Org.Mockito.Mockito.When(ctx.GetEventHandler()).ThenReturn(eventHandler); return(ctx); }
/// <exception cref="Org.Codehaus.Jettison.Json.JSONException"/> public virtual void VerifyHsJobConf(JSONObject info, Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job) { NUnit.Framework.Assert.AreEqual("incorrect number of elements", 2, info.Length()); WebServicesTestUtils.CheckStringMatch("path", job.GetConfFile().ToString(), info. GetString("path")); // just do simple verification of fields - not data is correct // in the fields JSONArray properties = info.GetJSONArray("property"); for (int i = 0; i < properties.Length(); i++) { JSONObject prop = properties.GetJSONObject(i); string name = prop.GetString("name"); string value = prop.GetString("value"); NUnit.Framework.Assert.IsTrue("name not set", (name != null && !name.IsEmpty())); NUnit.Framework.Assert.IsTrue("value not set", (value != null && !value.IsEmpty() )); } }
public static void VerifyHsJobGenericSecure(Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job, bool uberized, string diagnostics, long avgMapTime, long avgReduceTime, long avgShuffleTime, long avgMergeTime, int failedReduceAttempts, int killedReduceAttempts , int successfulReduceAttempts, int failedMapAttempts, int killedMapAttempts, int successfulMapAttempts) { string diagString = string.Empty; IList <string> diagList = job.GetDiagnostics(); if (diagList != null && !diagList.IsEmpty()) { StringBuilder b = new StringBuilder(); foreach (string diag in diagList) { b.Append(diag); } diagString = b.ToString(); } WebServicesTestUtils.CheckStringMatch("diagnostics", diagString, diagnostics); NUnit.Framework.Assert.AreEqual("isUber incorrect", job.IsUber(), uberized); // unfortunately the following fields are all calculated in JobInfo // so not easily accessible without doing all the calculations again. // For now just make sure they are present. NUnit.Framework.Assert.IsTrue("failedReduceAttempts not >= 0", failedReduceAttempts >= 0); NUnit.Framework.Assert.IsTrue("killedReduceAttempts not >= 0", killedReduceAttempts >= 0); NUnit.Framework.Assert.IsTrue("successfulReduceAttempts not >= 0", successfulReduceAttempts >= 0); NUnit.Framework.Assert.IsTrue("failedMapAttempts not >= 0", failedMapAttempts >= 0); NUnit.Framework.Assert.IsTrue("killedMapAttempts not >= 0", killedMapAttempts >= 0); NUnit.Framework.Assert.IsTrue("successfulMapAttempts not >= 0", successfulMapAttempts >= 0); NUnit.Framework.Assert.IsTrue("avgMapTime not >= 0", avgMapTime >= 0); NUnit.Framework.Assert.IsTrue("avgReduceTime not >= 0", avgReduceTime >= 0); NUnit.Framework.Assert.IsTrue("avgShuffleTime not >= 0", avgShuffleTime >= 0); NUnit.Framework.Assert.IsTrue("avgMergeTime not >= 0", avgMergeTime >= 0); }
/// <exception cref="System.Exception"/> public virtual void TestCommandLineWithLog4JConifg() { TestMapReduceChildJVM.MyMRApp app = new TestMapReduceChildJVM.MyMRApp(1, 0, true, this.GetType().FullName, true); Configuration conf = new Configuration(); conf.SetBoolean(MRConfig.MapreduceAppSubmissionCrossPlatform, true); string testLogPropertieFile = "test-log4j.properties"; string testLogPropertiePath = "../" + "test-log4j.properties"; conf.Set(MRJobConfig.MapreduceJobLog4jPropertiesFile, testLogPropertiePath); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(conf); app.WaitForState(job, JobState.Succeeded); app.VerifyCompleted(); NUnit.Framework.Assert.AreEqual("[" + MRApps.CrossPlatformify("JAVA_HOME") + "/bin/java" + " -Djava.net.preferIPv4Stack=true" + " -Dhadoop.metrics.log.level=WARN" + " -Xmx200m -Djava.io.tmpdir=" + MRApps.CrossPlatformify("PWD") + "/tmp" + " -Dlog4j.configuration=" + testLogPropertieFile + " -Dyarn.app.container.log.dir=<LOG_DIR>" + " -Dyarn.app.container.log.filesize=0" + " -Dhadoop.root.logger=INFO,CLA -Dhadoop.root.logfile=syslog" + " org.apache.hadoop.mapred.YarnChild 127.0.0.1" + " 54321" + " attempt_0_0000_m_000000_0" + " 0" + " 1><LOG_DIR>/stdout" + " 2><LOG_DIR>/stderr ]" , app.myCommandLine); }
private Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job LoadJob(HistoryFileManager.HistoryFileInfo fileInfo) { try { Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = fileInfo.LoadJob(); if (Log.IsDebugEnabled()) { Log.Debug("Adding " + job.GetID() + " to loaded job cache"); } // We can clobber results here, but that should be OK, because it only // means that we may have two identical copies of the same job floating // around for a while. loadedJobCache[job.GetID()] = job; return(job); } catch (IOException e) { throw new YarnRuntimeException("Could not find/load job: " + fileInfo.GetJobId(), e); } }
/// <exception cref="Org.Codehaus.Jettison.Json.JSONException"/> public static void VerifyHsJob(JSONObject info, Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job) { NUnit.Framework.Assert.AreEqual("incorrect number of elements", 25, info.Length() ); // everyone access fields VerifyHsJobGeneric(job, info.GetString("id"), info.GetString("user"), info.GetString ("name"), info.GetString("state"), info.GetString("queue"), info.GetLong("startTime" ), info.GetLong("finishTime"), info.GetInt("mapsTotal"), info.GetInt("mapsCompleted" ), info.GetInt("reducesTotal"), info.GetInt("reducesCompleted")); string diagnostics = string.Empty; if (info.Has("diagnostics")) { diagnostics = info.GetString("diagnostics"); } // restricted access fields - if security and acls set VerifyHsJobGenericSecure(job, info.GetBoolean("uberized"), diagnostics, info.GetLong ("avgMapTime"), info.GetLong("avgReduceTime"), info.GetLong("avgShuffleTime"), info .GetLong("avgMergeTime"), info.GetInt("failedReduceAttempts"), info.GetInt("killedReduceAttempts" ), info.GetInt("successfulReduceAttempts"), info.GetInt("failedMapAttempts"), info .GetInt("killedMapAttempts"), info.GetInt("successfulMapAttempts")); }
public virtual Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job GetFullJob(JobId jobId) { if (Log.IsDebugEnabled()) { Log.Debug("Looking for Job " + jobId); } try { HistoryFileManager.HistoryFileInfo fileInfo = hsManager.GetFileInfo(jobId); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job result = null; if (fileInfo != null) { result = loadedJobCache[jobId]; if (result == null) { result = LoadJob(fileInfo); } else { if (fileInfo.IsDeleted()) { Sharpen.Collections.Remove(loadedJobCache, jobId); result = null; } } } else { Sharpen.Collections.Remove(loadedJobCache, jobId); } return(result); } catch (IOException e) { throw new YarnRuntimeException(e); } }
public virtual void TestJobsQueryUser() { WebResource r = Resource(); ClientResponse response = r.Path("ws").Path("v1").Path("history").Path("mapreduce" ).Path("jobs").QueryParam("user", "mock").Accept(MediaType.ApplicationJson).Get < ClientResponse>(); NUnit.Framework.Assert.AreEqual(MediaType.ApplicationJsonType, response.GetType() ); JSONObject json = response.GetEntity <JSONObject>(); System.Console.Out.WriteLine(json.ToString()); NUnit.Framework.Assert.AreEqual("incorrect number of elements", 1, json.Length()); JSONObject jobs = json.GetJSONObject("jobs"); JSONArray arr = jobs.GetJSONArray("job"); NUnit.Framework.Assert.AreEqual("incorrect number of elements", 3, arr.Length()); // just verify one of them. JSONObject info = arr.GetJSONObject(0); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = appContext.GetPartialJob(MRApps. ToJobID(info.GetString("id"))); VerifyJobsUtils.VerifyHsJobPartial(info, job); }
protected override void Render(HtmlBlock.Block html) { string jid = $(AMParams.JobId); if (jid.IsEmpty()) { html.P().("Sorry, can't do anything without a JobID.").(); return; } JobId jobID = MRApps.ToJobID(jid); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = appContext.GetJob(jobID); if (job == null) { html.P().("Sorry, ", jid, " not found.").(); return; } IList <AMInfo> amInfos = job.GetAMInfos(); string amString = amInfos.Count == 1 ? "ApplicationMaster" : "ApplicationMasters"; JobInfo jinfo = new JobInfo(job, true); Info("Job Overview").("Job Name:", jinfo.GetName()).("State:", jinfo.GetState()). ("Uberized:", jinfo.IsUberized()).("Started:", Sharpen.Extensions.CreateDate(jinfo .GetStartTime())).("Elapsed:", StringUtils.FormatTime(jinfo.GetElapsedTime())); Hamlet.DIV <Org.Apache.Hadoop.Yarn.Webapp.Hamlet.Hamlet> div = html.(typeof(InfoBlock )).Div(JQueryUI.InfoWrap); // MRAppMasters Table Hamlet.TABLE <Hamlet.DIV <Org.Apache.Hadoop.Yarn.Webapp.Hamlet.Hamlet> > table = div .Table("#job"); table.Tr().Th(amString).().Tr().Th(JQueryUI.Th, "Attempt Number").Th(JQueryUI.Th, "Start Time").Th(JQueryUI.Th, "Node").Th(JQueryUI.Th, "Logs").(); foreach (AMInfo amInfo in amInfos) { AMAttemptInfo attempt = new AMAttemptInfo(amInfo, jinfo.GetId(), jinfo.GetUserName ()); table.Tr().Td(attempt.GetAttemptId().ToString()).Td(Sharpen.Extensions.CreateDate (attempt.GetStartTime()).ToString()).Td().A(".nodelink", Url(MRWebAppUtil.GetYARNWebappScheme (), attempt.GetNodeHttpAddress()), attempt.GetNodeHttpAddress()).().Td().A(".logslink" , Url(attempt.GetLogsLink()), "logs").().(); } table.(); div.(); html.Div(JQueryUI.InfoWrap).Table("#job").Tr().Th(JQueryUI.Th, "Task Type").Th(JQueryUI .Th, "Progress").Th(JQueryUI.Th, "Total").Th(JQueryUI.Th, "Pending").Th(JQueryUI .Th, "Running").Th(JQueryUI.Th, "Complete").().Tr(JQueryUI.Odd).Th("Map").Td().Div (JQueryUI.Progressbar).$title(StringHelper.Join(jinfo.GetMapProgressPercent(), '%' )).Div(JQueryUI.ProgressbarValue).$style(StringHelper.Join("width:", jinfo.GetMapProgressPercent (), '%')).().().().Td().A(Url("tasks", jid, "m", "ALL"), jinfo.GetMapsTotal().ToString ()).().Td().A(Url("tasks", jid, "m", "PENDING"), jinfo.GetMapsPending().ToString ()).().Td().A(Url("tasks", jid, "m", "RUNNING"), jinfo.GetMapsRunning().ToString ()).().Td().A(Url("tasks", jid, "m", "COMPLETED"), jinfo.GetMapsCompleted().ToString ()).().().Tr(JQueryUI.Even).Th("Reduce").Td().Div(JQueryUI.Progressbar).$title(StringHelper.Join (jinfo.GetReduceProgressPercent(), '%')).Div(JQueryUI.ProgressbarValue).$style(StringHelper.Join ("width:", jinfo.GetReduceProgressPercent(), '%')).().().().Td().A(Url("tasks", jid, "r", "ALL"), jinfo.GetReducesTotal().ToString()).().Td().A(Url("tasks", jid , "r", "PENDING"), jinfo.GetReducesPending().ToString()).().Td().A(Url("tasks", jid, "r", "RUNNING"), jinfo.GetReducesRunning().ToString()).().Td().A(Url("tasks" , jid, "r", "COMPLETED"), jinfo.GetReducesCompleted().ToString()).().().().Table ("#job").Tr().Th(JQueryUI.Th, "Attempt Type").Th(JQueryUI.Th, "New").Th(JQueryUI .Th, "Running").Th(JQueryUI.Th, "Failed").Th(JQueryUI.Th, "Killed").Th(JQueryUI. Th, "Successful").().Tr(JQueryUI.Odd).Th("Maps").Td().A(Url("attempts", jid, "m" , MRApps.TaskAttemptStateUI.New.ToString()), jinfo.GetNewMapAttempts().ToString( )).().Td().A(Url("attempts", jid, "m", MRApps.TaskAttemptStateUI.Running.ToString ()), jinfo.GetRunningMapAttempts().ToString()).().Td().A(Url("attempts", jid, "m" , MRApps.TaskAttemptStateUI.Failed.ToString()), jinfo.GetFailedMapAttempts().ToString ()).().Td().A(Url("attempts", jid, "m", MRApps.TaskAttemptStateUI.Killed.ToString ()), jinfo.GetKilledMapAttempts().ToString()).().Td().A(Url("attempts", jid, "m" , MRApps.TaskAttemptStateUI.Successful.ToString()), jinfo.GetSuccessfulMapAttempts ().ToString()).().().Tr(JQueryUI.Even).Th("Reduces").Td().A(Url("attempts", jid, "r", MRApps.TaskAttemptStateUI.New.ToString()), jinfo.GetNewReduceAttempts().ToString ()).().Td().A(Url("attempts", jid, "r", MRApps.TaskAttemptStateUI.Running.ToString ()), jinfo.GetRunningReduceAttempts().ToString()).().Td().A(Url("attempts", jid, "r", MRApps.TaskAttemptStateUI.Failed.ToString()), jinfo.GetFailedReduceAttempts ().ToString()).().Td().A(Url("attempts", jid, "r", MRApps.TaskAttemptStateUI.Killed .ToString()), jinfo.GetKilledReduceAttempts().ToString()).().Td().A(Url("attempts" , jid, "r", MRApps.TaskAttemptStateUI.Successful.ToString()), jinfo.GetSuccessfulReduceAttempts ().ToString()).().().().(); }
/* * (non-Javadoc) * @see org.apache.hadoop.yarn.webapp.view.HtmlBlock#render(org.apache.hadoop.yarn.webapp.view.HtmlBlock.Block) */ protected override void Render(HtmlBlock.Block html) { string jid = $(AMParams.JobId); if (jid.IsEmpty()) { html.P().("Sorry, can't do anything without a JobID.").(); return; } JobId jobID = MRApps.ToJobID(jid); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job j = appContext.GetJob(jobID); if (j == null) { html.P().("Sorry, ", jid, " not found.").(); return; } IList <AMInfo> amInfos = j.GetAMInfos(); JobInfo job = new JobInfo(j); ResponseInfo infoBlock = Info("Job Overview").("Job Name:", job.GetName()).("User Name:" , job.GetUserName()).("Queue:", job.GetQueueName()).("State:", job.GetState()).( "Uberized:", job.IsUber()).("Submitted:", Sharpen.Extensions.CreateDate(job.GetSubmitTime ())).("Started:", Sharpen.Extensions.CreateDate(job.GetStartTime())).("Finished:" , Sharpen.Extensions.CreateDate(job.GetFinishTime())).("Elapsed:", StringUtils.FormatTime (Times.Elapsed(job.GetStartTime(), job.GetFinishTime(), false))); string amString = amInfos.Count == 1 ? "ApplicationMaster" : "ApplicationMasters"; // todo - switch to use JobInfo IList <string> diagnostics = j.GetDiagnostics(); if (diagnostics != null && !diagnostics.IsEmpty()) { StringBuilder b = new StringBuilder(); foreach (string diag in diagnostics) { b.Append(diag); } infoBlock.("Diagnostics:", b.ToString()); } if (job.GetNumMaps() > 0) { infoBlock.("Average Map Time", StringUtils.FormatTime(job.GetAvgMapTime())); } if (job.GetNumReduces() > 0) { infoBlock.("Average Shuffle Time", StringUtils.FormatTime(job.GetAvgShuffleTime() )); infoBlock.("Average Merge Time", StringUtils.FormatTime(job.GetAvgMergeTime())); infoBlock.("Average Reduce Time", StringUtils.FormatTime(job.GetAvgReduceTime())); } foreach (ConfEntryInfo entry in job.GetAcls()) { infoBlock.("ACL " + entry.GetName() + ":", entry.GetValue()); } Hamlet.DIV <Org.Apache.Hadoop.Yarn.Webapp.Hamlet.Hamlet> div = html.(typeof(InfoBlock )).Div(JQueryUI.InfoWrap); // MRAppMasters Table Hamlet.TABLE <Hamlet.DIV <Org.Apache.Hadoop.Yarn.Webapp.Hamlet.Hamlet> > table = div .Table("#job"); table.Tr().Th(amString).().Tr().Th(JQueryUI.Th, "Attempt Number").Th(JQueryUI.Th, "Start Time").Th(JQueryUI.Th, "Node").Th(JQueryUI.Th, "Logs").(); bool odd = false; foreach (AMInfo amInfo in amInfos) { AMAttemptInfo attempt = new AMAttemptInfo(amInfo, job.GetId(), job.GetUserName(), string.Empty, string.Empty); table.Tr((odd = !odd) ? JQueryUI.Odd : JQueryUI.Even).Td(attempt.GetAttemptId().ToString ()).Td(Sharpen.Extensions.CreateDate(attempt.GetStartTime()).ToString()).Td().A( ".nodelink", Url(MRWebAppUtil.GetYARNWebappScheme(), attempt.GetNodeHttpAddress( )), attempt.GetNodeHttpAddress()).().Td().A(".logslink", Url(attempt.GetShortLogsLink ()), "logs").().(); } table.(); div.(); html.Div(JQueryUI.InfoWrap).Table("#job").Tr().Th(JQueryUI.Th, "Task Type").Th(JQueryUI .Th, "Total").Th(JQueryUI.Th, "Complete").().Tr(JQueryUI.Odd).Th().A(Url("tasks" , jid, "m"), "Map").().Td(job.GetMapsTotal().ToString().ToString()).Td(job.GetMapsCompleted ().ToString().ToString()).().Tr(JQueryUI.Even).Th().A(Url("tasks", jid, "r"), "Reduce" ).().Td(job.GetReducesTotal().ToString().ToString()).Td(job.GetReducesCompleted( ).ToString().ToString()).().().Table("#job").Tr().Th(JQueryUI.Th, "Attempt Type" ).Th(JQueryUI.Th, "Failed").Th(JQueryUI.Th, "Killed").Th(JQueryUI.Th, "Successful" ).().Tr(JQueryUI.Odd).Th("Maps").Td().A(Url("attempts", jid, "m", MRApps.TaskAttemptStateUI .Failed.ToString()), job.GetFailedMapAttempts().ToString()).().Td().A(Url("attempts" , jid, "m", MRApps.TaskAttemptStateUI.Killed.ToString()), job.GetKilledMapAttempts ().ToString()).().Td().A(Url("attempts", jid, "m", MRApps.TaskAttemptStateUI.Successful .ToString()), job.GetSuccessfulMapAttempts().ToString()).().().Tr(JQueryUI.Even) .Th("Reduces").Td().A(Url("attempts", jid, "r", MRApps.TaskAttemptStateUI.Failed .ToString()), job.GetFailedReduceAttempts().ToString()).().Td().A(Url("attempts" , jid, "r", MRApps.TaskAttemptStateUI.Killed.ToString()), job.GetKilledReduceAttempts ().ToString()).().Td().A(Url("attempts", jid, "r", MRApps.TaskAttemptStateUI.Successful .ToString()), job.GetSuccessfulReduceAttempts().ToString()).().().().(); }
public virtual void TestRefreshLoadedJobCache() { HistoryFileManager historyManager = Org.Mockito.Mockito.Mock <HistoryFileManager>( ); jobHistory = Org.Mockito.Mockito.Spy(new JobHistory()); Org.Mockito.Mockito.DoReturn(historyManager).When(jobHistory).CreateHistoryFileManager (); Configuration conf = new Configuration(); // Set the cache size to 2 conf.Set(JHAdminConfig.MrHistoryLoadedJobCacheSize, "2"); jobHistory.Init(conf); jobHistory.Start(); CachedHistoryStorage storage = Org.Mockito.Mockito.Spy((CachedHistoryStorage)jobHistory .GetHistoryStorage()); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job[] jobs = new Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job [3]; JobId[] jobIds = new JobId[3]; for (int i = 0; i < 3; i++) { jobs[i] = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job>(); jobIds[i] = Org.Mockito.Mockito.Mock <JobId>(); Org.Mockito.Mockito.When(jobs[i].GetID()).ThenReturn(jobIds[i]); } HistoryFileManager.HistoryFileInfo fileInfo = Org.Mockito.Mockito.Mock <HistoryFileManager.HistoryFileInfo >(); Org.Mockito.Mockito.When(historyManager.GetFileInfo(Any <JobId>())).ThenReturn(fileInfo ); Org.Mockito.Mockito.When(fileInfo.LoadJob()).ThenReturn(jobs[0]).ThenReturn(jobs[ 1]).ThenReturn(jobs[2]); // getFullJob will put the job in the cache if it isn't there for (int i_1 = 0; i_1 < 3; i_1++) { storage.GetFullJob(jobs[i_1].GetID()); } IDictionary <JobId, Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job> jobCache = storage .GetLoadedJobCache(); // job0 should have been purged since cache size is 2 NUnit.Framework.Assert.IsFalse(jobCache.Contains(jobs[0].GetID())); NUnit.Framework.Assert.IsTrue(jobCache.Contains(jobs[1].GetID()) && jobCache.Contains (jobs[2].GetID())); // Setting cache size to 3 conf.Set(JHAdminConfig.MrHistoryLoadedJobCacheSize, "3"); Org.Mockito.Mockito.DoReturn(conf).When(storage).CreateConf(); Org.Mockito.Mockito.When(fileInfo.LoadJob()).ThenReturn(jobs[0]).ThenReturn(jobs[ 1]).ThenReturn(jobs[2]); jobHistory.RefreshLoadedJobCache(); for (int i_2 = 0; i_2 < 3; i_2++) { storage.GetFullJob(jobs[i_2].GetID()); } jobCache = storage.GetLoadedJobCache(); // All three jobs should be in cache since its size is now 3 for (int i_3 = 0; i_3 < 3; i_3++) { NUnit.Framework.Assert.IsTrue(jobCache.Contains(jobs[i_3].GetID())); } }