// acls not being checked since
// we are using mock job instead of CompletedJob
public static void VerifyHsJobGeneric(Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job
job, string id, string user, string name, string state, string queue, long startTime
, long finishTime, int mapsTotal, int mapsCompleted, int reducesTotal, int reducesCompleted
)
{
JobReport report = job.GetReport();
WebServicesTestUtils.CheckStringMatch("id", MRApps.ToString(job.GetID()), id);
WebServicesTestUtils.CheckStringMatch("user", job.GetUserName().ToString(), user);
WebServicesTestUtils.CheckStringMatch("name", job.GetName(), name);
WebServicesTestUtils.CheckStringMatch("state", job.GetState().ToString(), state);
WebServicesTestUtils.CheckStringMatch("queue", job.GetQueueName(), queue);
NUnit.Framework.Assert.AreEqual("startTime incorrect", report.GetStartTime(), startTime
);
NUnit.Framework.Assert.AreEqual("finishTime incorrect", report.GetFinishTime(), finishTime
);
NUnit.Framework.Assert.AreEqual("mapsTotal incorrect", job.GetTotalMaps(), mapsTotal
);
NUnit.Framework.Assert.AreEqual("mapsCompleted incorrect", job.GetCompletedMaps()
, mapsCompleted);
NUnit.Framework.Assert.AreEqual("reducesTotal incorrect", job.GetTotalReduces(),
reducesTotal);
NUnit.Framework.Assert.AreEqual("reducesCompleted incorrect", job.GetCompletedReduces
(), reducesCompleted);
}
public virtual void TestJobsQueryState()
{
WebResource r = Resource();
// we only create 3 jobs and it cycles through states so we should have 3 unique states
IDictionary <JobId, Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job> jobsMap = appContext
.GetAllJobs();
string queryState = "BOGUS";
JobId jid = null;
foreach (KeyValuePair <JobId, Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job> entry in
jobsMap)
{
jid = entry.Value.GetID();
queryState = entry.Value.GetState().ToString();
break;
}
ClientResponse response = r.Path("ws").Path("v1").Path("history").Path("mapreduce"
).Path("jobs").QueryParam("state", queryState).Accept(MediaType.ApplicationJson)
.Get <ClientResponse>();
NUnit.Framework.Assert.AreEqual(MediaType.ApplicationJsonType, response.GetType()
);
JSONObject json = response.GetEntity <JSONObject>();
NUnit.Framework.Assert.AreEqual("incorrect number of elements", 1, json.Length());
JSONObject jobs = json.GetJSONObject("jobs");
JSONArray arr = jobs.GetJSONArray("job");
NUnit.Framework.Assert.AreEqual("incorrect number of elements", 1, arr.Length());
JSONObject info = arr.GetJSONObject(0);
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = appContext.GetPartialJob(jid);
VerifyJobsUtils.VerifyHsJobPartial(info, job);
}
public virtual void VerifyHsJobConfXML(NodeList nodes, Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job
job)
{
NUnit.Framework.Assert.AreEqual("incorrect number of elements", 1, nodes.GetLength
());
for (int i = 0; i < nodes.GetLength(); i++)
{
Element element = (Element)nodes.Item(i);
WebServicesTestUtils.CheckStringMatch("path", job.GetConfFile().ToString(), WebServicesTestUtils
.GetXmlString(element, "path"));
// just do simple verification of fields - not data is correct
// in the fields
NodeList properties = element.GetElementsByTagName("property");
for (int j = 0; j < properties.GetLength(); j++)
{
Element property = (Element)properties.Item(j);
NUnit.Framework.Assert.IsNotNull("should have counters in the web service info",
property);
string name = WebServicesTestUtils.GetXmlString(property, "name");
string value = WebServicesTestUtils.GetXmlString(property, "value");
NUnit.Framework.Assert.IsTrue("name not set", (name != null && !name.IsEmpty()));
NUnit.Framework.Assert.IsTrue("name not set", (value != null && !value.IsEmpty())
);
}
}
}
/// <summary>Ensure that a JOB_ID was passed into the page.</summary>
public virtual void RequireJob()
{
if ($(JobId).IsEmpty())
{
BadRequest("missing job ID");
throw new RuntimeException("Bad Request: Missing job ID");
}
JobId jobID = MRApps.ToJobID($(JobId));
app.SetJob(app.context.GetJob(jobID));
if (app.GetJob() == null)
{
NotFound($(JobId));
throw new RuntimeException("Not Found: " + $(JobId));
}
/* check for acl access */
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.context.GetJob(jobID);
if (!CheckAccess(job))
{
AccessDenied("User " + Request().GetRemoteUser() + " does not have " + " permission to view job "
+ $(JobId));
throw new RuntimeException("Access denied: User " + Request().GetRemoteUser() + " does not have permission to view job "
+ $(JobId));
}
}
/// <summary>Ensure that a TASK_ID was passed into the page.</summary>
public virtual void RequireTask()
{
if ($(TaskId).IsEmpty())
{
BadRequest("missing task ID");
throw new RuntimeException("missing task ID");
}
TaskId taskID = MRApps.ToTaskID($(TaskId));
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.context.GetJob(taskID.GetJobId
());
app.SetJob(job);
if (app.GetJob() == null)
{
NotFound(MRApps.ToString(taskID.GetJobId()));
throw new RuntimeException("Not Found: " + $(JobId));
}
else
{
app.SetTask(app.GetJob().GetTask(taskID));
if (app.GetTask() == null)
{
NotFound($(TaskId));
throw new RuntimeException("Not Found: " + $(TaskId));
}
}
if (!CheckAccess(job))
{
AccessDenied("User " + Request().GetRemoteUser() + " does not have " + " permission to view job "
+ $(JobId));
throw new RuntimeException("Access denied: User " + Request().GetRemoteUser() + " does not have permission to view job "
+ $(JobId));
}
}
/// <exception cref="System.Exception"/>
private void TestReduceCommandLine(Configuration conf)
{
TestMapReduceChildJVM.MyMRApp app = new TestMapReduceChildJVM.MyMRApp(0, 1, true,
this.GetType().FullName, true);
conf.SetBoolean(MRConfig.MapreduceAppSubmissionCrossPlatform, true);
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(conf);
app.WaitForState(job, JobState.Succeeded);
app.VerifyCompleted();
long shuffleLogSize = conf.GetLong(MRJobConfig.ShuffleLogKb, 0L) * 1024L;
int shuffleBackups = conf.GetInt(MRJobConfig.ShuffleLogBackups, 0);
string appenderName = shuffleLogSize > 0L && shuffleBackups > 0 ? "shuffleCRLA" :
"shuffleCLA";
NUnit.Framework.Assert.AreEqual("[" + MRApps.CrossPlatformify("JAVA_HOME") + "/bin/java"
+ " -Djava.net.preferIPv4Stack=true" + " -Dhadoop.metrics.log.level=WARN" + " -Xmx200m -Djava.io.tmpdir="
+ MRApps.CrossPlatformify("PWD") + "/tmp" + " -Dlog4j.configuration=container-log4j.properties"
+ " -Dyarn.app.container.log.dir=<LOG_DIR>" + " -Dyarn.app.container.log.filesize=0"
+ " -Dhadoop.root.logger=INFO,CLA -Dhadoop.root.logfile=syslog" + " -Dyarn.app.mapreduce.shuffle.logger=INFO,"
+ appenderName + " -Dyarn.app.mapreduce.shuffle.logfile=syslog.shuffle" + " -Dyarn.app.mapreduce.shuffle.log.filesize="
+ shuffleLogSize + " -Dyarn.app.mapreduce.shuffle.log.backups=" + shuffleBackups
+ " org.apache.hadoop.mapred.YarnChild 127.0.0.1" + " 54321" + " attempt_0_0000_r_000000_0"
+ " 0" + " 1><LOG_DIR>/stdout" + " 2><LOG_DIR>/stderr ]", app.myCommandLine);
NUnit.Framework.Assert.IsTrue("HADOOP_ROOT_LOGGER not set for job", app.cmdEnvironment
.Contains("HADOOP_ROOT_LOGGER"));
NUnit.Framework.Assert.AreEqual("INFO,console", app.cmdEnvironment["HADOOP_ROOT_LOGGER"
]);
NUnit.Framework.Assert.IsTrue("HADOOP_CLIENT_OPTS not set for job", app.cmdEnvironment
.Contains("HADOOP_CLIENT_OPTS"));
NUnit.Framework.Assert.AreEqual(string.Empty, app.cmdEnvironment["HADOOP_CLIENT_OPTS"
]);
}
/// <exception cref="System.Exception"/>
public virtual void TestCommandLine()
{
TestMapReduceChildJVM.MyMRApp app = new TestMapReduceChildJVM.MyMRApp(1, 0, true,
this.GetType().FullName, true);
Configuration conf = new Configuration();
conf.SetBoolean(MRConfig.MapreduceAppSubmissionCrossPlatform, true);
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(conf);
app.WaitForState(job, JobState.Succeeded);
app.VerifyCompleted();
NUnit.Framework.Assert.AreEqual("[" + MRApps.CrossPlatformify("JAVA_HOME") + "/bin/java"
+ " -Djava.net.preferIPv4Stack=true" + " -Dhadoop.metrics.log.level=WARN" + " -Xmx200m -Djava.io.tmpdir="
+ MRApps.CrossPlatformify("PWD") + "/tmp" + " -Dlog4j.configuration=container-log4j.properties"
+ " -Dyarn.app.container.log.dir=<LOG_DIR>" + " -Dyarn.app.container.log.filesize=0"
+ " -Dhadoop.root.logger=INFO,CLA -Dhadoop.root.logfile=syslog" + " org.apache.hadoop.mapred.YarnChild 127.0.0.1"
+ " 54321" + " attempt_0_0000_m_000000_0" + " 0" + " 1><LOG_DIR>/stdout" + " 2><LOG_DIR>/stderr ]"
, app.myCommandLine);
NUnit.Framework.Assert.IsTrue("HADOOP_ROOT_LOGGER not set for job", app.cmdEnvironment
.Contains("HADOOP_ROOT_LOGGER"));
NUnit.Framework.Assert.AreEqual("INFO,console", app.cmdEnvironment["HADOOP_ROOT_LOGGER"
]);
NUnit.Framework.Assert.IsTrue("HADOOP_CLIENT_OPTS not set for job", app.cmdEnvironment
.Contains("HADOOP_CLIENT_OPTS"));
NUnit.Framework.Assert.AreEqual(string.Empty, app.cmdEnvironment["HADOOP_CLIENT_OPTS"
]);
}
public virtual void TestEnvironmentVariables()
{
TestMapReduceChildJVM.MyMRApp app = new TestMapReduceChildJVM.MyMRApp(1, 0, true,
this.GetType().FullName, true);
Configuration conf = new Configuration();
conf.Set(JobConf.MapredMapTaskEnv, "HADOOP_CLIENT_OPTS=test");
conf.SetStrings(MRJobConfig.MapLogLevel, "WARN");
conf.SetBoolean(MRConfig.MapreduceAppSubmissionCrossPlatform, false);
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(conf);
app.WaitForState(job, JobState.Succeeded);
app.VerifyCompleted();
NUnit.Framework.Assert.IsTrue("HADOOP_ROOT_LOGGER not set for job", app.cmdEnvironment
.Contains("HADOOP_ROOT_LOGGER"));
NUnit.Framework.Assert.AreEqual("WARN,console", app.cmdEnvironment["HADOOP_ROOT_LOGGER"
]);
NUnit.Framework.Assert.IsTrue("HADOOP_CLIENT_OPTS not set for job", app.cmdEnvironment
.Contains("HADOOP_CLIENT_OPTS"));
NUnit.Framework.Assert.AreEqual("test", app.cmdEnvironment["HADOOP_CLIENT_OPTS"]);
// Try one more.
app = new TestMapReduceChildJVM.MyMRApp(1, 0, true, this.GetType().FullName, true
);
conf = new Configuration();
conf.Set(JobConf.MapredMapTaskEnv, "HADOOP_ROOT_LOGGER=trace");
job = app.Submit(conf);
app.WaitForState(job, JobState.Succeeded);
app.VerifyCompleted();
NUnit.Framework.Assert.IsTrue("HADOOP_ROOT_LOGGER not set for job", app.cmdEnvironment
.Contains("HADOOP_ROOT_LOGGER"));
NUnit.Framework.Assert.AreEqual("trace", app.cmdEnvironment["HADOOP_ROOT_LOGGER"]
);
}
/*
* (non-Javadoc)
* @see org.apache.hadoop.yarn.webapp.view.HtmlBlock#render(org.apache.hadoop.yarn.webapp.view.HtmlBlock.Block)
*/
protected override void Render(HtmlBlock.Block html)
{
string jid = $(AMParams.JobId);
if (jid.IsEmpty())
{
html.P().("Sorry, can't do anything without a JobID.").();
return;
}
JobId jobID = MRApps.ToJobID(jid);
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = appContext.GetJob(jobID);
if (job == null)
{
html.P().("Sorry, ", jid, " not found.").();
return;
}
Path confPath = job.GetConfFile();
try
{
ConfInfo info = new ConfInfo(job);
html.Div().H3(confPath.ToString()).();
Hamlet.TBODY <Hamlet.TABLE <Org.Apache.Hadoop.Yarn.Webapp.Hamlet.Hamlet> > tbody = html
.Table("#conf").Thead().Tr().Th(JQueryUI.Th, "key").Th(JQueryUI.Th, "value").Th(
JQueryUI.Th, "source chain").().().Tbody();
// Tasks table
foreach (ConfEntryInfo entry in info.GetProperties())
{
StringBuilder buffer = new StringBuilder();
string[] sources = entry.GetSource();
//Skip the last entry, because it is always the same HDFS file, and
// output them in reverse order so most recent is output first
bool first = true;
for (int i = (sources.Length - 2); i >= 0; i--)
{
if (!first)
{
// \u2B05 is an arrow <--
buffer.Append(" \u2B05 ");
}
first = false;
buffer.Append(sources[i]);
}
tbody.Tr().Td(entry.GetName()).Td(entry.GetValue()).Td(buffer.ToString()).();
}
tbody.().Tfoot().Tr().Th().Input("search_init").$type(HamletSpec.InputType.text).
$name("key").$value("key").().().Th().Input("search_init").$type(HamletSpec.InputType
.text).$name("value").$value("value").().().Th().Input("search_init").$type(HamletSpec.InputType
.text).$name("source chain").$value("source chain").().().().().();
}
catch (IOException e)
{
Log.Error("Error while reading " + confPath, e);
html.P().("Sorry got an error while reading conf file. ", confPath);
}
}
/// <exception cref="Org.Codehaus.Jettison.Json.JSONException"/>
public static void VerifyHsJobPartial(JSONObject info, Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job
job)
{
NUnit.Framework.Assert.AreEqual("incorrect number of elements", 12, info.Length()
);
// everyone access fields
VerifyHsJobGeneric(job, info.GetString("id"), info.GetString("user"), info.GetString
("name"), info.GetString("state"), info.GetString("queue"), info.GetLong("startTime"
), info.GetLong("finishTime"), info.GetInt("mapsTotal"), info.GetInt("mapsCompleted"
), info.GetInt("reducesTotal"), info.GetInt("reducesCompleted"));
}
/// <exception cref="System.Exception"/>
private void PrintStat(Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job, long startTime
)
{
long currentTime = Runtime.CurrentTimeMillis();
Runtime.GetRuntime().Gc();
long mem = Runtime.GetRuntime().TotalMemory() - Runtime.GetRuntime().FreeMemory();
System.Console.Out.WriteLine("JobState:" + job.GetState() + " CompletedMaps:" + job
.GetCompletedMaps() + " CompletedReduces:" + job.GetCompletedReduces() + " Memory(total-free)(KB):"
+ mem / 1024 + " ElapsedTime(ms):" + (currentTime - startTime));
}
/// <exception cref="System.IO.IOException"/>
public ConfInfo(Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job)
{
this.property = new AList <ConfEntryInfo>();
Configuration jobConf = job.LoadConfFile();
this.path = job.GetConfFile().ToString();
foreach (KeyValuePair <string, string> entry in jobConf)
{
this.property.AddItem(new ConfEntryInfo(entry.Key, entry.Value, jobConf.GetPropertySources
(entry.Key)));
}
}
public MockAppContext(int appid, int numTasks, int numAttempts, Path confPath)
{
appID = MockJobs.NewAppID(appid);
appAttemptID = ApplicationAttemptId.NewInstance(appID, 0);
IDictionary <JobId, Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job> map = Maps.NewHashMap
();
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = MockJobs.NewJob(appID, 0, numTasks
, numAttempts, confPath);
map[job.GetID()] = job;
jobs = map;
}
/// <summary>Runs memory and time benchmark with Mock MRApp.</summary>
/// <exception cref="System.Exception"/>
public virtual void Run(MRApp app)
{
Logger rootLogger = LogManager.GetRootLogger();
rootLogger.SetLevel(Level.Warn);
long startTime = Runtime.CurrentTimeMillis();
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(new Configuration());
while (!job.GetReport().GetJobState().Equals(JobState.Succeeded))
{
PrintStat(job, startTime);
Sharpen.Thread.Sleep(2000);
}
PrintStat(job, startTime);
}
/// <summary>check for job access.</summary>
/// <param name="job">the job that is being accessed</param>
/// <returns>True if the requesting user has permission to view the job</returns>
internal virtual bool CheckAccess(Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job)
{
string remoteUser = Request().GetRemoteUser();
UserGroupInformation callerUGI = null;
if (remoteUser != null)
{
callerUGI = UserGroupInformation.CreateRemoteUser(remoteUser);
}
if (callerUGI != null && !job.CheckAccess(callerUGI, JobACL.ViewJob))
{
return(false);
}
return(true);
}
private static AppContext CreateAppContext()
{
ApplicationId appId = ApplicationId.NewInstance(1, 1);
ApplicationAttemptId attemptId = ApplicationAttemptId.NewInstance(appId, 1);
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job
>();
EventHandler eventHandler = Org.Mockito.Mockito.Mock <EventHandler>();
AppContext ctx = Org.Mockito.Mockito.Mock <AppContext>();
Org.Mockito.Mockito.When(ctx.GetApplicationID()).ThenReturn(appId);
Org.Mockito.Mockito.When(ctx.GetApplicationAttemptId()).ThenReturn(attemptId);
Org.Mockito.Mockito.When(ctx.GetJob(Matchers.IsA <JobId>())).ThenReturn(job);
Org.Mockito.Mockito.When(ctx.GetClusterInfo()).ThenReturn(new ClusterInfo(Resource
.NewInstance(10240, 1)));
Org.Mockito.Mockito.When(ctx.GetEventHandler()).ThenReturn(eventHandler);
return(ctx);
}
/// <exception cref="Org.Codehaus.Jettison.Json.JSONException"/>
public virtual void VerifyHsJobConf(JSONObject info, Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job
job)
{
NUnit.Framework.Assert.AreEqual("incorrect number of elements", 2, info.Length());
WebServicesTestUtils.CheckStringMatch("path", job.GetConfFile().ToString(), info.
GetString("path"));
// just do simple verification of fields - not data is correct
// in the fields
JSONArray properties = info.GetJSONArray("property");
for (int i = 0; i < properties.Length(); i++)
{
JSONObject prop = properties.GetJSONObject(i);
string name = prop.GetString("name");
string value = prop.GetString("value");
NUnit.Framework.Assert.IsTrue("name not set", (name != null && !name.IsEmpty()));
NUnit.Framework.Assert.IsTrue("value not set", (value != null && !value.IsEmpty()
));
}
}
public static void VerifyHsJobGenericSecure(Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job
job, bool uberized, string diagnostics, long avgMapTime, long avgReduceTime, long
avgShuffleTime, long avgMergeTime, int failedReduceAttempts, int killedReduceAttempts
, int successfulReduceAttempts, int failedMapAttempts, int killedMapAttempts, int
successfulMapAttempts)
{
string diagString = string.Empty;
IList <string> diagList = job.GetDiagnostics();
if (diagList != null && !diagList.IsEmpty())
{
StringBuilder b = new StringBuilder();
foreach (string diag in diagList)
{
b.Append(diag);
}
diagString = b.ToString();
}
WebServicesTestUtils.CheckStringMatch("diagnostics", diagString, diagnostics);
NUnit.Framework.Assert.AreEqual("isUber incorrect", job.IsUber(), uberized);
// unfortunately the following fields are all calculated in JobInfo
// so not easily accessible without doing all the calculations again.
// For now just make sure they are present.
NUnit.Framework.Assert.IsTrue("failedReduceAttempts not >= 0", failedReduceAttempts
>= 0);
NUnit.Framework.Assert.IsTrue("killedReduceAttempts not >= 0", killedReduceAttempts
>= 0);
NUnit.Framework.Assert.IsTrue("successfulReduceAttempts not >= 0", successfulReduceAttempts
>= 0);
NUnit.Framework.Assert.IsTrue("failedMapAttempts not >= 0", failedMapAttempts >=
0);
NUnit.Framework.Assert.IsTrue("killedMapAttempts not >= 0", killedMapAttempts >=
0);
NUnit.Framework.Assert.IsTrue("successfulMapAttempts not >= 0", successfulMapAttempts
>= 0);
NUnit.Framework.Assert.IsTrue("avgMapTime not >= 0", avgMapTime >= 0);
NUnit.Framework.Assert.IsTrue("avgReduceTime not >= 0", avgReduceTime >= 0);
NUnit.Framework.Assert.IsTrue("avgShuffleTime not >= 0", avgShuffleTime >= 0);
NUnit.Framework.Assert.IsTrue("avgMergeTime not >= 0", avgMergeTime >= 0);
}
/// <exception cref="System.Exception"/>
public virtual void TestCommandLineWithLog4JConifg()
{
TestMapReduceChildJVM.MyMRApp app = new TestMapReduceChildJVM.MyMRApp(1, 0, true,
this.GetType().FullName, true);
Configuration conf = new Configuration();
conf.SetBoolean(MRConfig.MapreduceAppSubmissionCrossPlatform, true);
string testLogPropertieFile = "test-log4j.properties";
string testLogPropertiePath = "../" + "test-log4j.properties";
conf.Set(MRJobConfig.MapreduceJobLog4jPropertiesFile, testLogPropertiePath);
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(conf);
app.WaitForState(job, JobState.Succeeded);
app.VerifyCompleted();
NUnit.Framework.Assert.AreEqual("[" + MRApps.CrossPlatformify("JAVA_HOME") + "/bin/java"
+ " -Djava.net.preferIPv4Stack=true" + " -Dhadoop.metrics.log.level=WARN" + " -Xmx200m -Djava.io.tmpdir="
+ MRApps.CrossPlatformify("PWD") + "/tmp" + " -Dlog4j.configuration=" + testLogPropertieFile
+ " -Dyarn.app.container.log.dir=<LOG_DIR>" + " -Dyarn.app.container.log.filesize=0"
+ " -Dhadoop.root.logger=INFO,CLA -Dhadoop.root.logfile=syslog" + " org.apache.hadoop.mapred.YarnChild 127.0.0.1"
+ " 54321" + " attempt_0_0000_m_000000_0" + " 0" + " 1><LOG_DIR>/stdout" + " 2><LOG_DIR>/stderr ]"
, app.myCommandLine);
}
private Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job LoadJob(HistoryFileManager.HistoryFileInfo
fileInfo)
{
try
{
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = fileInfo.LoadJob();
if (Log.IsDebugEnabled())
{
Log.Debug("Adding " + job.GetID() + " to loaded job cache");
}
// We can clobber results here, but that should be OK, because it only
// means that we may have two identical copies of the same job floating
// around for a while.
loadedJobCache[job.GetID()] = job;
return(job);
}
catch (IOException e)
{
throw new YarnRuntimeException("Could not find/load job: " + fileInfo.GetJobId(),
e);
}
}
/// <exception cref="Org.Codehaus.Jettison.Json.JSONException"/>
public static void VerifyHsJob(JSONObject info, Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job
job)
{
NUnit.Framework.Assert.AreEqual("incorrect number of elements", 25, info.Length()
);
// everyone access fields
VerifyHsJobGeneric(job, info.GetString("id"), info.GetString("user"), info.GetString
("name"), info.GetString("state"), info.GetString("queue"), info.GetLong("startTime"
), info.GetLong("finishTime"), info.GetInt("mapsTotal"), info.GetInt("mapsCompleted"
), info.GetInt("reducesTotal"), info.GetInt("reducesCompleted"));
string diagnostics = string.Empty;
if (info.Has("diagnostics"))
{
diagnostics = info.GetString("diagnostics");
}
// restricted access fields - if security and acls set
VerifyHsJobGenericSecure(job, info.GetBoolean("uberized"), diagnostics, info.GetLong
("avgMapTime"), info.GetLong("avgReduceTime"), info.GetLong("avgShuffleTime"), info
.GetLong("avgMergeTime"), info.GetInt("failedReduceAttempts"), info.GetInt("killedReduceAttempts"
), info.GetInt("successfulReduceAttempts"), info.GetInt("failedMapAttempts"), info
.GetInt("killedMapAttempts"), info.GetInt("successfulMapAttempts"));
}
public virtual Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job GetFullJob(JobId jobId)
{
if (Log.IsDebugEnabled())
{
Log.Debug("Looking for Job " + jobId);
}
try
{
HistoryFileManager.HistoryFileInfo fileInfo = hsManager.GetFileInfo(jobId);
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job result = null;
if (fileInfo != null)
{
result = loadedJobCache[jobId];
if (result == null)
{
result = LoadJob(fileInfo);
}
else
{
if (fileInfo.IsDeleted())
{
Sharpen.Collections.Remove(loadedJobCache, jobId);
result = null;
}
}
}
else
{
Sharpen.Collections.Remove(loadedJobCache, jobId);
}
return(result);
}
catch (IOException e)
{
throw new YarnRuntimeException(e);
}
}
public virtual void TestJobsQueryUser()
{
WebResource r = Resource();
ClientResponse response = r.Path("ws").Path("v1").Path("history").Path("mapreduce"
).Path("jobs").QueryParam("user", "mock").Accept(MediaType.ApplicationJson).Get <
ClientResponse>();
NUnit.Framework.Assert.AreEqual(MediaType.ApplicationJsonType, response.GetType()
);
JSONObject json = response.GetEntity <JSONObject>();
System.Console.Out.WriteLine(json.ToString());
NUnit.Framework.Assert.AreEqual("incorrect number of elements", 1, json.Length());
JSONObject jobs = json.GetJSONObject("jobs");
JSONArray arr = jobs.GetJSONArray("job");
NUnit.Framework.Assert.AreEqual("incorrect number of elements", 3, arr.Length());
// just verify one of them.
JSONObject info = arr.GetJSONObject(0);
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = appContext.GetPartialJob(MRApps.
ToJobID(info.GetString("id")));
VerifyJobsUtils.VerifyHsJobPartial(info, job);
}
protected override void Render(HtmlBlock.Block html)
{
string jid = $(AMParams.JobId);
if (jid.IsEmpty())
{
html.P().("Sorry, can't do anything without a JobID.").();
return;
}
JobId jobID = MRApps.ToJobID(jid);
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = appContext.GetJob(jobID);
if (job == null)
{
html.P().("Sorry, ", jid, " not found.").();
return;
}
IList <AMInfo> amInfos = job.GetAMInfos();
string amString = amInfos.Count == 1 ? "ApplicationMaster" : "ApplicationMasters";
JobInfo jinfo = new JobInfo(job, true);
Info("Job Overview").("Job Name:", jinfo.GetName()).("State:", jinfo.GetState()).
("Uberized:", jinfo.IsUberized()).("Started:", Sharpen.Extensions.CreateDate(jinfo
.GetStartTime())).("Elapsed:", StringUtils.FormatTime(jinfo.GetElapsedTime()));
Hamlet.DIV <Org.Apache.Hadoop.Yarn.Webapp.Hamlet.Hamlet> div = html.(typeof(InfoBlock
)).Div(JQueryUI.InfoWrap);
// MRAppMasters Table
Hamlet.TABLE <Hamlet.DIV <Org.Apache.Hadoop.Yarn.Webapp.Hamlet.Hamlet> > table = div
.Table("#job");
table.Tr().Th(amString).().Tr().Th(JQueryUI.Th, "Attempt Number").Th(JQueryUI.Th,
"Start Time").Th(JQueryUI.Th, "Node").Th(JQueryUI.Th, "Logs").();
foreach (AMInfo amInfo in amInfos)
{
AMAttemptInfo attempt = new AMAttemptInfo(amInfo, jinfo.GetId(), jinfo.GetUserName
());
table.Tr().Td(attempt.GetAttemptId().ToString()).Td(Sharpen.Extensions.CreateDate
(attempt.GetStartTime()).ToString()).Td().A(".nodelink", Url(MRWebAppUtil.GetYARNWebappScheme
(), attempt.GetNodeHttpAddress()), attempt.GetNodeHttpAddress()).().Td().A(".logslink"
, Url(attempt.GetLogsLink()), "logs").().();
}
table.();
div.();
html.Div(JQueryUI.InfoWrap).Table("#job").Tr().Th(JQueryUI.Th, "Task Type").Th(JQueryUI
.Th, "Progress").Th(JQueryUI.Th, "Total").Th(JQueryUI.Th, "Pending").Th(JQueryUI
.Th, "Running").Th(JQueryUI.Th, "Complete").().Tr(JQueryUI.Odd).Th("Map").Td().Div
(JQueryUI.Progressbar).$title(StringHelper.Join(jinfo.GetMapProgressPercent(), '%'
)).Div(JQueryUI.ProgressbarValue).$style(StringHelper.Join("width:", jinfo.GetMapProgressPercent
(), '%')).().().().Td().A(Url("tasks", jid, "m", "ALL"), jinfo.GetMapsTotal().ToString
()).().Td().A(Url("tasks", jid, "m", "PENDING"), jinfo.GetMapsPending().ToString
()).().Td().A(Url("tasks", jid, "m", "RUNNING"), jinfo.GetMapsRunning().ToString
()).().Td().A(Url("tasks", jid, "m", "COMPLETED"), jinfo.GetMapsCompleted().ToString
()).().().Tr(JQueryUI.Even).Th("Reduce").Td().Div(JQueryUI.Progressbar).$title(StringHelper.Join
(jinfo.GetReduceProgressPercent(), '%')).Div(JQueryUI.ProgressbarValue).$style(StringHelper.Join
("width:", jinfo.GetReduceProgressPercent(), '%')).().().().Td().A(Url("tasks",
jid, "r", "ALL"), jinfo.GetReducesTotal().ToString()).().Td().A(Url("tasks", jid
, "r", "PENDING"), jinfo.GetReducesPending().ToString()).().Td().A(Url("tasks",
jid, "r", "RUNNING"), jinfo.GetReducesRunning().ToString()).().Td().A(Url("tasks"
, jid, "r", "COMPLETED"), jinfo.GetReducesCompleted().ToString()).().().().Table
("#job").Tr().Th(JQueryUI.Th, "Attempt Type").Th(JQueryUI.Th, "New").Th(JQueryUI
.Th, "Running").Th(JQueryUI.Th, "Failed").Th(JQueryUI.Th, "Killed").Th(JQueryUI.
Th, "Successful").().Tr(JQueryUI.Odd).Th("Maps").Td().A(Url("attempts", jid, "m"
, MRApps.TaskAttemptStateUI.New.ToString()), jinfo.GetNewMapAttempts().ToString(
)).().Td().A(Url("attempts", jid, "m", MRApps.TaskAttemptStateUI.Running.ToString
()), jinfo.GetRunningMapAttempts().ToString()).().Td().A(Url("attempts", jid, "m"
, MRApps.TaskAttemptStateUI.Failed.ToString()), jinfo.GetFailedMapAttempts().ToString
()).().Td().A(Url("attempts", jid, "m", MRApps.TaskAttemptStateUI.Killed.ToString
()), jinfo.GetKilledMapAttempts().ToString()).().Td().A(Url("attempts", jid, "m"
, MRApps.TaskAttemptStateUI.Successful.ToString()), jinfo.GetSuccessfulMapAttempts
().ToString()).().().Tr(JQueryUI.Even).Th("Reduces").Td().A(Url("attempts", jid,
"r", MRApps.TaskAttemptStateUI.New.ToString()), jinfo.GetNewReduceAttempts().ToString
()).().Td().A(Url("attempts", jid, "r", MRApps.TaskAttemptStateUI.Running.ToString
()), jinfo.GetRunningReduceAttempts().ToString()).().Td().A(Url("attempts", jid,
"r", MRApps.TaskAttemptStateUI.Failed.ToString()), jinfo.GetFailedReduceAttempts
().ToString()).().Td().A(Url("attempts", jid, "r", MRApps.TaskAttemptStateUI.Killed
.ToString()), jinfo.GetKilledReduceAttempts().ToString()).().Td().A(Url("attempts"
, jid, "r", MRApps.TaskAttemptStateUI.Successful.ToString()), jinfo.GetSuccessfulReduceAttempts
().ToString()).().().().();
}
/*
* (non-Javadoc)
* @see org.apache.hadoop.yarn.webapp.view.HtmlBlock#render(org.apache.hadoop.yarn.webapp.view.HtmlBlock.Block)
*/
protected override void Render(HtmlBlock.Block html)
{
string jid = $(AMParams.JobId);
if (jid.IsEmpty())
{
html.P().("Sorry, can't do anything without a JobID.").();
return;
}
JobId jobID = MRApps.ToJobID(jid);
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job j = appContext.GetJob(jobID);
if (j == null)
{
html.P().("Sorry, ", jid, " not found.").();
return;
}
IList <AMInfo> amInfos = j.GetAMInfos();
JobInfo job = new JobInfo(j);
ResponseInfo infoBlock = Info("Job Overview").("Job Name:", job.GetName()).("User Name:"
, job.GetUserName()).("Queue:", job.GetQueueName()).("State:", job.GetState()).(
"Uberized:", job.IsUber()).("Submitted:", Sharpen.Extensions.CreateDate(job.GetSubmitTime
())).("Started:", Sharpen.Extensions.CreateDate(job.GetStartTime())).("Finished:"
, Sharpen.Extensions.CreateDate(job.GetFinishTime())).("Elapsed:", StringUtils.FormatTime
(Times.Elapsed(job.GetStartTime(), job.GetFinishTime(), false)));
string amString = amInfos.Count == 1 ? "ApplicationMaster" : "ApplicationMasters";
// todo - switch to use JobInfo
IList <string> diagnostics = j.GetDiagnostics();
if (diagnostics != null && !diagnostics.IsEmpty())
{
StringBuilder b = new StringBuilder();
foreach (string diag in diagnostics)
{
b.Append(diag);
}
infoBlock.("Diagnostics:", b.ToString());
}
if (job.GetNumMaps() > 0)
{
infoBlock.("Average Map Time", StringUtils.FormatTime(job.GetAvgMapTime()));
}
if (job.GetNumReduces() > 0)
{
infoBlock.("Average Shuffle Time", StringUtils.FormatTime(job.GetAvgShuffleTime()
));
infoBlock.("Average Merge Time", StringUtils.FormatTime(job.GetAvgMergeTime()));
infoBlock.("Average Reduce Time", StringUtils.FormatTime(job.GetAvgReduceTime()));
}
foreach (ConfEntryInfo entry in job.GetAcls())
{
infoBlock.("ACL " + entry.GetName() + ":", entry.GetValue());
}
Hamlet.DIV <Org.Apache.Hadoop.Yarn.Webapp.Hamlet.Hamlet> div = html.(typeof(InfoBlock
)).Div(JQueryUI.InfoWrap);
// MRAppMasters Table
Hamlet.TABLE <Hamlet.DIV <Org.Apache.Hadoop.Yarn.Webapp.Hamlet.Hamlet> > table = div
.Table("#job");
table.Tr().Th(amString).().Tr().Th(JQueryUI.Th, "Attempt Number").Th(JQueryUI.Th,
"Start Time").Th(JQueryUI.Th, "Node").Th(JQueryUI.Th, "Logs").();
bool odd = false;
foreach (AMInfo amInfo in amInfos)
{
AMAttemptInfo attempt = new AMAttemptInfo(amInfo, job.GetId(), job.GetUserName(),
string.Empty, string.Empty);
table.Tr((odd = !odd) ? JQueryUI.Odd : JQueryUI.Even).Td(attempt.GetAttemptId().ToString
()).Td(Sharpen.Extensions.CreateDate(attempt.GetStartTime()).ToString()).Td().A(
".nodelink", Url(MRWebAppUtil.GetYARNWebappScheme(), attempt.GetNodeHttpAddress(
)), attempt.GetNodeHttpAddress()).().Td().A(".logslink", Url(attempt.GetShortLogsLink
()), "logs").().();
}
table.();
div.();
html.Div(JQueryUI.InfoWrap).Table("#job").Tr().Th(JQueryUI.Th, "Task Type").Th(JQueryUI
.Th, "Total").Th(JQueryUI.Th, "Complete").().Tr(JQueryUI.Odd).Th().A(Url("tasks"
, jid, "m"), "Map").().Td(job.GetMapsTotal().ToString().ToString()).Td(job.GetMapsCompleted
().ToString().ToString()).().Tr(JQueryUI.Even).Th().A(Url("tasks", jid, "r"), "Reduce"
).().Td(job.GetReducesTotal().ToString().ToString()).Td(job.GetReducesCompleted(
).ToString().ToString()).().().Table("#job").Tr().Th(JQueryUI.Th, "Attempt Type"
).Th(JQueryUI.Th, "Failed").Th(JQueryUI.Th, "Killed").Th(JQueryUI.Th, "Successful"
).().Tr(JQueryUI.Odd).Th("Maps").Td().A(Url("attempts", jid, "m", MRApps.TaskAttemptStateUI
.Failed.ToString()), job.GetFailedMapAttempts().ToString()).().Td().A(Url("attempts"
, jid, "m", MRApps.TaskAttemptStateUI.Killed.ToString()), job.GetKilledMapAttempts
().ToString()).().Td().A(Url("attempts", jid, "m", MRApps.TaskAttemptStateUI.Successful
.ToString()), job.GetSuccessfulMapAttempts().ToString()).().().Tr(JQueryUI.Even)
.Th("Reduces").Td().A(Url("attempts", jid, "r", MRApps.TaskAttemptStateUI.Failed
.ToString()), job.GetFailedReduceAttempts().ToString()).().Td().A(Url("attempts"
, jid, "r", MRApps.TaskAttemptStateUI.Killed.ToString()), job.GetKilledReduceAttempts
().ToString()).().Td().A(Url("attempts", jid, "r", MRApps.TaskAttemptStateUI.Successful
.ToString()), job.GetSuccessfulReduceAttempts().ToString()).().().().();
}
public virtual void TestRefreshLoadedJobCache()
{
HistoryFileManager historyManager = Org.Mockito.Mockito.Mock <HistoryFileManager>(
);
jobHistory = Org.Mockito.Mockito.Spy(new JobHistory());
Org.Mockito.Mockito.DoReturn(historyManager).When(jobHistory).CreateHistoryFileManager
();
Configuration conf = new Configuration();
// Set the cache size to 2
conf.Set(JHAdminConfig.MrHistoryLoadedJobCacheSize, "2");
jobHistory.Init(conf);
jobHistory.Start();
CachedHistoryStorage storage = Org.Mockito.Mockito.Spy((CachedHistoryStorage)jobHistory
.GetHistoryStorage());
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job[] jobs = new Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job
[3];
JobId[] jobIds = new JobId[3];
for (int i = 0; i < 3; i++)
{
jobs[i] = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job>();
jobIds[i] = Org.Mockito.Mockito.Mock <JobId>();
Org.Mockito.Mockito.When(jobs[i].GetID()).ThenReturn(jobIds[i]);
}
HistoryFileManager.HistoryFileInfo fileInfo = Org.Mockito.Mockito.Mock <HistoryFileManager.HistoryFileInfo
>();
Org.Mockito.Mockito.When(historyManager.GetFileInfo(Any <JobId>())).ThenReturn(fileInfo
);
Org.Mockito.Mockito.When(fileInfo.LoadJob()).ThenReturn(jobs[0]).ThenReturn(jobs[
1]).ThenReturn(jobs[2]);
// getFullJob will put the job in the cache if it isn't there
for (int i_1 = 0; i_1 < 3; i_1++)
{
storage.GetFullJob(jobs[i_1].GetID());
}
IDictionary <JobId, Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job> jobCache = storage
.GetLoadedJobCache();
// job0 should have been purged since cache size is 2
NUnit.Framework.Assert.IsFalse(jobCache.Contains(jobs[0].GetID()));
NUnit.Framework.Assert.IsTrue(jobCache.Contains(jobs[1].GetID()) && jobCache.Contains
(jobs[2].GetID()));
// Setting cache size to 3
conf.Set(JHAdminConfig.MrHistoryLoadedJobCacheSize, "3");
Org.Mockito.Mockito.DoReturn(conf).When(storage).CreateConf();
Org.Mockito.Mockito.When(fileInfo.LoadJob()).ThenReturn(jobs[0]).ThenReturn(jobs[
1]).ThenReturn(jobs[2]);
jobHistory.RefreshLoadedJobCache();
for (int i_2 = 0; i_2 < 3; i_2++)
{
storage.GetFullJob(jobs[i_2].GetID());
}
jobCache = storage.GetLoadedJobCache();
// All three jobs should be in cache since its size is now 3
for (int i_3 = 0; i_3 < 3; i_3++)
{
NUnit.Framework.Assert.IsTrue(jobCache.Contains(jobs[i_3].GetID()));
}
}
