// acls not being checked since
// we are using mock job instead of CompletedJob
public static void VerifyHsJobGeneric(Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job
job, string id, string user, string name, string state, string queue, long startTime
, long finishTime, int mapsTotal, int mapsCompleted, int reducesTotal, int reducesCompleted
)
{
JobReport report = job.GetReport();
WebServicesTestUtils.CheckStringMatch("id", MRApps.ToString(job.GetID()), id);
WebServicesTestUtils.CheckStringMatch("user", job.GetUserName().ToString(), user);
WebServicesTestUtils.CheckStringMatch("name", job.GetName(), name);
WebServicesTestUtils.CheckStringMatch("state", job.GetState().ToString(), state);
WebServicesTestUtils.CheckStringMatch("queue", job.GetQueueName(), queue);
NUnit.Framework.Assert.AreEqual("startTime incorrect", report.GetStartTime(), startTime
);
NUnit.Framework.Assert.AreEqual("finishTime incorrect", report.GetFinishTime(), finishTime
);
NUnit.Framework.Assert.AreEqual("mapsTotal incorrect", job.GetTotalMaps(), mapsTotal
);
NUnit.Framework.Assert.AreEqual("mapsCompleted incorrect", job.GetCompletedMaps()
, mapsCompleted);
NUnit.Framework.Assert.AreEqual("reducesTotal incorrect", job.GetTotalReduces(),
reducesTotal);
NUnit.Framework.Assert.AreEqual("reducesCompleted incorrect", job.GetCompletedReduces
(), reducesCompleted);
}
public virtual void VerifyHsJobConfXML(NodeList nodes, Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job
job)
{
NUnit.Framework.Assert.AreEqual("incorrect number of elements", 1, nodes.GetLength
());
for (int i = 0; i < nodes.GetLength(); i++)
{
Element element = (Element)nodes.Item(i);
WebServicesTestUtils.CheckStringMatch("path", job.GetConfFile().ToString(), WebServicesTestUtils
.GetXmlString(element, "path"));
// just do simple verification of fields - not data is correct
// in the fields
NodeList properties = element.GetElementsByTagName("property");
for (int j = 0; j < properties.GetLength(); j++)
{
Element property = (Element)properties.Item(j);
NUnit.Framework.Assert.IsNotNull("should have counters in the web service info",
property);
string name = WebServicesTestUtils.GetXmlString(property, "name");
string value = WebServicesTestUtils.GetXmlString(property, "value");
NUnit.Framework.Assert.IsTrue("name not set", (name != null && !name.IsEmpty()));
NUnit.Framework.Assert.IsTrue("name not set", (value != null && !value.IsEmpty())
);
}
}
}
public virtual void TestSingleNodeQueryStateLost()
{
WebResource r = Resource();
MockNM nm1 = rm.RegisterNode("h1:1234", 5120);
MockNM nm2 = rm.RegisterNode("h2:1234", 5120);
rm.SendNodeStarted(nm1);
rm.SendNodeStarted(nm2);
rm.NMwaitForState(nm1.GetNodeId(), NodeState.Running);
rm.NMwaitForState(nm2.GetNodeId(), NodeState.Running);
rm.SendNodeLost(nm1);
rm.SendNodeLost(nm2);
ClientResponse response = r.Path("ws").Path("v1").Path("cluster").Path("nodes").Path
("h2:1234").Accept(MediaType.ApplicationJson).Get <ClientResponse>();
NUnit.Framework.Assert.AreEqual(MediaType.ApplicationJsonType, response.GetType()
);
JSONObject json = response.GetEntity <JSONObject>();
JSONObject info = json.GetJSONObject("node");
string id = info.Get("id").ToString();
NUnit.Framework.Assert.AreEqual("Incorrect Node Information.", "h2:1234", id);
RMNode rmNode = rm.GetRMContext().GetInactiveRMNodes()["h2"];
WebServicesTestUtils.CheckStringMatch("nodeHTTPAddress", string.Empty, info.GetString
("nodeHTTPAddress"));
WebServicesTestUtils.CheckStringMatch("state", rmNode.GetState().ToString(), info
.GetString("state"));
}
public virtual void TestJobsQueryStateInvalid()
{
WebResource r = Resource();
ClientResponse response = r.Path("ws").Path("v1").Path("history").Path("mapreduce"
).Path("jobs").QueryParam("state", "InvalidState").Accept(MediaType.ApplicationJson
).Get <ClientResponse>();
NUnit.Framework.Assert.AreEqual(ClientResponse.Status.BadRequest, response.GetClientResponseStatus
());
NUnit.Framework.Assert.AreEqual(MediaType.ApplicationJsonType, response.GetType()
);
JSONObject msg = response.GetEntity <JSONObject>();
JSONObject exception = msg.GetJSONObject("RemoteException");
NUnit.Framework.Assert.AreEqual("incorrect number of elements", 3, exception.Length
());
string message = exception.GetString("message");
string type = exception.GetString("exception");
string classname = exception.GetString("javaClassName");
WebServicesTestUtils.CheckStringContains("exception message", "org.apache.hadoop.mapreduce.v2.api.records.JobState.InvalidState"
, message);
WebServicesTestUtils.CheckStringMatch("exception type", "IllegalArgumentException"
, type);
WebServicesTestUtils.CheckStringMatch("exception classname", "java.lang.IllegalArgumentException"
, classname);
}
public virtual void VerifyHsTaskCountersXML(NodeList nodes, TaskAttempt att)
{
for (int i = 0; i < nodes.GetLength(); i++)
{
Element element = (Element)nodes.Item(i);
WebServicesTestUtils.CheckStringMatch("id", MRApps.ToString(att.GetID()), WebServicesTestUtils
.GetXmlString(element, "id"));
// just do simple verification of fields - not data is correct
// in the fields
NodeList groups = element.GetElementsByTagName("taskAttemptCounterGroup");
for (int j = 0; j < groups.GetLength(); j++)
{
Element counters = (Element)groups.Item(j);
NUnit.Framework.Assert.IsNotNull("should have counters in the web service info",
counters);
string name = WebServicesTestUtils.GetXmlString(counters, "counterGroupName");
NUnit.Framework.Assert.IsTrue("name not set", (name != null && !name.IsEmpty()));
NodeList counterArr = counters.GetElementsByTagName("counter");
for (int z = 0; z < counterArr.GetLength(); z++)
{
Element counter = (Element)counterArr.Item(z);
string counterName = WebServicesTestUtils.GetXmlString(counter, "name");
NUnit.Framework.Assert.IsTrue("counter name not set", (counterName != null && !counterName
.IsEmpty()));
long value = WebServicesTestUtils.GetXmlLong(counter, "value");
NUnit.Framework.Assert.IsTrue("value not >= 0", value >= 0);
}
}
}
}
public virtual void TestJobsQueryStartTimeNegative()
{
WebResource r = Resource();
ClientResponse response = r.Path("ws").Path("v1").Path("history").Path("mapreduce"
).Path("jobs").QueryParam("startedTimeBegin", (-1000).ToString()).Accept(MediaType
.ApplicationJson).Get <ClientResponse>();
NUnit.Framework.Assert.AreEqual(ClientResponse.Status.BadRequest, response.GetClientResponseStatus
());
NUnit.Framework.Assert.AreEqual(MediaType.ApplicationJsonType, response.GetType()
);
JSONObject msg = response.GetEntity <JSONObject>();
JSONObject exception = msg.GetJSONObject("RemoteException");
NUnit.Framework.Assert.AreEqual("incorrect number of elements", 3, exception.Length
());
string message = exception.GetString("message");
string type = exception.GetString("exception");
string classname = exception.GetString("javaClassName");
WebServicesTestUtils.CheckStringMatch("exception message", "java.lang.Exception: startedTimeBegin must be greater than 0"
, message);
WebServicesTestUtils.CheckStringMatch("exception type", "BadRequestException", type
);
WebServicesTestUtils.CheckStringMatch("exception classname", "org.apache.hadoop.yarn.webapp.BadRequestException"
, classname);
}
/// <exception cref="Org.Codehaus.Jettison.Json.JSONException"/>
public virtual void VerifyHsJobTaskAttemptCounters(JSONObject info, TaskAttempt att
)
{
NUnit.Framework.Assert.AreEqual("incorrect number of elements", 2, info.Length());
WebServicesTestUtils.CheckStringMatch("id", MRApps.ToString(att.GetID()), info.GetString
("id"));
// just do simple verification of fields - not data is correct
// in the fields
JSONArray counterGroups = info.GetJSONArray("taskAttemptCounterGroup");
for (int i = 0; i < counterGroups.Length(); i++)
{
JSONObject counterGroup = counterGroups.GetJSONObject(i);
string name = counterGroup.GetString("counterGroupName");
NUnit.Framework.Assert.IsTrue("name not set", (name != null && !name.IsEmpty()));
JSONArray counters = counterGroup.GetJSONArray("counter");
for (int j = 0; j < counters.Length(); j++)
{
JSONObject counter = counters.GetJSONObject(j);
string counterName = counter.GetString("name");
NUnit.Framework.Assert.IsTrue("name not set", (counterName != null && !counterName
.IsEmpty()));
long value = counter.GetLong("value");
NUnit.Framework.Assert.IsTrue("value >= 0", value >= 0);
}
}
}
public virtual void VerifyClusterGeneric(long clusterid, long startedon, string state
, string haState, string haZooKeeperConnectionState, string hadoopVersionBuiltOn
, string hadoopBuildVersion, string hadoopVersion, string resourceManagerVersionBuiltOn
, string resourceManagerBuildVersion, string resourceManagerVersion)
{
NUnit.Framework.Assert.AreEqual("clusterId doesn't match: ", ResourceManager.GetClusterTimeStamp
(), clusterid);
NUnit.Framework.Assert.AreEqual("startedOn doesn't match: ", ResourceManager.GetClusterTimeStamp
(), startedon);
NUnit.Framework.Assert.IsTrue("stated doesn't match: " + state, state.Matches(Service.STATE
.Inited.ToString()));
NUnit.Framework.Assert.IsTrue("HA state doesn't match: " + haState, haState.Matches
("INITIALIZING"));
WebServicesTestUtils.CheckStringMatch("hadoopVersionBuiltOn", VersionInfo.GetDate
(), hadoopVersionBuiltOn);
WebServicesTestUtils.CheckStringEqual("hadoopBuildVersion", VersionInfo.GetBuildVersion
(), hadoopBuildVersion);
WebServicesTestUtils.CheckStringMatch("hadoopVersion", VersionInfo.GetVersion(),
hadoopVersion);
WebServicesTestUtils.CheckStringMatch("resourceManagerVersionBuiltOn", YarnVersionInfo
.GetDate(), resourceManagerVersionBuiltOn);
WebServicesTestUtils.CheckStringEqual("resourceManagerBuildVersion", YarnVersionInfo
.GetBuildVersion(), resourceManagerBuildVersion);
WebServicesTestUtils.CheckStringMatch("resourceManagerVersion", YarnVersionInfo.GetVersion
(), resourceManagerVersion);
}
public virtual void TestJobsQueryFinishTimeEndInvalidformat()
{
WebResource r = Resource();
ClientResponse response = r.Path("ws").Path("v1").Path("history").Path("mapreduce"
).Path("jobs").QueryParam("finishedTimeEnd", "efsd").Accept(MediaType.ApplicationJson
).Get <ClientResponse>();
NUnit.Framework.Assert.AreEqual(ClientResponse.Status.BadRequest, response.GetClientResponseStatus
());
NUnit.Framework.Assert.AreEqual(MediaType.ApplicationJsonType, response.GetType()
);
JSONObject msg = response.GetEntity <JSONObject>();
JSONObject exception = msg.GetJSONObject("RemoteException");
NUnit.Framework.Assert.AreEqual("incorrect number of elements", 3, exception.Length
());
string message = exception.GetString("message");
string type = exception.GetString("exception");
string classname = exception.GetString("javaClassName");
WebServicesTestUtils.CheckStringMatch("exception message", "java.lang.Exception: Invalid number format: For input string: \"efsd\""
, message);
WebServicesTestUtils.CheckStringMatch("exception type", "BadRequestException", type
);
WebServicesTestUtils.CheckStringMatch("exception classname", "org.apache.hadoop.yarn.webapp.BadRequestException"
, classname);
}
/// <exception cref="Org.Codehaus.Jettison.Json.JSONException"/>
/// <exception cref="System.Exception"/>
public virtual void VerifyNodeAppInfoGeneric(Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
app, string id, string state, string user)
{
WebServicesTestUtils.CheckStringMatch("id", app.GetAppId().ToString(), id);
WebServicesTestUtils.CheckStringMatch("state", app.GetApplicationState().ToString
(), state);
WebServicesTestUtils.CheckStringMatch("user", app.GetUser().ToString(), user);
}
private void VerifyStateInvalidException(string message, string type, string classname
)
{
WebServicesTestUtils.CheckStringContains("exception message", "org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationState.FOO_STATE"
, message);
WebServicesTestUtils.CheckStringMatch("exception type", "IllegalArgumentException"
, type);
WebServicesTestUtils.CheckStringMatch("exception classname", "java.lang.IllegalArgumentException"
, classname);
}
public virtual void VerifyAMInfoGeneric(AppContext ctx, string id, string user, string
name, long startedOn, long elapsedTime)
{
WebServicesTestUtils.CheckStringMatch("id", ctx.GetApplicationID().ToString(), id
);
WebServicesTestUtils.CheckStringMatch("user", ctx.GetUser().ToString(), user);
WebServicesTestUtils.CheckStringMatch("name", ctx.GetApplicationName(), name);
NUnit.Framework.Assert.AreEqual("startedOn incorrect", ctx.GetStartTime(), startedOn
);
NUnit.Framework.Assert.IsTrue("elapsedTime not greater then 0", (elapsedTime > 0)
);
}
public virtual void VerifyHsInfoGeneric(string hadoopVersionBuiltOn, string hadoopBuildVersion
, string hadoopVersion, long startedon)
{
WebServicesTestUtils.CheckStringMatch("hadoopVersionBuiltOn", VersionInfo.GetDate
(), hadoopVersionBuiltOn);
WebServicesTestUtils.CheckStringEqual("hadoopBuildVersion", VersionInfo.GetBuildVersion
(), hadoopBuildVersion);
WebServicesTestUtils.CheckStringMatch("hadoopVersion", VersionInfo.GetVersion(),
hadoopVersion);
NUnit.Framework.Assert.AreEqual("startedOn doesn't match: ", JobHistoryServer.historyServerTimeStamp
, startedon);
}
public virtual void TestInvalidUri2()
{
WebResource r = Resource();
string responseStr = string.Empty;
try
{
responseStr = r.Accept(MediaType.ApplicationJson).Get <string>();
NUnit.Framework.Assert.Fail("should have thrown exception on invalid uri");
}
catch (UniformInterfaceException ue)
{
ClientResponse response = ue.GetResponse();
NUnit.Framework.Assert.AreEqual(ClientResponse.Status.NotFound, response.GetClientResponseStatus
());
WebServicesTestUtils.CheckStringMatch("error string exists and shouldn't", string.Empty
, responseStr);
}
}
/// <exception cref="Org.Codehaus.Jettison.Json.JSONException"/>
/// <exception cref="System.Exception"/>
private void TestTaskAttemptIdErrorGeneric(string attid, string error)
{
WebResource r = Resource();
IDictionary <JobId, Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job> jobsMap = appContext
.GetAllJobs();
foreach (JobId id in jobsMap.Keys)
{
string jobId = MRApps.ToString(id);
foreach (Task task in jobsMap[id].GetTasks().Values)
{
string tid = MRApps.ToString(task.GetID());
try
{
r.Path("ws").Path("v1").Path("history").Path("mapreduce").Path("jobs").Path(jobId
).Path("tasks").Path(tid).Path("attempts").Path(attid).Accept(MediaType.ApplicationJson
).Get <JSONObject>();
NUnit.Framework.Assert.Fail("should have thrown exception on invalid uri");
}
catch (UniformInterfaceException ue)
{
ClientResponse response = ue.GetResponse();
NUnit.Framework.Assert.AreEqual(ClientResponse.Status.NotFound, response.GetClientResponseStatus
());
NUnit.Framework.Assert.AreEqual(MediaType.ApplicationJsonType, response.GetType()
);
JSONObject msg = response.GetEntity <JSONObject>();
JSONObject exception = msg.GetJSONObject("RemoteException");
NUnit.Framework.Assert.AreEqual("incorrect number of elements", 3, exception.Length
());
string message = exception.GetString("message");
string type = exception.GetString("exception");
string classname = exception.GetString("javaClassName");
WebServicesTestUtils.CheckStringMatch("exception message", error, message);
WebServicesTestUtils.CheckStringMatch("exception type", "NotFoundException", type
);
WebServicesTestUtils.CheckStringMatch("exception classname", "org.apache.hadoop.yarn.webapp.NotFoundException"
, classname);
}
}
}
}
/// <exception cref="Org.Codehaus.Jettison.Json.JSONException"/>
public virtual void VerifyHsJobConf(JSONObject info, Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job
job)
{
NUnit.Framework.Assert.AreEqual("incorrect number of elements", 2, info.Length());
WebServicesTestUtils.CheckStringMatch("path", job.GetConfFile().ToString(), info.
GetString("path"));
// just do simple verification of fields - not data is correct
// in the fields
JSONArray properties = info.GetJSONArray("property");
for (int i = 0; i < properties.Length(); i++)
{
JSONObject prop = properties.GetJSONObject(i);
string name = prop.GetString("name");
string value = prop.GetString("value");
NUnit.Framework.Assert.IsTrue("name not set", (name != null && !name.IsEmpty()));
NUnit.Framework.Assert.IsTrue("value not set", (value != null && !value.IsEmpty()
));
}
}
public virtual void TestInvalidAccept()
{
WebResource r = Resource();
string responseStr = string.Empty;
try
{
responseStr = r.Path("ws").Path("v1").Path("applicationhistory").QueryParam("user.name"
, Users[round]).Accept(MediaType.TextPlain).Get <string>();
NUnit.Framework.Assert.Fail("should have thrown exception on invalid uri");
}
catch (UniformInterfaceException ue)
{
ClientResponse response = ue.GetResponse();
NUnit.Framework.Assert.AreEqual(ClientResponse.Status.InternalServerError, response
.GetClientResponseStatus());
WebServicesTestUtils.CheckStringMatch("error string exists and shouldn't", string.Empty
, responseStr);
}
}
public virtual void TestNodeAppsUserEmpty()
{
WebResource r = Resource();
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
app = new MockApp(1);
nmContext.GetApplications()[app.GetAppId()] = app;
AddAppContainers(app);
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
app2 = new MockApp("foo", 1234, 2);
nmContext.GetApplications()[app2.GetAppId()] = app2;
AddAppContainers(app2);
try
{
r.Path("ws").Path("v1").Path("node").Path("apps").QueryParam("user", string.Empty
).Accept(MediaType.ApplicationJson).Get <JSONObject>();
NUnit.Framework.Assert.Fail("should have thrown exception on invalid user query");
}
catch (UniformInterfaceException ue)
{
ClientResponse response = ue.GetResponse();
NUnit.Framework.Assert.AreEqual(ClientResponse.Status.BadRequest, response.GetClientResponseStatus
());
NUnit.Framework.Assert.AreEqual(MediaType.ApplicationJsonType, response.GetType()
);
JSONObject msg = response.GetEntity <JSONObject>();
JSONObject exception = msg.GetJSONObject("RemoteException");
NUnit.Framework.Assert.AreEqual("incorrect number of elements", 3, exception.Length
());
string message = exception.GetString("message");
string type = exception.GetString("exception");
string classname = exception.GetString("javaClassName");
WebServicesTestUtils.CheckStringMatch("exception message", "java.lang.Exception: Error: You must specify a non-empty string for the user"
, message);
WebServicesTestUtils.CheckStringMatch("exception type", "BadRequestException", type
);
WebServicesTestUtils.CheckStringMatch("exception classname", "org.apache.hadoop.yarn.webapp.BadRequestException"
, classname);
}
}
public static void VerifyHsJobGenericSecure(Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job
job, bool uberized, string diagnostics, long avgMapTime, long avgReduceTime, long
avgShuffleTime, long avgMergeTime, int failedReduceAttempts, int killedReduceAttempts
, int successfulReduceAttempts, int failedMapAttempts, int killedMapAttempts, int
successfulMapAttempts)
{
string diagString = string.Empty;
IList <string> diagList = job.GetDiagnostics();
if (diagList != null && !diagList.IsEmpty())
{
StringBuilder b = new StringBuilder();
foreach (string diag in diagList)
{
b.Append(diag);
}
diagString = b.ToString();
}
WebServicesTestUtils.CheckStringMatch("diagnostics", diagString, diagnostics);
NUnit.Framework.Assert.AreEqual("isUber incorrect", job.IsUber(), uberized);
// unfortunately the following fields are all calculated in JobInfo
// so not easily accessible without doing all the calculations again.
// For now just make sure they are present.
NUnit.Framework.Assert.IsTrue("failedReduceAttempts not >= 0", failedReduceAttempts
>= 0);
NUnit.Framework.Assert.IsTrue("killedReduceAttempts not >= 0", killedReduceAttempts
>= 0);
NUnit.Framework.Assert.IsTrue("successfulReduceAttempts not >= 0", successfulReduceAttempts
>= 0);
NUnit.Framework.Assert.IsTrue("failedMapAttempts not >= 0", failedMapAttempts >=
0);
NUnit.Framework.Assert.IsTrue("killedMapAttempts not >= 0", killedMapAttempts >=
0);
NUnit.Framework.Assert.IsTrue("successfulMapAttempts not >= 0", successfulMapAttempts
>= 0);
NUnit.Framework.Assert.IsTrue("avgMapTime not >= 0", avgMapTime >= 0);
NUnit.Framework.Assert.IsTrue("avgReduceTime not >= 0", avgReduceTime >= 0);
NUnit.Framework.Assert.IsTrue("avgShuffleTime not >= 0", avgShuffleTime >= 0);
NUnit.Framework.Assert.IsTrue("avgMergeTime not >= 0", avgMergeTime >= 0);
}
public virtual void TestSingleContainerWrong()
{
WebResource r = Resource();
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
app = new MockApp(1);
nmContext.GetApplications()[app.GetAppId()] = app;
AddAppContainers(app);
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
app2 = new MockApp(2);
nmContext.GetApplications()[app2.GetAppId()] = app2;
AddAppContainers(app2);
try
{
r.Path("ws").Path("v1").Path("node").Path("containers").Path("container_1234_0001_01_000005"
).Accept(MediaType.ApplicationJson).Get <JSONObject>();
NUnit.Framework.Assert.Fail("should have thrown exception on invalid user query");
}
catch (UniformInterfaceException ue)
{
ClientResponse response = ue.GetResponse();
NUnit.Framework.Assert.AreEqual(ClientResponse.Status.NotFound, response.GetClientResponseStatus
());
NUnit.Framework.Assert.AreEqual(MediaType.ApplicationJsonType, response.GetType()
);
JSONObject msg = response.GetEntity <JSONObject>();
JSONObject exception = msg.GetJSONObject("RemoteException");
NUnit.Framework.Assert.AreEqual("incorrect number of elements", 3, exception.Length
());
string message = exception.GetString("message");
string type = exception.GetString("exception");
string classname = exception.GetString("javaClassName");
WebServicesTestUtils.CheckStringMatch("exception message", "java.lang.Exception: container with id, container_1234_0001_01_000005, not found"
, message);
WebServicesTestUtils.CheckStringMatch("exception type", "NotFoundException", type
);
WebServicesTestUtils.CheckStringMatch("exception classname", "org.apache.hadoop.yarn.webapp.NotFoundException"
, classname);
}
}
public virtual void TestTasksQueryInvalid()
{
WebResource r = Resource();
IDictionary <JobId, Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job> jobsMap = appContext
.GetAllJobs();
foreach (JobId id in jobsMap.Keys)
{
string jobId = MRApps.ToString(id);
// tasktype must be exactly either "m" or "r"
string tasktype = "reduce";
try
{
r.Path("ws").Path("v1").Path("mapreduce").Path("jobs").Path(jobId).Path("tasks").
QueryParam("type", tasktype).Accept(MediaType.ApplicationJson).Get <JSONObject>();
NUnit.Framework.Assert.Fail("should have thrown exception on invalid uri");
}
catch (UniformInterfaceException ue)
{
ClientResponse response = ue.GetResponse();
NUnit.Framework.Assert.AreEqual(ClientResponse.Status.BadRequest, response.GetClientResponseStatus
());
NUnit.Framework.Assert.AreEqual(MediaType.ApplicationJsonType, response.GetType()
);
JSONObject msg = response.GetEntity <JSONObject>();
JSONObject exception = msg.GetJSONObject("RemoteException");
NUnit.Framework.Assert.AreEqual("incorrect number of elements", 3, exception.Length
());
string message = exception.GetString("message");
string type = exception.GetString("exception");
string classname = exception.GetString("javaClassName");
WebServicesTestUtils.CheckStringMatch("exception message", "java.lang.Exception: tasktype must be either m or r"
, message);
WebServicesTestUtils.CheckStringMatch("exception type", "BadRequestException", type
);
WebServicesTestUtils.CheckStringMatch("exception classname", "org.apache.hadoop.yarn.webapp.BadRequestException"
, classname);
}
}
}
/// <exception cref="Org.Codehaus.Jettison.Json.JSONException"/>
/// <exception cref="System.Exception"/>
public virtual void VerifyNodeInfoGeneric(MockNM nm, string state, string rack, string
id, string nodeHostName, string nodeHTTPAddress, long lastHealthUpdate, string
healthReport, int numContainers, long usedMemoryMB, long availMemoryMB, long usedVirtualCores
, long availVirtualCores, string version)
{
RMNode node = rm.GetRMContext().GetRMNodes()[nm.GetNodeId()];
ResourceScheduler sched = rm.GetResourceScheduler();
SchedulerNodeReport report = sched.GetNodeReport(nm.GetNodeId());
WebServicesTestUtils.CheckStringMatch("state", node.GetState().ToString(), state);
WebServicesTestUtils.CheckStringMatch("rack", node.GetRackName(), rack);
WebServicesTestUtils.CheckStringMatch("id", nm.GetNodeId().ToString(), id);
WebServicesTestUtils.CheckStringMatch("nodeHostName", nm.GetNodeId().GetHost(), nodeHostName
);
WebServicesTestUtils.CheckStringMatch("healthReport", node.GetHealthReport().ToString
(), healthReport);
string expectedHttpAddress = nm.GetNodeId().GetHost() + ":" + nm.GetHttpPort();
WebServicesTestUtils.CheckStringMatch("nodeHTTPAddress", expectedHttpAddress, nodeHTTPAddress
);
WebServicesTestUtils.CheckStringMatch("version", node.GetNodeManagerVersion(), version
);
long expectedHealthUpdate = node.GetLastHealthReportTime();
NUnit.Framework.Assert.AreEqual("lastHealthUpdate doesn't match, got: " + lastHealthUpdate
+ " expected: " + expectedHealthUpdate, expectedHealthUpdate, lastHealthUpdate);
if (report != null)
{
NUnit.Framework.Assert.AreEqual("numContainers doesn't match: " + numContainers,
report.GetNumContainers(), numContainers);
NUnit.Framework.Assert.AreEqual("usedMemoryMB doesn't match: " + usedMemoryMB, report
.GetUsedResource().GetMemory(), usedMemoryMB);
NUnit.Framework.Assert.AreEqual("availMemoryMB doesn't match: " + availMemoryMB,
report.GetAvailableResource().GetMemory(), availMemoryMB);
NUnit.Framework.Assert.AreEqual("usedVirtualCores doesn't match: " + usedVirtualCores
, report.GetUsedResource().GetVirtualCores(), usedVirtualCores);
NUnit.Framework.Assert.AreEqual("availVirtualCores doesn't match: " + availVirtualCores
, report.GetAvailableResource().GetVirtualCores(), availVirtualCores);
}
}
public virtual void TestTaskIdInvalid3()
{
WebResource r = Resource();
IDictionary <JobId, Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job> jobsMap = appContext
.GetAllJobs();
foreach (JobId id in jobsMap.Keys)
{
string jobId = MRApps.ToString(id);
string tid = "task_0_0000_m";
try
{
r.Path("ws").Path("v1").Path("mapreduce").Path("jobs").Path(jobId).Path("tasks").
Path(tid).Get <JSONObject>();
NUnit.Framework.Assert.Fail("should have thrown exception on invalid uri");
}
catch (UniformInterfaceException ue)
{
ClientResponse response = ue.GetResponse();
NUnit.Framework.Assert.AreEqual(ClientResponse.Status.NotFound, response.GetClientResponseStatus
());
NUnit.Framework.Assert.AreEqual(MediaType.ApplicationJsonType, response.GetType()
);
JSONObject msg = response.GetEntity <JSONObject>();
JSONObject exception = msg.GetJSONObject("RemoteException");
NUnit.Framework.Assert.AreEqual("incorrect number of elements", 3, exception.Length
());
string message = exception.GetString("message");
string type = exception.GetString("exception");
string classname = exception.GetString("javaClassName");
WebServicesTestUtils.CheckStringMatch("exception message", "java.lang.Exception: TaskId string : "
+ "task_0_0000_m is not properly formed", message);
WebServicesTestUtils.CheckStringMatch("exception type", "NotFoundException", type
);
WebServicesTestUtils.CheckStringMatch("exception classname", "org.apache.hadoop.yarn.webapp.NotFoundException"
, classname);
}
}
}
/// <exception cref="Org.Codehaus.Jettison.Json.JSONException"/>
/// <exception cref="System.Exception"/>
public virtual void VerifyNodeContainerInfoGeneric(Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container
cont, string id, string state, string user, int exitCode, string diagnostics, string
nodeId, int totalMemoryNeededMB, int totalVCoresNeeded, string logsLink)
{
WebServicesTestUtils.CheckStringMatch("id", cont.GetContainerId().ToString(), id);
WebServicesTestUtils.CheckStringMatch("state", cont.GetContainerState().ToString(
), state);
WebServicesTestUtils.CheckStringMatch("user", cont.GetUser().ToString(), user);
NUnit.Framework.Assert.AreEqual("exitCode wrong", 0, exitCode);
WebServicesTestUtils.CheckStringMatch("diagnostics", "testing", diagnostics);
WebServicesTestUtils.CheckStringMatch("nodeId", nmContext.GetNodeId().ToString(),
nodeId);
NUnit.Framework.Assert.AreEqual("totalMemoryNeededMB wrong", YarnConfiguration.DefaultRmSchedulerMinimumAllocationMb
, totalMemoryNeededMB);
NUnit.Framework.Assert.AreEqual("totalVCoresNeeded wrong", YarnConfiguration.DefaultRmSchedulerMinimumAllocationVcores
, totalVCoresNeeded);
string shortLink = StringHelper.Ujoin("containerlogs", cont.GetContainerId().ToString
(), cont.GetUser());
NUnit.Framework.Assert.IsTrue("containerLogsLink wrong", logsLink.Contains(shortLink
));
}
public virtual void TestNodesQueryStateInvalid()
{
WebResource r = Resource();
rm.RegisterNode("h1:1234", 5120);
rm.RegisterNode("h2:1235", 5121);
try
{
r.Path("ws").Path("v1").Path("cluster").Path("nodes").QueryParam("states", "BOGUSSTATE"
).Accept(MediaType.ApplicationJson).Get <JSONObject>();
NUnit.Framework.Assert.Fail("should have thrown exception querying invalid state"
);
}
catch (UniformInterfaceException ue)
{
ClientResponse response = ue.GetResponse();
NUnit.Framework.Assert.AreEqual(ClientResponse.Status.BadRequest, response.GetClientResponseStatus
());
NUnit.Framework.Assert.AreEqual(MediaType.ApplicationJsonType, response.GetType()
);
JSONObject msg = response.GetEntity <JSONObject>();
JSONObject exception = msg.GetJSONObject("RemoteException");
NUnit.Framework.Assert.AreEqual("incorrect number of elements", 3, exception.Length
());
string message = exception.GetString("message");
string type = exception.GetString("exception");
string classname = exception.GetString("javaClassName");
WebServicesTestUtils.CheckStringContains("exception message", "org.apache.hadoop.yarn.api.records.NodeState.BOGUSSTATE"
, message);
WebServicesTestUtils.CheckStringMatch("exception type", "IllegalArgumentException"
, type);
WebServicesTestUtils.CheckStringMatch("exception classname", "java.lang.IllegalArgumentException"
, classname);
}
finally
{
rm.Stop();
}
}
public virtual void TestInvalidNode()
{
rm.RegisterNode("h1:1234", 5120);
rm.RegisterNode("h2:1235", 5121);
WebResource r = Resource();
try
{
r.Path("ws").Path("v1").Path("cluster").Path("nodes").Path("node_invalid_foo").Accept
(MediaType.ApplicationJson).Get <JSONObject>();
NUnit.Framework.Assert.Fail("should have thrown exception on non-existent nodeid"
);
}
catch (UniformInterfaceException ue)
{
ClientResponse response = ue.GetResponse();
NUnit.Framework.Assert.AreEqual(ClientResponse.Status.BadRequest, response.GetClientResponseStatus
());
NUnit.Framework.Assert.AreEqual(MediaType.ApplicationJsonType, response.GetType()
);
JSONObject msg = response.GetEntity <JSONObject>();
JSONObject exception = msg.GetJSONObject("RemoteException");
NUnit.Framework.Assert.AreEqual("incorrect number of elements", 3, exception.Length
());
string message = exception.GetString("message");
string type = exception.GetString("exception");
string classname = exception.GetString("javaClassName");
WebServicesTestUtils.CheckStringMatch("exception message", "Invalid NodeId \\[node_invalid_foo\\]. Expected host:port"
, message);
WebServicesTestUtils.CheckStringMatch("exception type", "IllegalArgumentException"
, type);
WebServicesTestUtils.CheckStringMatch("exception classname", "java.lang.IllegalArgumentException"
, classname);
}
finally
{
rm.Stop();
}
}
public virtual void TestNodesQueryStateLost()
{
WebResource r = Resource();
MockNM nm1 = rm.RegisterNode("h1:1234", 5120);
MockNM nm2 = rm.RegisterNode("h2:1234", 5120);
rm.SendNodeStarted(nm1);
rm.SendNodeStarted(nm2);
rm.NMwaitForState(nm1.GetNodeId(), NodeState.Running);
rm.NMwaitForState(nm2.GetNodeId(), NodeState.Running);
rm.SendNodeLost(nm1);
rm.SendNodeLost(nm2);
ClientResponse response = r.Path("ws").Path("v1").Path("cluster").Path("nodes").QueryParam
("states", NodeState.Lost.ToString()).Accept(MediaType.ApplicationJson).Get <ClientResponse
>();
NUnit.Framework.Assert.AreEqual(MediaType.ApplicationJsonType, response.GetType()
);
JSONObject json = response.GetEntity <JSONObject>();
JSONObject nodes = json.GetJSONObject("nodes");
NUnit.Framework.Assert.AreEqual("incorrect number of elements", 1, nodes.Length()
);
JSONArray nodeArray = nodes.GetJSONArray("node");
NUnit.Framework.Assert.AreEqual("incorrect number of elements", 2, nodeArray.Length
());
for (int i = 0; i < nodeArray.Length(); ++i)
{
JSONObject info = nodeArray.GetJSONObject(i);
string host = info.Get("id").ToString().Split(":")[0];
RMNode rmNode = rm.GetRMContext().GetInactiveRMNodes()[host];
WebServicesTestUtils.CheckStringMatch("nodeHTTPAddress", string.Empty, info.GetString
("nodeHTTPAddress"));
WebServicesTestUtils.CheckStringMatch("state", rmNode.GetState().ToString(), info
.GetString("state"));
}
}
public virtual void VerifyTaskAttemptGeneric(TaskAttempt ta, TaskType ttype, string
id, string state, string type, string rack, string nodeHttpAddress, string diagnostics
, string assignedContainerId, long startTime, long finishTime, long elapsedTime,
float progress)
{
TaskAttemptId attid = ta.GetID();
string attemptId = MRApps.ToString(attid);
WebServicesTestUtils.CheckStringMatch("id", attemptId, id);
WebServicesTestUtils.CheckStringMatch("type", ttype.ToString(), type);
WebServicesTestUtils.CheckStringMatch("state", ta.GetState().ToString(), state);
WebServicesTestUtils.CheckStringMatch("rack", ta.GetNodeRackName(), rack);
WebServicesTestUtils.CheckStringMatch("nodeHttpAddress", ta.GetNodeHttpAddress(),
nodeHttpAddress);
string expectDiag = string.Empty;
IList <string> diagnosticsList = ta.GetDiagnostics();
if (diagnosticsList != null && !diagnostics.IsEmpty())
{
StringBuilder b = new StringBuilder();
foreach (string diag in diagnosticsList)
{
b.Append(diag);
}
expectDiag = b.ToString();
}
WebServicesTestUtils.CheckStringMatch("diagnostics", expectDiag, diagnostics);
WebServicesTestUtils.CheckStringMatch("assignedContainerId", ConverterUtils.ToString
(ta.GetAssignedContainerID()), assignedContainerId);
NUnit.Framework.Assert.AreEqual("startTime wrong", ta.GetLaunchTime(), startTime);
NUnit.Framework.Assert.AreEqual("finishTime wrong", ta.GetFinishTime(), finishTime
);
NUnit.Framework.Assert.AreEqual("elapsedTime wrong", finishTime - startTime, elapsedTime
);
NUnit.Framework.Assert.AreEqual("progress wrong", ta.GetProgress() * 100, progress
, 1e-3f);
}
public virtual void VerifyNodeInfoGeneric(string id, string healthReport, long totalVmemAllocatedContainersMB
, long totalPmemAllocatedContainersMB, long totalVCoresAllocatedContainers, bool
vmemCheckEnabled, bool pmemCheckEnabled, long lastNodeUpdateTime, bool nodeHealthy
, string nodeHostName, string hadoopVersionBuiltOn, string hadoopBuildVersion, string
hadoopVersion, string resourceManagerVersionBuiltOn, string resourceManagerBuildVersion
, string resourceManagerVersion)
{
WebServicesTestUtils.CheckStringMatch("id", "testhost.foo.com:8042", id);
WebServicesTestUtils.CheckStringMatch("healthReport", "Healthy", healthReport);
NUnit.Framework.Assert.AreEqual("totalVmemAllocatedContainersMB incorrect", 15872
, totalVmemAllocatedContainersMB);
NUnit.Framework.Assert.AreEqual("totalPmemAllocatedContainersMB incorrect", 16384
, totalPmemAllocatedContainersMB);
NUnit.Framework.Assert.AreEqual("totalVCoresAllocatedContainers incorrect", 4000,
totalVCoresAllocatedContainers);
NUnit.Framework.Assert.AreEqual("vmemCheckEnabled incorrect", true, vmemCheckEnabled
);
NUnit.Framework.Assert.AreEqual("pmemCheckEnabled incorrect", true, pmemCheckEnabled
);
NUnit.Framework.Assert.IsTrue("lastNodeUpdateTime incorrect", lastNodeUpdateTime
== nmContext.GetNodeHealthStatus().GetLastHealthReportTime());
NUnit.Framework.Assert.IsTrue("nodeHealthy isn't true", nodeHealthy);
WebServicesTestUtils.CheckStringMatch("nodeHostName", "testhost.foo.com", nodeHostName
);
WebServicesTestUtils.CheckStringMatch("hadoopVersionBuiltOn", VersionInfo.GetDate
(), hadoopVersionBuiltOn);
WebServicesTestUtils.CheckStringEqual("hadoopBuildVersion", VersionInfo.GetBuildVersion
(), hadoopBuildVersion);
WebServicesTestUtils.CheckStringMatch("hadoopVersion", VersionInfo.GetVersion(),
hadoopVersion);
WebServicesTestUtils.CheckStringMatch("resourceManagerVersionBuiltOn", YarnVersionInfo
.GetDate(), resourceManagerVersionBuiltOn);
WebServicesTestUtils.CheckStringEqual("resourceManagerBuildVersion", YarnVersionInfo
.GetBuildVersion(), resourceManagerBuildVersion);
WebServicesTestUtils.CheckStringMatch("resourceManagerVersion", YarnVersionInfo.GetVersion
(), resourceManagerVersion);
}
public virtual void VerifyTaskGeneric(Task task, string id, string state, string
type, string successfulAttempt, long startTime, long finishTime, long elapsedTime
, float progress, string status)
{
TaskId taskid = task.GetID();
string tid = MRApps.ToString(taskid);
TaskReport report = task.GetReport();
WebServicesTestUtils.CheckStringMatch("id", tid, id);
WebServicesTestUtils.CheckStringMatch("type", task.GetType().ToString(), type);
WebServicesTestUtils.CheckStringMatch("state", report.GetTaskState().ToString(),
state);
// not easily checked without duplicating logic, just make sure its here
NUnit.Framework.Assert.IsNotNull("successfulAttempt null", successfulAttempt);
NUnit.Framework.Assert.AreEqual("startTime wrong", report.GetStartTime(), startTime
);
NUnit.Framework.Assert.AreEqual("finishTime wrong", report.GetFinishTime(), finishTime
);
NUnit.Framework.Assert.AreEqual("elapsedTime wrong", finishTime - startTime, elapsedTime
);
NUnit.Framework.Assert.AreEqual("progress wrong", report.GetProgress() * 100, progress
, 1e-3f);
NUnit.Framework.Assert.AreEqual("status wrong", report.GetStatus(), status);
}