/// <exception cref="Org.Codehaus.Jettison.Json.JSONException"/>
/// <exception cref="System.Exception"/>
public virtual void VerifySchedulerFifoXML(string xml)
{
DocumentBuilderFactory dbf = DocumentBuilderFactory.NewInstance();
DocumentBuilder db = dbf.NewDocumentBuilder();
InputSource @is = new InputSource();
@is.SetCharacterStream(new StringReader(xml));
Document dom = db.Parse(@is);
NodeList nodesSched = dom.GetElementsByTagName("scheduler");
NUnit.Framework.Assert.AreEqual("incorrect number of elements", 1, nodesSched.GetLength
());
NodeList nodes = dom.GetElementsByTagName("schedulerInfo");
NUnit.Framework.Assert.AreEqual("incorrect number of elements", 1, nodes.GetLength
());
for (int i = 0; i < nodes.GetLength(); i++)
{
Element element = (Element)nodes.Item(i);
VerifyClusterSchedulerFifoGeneric(WebServicesTestUtils.GetXmlAttrString(element,
"xsi:type"), WebServicesTestUtils.GetXmlString(element, "qstate"), WebServicesTestUtils
.GetXmlFloat(element, "capacity"), WebServicesTestUtils.GetXmlFloat(element, "usedCapacity"
), WebServicesTestUtils.GetXmlInt(element, "minQueueMemoryCapacity"), WebServicesTestUtils
.GetXmlInt(element, "maxQueueMemoryCapacity"), WebServicesTestUtils.GetXmlInt(element
, "numNodes"), WebServicesTestUtils.GetXmlInt(element, "usedNodeCapacity"), WebServicesTestUtils
.GetXmlInt(element, "availNodeCapacity"), WebServicesTestUtils.GetXmlInt(element
, "totalNodeCapacity"), WebServicesTestUtils.GetXmlInt(element, "numContainers")
);
}
}
/// <exception cref="Org.Codehaus.Jettison.Json.JSONException"/>
/// <exception cref="System.Exception"/>
public virtual void VerifyClusterMetricsXML(string xml)
{
DocumentBuilderFactory dbf = DocumentBuilderFactory.NewInstance();
DocumentBuilder db = dbf.NewDocumentBuilder();
InputSource @is = new InputSource();
@is.SetCharacterStream(new StringReader(xml));
Document dom = db.Parse(@is);
NodeList nodes = dom.GetElementsByTagName("clusterMetrics");
NUnit.Framework.Assert.AreEqual("incorrect number of elements", 1, nodes.GetLength
());
for (int i = 0; i < nodes.GetLength(); i++)
{
Element element = (Element)nodes.Item(i);
VerifyClusterMetrics(WebServicesTestUtils.GetXmlInt(element, "appsSubmitted"), WebServicesTestUtils
.GetXmlInt(element, "appsCompleted"), WebServicesTestUtils.GetXmlInt(element, "reservedMB"
), WebServicesTestUtils.GetXmlInt(element, "availableMB"), WebServicesTestUtils.
GetXmlInt(element, "allocatedMB"), WebServicesTestUtils.GetXmlInt(element, "reservedVirtualCores"
), WebServicesTestUtils.GetXmlInt(element, "availableVirtualCores"), WebServicesTestUtils
.GetXmlInt(element, "allocatedVirtualCores"), WebServicesTestUtils.GetXmlInt(element
, "totalVirtualCores"), WebServicesTestUtils.GetXmlInt(element, "containersAllocated"
), WebServicesTestUtils.GetXmlInt(element, "totalMB"), WebServicesTestUtils.GetXmlInt
(element, "totalNodes"), WebServicesTestUtils.GetXmlInt(element, "lostNodes"), WebServicesTestUtils
.GetXmlInt(element, "unhealthyNodes"), WebServicesTestUtils.GetXmlInt(element, "decommissionedNodes"
), WebServicesTestUtils.GetXmlInt(element, "rebootedNodes"), WebServicesTestUtils
.GetXmlInt(element, "activeNodes"));
}
}
public virtual void TestTaskAttemptIdXMLCounters()
{
WebResource r = Resource();
IDictionary <JobId, Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job> jobsMap = appContext
.GetAllJobs();
foreach (JobId id in jobsMap.Keys)
{
string jobId = MRApps.ToString(id);
foreach (Task task in jobsMap[id].GetTasks().Values)
{
string tid = MRApps.ToString(task.GetID());
foreach (TaskAttempt att in task.GetAttempts().Values)
{
TaskAttemptId attemptid = att.GetID();
string attid = MRApps.ToString(attemptid);
ClientResponse response = r.Path("ws").Path("v1").Path("history").Path("mapreduce"
).Path("jobs").Path(jobId).Path("tasks").Path(tid).Path("attempts").Path(attid).
Path("counters").Accept(MediaType.ApplicationXml).Get <ClientResponse>();
NUnit.Framework.Assert.AreEqual(MediaType.ApplicationXmlType, response.GetType());
string xml = response.GetEntity <string>();
DocumentBuilderFactory dbf = DocumentBuilderFactory.NewInstance();
DocumentBuilder db = dbf.NewDocumentBuilder();
InputSource @is = new InputSource();
@is.SetCharacterStream(new StringReader(xml));
Document dom = db.Parse(@is);
NodeList nodes = dom.GetElementsByTagName("jobTaskAttemptCounters");
VerifyHsTaskCountersXML(nodes, att);
}
}
}
}
/// <exception cref="Org.Codehaus.Jettison.Json.JSONException"/>
/// <exception cref="System.Exception"/>
public virtual void VerifyClusterInfoXML(string xml)
{
DocumentBuilderFactory dbf = DocumentBuilderFactory.NewInstance();
DocumentBuilder db = dbf.NewDocumentBuilder();
InputSource @is = new InputSource();
@is.SetCharacterStream(new StringReader(xml));
Document dom = db.Parse(@is);
NodeList nodes = dom.GetElementsByTagName("clusterInfo");
NUnit.Framework.Assert.AreEqual("incorrect number of elements", 1, nodes.GetLength
());
for (int i = 0; i < nodes.GetLength(); i++)
{
Element element = (Element)nodes.Item(i);
VerifyClusterGeneric(WebServicesTestUtils.GetXmlLong(element, "id"), WebServicesTestUtils
.GetXmlLong(element, "startedOn"), WebServicesTestUtils.GetXmlString(element, "state"
), WebServicesTestUtils.GetXmlString(element, "haState"), WebServicesTestUtils.GetXmlString
(element, "haZooKeeperConnectionState"), WebServicesTestUtils.GetXmlString(element
, "hadoopVersionBuiltOn"), WebServicesTestUtils.GetXmlString(element, "hadoopBuildVersion"
), WebServicesTestUtils.GetXmlString(element, "hadoopVersion"), WebServicesTestUtils
.GetXmlString(element, "resourceManagerVersionBuiltOn"), WebServicesTestUtils.GetXmlString
(element, "resourceManagerBuildVersion"), WebServicesTestUtils.GetXmlString(element
, "resourceManagerVersion"));
}
}
/// <summary>test xml generation</summary>
/// <exception cref="Javax.Xml.Parsers.ParserConfigurationException"/>
/// <exception cref="System.Exception"></exception>
public virtual void TestQueueConfigurationParser()
{
JobQueueInfo info = new JobQueueInfo("root", "rootInfo");
JobQueueInfo infoChild1 = new JobQueueInfo("child1", "child1Info");
JobQueueInfo infoChild2 = new JobQueueInfo("child2", "child1Info");
info.AddChild(infoChild1);
info.AddChild(infoChild2);
DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory.NewInstance();
DocumentBuilder builder = docBuilderFactory.NewDocumentBuilder();
Document document = builder.NewDocument();
// test QueueConfigurationParser.getQueueElement
Element e = QueueConfigurationParser.GetQueueElement(document, info);
// transform result to string for check
DOMSource domSource = new DOMSource(e);
StringWriter writer = new StringWriter();
StreamResult result = new StreamResult(writer);
TransformerFactory tf = TransformerFactory.NewInstance();
Transformer transformer = tf.NewTransformer();
transformer.Transform(domSource, result);
string str = writer.ToString();
NUnit.Framework.Assert.IsTrue(str.EndsWith("<queue><name>root</name><properties/><state>running</state><queue><name>child1</name><properties/><state>running</state></queue><queue><name>child2</name><properties/><state>running</state></queue></queue>"
));
}
public virtual void TestTasksXML()
{
WebResource r = Resource();
IDictionary <JobId, Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job> jobsMap = appContext
.GetAllJobs();
foreach (JobId id in jobsMap.Keys)
{
string jobId = MRApps.ToString(id);
ClientResponse response = r.Path("ws").Path("v1").Path("mapreduce").Path("jobs").
Path(jobId).Path("tasks").Accept(MediaType.ApplicationXml).Get <ClientResponse>();
NUnit.Framework.Assert.AreEqual(MediaType.ApplicationXmlType, response.GetType());
string xml = response.GetEntity <string>();
DocumentBuilderFactory dbf = DocumentBuilderFactory.NewInstance();
DocumentBuilder db = dbf.NewDocumentBuilder();
InputSource @is = new InputSource();
@is.SetCharacterStream(new StringReader(xml));
Document dom = db.Parse(@is);
NodeList tasks = dom.GetElementsByTagName("tasks");
NUnit.Framework.Assert.AreEqual("incorrect number of elements", 1, tasks.GetLength
());
NodeList task = dom.GetElementsByTagName("task");
VerifyAMTaskXML(task, jobsMap[id]);
}
}
/// <summary>Method to load the resource file.</summary>
/// <remarks>
/// Method to load the resource file.
/// generates the root.
/// </remarks>
/// <param name="resourceInput">InputStream that provides the XML to parse</param>
/// <returns/>
/// <exception cref="Javax.Xml.Parsers.ParserConfigurationException"/>
/// <exception cref="Org.Xml.Sax.SAXException"/>
/// <exception cref="System.IO.IOException"/>
protected internal virtual Queue LoadResource(InputStream resourceInput)
{
DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory.NewInstance();
//ignore all comments inside the xml file
docBuilderFactory.SetIgnoringComments(true);
//allow includes in the xml file
docBuilderFactory.SetNamespaceAware(true);
try
{
docBuilderFactory.SetXIncludeAware(true);
}
catch (NotSupportedException e)
{
Log.Info("Failed to set setXIncludeAware(true) for parser " + docBuilderFactory +
NameSeparator + e);
}
DocumentBuilder builder = docBuilderFactory.NewDocumentBuilder();
Document doc = null;
Element queuesNode = null;
doc = builder.Parse(resourceInput);
queuesNode = doc.GetDocumentElement();
return(this.ParseResource(queuesNode));
}
public virtual void TestWriteXml()
{
StringWriter sw = new StringWriter();
ConfServlet.WriteResponse(GetTestConf(), sw, "xml");
string xml = sw.ToString();
DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory.NewInstance();
DocumentBuilder builder = docBuilderFactory.NewDocumentBuilder();
Document doc = builder.Parse(new InputSource(new StringReader(xml)));
NodeList nameNodes = doc.GetElementsByTagName("name");
bool foundSetting = false;
for (int i = 0; i < nameNodes.GetLength(); i++)
{
Node nameNode = nameNodes.Item(i);
string key = nameNode.GetTextContent();
System.Console.Error.WriteLine("xml key: " + key);
if (TestKey.Equals(key))
{
foundSetting = true;
Element propertyElem = (Element)nameNode.GetParentNode();
string val = propertyElem.GetElementsByTagName("value").Item(0).GetTextContent();
Assert.Equal(TestVal, val);
}
}
Assert.True(foundSetting);
}
/// <summary>
/// Given a path to a file containing a list of SsurgeonPatterns, returns
/// TODO: deal with resources
/// </summary>
/// <exception cref="System.Exception"/>
public virtual IList <SsurgeonPattern> ReadFromFile(File file)
{
IList <SsurgeonPattern> retList = new List <SsurgeonPattern>();
IDocument doc = DocumentBuilderFactory.NewInstance().NewDocumentBuilder().Parse(file);
INodeList patternNodes = doc.GetElementsByTagName(SsurgeonPattern.SsurgeonElemTag);
for (int i = 0; i < patternNodes.GetLength(); i++)
{
INode node = patternNodes.Item(i);
if (node.GetNodeType() == NodeConstants.ElementNode)
{
IElement elt = (IElement)node;
SsurgeonPattern pattern = SsurgeonPatternFromXML(elt);
retList.Add(pattern);
}
}
INodeList resourceNodes = doc.GetElementsByTagName(SsurgeonPattern.ResourceTag);
for (int i_1 = 0; i_1 < resourceNodes.GetLength(); i_1++)
{
INode node = patternNodes.Item(i_1);
if (node.GetNodeType() == NodeConstants.ElementNode)
{
IElement resourceElt = (IElement)node;
SsurgeonWordlist wlRsrc = new SsurgeonWordlist(resourceElt);
AddResource(wlRsrc);
}
}
return(retList);
}
public virtual void TestNodes2XML()
{
rm.Start();
WebResource r = Resource();
rm.RegisterNode("h1:1234", 5120);
rm.RegisterNode("h2:1235", 5121);
ClientResponse response = r.Path("ws").Path("v1").Path("cluster").Path("nodes").Accept
(MediaType.ApplicationXml).Get <ClientResponse>();
NUnit.Framework.Assert.AreEqual(MediaType.ApplicationXmlType, response.GetType());
string xml = response.GetEntity <string>();
DocumentBuilderFactory dbf = DocumentBuilderFactory.NewInstance();
DocumentBuilder db = dbf.NewDocumentBuilder();
InputSource @is = new InputSource();
@is.SetCharacterStream(new StringReader(xml));
Document dom = db.Parse(@is);
NodeList nodesApps = dom.GetElementsByTagName("nodes");
NUnit.Framework.Assert.AreEqual("incorrect number of elements", 1, nodesApps.GetLength
());
NodeList nodes = dom.GetElementsByTagName("node");
NUnit.Framework.Assert.AreEqual("incorrect number of elements", 2, nodes.GetLength
());
rm.Stop();
}
public static Node Parse(InputStream input)
{
DocumentBuilderFactory dbFactory = DocumentBuilderFactory.NewInstance();
dbFactory.IgnoringComments = true;
dbFactory.IgnoringElementContentWhitespace = true;
dbFactory.NamespaceAware = true;
dbFactory.Validating = false;
try
{
DocumentBuilder dber = dbFactory.NewDocumentBuilder();
Document doc = dber.parse(input);
string tagname = doc.ChildNodes.Item(0).NodeName;
NodeList nodes = doc.getElementsByTagName(tagname);
return(nodes.Item(0));
}
catch (SAXException)
{
throw new Indeterminate(Indeterminate.IndeterminateProcessingError);
}
catch (IOException)
{
throw new Indeterminate(Indeterminate.IndeterminateProcessingError);
}
catch (ParserConfigurationException)
{
throw new Indeterminate(Indeterminate.IndeterminateProcessingError);
}
}
public virtual void TestTaskIdXML()
{
WebResource r = Resource();
IDictionary <JobId, Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job> jobsMap = appContext
.GetAllJobs();
foreach (JobId id in jobsMap.Keys)
{
string jobId = MRApps.ToString(id);
foreach (Task task in jobsMap[id].GetTasks().Values)
{
string tid = MRApps.ToString(task.GetID());
ClientResponse response = r.Path("ws").Path("v1").Path("mapreduce").Path("jobs").
Path(jobId).Path("tasks").Path(tid).Accept(MediaType.ApplicationXml).Get <ClientResponse
>();
NUnit.Framework.Assert.AreEqual(MediaType.ApplicationXmlType, response.GetType());
string xml = response.GetEntity <string>();
DocumentBuilderFactory dbf = DocumentBuilderFactory.NewInstance();
DocumentBuilder db = dbf.NewDocumentBuilder();
InputSource @is = new InputSource();
@is.SetCharacterStream(new StringReader(xml));
Document dom = db.Parse(@is);
NodeList nodes = dom.GetElementsByTagName("task");
for (int i = 0; i < nodes.GetLength(); i++)
{
Element element = (Element)nodes.Item(i);
VerifyAMSingleTaskXML(element, task);
}
}
}
}
public virtual void TestNodeContainerXML()
{
WebResource r = Resource();
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
app = new MockApp(1);
nmContext.GetApplications()[app.GetAppId()] = app;
AddAppContainers(app);
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
app2 = new MockApp(2);
nmContext.GetApplications()[app2.GetAppId()] = app2;
AddAppContainers(app2);
ClientResponse response = r.Path("ws").Path("v1").Path("node").Path("containers")
.Accept(MediaType.ApplicationXml).Get <ClientResponse>();
NUnit.Framework.Assert.AreEqual(MediaType.ApplicationXmlType, response.GetType());
string xml = response.GetEntity <string>();
DocumentBuilderFactory dbf = DocumentBuilderFactory.NewInstance();
DocumentBuilder db = dbf.NewDocumentBuilder();
InputSource @is = new InputSource();
@is.SetCharacterStream(new StringReader(xml));
Document dom = db.Parse(@is);
NodeList nodes = dom.GetElementsByTagName("container");
NUnit.Framework.Assert.AreEqual("incorrect number of elements", 4, nodes.GetLength
());
}
/// <exception cref="System.Exception"/>
private QueuePlacementPolicy Parse(string str)
{
// Read and parse the allocations file.
DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory.NewInstance();
docBuilderFactory.SetIgnoringComments(true);
DocumentBuilder builder = docBuilderFactory.NewDocumentBuilder();
Document doc = builder.Parse(IOUtils.ToInputStream(str));
Element root = doc.GetDocumentElement();
return(QueuePlacementPolicy.FromXml(root, configuredQueues, conf));
}
private static IDocument CreatePatternXMLDoc(IList <SsurgeonPattern> patterns)
{
try
{
DocumentBuilderFactory dbf = DocumentBuilderFactory.NewInstance();
DocumentBuilder db = dbf.NewDocumentBuilder();
IDocument domDoc = db.NewDocument();
IElement rootElt = domDoc.CreateElement(SsurgeonPattern.EltListTag);
domDoc.AppendChild(rootElt);
int ordinal = 1;
foreach (SsurgeonPattern pattern in patterns)
{
IElement patElt = domDoc.CreateElement(SsurgeonPattern.SsurgeonElemTag);
patElt.SetAttribute(SsurgeonPattern.OrdinalAttr, ordinal.ToString());
IElement semgrexElt = domDoc.CreateElement(SsurgeonPattern.SemgrexElemTag);
semgrexElt.AppendChild(domDoc.CreateTextNode(pattern.GetSemgrexPattern().Pattern()));
patElt.AppendChild(semgrexElt);
IElement uidElem = domDoc.CreateElement(SsurgeonPattern.UidElemTag);
uidElem.AppendChild(domDoc.CreateTextNode(pattern.GetUID()));
patElt.AppendChild(uidElem);
IElement notesElem = domDoc.CreateElement(SsurgeonPattern.NotesElemTag);
notesElem.AppendChild(domDoc.CreateTextNode(pattern.GetNotes()));
patElt.AppendChild(notesElem);
SemanticGraph semgrexGraph = pattern.GetSemgrexGraph();
if (semgrexGraph != null)
{
IElement patNode = domDoc.CreateElement(SsurgeonPattern.SemgrexGraphElemTag);
patNode.AppendChild(domDoc.CreateTextNode(semgrexGraph.ToCompactString()));
}
IElement editList = domDoc.CreateElement(SsurgeonPattern.EditListElemTag);
patElt.AppendChild(editList);
int editOrdinal = 1;
foreach (SsurgeonEdit edit in pattern.GetEditScript())
{
IElement editElem = domDoc.CreateElement(SsurgeonPattern.EditElemTag);
editElem.SetAttribute(SsurgeonPattern.OrdinalAttr, editOrdinal.ToString());
editElem.AppendChild(domDoc.CreateTextNode(edit.ToEditString()));
editList.AppendChild(editElem);
editOrdinal++;
}
rootElt.AppendChild(patElt);
ordinal++;
}
return(domDoc);
}
catch (Exception e)
{
log.Error(typeof(Edu.Stanford.Nlp.Semgraph.Semgrex.Ssurgeon.Ssurgeon).FullName, "createPatternXML");
log.Error(e);
return(null);
}
}
public static IElement ParseElement(string xml)
{
try
{
DocumentBuilderFactory dbFactory = DocumentBuilderFactory.NewInstance();
DocumentBuilder docBuilder = dbFactory.NewDocumentBuilder();
IDocument doc = docBuilder.Parse(new ByteArrayInputStream(Sharpen.Runtime.GetBytesForString(xml)));
return(doc.GetDocumentElement());
}
catch (Exception e)
{
throw new Exception(e);
}
}
public static IDocument CreateDocument()
{
try
{
DocumentBuilderFactory dbFactory = DocumentBuilderFactory.NewInstance();
DocumentBuilder docBuilder = dbFactory.NewDocumentBuilder();
IDocument doc = docBuilder.NewDocument();
return(doc);
}
catch (Exception e)
{
throw new Exception(e);
}
}
/// <summary>
/// TODO: Extract method logic into generic method to keep the code DRY.
/// </summary>
/// <param name="stream"></param>
/// <param name="props"></param>
/// <returns></returns>
public IRuleExecutionSet CreateRuleExecutionSet(Stream stream, IDictionary <object, object> props)
{
XmlDocument doc = null;
try
{
IDocumentBuilderFactory dbf = DocumentBuilderFactory.NewInstance();
IDocumentBuilder db = DocumentBuilder.NewInstance();
doc = db.Parse(new InputSource(stream));
}
catch (Exception e)
{
throw new RuleExecutionSetCreateException("Parse error.", e);
}
return(this.CreateRuleExecutionSet(doc.DocumentElement, props));
}
// Canibalized from FileSystemAccess <code>Configuration.loadResource()</code>.
/// <summary>Create a configuration from an InputStream.</summary>
/// <remarks>
/// Create a configuration from an InputStream.
/// <p>
/// ERROR canibalized from <code>Configuration.loadResource()</code>.
/// </remarks>
/// <param name="is">inputstream to read the configuration from.</param>
/// <exception cref="System.IO.IOException">thrown if the configuration could not be read.
/// </exception>
public static void Load(Configuration conf, InputStream @is)
{
try
{
DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory.NewInstance();
// ignore all comments inside the xml file
docBuilderFactory.SetIgnoringComments(true);
DocumentBuilder builder = docBuilderFactory.NewDocumentBuilder();
Document doc = builder.Parse(@is);
ParseDocument(conf, doc);
}
catch (SAXException e)
{
throw new IOException(e);
}
catch (ParserConfigurationException e)
{
throw new IOException(e);
}
}
public virtual void TestSingleNodesXML()
{
WebResource r = Resource();
ClientResponse response = r.Path("ws").Path("v1").Path("node").Path("info/").Accept
(MediaType.ApplicationXml).Get <ClientResponse>();
NUnit.Framework.Assert.AreEqual(MediaType.ApplicationXmlType, response.GetType());
string xml = response.GetEntity <string>();
DocumentBuilderFactory dbf = DocumentBuilderFactory.NewInstance();
DocumentBuilder db = dbf.NewDocumentBuilder();
InputSource @is = new InputSource();
@is.SetCharacterStream(new StringReader(xml));
Document dom = db.Parse(@is);
NodeList nodes = dom.GetElementsByTagName("nodeInfo");
NUnit.Framework.Assert.AreEqual("incorrect number of elements", 1, nodes.GetLength
());
VerifyNodesXML(nodes);
}
/// <exception cref="Org.Codehaus.Jettison.Json.JSONException"/>
/// <exception cref="System.Exception"/>
public virtual void VerifyAMInfoXML(string xml, AppContext ctx)
{
DocumentBuilderFactory dbf = DocumentBuilderFactory.NewInstance();
DocumentBuilder db = dbf.NewDocumentBuilder();
InputSource @is = new InputSource();
@is.SetCharacterStream(new StringReader(xml));
Document dom = db.Parse(@is);
NodeList nodes = dom.GetElementsByTagName("info");
NUnit.Framework.Assert.AreEqual("incorrect number of elements", 1, nodes.GetLength
());
for (int i = 0; i < nodes.GetLength(); i++)
{
Element element = (Element)nodes.Item(i);
VerifyAMInfoGeneric(ctx, WebServicesTestUtils.GetXmlString(element, "appId"), WebServicesTestUtils
.GetXmlString(element, "user"), WebServicesTestUtils.GetXmlString(element, "name"
), WebServicesTestUtils.GetXmlLong(element, "startedOn"), WebServicesTestUtils.GetXmlLong
(element, "elapsedTime"));
}
}
public virtual void TestNonexistNodeXML()
{
rm.RegisterNode("h1:1234", 5120);
rm.RegisterNode("h2:1235", 5121);
WebResource r = Resource();
try
{
r.Path("ws").Path("v1").Path("cluster").Path("nodes").Path("node_invalid:99").Accept
(MediaType.ApplicationXml).Get <JSONObject>();
NUnit.Framework.Assert.Fail("should have thrown exception on non-existent nodeid"
);
}
catch (UniformInterfaceException ue)
{
ClientResponse response = ue.GetResponse();
NUnit.Framework.Assert.AreEqual(ClientResponse.Status.NotFound, response.GetClientResponseStatus
());
NUnit.Framework.Assert.AreEqual(MediaType.ApplicationXmlType, response.GetType());
string msg = response.GetEntity <string>();
System.Console.Out.WriteLine(msg);
DocumentBuilderFactory dbf = DocumentBuilderFactory.NewInstance();
DocumentBuilder db = dbf.NewDocumentBuilder();
InputSource @is = new InputSource();
@is.SetCharacterStream(new StringReader(msg));
Document dom = db.Parse(@is);
NodeList nodes = dom.GetElementsByTagName("RemoteException");
Element element = (Element)nodes.Item(0);
string message = WebServicesTestUtils.GetXmlString(element, "message");
string type = WebServicesTestUtils.GetXmlString(element, "exception");
string classname = WebServicesTestUtils.GetXmlString(element, "javaClassName");
VerifyNonexistNodeException(message, type, classname);
}
finally
{
rm.Stop();
}
}
/// <summary>Constructs one Document from an XML file</summary>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Org.Xml.Sax.SAXException"/>
/// <exception cref="Javax.Xml.Parsers.ParserConfigurationException"/>
public static IDocument ReadDocument(File f)
{
IDocument document = null;
DocumentBuilderFactory factory = DocumentBuilderFactory.NewInstance();
// factory.setValidating(true);
// factory.setNamespaceAware(true);
try
{
DocumentBuilder builder = factory.NewDocumentBuilder();
document = builder.Parse(f);
}
catch (SAXException sxe)
{
// displayDocument(document);
// Error generated during parsing)
Exception x = sxe;
if (sxe.GetException() != null)
{
x = sxe.GetException();
}
Sharpen.Runtime.PrintStackTrace(x);
throw;
}
catch (ParserConfigurationException pce)
{
// Parser with specified options can't be built
Sharpen.Runtime.PrintStackTrace(pce);
throw;
}
catch (IOException ioe)
{
// I/O error
Sharpen.Runtime.PrintStackTrace(ioe);
throw;
}
return(document);
}
public virtual void TestNodeAppsStateInvalidXML()
{
WebResource r = Resource();
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
app = new MockApp(1);
nmContext.GetApplications()[app.GetAppId()] = app;
AddAppContainers(app);
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
app2 = new MockApp("foo", 1234, 2);
nmContext.GetApplications()[app2.GetAppId()] = app2;
AddAppContainers(app2);
try
{
r.Path("ws").Path("v1").Path("node").Path("apps").QueryParam("state", "FOO_STATE"
).Accept(MediaType.ApplicationXml).Get <JSONObject>();
NUnit.Framework.Assert.Fail("should have thrown exception on invalid user query");
}
catch (UniformInterfaceException ue)
{
ClientResponse response = ue.GetResponse();
NUnit.Framework.Assert.AreEqual(ClientResponse.Status.BadRequest, response.GetClientResponseStatus
());
NUnit.Framework.Assert.AreEqual(MediaType.ApplicationXmlType, response.GetType());
string msg = response.GetEntity <string>();
DocumentBuilderFactory dbf = DocumentBuilderFactory.NewInstance();
DocumentBuilder db = dbf.NewDocumentBuilder();
InputSource @is = new InputSource();
@is.SetCharacterStream(new StringReader(msg));
Document dom = db.Parse(@is);
NodeList nodes = dom.GetElementsByTagName("RemoteException");
Element element = (Element)nodes.Item(0);
string message = WebServicesTestUtils.GetXmlString(element, "message");
string type = WebServicesTestUtils.GetXmlString(element, "exception");
string classname = WebServicesTestUtils.GetXmlString(element, "javaClassName");
VerifyStateInvalidException(message, type, classname);
}
}
/// <exception cref="Org.Codehaus.Jettison.Json.JSONException"/>
/// <exception cref="System.Exception"/>
public virtual void VerifyBlacklistedNodesInfoXML(string xml, AppContext ctx)
{
DocumentBuilderFactory dbf = DocumentBuilderFactory.NewInstance();
DocumentBuilder db = dbf.NewDocumentBuilder();
InputSource @is = new InputSource();
@is.SetCharacterStream(new StringReader(xml));
Document dom = db.Parse(@is);
NodeList infonodes = dom.GetElementsByTagName("blacklistednodesinfo");
NUnit.Framework.Assert.AreEqual("incorrect number of elements", 1, infonodes.GetLength
());
NodeList nodes = dom.GetElementsByTagName("blacklistedNodes");
ICollection <string> blacklistedNodes = ctx.GetBlacklistedNodes();
NUnit.Framework.Assert.AreEqual("incorrect number of elements", blacklistedNodes.
Count, nodes.GetLength());
for (int i = 0; i < nodes.GetLength(); i++)
{
Element element = (Element)nodes.Item(i);
NUnit.Framework.Assert.IsTrue(blacklistedNodes.Contains(element.GetFirstChild().GetNodeValue
()));
}
}
public virtual void TestPerUserResourcesXML()
{
//Start RM so that it accepts app submissions
rm.Start();
try
{
rm.SubmitApp(10, "app1", "user1", null, "b1");
rm.SubmitApp(20, "app2", "user2", null, "b1");
//Get the XML from ws/v1/cluster/scheduler
WebResource r = Resource();
ClientResponse response = r.Path("ws/v1/cluster/scheduler").Accept(MediaType.ApplicationXml
).Get <ClientResponse>();
NUnit.Framework.Assert.AreEqual(MediaType.ApplicationXmlType, response.GetType());
string xml = response.GetEntity <string>();
DocumentBuilder db = DocumentBuilderFactory.NewInstance().NewDocumentBuilder();
InputSource @is = new InputSource();
@is.SetCharacterStream(new StringReader(xml));
//Parse the XML we got
Document dom = db.Parse(@is);
//Get all users elements (1 for each leaf queue)
NodeList allUsers = dom.GetElementsByTagName("users");
for (int i = 0; i < allUsers.GetLength(); ++i)
{
Node perUserResources = allUsers.Item(i);
string queueName = GetChildNodeByName(perUserResources.GetParentNode(), "queueName"
).GetTextContent();
if (queueName.Equals("b1"))
{
//b1 should have two users (user1 and user2) which submitted jobs
NUnit.Framework.Assert.AreEqual(2, perUserResources.GetChildNodes().GetLength());
NodeList users = perUserResources.GetChildNodes();
for (int j = 0; j < users.GetLength(); ++j)
{
Node user = users.Item(j);
string username = GetChildNodeByName(user, "username").GetTextContent();
NUnit.Framework.Assert.IsTrue(username.Equals("user1") || username.Equals("user2"
));
//Should be a parsable integer
System.Convert.ToInt32(GetChildNodeByName(GetChildNodeByName(user, "resourcesUsed"
), "memory").GetTextContent());
System.Convert.ToInt32(GetChildNodeByName(user, "numActiveApplications").GetTextContent
());
System.Convert.ToInt32(GetChildNodeByName(user, "numPendingApplications").GetTextContent
());
}
}
else
{
//Queues other than b1 should have 0 users
NUnit.Framework.Assert.AreEqual(0, perUserResources.GetChildNodes().GetLength());
}
}
NodeList allResourcesUsed = dom.GetElementsByTagName("resourcesUsed");
for (int i_1 = 0; i_1 < allResourcesUsed.GetLength(); ++i_1)
{
Node resourcesUsed = allResourcesUsed.Item(i_1);
System.Convert.ToInt32(GetChildNodeByName(resourcesUsed, "memory").GetTextContent
());
System.Convert.ToInt32(GetChildNodeByName(resourcesUsed, "vCores").GetTextContent
());
}
}
finally
{
rm.Stop();
}
}
/// <summary>Updates the allocation list from the allocation config file.</summary>
/// <remarks>
/// Updates the allocation list from the allocation config file. This file is
/// expected to be in the XML format specified in the design doc.
/// </remarks>
/// <exception cref="System.IO.IOException">if the config file cannot be read.</exception>
/// <exception cref="AllocationConfigurationException">if allocations are invalid.</exception>
/// <exception cref="Javax.Xml.Parsers.ParserConfigurationException">if XML parser is misconfigured.
/// </exception>
/// <exception cref="Org.Xml.Sax.SAXException">if config file is malformed.</exception>
/// <exception cref="Org.Apache.Hadoop.Yarn.Server.Resourcemanager.Scheduler.Fair.AllocationConfigurationException
/// "/>
public virtual void ReloadAllocations()
{
lock (this)
{
if (allocFile == null)
{
return;
}
Log.Info("Loading allocation file " + allocFile);
// Create some temporary hashmaps to hold the new allocs, and we only save
// them in our fields if we have parsed the entire allocs file successfully.
IDictionary <string, Resource> minQueueResources = new Dictionary <string, Resource
>();
IDictionary <string, Resource> maxQueueResources = new Dictionary <string, Resource
>();
IDictionary <string, int> queueMaxApps = new Dictionary <string, int>();
IDictionary <string, int> userMaxApps = new Dictionary <string, int>();
IDictionary <string, float> queueMaxAMShares = new Dictionary <string, float>();
IDictionary <string, ResourceWeights> queueWeights = new Dictionary <string, ResourceWeights
>();
IDictionary <string, SchedulingPolicy> queuePolicies = new Dictionary <string, SchedulingPolicy
>();
IDictionary <string, long> minSharePreemptionTimeouts = new Dictionary <string, long
>();
IDictionary <string, long> fairSharePreemptionTimeouts = new Dictionary <string, long
>();
IDictionary <string, float> fairSharePreemptionThresholds = new Dictionary <string,
float>();
IDictionary <string, IDictionary <QueueACL, AccessControlList> > queueAcls = new Dictionary
<string, IDictionary <QueueACL, AccessControlList> >();
ICollection <string> reservableQueues = new HashSet <string>();
int userMaxAppsDefault = int.MaxValue;
int queueMaxAppsDefault = int.MaxValue;
float queueMaxAMShareDefault = 0.5f;
long defaultFairSharePreemptionTimeout = long.MaxValue;
long defaultMinSharePreemptionTimeout = long.MaxValue;
float defaultFairSharePreemptionThreshold = 0.5f;
SchedulingPolicy defaultSchedPolicy = SchedulingPolicy.DefaultPolicy;
// Reservation global configuration knobs
string planner = null;
string reservationAgent = null;
string reservationAdmissionPolicy = null;
QueuePlacementPolicy newPlacementPolicy = null;
// Remember all queue names so we can display them on web UI, etc.
// configuredQueues is segregated based on whether it is a leaf queue
// or a parent queue. This information is used for creating queues
// and also for making queue placement decisions(QueuePlacementRule.java).
IDictionary <FSQueueType, ICollection <string> > configuredQueues = new Dictionary <FSQueueType
, ICollection <string> >();
foreach (FSQueueType queueType in FSQueueType.Values())
{
configuredQueues[queueType] = new HashSet <string>();
}
// Read and parse the allocations file.
DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory.NewInstance();
docBuilderFactory.SetIgnoringComments(true);
DocumentBuilder builder = docBuilderFactory.NewDocumentBuilder();
Document doc = builder.Parse(allocFile);
Element root = doc.GetDocumentElement();
if (!"allocations".Equals(root.GetTagName()))
{
throw new AllocationConfigurationException("Bad fair scheduler config " + "file: top-level element not <allocations>"
);
}
NodeList elements = root.GetChildNodes();
IList <Element> queueElements = new AList <Element>();
Element placementPolicyElement = null;
for (int i = 0; i < elements.GetLength(); i++)
{
Node node = elements.Item(i);
if (node is Element)
{
Element element = (Element)node;
if ("queue".Equals(element.GetTagName()) || "pool".Equals(element.GetTagName()))
{
queueElements.AddItem(element);
}
else
{
if ("user".Equals(element.GetTagName()))
{
string userName = element.GetAttribute("name");
NodeList fields = element.GetChildNodes();
for (int j = 0; j < fields.GetLength(); j++)
{
Node fieldNode = fields.Item(j);
if (!(fieldNode is Element))
{
continue;
}
Element field = (Element)fieldNode;
if ("maxRunningApps".Equals(field.GetTagName()))
{
string text = ((Text)field.GetFirstChild()).GetData().Trim();
int val = System.Convert.ToInt32(text);
userMaxApps[userName] = val;
}
}
}
else
{
if ("userMaxAppsDefault".Equals(element.GetTagName()))
{
string text = ((Text)element.GetFirstChild()).GetData().Trim();
int val = System.Convert.ToInt32(text);
userMaxAppsDefault = val;
}
else
{
if ("defaultFairSharePreemptionTimeout".Equals(element.GetTagName()))
{
string text = ((Text)element.GetFirstChild()).GetData().Trim();
long val = long.Parse(text) * 1000L;
defaultFairSharePreemptionTimeout = val;
}
else
{
if ("fairSharePreemptionTimeout".Equals(element.GetTagName()))
{
if (defaultFairSharePreemptionTimeout == long.MaxValue)
{
string text = ((Text)element.GetFirstChild()).GetData().Trim();
long val = long.Parse(text) * 1000L;
defaultFairSharePreemptionTimeout = val;
}
}
else
{
if ("defaultMinSharePreemptionTimeout".Equals(element.GetTagName()))
{
string text = ((Text)element.GetFirstChild()).GetData().Trim();
long val = long.Parse(text) * 1000L;
defaultMinSharePreemptionTimeout = val;
}
else
{
if ("defaultFairSharePreemptionThreshold".Equals(element.GetTagName()))
{
string text = ((Text)element.GetFirstChild()).GetData().Trim();
float val = float.ParseFloat(text);
val = Math.Max(Math.Min(val, 1.0f), 0.0f);
defaultFairSharePreemptionThreshold = val;
}
else
{
if ("queueMaxAppsDefault".Equals(element.GetTagName()))
{
string text = ((Text)element.GetFirstChild()).GetData().Trim();
int val = System.Convert.ToInt32(text);
queueMaxAppsDefault = val;
}
else
{
if ("queueMaxAMShareDefault".Equals(element.GetTagName()))
{
string text = ((Text)element.GetFirstChild()).GetData().Trim();
float val = float.ParseFloat(text);
val = Math.Min(val, 1.0f);
queueMaxAMShareDefault = val;
}
else
{
if ("defaultQueueSchedulingPolicy".Equals(element.GetTagName()) || "defaultQueueSchedulingMode"
.Equals(element.GetTagName()))
{
string text = ((Text)element.GetFirstChild()).GetData().Trim();
defaultSchedPolicy = SchedulingPolicy.Parse(text);
}
else
{
if ("queuePlacementPolicy".Equals(element.GetTagName()))
{
placementPolicyElement = element;
}
else
{
if ("reservation-planner".Equals(element.GetTagName()))
{
string text = ((Text)element.GetFirstChild()).GetData().Trim();
planner = text;
}
else
{
if ("reservation-agent".Equals(element.GetTagName()))
{
string text = ((Text)element.GetFirstChild()).GetData().Trim();
reservationAgent = text;
}
else
{
if ("reservation-policy".Equals(element.GetTagName()))
{
string text = ((Text)element.GetFirstChild()).GetData().Trim();
reservationAdmissionPolicy = text;
}
else
{
Log.Warn("Bad element in allocations file: " + element.GetTagName());
}
}
}
}
}
}
}
}
}
}
}
}
}
}
}
}
// Load queue elements. A root queue can either be included or omitted. If
// it's included, all other queues must be inside it.
foreach (Element element_1 in queueElements)
{
string parent = "root";
if (Sharpen.Runtime.EqualsIgnoreCase(element_1.GetAttribute("name"), "root"))
{
if (queueElements.Count > 1)
{
throw new AllocationConfigurationException("If configuring root queue," + " no other queues can be placed alongside it."
);
}
parent = null;
}
LoadQueue(parent, element_1, minQueueResources, maxQueueResources, queueMaxApps,
userMaxApps, queueMaxAMShares, queueWeights, queuePolicies, minSharePreemptionTimeouts
, fairSharePreemptionTimeouts, fairSharePreemptionThresholds, queueAcls, configuredQueues
, reservableQueues);
}
// Load placement policy and pass it configured queues
Configuration conf = GetConfig();
if (placementPolicyElement != null)
{
newPlacementPolicy = QueuePlacementPolicy.FromXml(placementPolicyElement, configuredQueues
, conf);
}
else
{
newPlacementPolicy = QueuePlacementPolicy.FromConfiguration(conf, configuredQueues
);
}
// Set the min/fair share preemption timeout for the root queue
if (!minSharePreemptionTimeouts.Contains(QueueManager.RootQueue))
{
minSharePreemptionTimeouts[QueueManager.RootQueue] = defaultMinSharePreemptionTimeout;
}
if (!fairSharePreemptionTimeouts.Contains(QueueManager.RootQueue))
{
fairSharePreemptionTimeouts[QueueManager.RootQueue] = defaultFairSharePreemptionTimeout;
}
// Set the fair share preemption threshold for the root queue
if (!fairSharePreemptionThresholds.Contains(QueueManager.RootQueue))
{
fairSharePreemptionThresholds[QueueManager.RootQueue] = defaultFairSharePreemptionThreshold;
}
ReservationQueueConfiguration globalReservationQueueConfig = new ReservationQueueConfiguration
();
if (planner != null)
{
globalReservationQueueConfig.SetPlanner(planner);
}
if (reservationAdmissionPolicy != null)
{
globalReservationQueueConfig.SetReservationAdmissionPolicy(reservationAdmissionPolicy
);
}
if (reservationAgent != null)
{
globalReservationQueueConfig.SetReservationAgent(reservationAgent);
}
AllocationConfiguration info = new AllocationConfiguration(minQueueResources, maxQueueResources
, queueMaxApps, userMaxApps, queueWeights, queueMaxAMShares, userMaxAppsDefault,
queueMaxAppsDefault, queueMaxAMShareDefault, queuePolicies, defaultSchedPolicy,
minSharePreemptionTimeouts, fairSharePreemptionTimeouts, fairSharePreemptionThresholds
, queueAcls, newPlacementPolicy, configuredQueues, globalReservationQueueConfig,
reservableQueues);
lastSuccessfulReload = clock.GetTime();
lastReloadAttemptFailed = false;
reloadListener.OnReload(info);
}
}