// format and print information about the passed in job queue.
/// <exception cref="System.IO.IOException"/>
internal virtual void PrintJobQueueInfo(JobQueueInfo jobQueueInfo, TextWriter writer
, string prefix)
{
if (jobQueueInfo == null)
{
writer.Write("No queue found.\n");
writer.Flush();
return;
}
writer.Write(string.Format(prefix + "======================\n"));
writer.Write(string.Format(prefix + "Queue Name : %s \n", jobQueueInfo.GetQueueName
()));
writer.Write(string.Format(prefix + "Queue State : %s \n", jobQueueInfo.GetQueueState
()));
writer.Write(string.Format(prefix + "Scheduling Info : %s \n", jobQueueInfo.GetSchedulingInfo
()));
IList <JobQueueInfo> childQueues = jobQueueInfo.GetChildren();
if (childQueues != null && childQueues.Count > 0)
{
for (int i = 0; i < childQueues.Count; i++)
{
PrintJobQueueInfo(childQueues[i], writer, " " + prefix);
}
}
writer.Flush();
}
/// <summary>test xml generation</summary>
/// <exception cref="Javax.Xml.Parsers.ParserConfigurationException"/>
/// <exception cref="System.Exception"></exception>
public virtual void TestQueueConfigurationParser()
{
JobQueueInfo info = new JobQueueInfo("root", "rootInfo");
JobQueueInfo infoChild1 = new JobQueueInfo("child1", "child1Info");
JobQueueInfo infoChild2 = new JobQueueInfo("child2", "child1Info");
info.AddChild(infoChild1);
info.AddChild(infoChild2);
DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory.NewInstance();
DocumentBuilder builder = docBuilderFactory.NewDocumentBuilder();
Document document = builder.NewDocument();
// test QueueConfigurationParser.getQueueElement
Element e = QueueConfigurationParser.GetQueueElement(document, info);
// transform result to string for check
DOMSource domSource = new DOMSource(e);
StringWriter writer = new StringWriter();
StreamResult result = new StreamResult(writer);
TransformerFactory tf = TransformerFactory.NewInstance();
Transformer transformer = tf.NewTransformer();
transformer.Transform(domSource, result);
string str = writer.ToString();
NUnit.Framework.Assert.IsTrue(str.EndsWith("<queue><name>root</name><properties/><state>running</state><queue><name>child1</name><properties/><state>running</state></queue><queue><name>child2</name><properties/><state>running</state></queue></queue>"
));
}
/// <summary>
/// Return hierarchy of
/// <see cref="JobQueueInfo"/>
/// objects
/// under this Queue.
/// </summary>
/// <returns>JobQueueInfo[]</returns>
internal virtual JobQueueInfo GetJobQueueInfo()
{
JobQueueInfo queueInfo = new JobQueueInfo();
queueInfo.SetQueueName(name);
Log.Debug("created jobQInfo " + queueInfo.GetQueueName());
queueInfo.SetQueueState(state.GetStateName());
if (schedulingInfo != null)
{
queueInfo.SetSchedulingInfo(schedulingInfo.ToString());
}
if (props != null)
{
//Create deep copy of properties.
Properties newProps = new Properties();
foreach (object key in props.Keys)
{
newProps.SetProperty(key.ToString(), props.GetProperty(key.ToString()));
}
queueInfo.SetProperties(newProps);
}
if (children != null && children.Count > 0)
{
IList <JobQueueInfo> list = new AList <JobQueueInfo>();
foreach (Org.Apache.Hadoop.Mapred.Queue child in children)
{
list.AddItem(child.GetJobQueueInfo());
}
queueInfo.SetChildren(list);
}
return(queueInfo);
}
/// <summary>
/// Construct an
/// <see cref="Org.W3c.Dom.Element"/>
/// for a single queue, constructing the inner
/// queue <name/>, <properties/>, <state/> and the inner
/// <queue> elements recursively.
/// </summary>
/// <param name="document"/>
/// <param name="jqi"/>
/// <returns/>
internal static Element GetQueueElement(Document document, JobQueueInfo jqi)
{
// Queue
Element q = document.CreateElement(QueueTag);
// Queue-name
Element qName = document.CreateElement(QueueNameTag);
qName.SetTextContent(GetSimpleQueueName(jqi.GetQueueName()));
q.AppendChild(qName);
// Queue-properties
Properties props = jqi.GetProperties();
Element propsElement = document.CreateElement(PropertiesTag);
if (props != null)
{
ICollection <string> propList = props.StringPropertyNames();
foreach (string prop in propList)
{
Element propertyElement = document.CreateElement(PropertyTag);
propertyElement.SetAttribute(KeyTag, prop);
propertyElement.SetAttribute(ValueTag, (string)props[prop]);
propsElement.AppendChild(propertyElement);
}
}
q.AppendChild(propsElement);
// Queue-state
string queueState = jqi.GetState().GetStateName();
if (queueState != null && !queueState.Equals(QueueState.Undefined.GetStateName()))
{
Element qStateElement = document.CreateElement(StateTag);
qStateElement.SetTextContent(queueState);
q.AppendChild(qStateElement);
}
// Queue-children
IList <JobQueueInfo> children = jqi.GetChildren();
if (children != null)
{
foreach (JobQueueInfo child in children)
{
q.AppendChild(GetQueueElement(document, child));
}
}
return(q);
}
public virtual void TestPrintJobQueueInfo()
{
JobQueueClient queueClient = new JobQueueClient();
JobQueueInfo parent = new JobQueueInfo();
JobQueueInfo child = new JobQueueInfo();
JobQueueInfo grandChild = new JobQueueInfo();
child.AddChild(grandChild);
parent.AddChild(child);
grandChild.SetQueueName("GrandChildQueue");
ByteArrayOutputStream bbos = new ByteArrayOutputStream();
PrintWriter writer = new PrintWriter(bbos);
queueClient.PrintJobQueueInfo(parent, writer);
NUnit.Framework.Assert.IsTrue("printJobQueueInfo did not print grandchild's name"
, bbos.ToString().Contains("GrandChildQueue"));
}
/// <summary>
/// Return an array of
/// <see cref="JobQueueInfo"/>
/// objects for all the
/// queues configurated in the system.
/// </summary>
/// <returns>array of JobQueueInfo objects.</returns>
internal virtual JobQueueInfo[] GetJobQueueInfos()
{
lock (this)
{
AList <JobQueueInfo> queueInfoList = new AList <JobQueueInfo>();
foreach (string queue in allQueues.Keys)
{
JobQueueInfo queueInfo = GetJobQueueInfo(queue);
if (queueInfo != null)
{
queueInfoList.AddItem(queueInfo);
}
}
return(Sharpen.Collections.ToArray(queueInfoList, new JobQueueInfo[queueInfoList.
Count]));
}
}
/// <summary>
/// Method used to display information pertaining to a Single JobQueue
/// registered with the
/// <see cref="QueueManager"/>
/// . Display of the Jobs is determine
/// by the boolean
/// </summary>
/// <exception cref="System.IO.IOException">, InterruptedException</exception>
/// <exception cref="System.Exception"/>
private void DisplayQueueInfo(string queue, bool showJobs)
{
JobQueueInfo jobQueueInfo = jc.GetQueueInfo(queue);
if (jobQueueInfo == null)
{
System.Console.Out.WriteLine("Queue \"" + queue + "\" does not exist.");
return;
}
PrintJobQueueInfo(jobQueueInfo, new PrintWriter(new OutputStreamWriter(System.Console.Out
, Charsets.Utf8)));
if (showJobs && (jobQueueInfo.GetChildren() == null || jobQueueInfo.GetChildren()
.Count == 0))
{
JobStatus[] jobs = jobQueueInfo.GetJobStatuses();
if (jobs == null)
{
jobs = new JobStatus[0];
}
jc.DisplayJobList(jobs);
}
}
// format and print information about the passed in job queue.
/// <exception cref="System.IO.IOException"/>
internal virtual void PrintJobQueueInfo(JobQueueInfo jobQueueInfo, TextWriter writer
)
{
PrintJobQueueInfo(jobQueueInfo, writer, string.Empty);
}
/// <summary>test JobConf</summary>
/// <exception cref="System.Exception"/>
public virtual void TestNetworkedJob()
{
// mock creation
MiniMRClientCluster mr = null;
FileSystem fileSys = null;
try
{
mr = CreateMiniClusterWithCapacityScheduler();
JobConf job = new JobConf(mr.GetConfig());
fileSys = FileSystem.Get(job);
fileSys.Delete(testDir, true);
FSDataOutputStream @out = fileSys.Create(inFile, true);
@out.WriteBytes("This is a test file");
@out.Close();
FileInputFormat.SetInputPaths(job, inFile);
FileOutputFormat.SetOutputPath(job, outDir);
job.SetInputFormat(typeof(TextInputFormat));
job.SetOutputFormat(typeof(TextOutputFormat));
job.SetMapperClass(typeof(IdentityMapper));
job.SetReducerClass(typeof(IdentityReducer));
job.SetNumReduceTasks(0);
JobClient client = new JobClient(mr.GetConfig());
RunningJob rj = client.SubmitJob(job);
JobID jobId = rj.GetID();
JobClient.NetworkedJob runningJob = (JobClient.NetworkedJob)client.GetJob(jobId);
runningJob.SetJobPriority(JobPriority.High.ToString());
// test getters
NUnit.Framework.Assert.IsTrue(runningJob.GetConfiguration().ToString().EndsWith("0001/job.xml"
));
NUnit.Framework.Assert.AreEqual(runningJob.GetID(), jobId);
NUnit.Framework.Assert.AreEqual(runningJob.GetJobID(), jobId.ToString());
NUnit.Framework.Assert.AreEqual(runningJob.GetJobName(), "N/A");
NUnit.Framework.Assert.IsTrue(runningJob.GetJobFile().EndsWith(".staging/" + runningJob
.GetJobID() + "/job.xml"));
NUnit.Framework.Assert.IsTrue(runningJob.GetTrackingURL().Length > 0);
NUnit.Framework.Assert.IsTrue(runningJob.MapProgress() == 0.0f);
NUnit.Framework.Assert.IsTrue(runningJob.ReduceProgress() == 0.0f);
NUnit.Framework.Assert.IsTrue(runningJob.CleanupProgress() == 0.0f);
NUnit.Framework.Assert.IsTrue(runningJob.SetupProgress() == 0.0f);
TaskCompletionEvent[] tce = runningJob.GetTaskCompletionEvents(0);
NUnit.Framework.Assert.AreEqual(tce.Length, 0);
NUnit.Framework.Assert.AreEqual(runningJob.GetHistoryUrl(), string.Empty);
NUnit.Framework.Assert.IsFalse(runningJob.IsRetired());
NUnit.Framework.Assert.AreEqual(runningJob.GetFailureInfo(), string.Empty);
NUnit.Framework.Assert.AreEqual(runningJob.GetJobStatus().GetJobName(), "N/A");
NUnit.Framework.Assert.AreEqual(client.GetMapTaskReports(jobId).Length, 0);
try
{
client.GetSetupTaskReports(jobId);
}
catch (YarnRuntimeException e)
{
NUnit.Framework.Assert.AreEqual(e.Message, "Unrecognized task type: JOB_SETUP");
}
try
{
client.GetCleanupTaskReports(jobId);
}
catch (YarnRuntimeException e)
{
NUnit.Framework.Assert.AreEqual(e.Message, "Unrecognized task type: JOB_CLEANUP");
}
NUnit.Framework.Assert.AreEqual(client.GetReduceTaskReports(jobId).Length, 0);
// test ClusterStatus
ClusterStatus status = client.GetClusterStatus(true);
NUnit.Framework.Assert.AreEqual(status.GetActiveTrackerNames().Count, 2);
// it method does not implemented and always return empty array or null;
NUnit.Framework.Assert.AreEqual(status.GetBlacklistedTrackers(), 0);
NUnit.Framework.Assert.AreEqual(status.GetBlacklistedTrackerNames().Count, 0);
NUnit.Framework.Assert.AreEqual(status.GetBlackListedTrackersInfo().Count, 0);
NUnit.Framework.Assert.AreEqual(status.GetJobTrackerStatus(), Cluster.JobTrackerStatus
.Running);
NUnit.Framework.Assert.AreEqual(status.GetMapTasks(), 1);
NUnit.Framework.Assert.AreEqual(status.GetMaxMapTasks(), 20);
NUnit.Framework.Assert.AreEqual(status.GetMaxReduceTasks(), 4);
NUnit.Framework.Assert.AreEqual(status.GetNumExcludedNodes(), 0);
NUnit.Framework.Assert.AreEqual(status.GetReduceTasks(), 1);
NUnit.Framework.Assert.AreEqual(status.GetTaskTrackers(), 2);
NUnit.Framework.Assert.AreEqual(status.GetTTExpiryInterval(), 0);
NUnit.Framework.Assert.AreEqual(status.GetJobTrackerStatus(), Cluster.JobTrackerStatus
.Running);
NUnit.Framework.Assert.AreEqual(status.GetGraylistedTrackers(), 0);
// test read and write
ByteArrayOutputStream dataOut = new ByteArrayOutputStream();
status.Write(new DataOutputStream(dataOut));
ClusterStatus status2 = new ClusterStatus();
status2.ReadFields(new DataInputStream(new ByteArrayInputStream(dataOut.ToByteArray
())));
NUnit.Framework.Assert.AreEqual(status.GetActiveTrackerNames(), status2.GetActiveTrackerNames
());
NUnit.Framework.Assert.AreEqual(status.GetBlackListedTrackersInfo(), status2.GetBlackListedTrackersInfo
());
NUnit.Framework.Assert.AreEqual(status.GetMapTasks(), status2.GetMapTasks());
try
{
}
catch (RuntimeException e)
{
NUnit.Framework.Assert.IsTrue(e.Message.EndsWith("not found on CLASSPATH"));
}
// test taskStatusfilter
JobClient.SetTaskOutputFilter(job, JobClient.TaskStatusFilter.All);
NUnit.Framework.Assert.AreEqual(JobClient.GetTaskOutputFilter(job), JobClient.TaskStatusFilter
.All);
// runningJob.setJobPriority(JobPriority.HIGH.name());
// test default map
NUnit.Framework.Assert.AreEqual(client.GetDefaultMaps(), 20);
NUnit.Framework.Assert.AreEqual(client.GetDefaultReduces(), 4);
NUnit.Framework.Assert.AreEqual(client.GetSystemDir().GetName(), "jobSubmitDir");
// test queue information
JobQueueInfo[] rootQueueInfo = client.GetRootQueues();
NUnit.Framework.Assert.AreEqual(rootQueueInfo.Length, 1);
NUnit.Framework.Assert.AreEqual(rootQueueInfo[0].GetQueueName(), "default");
JobQueueInfo[] qinfo = client.GetQueues();
NUnit.Framework.Assert.AreEqual(qinfo.Length, 1);
NUnit.Framework.Assert.AreEqual(qinfo[0].GetQueueName(), "default");
NUnit.Framework.Assert.AreEqual(client.GetChildQueues("default").Length, 0);
NUnit.Framework.Assert.AreEqual(client.GetJobsFromQueue("default").Length, 1);
NUnit.Framework.Assert.IsTrue(client.GetJobsFromQueue("default")[0].GetJobFile().
EndsWith("/job.xml"));
JobQueueInfo qi = client.GetQueueInfo("default");
NUnit.Framework.Assert.AreEqual(qi.GetQueueName(), "default");
NUnit.Framework.Assert.AreEqual(qi.GetQueueState(), "running");
QueueAclsInfo[] aai = client.GetQueueAclsForCurrentUser();
NUnit.Framework.Assert.AreEqual(aai.Length, 2);
NUnit.Framework.Assert.AreEqual(aai[0].GetQueueName(), "root");
NUnit.Framework.Assert.AreEqual(aai[1].GetQueueName(), "default");
// test token
Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier> token = client.
GetDelegationToken(new Text(UserGroupInformation.GetCurrentUser().GetShortUserName
()));
NUnit.Framework.Assert.AreEqual(token.GetKind().ToString(), "RM_DELEGATION_TOKEN"
);
// test JobClient
// The following asserts read JobStatus twice and ensure the returned
// JobStatus objects correspond to the same Job.
NUnit.Framework.Assert.AreEqual("Expected matching JobIDs", jobId, ((JobID)client
.GetJob(jobId).GetJobStatus().GetJobID()));
NUnit.Framework.Assert.AreEqual("Expected matching startTimes", rj.GetJobStatus()
.GetStartTime(), client.GetJob(jobId).GetJobStatus().GetStartTime());
}
finally
{
if (fileSys != null)
{
fileSys.Delete(testDir, true);
}
if (mr != null)
{
mr.Stop();
}
}
}