/// <summary>Launches a MR job and tests the job counters against the expected values.
/// </summary>
/// <param name="testName">The name for the job</param>
/// <param name="mr">The MR cluster</param>
/// <param name="fileSys">The FileSystem</param>
/// <param name="in">Input path</param>
/// <param name="out">Output path</param>
/// <param name="numMaps">Number of maps</param>
/// <param name="otherLocalMaps">Expected value of other local maps</param>
/// <param name="datalocalMaps">Expected value of data(node) local maps</param>
/// <param name="racklocalMaps">Expected value of rack local maps</param>
/// <exception cref="System.IO.IOException"/>
internal static void LaunchJobAndTestCounters(string jobName, MiniMRCluster mr, FileSystem
fileSys, Path @in, Path @out, int numMaps, int otherLocalMaps, int dataLocalMaps
, int rackLocalMaps)
{
JobConf jobConf = mr.CreateJobConf();
if (fileSys.Exists(@out))
{
fileSys.Delete(@out, true);
}
RunningJob job = LaunchJob(jobConf, @in, @out, numMaps, jobName);
Counters counters = job.GetCounters();
NUnit.Framework.Assert.AreEqual("Number of local maps", counters.GetCounter(JobCounter
.OtherLocalMaps), otherLocalMaps);
NUnit.Framework.Assert.AreEqual("Number of Data-local maps", counters.GetCounter(
JobCounter.DataLocalMaps), dataLocalMaps);
NUnit.Framework.Assert.AreEqual("Number of Rack-local maps", counters.GetCounter(
JobCounter.RackLocalMaps), rackLocalMaps);
mr.WaitUntilIdle();
mr.Shutdown();
}
// run a job which gets stuck in mapper and kill it.
/// <exception cref="System.IO.IOException"/>
private void TestKilledJob(string fileName, Type committer, string[] exclude)
{
JobConf jc = mr.CreateJobConf();
Path outDir = GetNewOutputDir();
ConfigureJob(jc, "kill job with abort()", 1, 0, outDir);
// set the job to wait for long
jc.SetMapperClass(typeof(UtilsForTests.KillMapper));
jc.SetOutputCommitter(committer);
JobClient jobClient = new JobClient(jc);
RunningJob job = jobClient.SubmitJob(jc);
JobID id = job.GetID();
Counters counters = job.GetCounters();
// wait for the map to be launched
while (true)
{
if (counters.GetCounter(JobCounter.TotalLaunchedMaps) == 1)
{
break;
}
Log.Info("Waiting for a map task to be launched");
UtilsForTests.WaitFor(100);
counters = job.GetCounters();
}
job.KillJob();
// kill the job
job.WaitForCompletion();
// wait for the job to complete
NUnit.Framework.Assert.AreEqual("Job was not killed", JobStatus.Killed, job.GetJobState
());
if (fileName != null)
{
Path testFile = new Path(outDir, fileName);
NUnit.Framework.Assert.IsTrue("File " + testFile + " missing for job " + id, fileSys
.Exists(testFile));
}
// check if the files from the missing set exists
foreach (string ex in exclude)
{
Path file = new Path(outDir, ex);
NUnit.Framework.Assert.IsFalse("File " + file + " should not be present for killed job "
+ id, fileSys.Exists(file));
}
}
/// <exception cref="System.Exception"/>
private long GetTaskCounterUsage(JobClient client, JobID id, int numReports, int
taskId, TaskType type)
{
TaskReport[] reports = null;
if (TaskType.Map.Equals(type))
{
reports = client.GetMapTaskReports(id);
}
else
{
if (TaskType.Reduce.Equals(type))
{
reports = client.GetReduceTaskReports(id);
}
}
NUnit.Framework.Assert.IsNotNull("No reports found for task type '" + type.ToString
() + "' in job " + id, reports);
// make sure that the total number of reports match the expected
NUnit.Framework.Assert.AreEqual("Mismatch in task id", numReports, reports.Length
);
Counters counters = reports[taskId].GetCounters();
return(counters.GetCounter(TaskCounter.CommittedHeapBytes));
}