public virtual void TestMRTimelineEventHandling()
{
Configuration conf = new YarnConfiguration();
conf.SetBoolean(YarnConfiguration.TimelineServiceEnabled, true);
conf.SetBoolean(MRJobConfig.MapreduceJobEmitTimelineData, true);
MiniMRYarnCluster cluster = null;
try
{
cluster = new MiniMRYarnCluster(typeof(TestJobHistoryEventHandler).Name, 1);
cluster.Init(conf);
cluster.Start();
conf.Set(YarnConfiguration.TimelineServiceWebappAddress, MiniYARNCluster.GetHostname
() + ":" + cluster.GetApplicationHistoryServer().GetPort());
TimelineStore ts = cluster.GetApplicationHistoryServer().GetTimelineStore();
Path inDir = new Path("input");
Path outDir = new Path("output");
RunningJob job = UtilsForTests.RunJobSucceed(new JobConf(conf), inDir, outDir);
NUnit.Framework.Assert.AreEqual(JobStatus.Succeeded, job.GetJobStatus().GetState(
).GetValue());
TimelineEntities entities = ts.GetEntities("MAPREDUCE_JOB", null, null, null, null
, null, null, null, null, null);
NUnit.Framework.Assert.AreEqual(1, entities.GetEntities().Count);
TimelineEntity tEntity = entities.GetEntities()[0];
NUnit.Framework.Assert.AreEqual(job.GetID().ToString(), tEntity.GetEntityId());
NUnit.Framework.Assert.AreEqual("MAPREDUCE_JOB", tEntity.GetEntityType());
NUnit.Framework.Assert.AreEqual(EventType.AmStarted.ToString(), tEntity.GetEvents
()[tEntity.GetEvents().Count - 1].GetEventType());
NUnit.Framework.Assert.AreEqual(EventType.JobFinished.ToString(), tEntity.GetEvents
()[0].GetEventType());
job = UtilsForTests.RunJobFail(new JobConf(conf), inDir, outDir);
NUnit.Framework.Assert.AreEqual(JobStatus.Failed, job.GetJobStatus().GetState().GetValue
());
entities = ts.GetEntities("MAPREDUCE_JOB", null, null, null, null, null, null, null
, null, null);
NUnit.Framework.Assert.AreEqual(2, entities.GetEntities().Count);
tEntity = entities.GetEntities()[0];
NUnit.Framework.Assert.AreEqual(job.GetID().ToString(), tEntity.GetEntityId());
NUnit.Framework.Assert.AreEqual("MAPREDUCE_JOB", tEntity.GetEntityType());
NUnit.Framework.Assert.AreEqual(EventType.AmStarted.ToString(), tEntity.GetEvents
()[tEntity.GetEvents().Count - 1].GetEventType());
NUnit.Framework.Assert.AreEqual(EventType.JobFailed.ToString(), tEntity.GetEvents
()[0].GetEventType());
}
finally
{
if (cluster != null)
{
cluster.Stop();
}
}
}
// run a job for which all the attempts simply fail.
/// <exception cref="System.IO.IOException"/>
private void TestFailedJob(string fileName, Type committer, string[] exclude)
{
JobConf jc = mr.CreateJobConf();
Path outDir = GetNewOutputDir();
ConfigureJob(jc, "fail job with abort()", 1, 0, outDir);
jc.SetMaxMapAttempts(1);
// set the job to fail
jc.SetMapperClass(typeof(UtilsForTests.FailMapper));
jc.SetOutputCommitter(committer);
JobClient jobClient = new JobClient(jc);
RunningJob job = jobClient.SubmitJob(jc);
JobID id = job.GetID();
job.WaitForCompletion();
NUnit.Framework.Assert.AreEqual("Job did not fail", JobStatus.Failed, job.GetJobState
());
if (fileName != null)
{
Path testFile = new Path(outDir, fileName);
NUnit.Framework.Assert.IsTrue("File " + testFile + " missing for failed job " + id
, fileSys.Exists(testFile));
}
// check if the files from the missing set exists
foreach (string ex in exclude)
{
Path file = new Path(outDir, ex);
NUnit.Framework.Assert.IsFalse("File " + file + " should not be present for failed job "
+ id, fileSys.Exists(file));
}
}
// run a job with 1 map and let it run to completion
/// <exception cref="System.IO.IOException"/>
private void TestSuccessfulJob(string filename, Type committer, string[] exclude)
{
JobConf jc = mr.CreateJobConf();
Path outDir = GetNewOutputDir();
ConfigureJob(jc, "job with cleanup()", 1, 0, outDir);
jc.SetOutputCommitter(committer);
JobClient jobClient = new JobClient(jc);
RunningJob job = jobClient.SubmitJob(jc);
JobID id = job.GetID();
job.WaitForCompletion();
Log.Info("Job finished : " + job.IsComplete());
Path testFile = new Path(outDir, filename);
NUnit.Framework.Assert.IsTrue("Done file \"" + testFile + "\" missing for job " +
id, fileSys.Exists(testFile));
// check if the files from the missing set exists
foreach (string ex in exclude)
{
Path file = new Path(outDir, ex);
NUnit.Framework.Assert.IsFalse("File " + file + " should not be present for successful job "
+ id, fileSys.Exists(file));
}
}
// run a job which gets stuck in mapper and kill it.
/// <exception cref="System.IO.IOException"/>
private void TestKilledJob(string fileName, Type committer, string[] exclude)
{
JobConf jc = mr.CreateJobConf();
Path outDir = GetNewOutputDir();
ConfigureJob(jc, "kill job with abort()", 1, 0, outDir);
// set the job to wait for long
jc.SetMapperClass(typeof(UtilsForTests.KillMapper));
jc.SetOutputCommitter(committer);
JobClient jobClient = new JobClient(jc);
RunningJob job = jobClient.SubmitJob(jc);
JobID id = job.GetID();
Counters counters = job.GetCounters();
// wait for the map to be launched
while (true)
{
if (counters.GetCounter(JobCounter.TotalLaunchedMaps) == 1)
{
break;
}
Log.Info("Waiting for a map task to be launched");
UtilsForTests.WaitFor(100);
counters = job.GetCounters();
}
job.KillJob();
// kill the job
job.WaitForCompletion();
// wait for the job to complete
NUnit.Framework.Assert.AreEqual("Job was not killed", JobStatus.Killed, job.GetJobState
());
if (fileName != null)
{
Path testFile = new Path(outDir, fileName);
NUnit.Framework.Assert.IsTrue("File " + testFile + " missing for job " + id, fileSys
.Exists(testFile));
}
// check if the files from the missing set exists
foreach (string ex in exclude)
{
Path file = new Path(outDir, ex);
NUnit.Framework.Assert.IsFalse("File " + file + " should not be present for killed job "
+ id, fileSys.Exists(file));
}
}
public virtual void TestGetRunningJobFromJobClient()
{
JobConf conf = new JobConf();
conf.Set("mapreduce.framework.name", "local");
FileInputFormat.AddInputPath(conf, CreateTempFile("in", "hello"));
Path outputDir = new Path(TestRootDir, GetType().Name);
outputDir.GetFileSystem(conf).Delete(outputDir, true);
FileOutputFormat.SetOutputPath(conf, outputDir);
JobClient jc = new JobClient(conf);
RunningJob runningJob = jc.SubmitJob(conf);
NUnit.Framework.Assert.IsNotNull("Running job", runningJob);
// Check that the running job can be retrieved by ID
RunningJob newRunningJob = jc.GetJob(runningJob.GetID());
NUnit.Framework.Assert.IsNotNull("New running job", newRunningJob);
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
/// <exception cref="System.TypeLoadException"/>
public virtual void TestGetJobStatus()
{
MiniMRClientCluster mr = null;
FileSystem fileSys = null;
try
{
mr = CreateMiniClusterWithCapacityScheduler();
JobConf job = new JobConf(mr.GetConfig());
fileSys = FileSystem.Get(job);
fileSys.Delete(testDir, true);
FSDataOutputStream @out = fileSys.Create(inFile, true);
@out.WriteBytes("This is a test file");
@out.Close();
FileInputFormat.SetInputPaths(job, inFile);
FileOutputFormat.SetOutputPath(job, outDir);
job.SetInputFormat(typeof(TextInputFormat));
job.SetOutputFormat(typeof(TextOutputFormat));
job.SetMapperClass(typeof(IdentityMapper));
job.SetReducerClass(typeof(IdentityReducer));
job.SetNumReduceTasks(0);
JobClient client = new JobClient(mr.GetConfig());
RunningJob rj = client.SubmitJob(job);
JobID jobId = rj.GetID();
// The following asserts read JobStatus twice and ensure the returned
// JobStatus objects correspond to the same Job.
NUnit.Framework.Assert.AreEqual("Expected matching JobIDs", jobId, ((JobID)client
.GetJob(jobId).GetJobStatus().GetJobID()));
NUnit.Framework.Assert.AreEqual("Expected matching startTimes", rj.GetJobStatus()
.GetStartTime(), client.GetJob(jobId).GetJobStatus().GetStartTime());
}
finally
{
if (fileSys != null)
{
fileSys.Delete(testDir, true);
}
if (mr != null)
{
mr.Stop();
}
}
}
// set up heap options, target value for memory loader and the output
// directory before running the job
/// <exception cref="System.IO.IOException"/>
private static RunningJob RunHeapUsageTestJob(JobConf conf, Path testRootDir, string
heapOptions, long targetMapValue, long targetReduceValue, FileSystem fs, JobClient
client, Path inDir)
{
// define a job
JobConf jobConf = new JobConf(conf);
// configure the jobs
jobConf.SetNumMapTasks(1);
jobConf.SetNumReduceTasks(1);
jobConf.SetMapperClass(typeof(TestJobCounters.MemoryLoaderMapper));
jobConf.SetReducerClass(typeof(TestJobCounters.MemoryLoaderReducer));
jobConf.SetInputFormat(typeof(TextInputFormat));
jobConf.SetOutputKeyClass(typeof(LongWritable));
jobConf.SetOutputValueClass(typeof(Org.Apache.Hadoop.IO.Text));
jobConf.SetMaxMapAttempts(1);
jobConf.SetMaxReduceAttempts(1);
jobConf.Set(JobConf.MapredTaskJavaOpts, heapOptions);
// set the targets
jobConf.SetLong(TestJobCounters.MemoryLoaderMapper.TargetValue, targetMapValue);
jobConf.SetLong(TestJobCounters.MemoryLoaderReducer.TargetValue, targetReduceValue
);
// set the input directory for the job
FileInputFormat.SetInputPaths(jobConf, inDir);
// define job output folder
Path outDir = new Path(testRootDir, "out");
fs.Delete(outDir, true);
FileOutputFormat.SetOutputPath(jobConf, outDir);
// run the job
RunningJob job = client.SubmitJob(jobConf);
job.WaitForCompletion();
JobID jobID = job.GetID();
NUnit.Framework.Assert.IsTrue("Job " + jobID + " failed!", job.IsSuccessful());
return(job);
}
public virtual void TestHeapUsageCounter()
{
JobConf conf = new JobConf();
// create a local filesystem handle
FileSystem fileSystem = FileSystem.GetLocal(conf);
// define test root directories
Path rootDir = new Path(Runtime.GetProperty("test.build.data", "/tmp"));
Path testRootDir = new Path(rootDir, "testHeapUsageCounter");
// cleanup the test root directory
fileSystem.Delete(testRootDir, true);
// set the current working directory
fileSystem.SetWorkingDirectory(testRootDir);
fileSystem.DeleteOnExit(testRootDir);
// create a mini cluster using the local file system
MiniMRCluster mrCluster = new MiniMRCluster(1, fileSystem.GetUri().ToString(), 1);
try
{
conf = mrCluster.CreateJobConf();
JobClient jobClient = new JobClient(conf);
// define job input
Path inDir = new Path(testRootDir, "in");
// create input data
CreateWordsFile(inDir, conf);
// configure and run a low memory job which will run without loading the
// jvm's heap
RunningJob lowMemJob = RunHeapUsageTestJob(conf, testRootDir, "-Xms32m -Xmx1G", 0
, 0, fileSystem, jobClient, inDir);
JobID lowMemJobID = lowMemJob.GetID();
long lowMemJobMapHeapUsage = GetTaskCounterUsage(jobClient, lowMemJobID, 1, 0, TaskType
.Map);
System.Console.Out.WriteLine("Job1 (low memory job) map task heap usage: " + lowMemJobMapHeapUsage
);
long lowMemJobReduceHeapUsage = GetTaskCounterUsage(jobClient, lowMemJobID, 1, 0,
TaskType.Reduce);
System.Console.Out.WriteLine("Job1 (low memory job) reduce task heap usage: " + lowMemJobReduceHeapUsage
);
// configure and run a high memory job which will load the jvm's heap
RunningJob highMemJob = RunHeapUsageTestJob(conf, testRootDir, "-Xms32m -Xmx1G",
lowMemJobMapHeapUsage + 256 * 1024 * 1024, lowMemJobReduceHeapUsage + 256 * 1024
* 1024, fileSystem, jobClient, inDir);
JobID highMemJobID = highMemJob.GetID();
long highMemJobMapHeapUsage = GetTaskCounterUsage(jobClient, highMemJobID, 1, 0,
TaskType.Map);
System.Console.Out.WriteLine("Job2 (high memory job) map task heap usage: " + highMemJobMapHeapUsage
);
long highMemJobReduceHeapUsage = GetTaskCounterUsage(jobClient, highMemJobID, 1,
0, TaskType.Reduce);
System.Console.Out.WriteLine("Job2 (high memory job) reduce task heap usage: " +
highMemJobReduceHeapUsage);
NUnit.Framework.Assert.IsTrue("Incorrect map heap usage reported by the map task"
, lowMemJobMapHeapUsage < highMemJobMapHeapUsage);
NUnit.Framework.Assert.IsTrue("Incorrect reduce heap usage reported by the reduce task"
, lowMemJobReduceHeapUsage < highMemJobReduceHeapUsage);
}
finally
{
// shutdown the mr cluster
mrCluster.Shutdown();
try
{
fileSystem.Delete(testRootDir, true);
}
catch (IOException)
{
}
}
}
/// <summary>test JobConf</summary>
/// <exception cref="System.Exception"/>
public virtual void TestNetworkedJob()
{
// mock creation
MiniMRClientCluster mr = null;
FileSystem fileSys = null;
try
{
mr = CreateMiniClusterWithCapacityScheduler();
JobConf job = new JobConf(mr.GetConfig());
fileSys = FileSystem.Get(job);
fileSys.Delete(testDir, true);
FSDataOutputStream @out = fileSys.Create(inFile, true);
@out.WriteBytes("This is a test file");
@out.Close();
FileInputFormat.SetInputPaths(job, inFile);
FileOutputFormat.SetOutputPath(job, outDir);
job.SetInputFormat(typeof(TextInputFormat));
job.SetOutputFormat(typeof(TextOutputFormat));
job.SetMapperClass(typeof(IdentityMapper));
job.SetReducerClass(typeof(IdentityReducer));
job.SetNumReduceTasks(0);
JobClient client = new JobClient(mr.GetConfig());
RunningJob rj = client.SubmitJob(job);
JobID jobId = rj.GetID();
JobClient.NetworkedJob runningJob = (JobClient.NetworkedJob)client.GetJob(jobId);
runningJob.SetJobPriority(JobPriority.High.ToString());
// test getters
NUnit.Framework.Assert.IsTrue(runningJob.GetConfiguration().ToString().EndsWith("0001/job.xml"
));
NUnit.Framework.Assert.AreEqual(runningJob.GetID(), jobId);
NUnit.Framework.Assert.AreEqual(runningJob.GetJobID(), jobId.ToString());
NUnit.Framework.Assert.AreEqual(runningJob.GetJobName(), "N/A");
NUnit.Framework.Assert.IsTrue(runningJob.GetJobFile().EndsWith(".staging/" + runningJob
.GetJobID() + "/job.xml"));
NUnit.Framework.Assert.IsTrue(runningJob.GetTrackingURL().Length > 0);
NUnit.Framework.Assert.IsTrue(runningJob.MapProgress() == 0.0f);
NUnit.Framework.Assert.IsTrue(runningJob.ReduceProgress() == 0.0f);
NUnit.Framework.Assert.IsTrue(runningJob.CleanupProgress() == 0.0f);
NUnit.Framework.Assert.IsTrue(runningJob.SetupProgress() == 0.0f);
TaskCompletionEvent[] tce = runningJob.GetTaskCompletionEvents(0);
NUnit.Framework.Assert.AreEqual(tce.Length, 0);
NUnit.Framework.Assert.AreEqual(runningJob.GetHistoryUrl(), string.Empty);
NUnit.Framework.Assert.IsFalse(runningJob.IsRetired());
NUnit.Framework.Assert.AreEqual(runningJob.GetFailureInfo(), string.Empty);
NUnit.Framework.Assert.AreEqual(runningJob.GetJobStatus().GetJobName(), "N/A");
NUnit.Framework.Assert.AreEqual(client.GetMapTaskReports(jobId).Length, 0);
try
{
client.GetSetupTaskReports(jobId);
}
catch (YarnRuntimeException e)
{
NUnit.Framework.Assert.AreEqual(e.Message, "Unrecognized task type: JOB_SETUP");
}
try
{
client.GetCleanupTaskReports(jobId);
}
catch (YarnRuntimeException e)
{
NUnit.Framework.Assert.AreEqual(e.Message, "Unrecognized task type: JOB_CLEANUP");
}
NUnit.Framework.Assert.AreEqual(client.GetReduceTaskReports(jobId).Length, 0);
// test ClusterStatus
ClusterStatus status = client.GetClusterStatus(true);
NUnit.Framework.Assert.AreEqual(status.GetActiveTrackerNames().Count, 2);
// it method does not implemented and always return empty array or null;
NUnit.Framework.Assert.AreEqual(status.GetBlacklistedTrackers(), 0);
NUnit.Framework.Assert.AreEqual(status.GetBlacklistedTrackerNames().Count, 0);
NUnit.Framework.Assert.AreEqual(status.GetBlackListedTrackersInfo().Count, 0);
NUnit.Framework.Assert.AreEqual(status.GetJobTrackerStatus(), Cluster.JobTrackerStatus
.Running);
NUnit.Framework.Assert.AreEqual(status.GetMapTasks(), 1);
NUnit.Framework.Assert.AreEqual(status.GetMaxMapTasks(), 20);
NUnit.Framework.Assert.AreEqual(status.GetMaxReduceTasks(), 4);
NUnit.Framework.Assert.AreEqual(status.GetNumExcludedNodes(), 0);
NUnit.Framework.Assert.AreEqual(status.GetReduceTasks(), 1);
NUnit.Framework.Assert.AreEqual(status.GetTaskTrackers(), 2);
NUnit.Framework.Assert.AreEqual(status.GetTTExpiryInterval(), 0);
NUnit.Framework.Assert.AreEqual(status.GetJobTrackerStatus(), Cluster.JobTrackerStatus
.Running);
NUnit.Framework.Assert.AreEqual(status.GetGraylistedTrackers(), 0);
// test read and write
ByteArrayOutputStream dataOut = new ByteArrayOutputStream();
status.Write(new DataOutputStream(dataOut));
ClusterStatus status2 = new ClusterStatus();
status2.ReadFields(new DataInputStream(new ByteArrayInputStream(dataOut.ToByteArray
())));
NUnit.Framework.Assert.AreEqual(status.GetActiveTrackerNames(), status2.GetActiveTrackerNames
());
NUnit.Framework.Assert.AreEqual(status.GetBlackListedTrackersInfo(), status2.GetBlackListedTrackersInfo
());
NUnit.Framework.Assert.AreEqual(status.GetMapTasks(), status2.GetMapTasks());
try
{
}
catch (RuntimeException e)
{
NUnit.Framework.Assert.IsTrue(e.Message.EndsWith("not found on CLASSPATH"));
}
// test taskStatusfilter
JobClient.SetTaskOutputFilter(job, JobClient.TaskStatusFilter.All);
NUnit.Framework.Assert.AreEqual(JobClient.GetTaskOutputFilter(job), JobClient.TaskStatusFilter
.All);
// runningJob.setJobPriority(JobPriority.HIGH.name());
// test default map
NUnit.Framework.Assert.AreEqual(client.GetDefaultMaps(), 20);
NUnit.Framework.Assert.AreEqual(client.GetDefaultReduces(), 4);
NUnit.Framework.Assert.AreEqual(client.GetSystemDir().GetName(), "jobSubmitDir");
// test queue information
JobQueueInfo[] rootQueueInfo = client.GetRootQueues();
NUnit.Framework.Assert.AreEqual(rootQueueInfo.Length, 1);
NUnit.Framework.Assert.AreEqual(rootQueueInfo[0].GetQueueName(), "default");
JobQueueInfo[] qinfo = client.GetQueues();
NUnit.Framework.Assert.AreEqual(qinfo.Length, 1);
NUnit.Framework.Assert.AreEqual(qinfo[0].GetQueueName(), "default");
NUnit.Framework.Assert.AreEqual(client.GetChildQueues("default").Length, 0);
NUnit.Framework.Assert.AreEqual(client.GetJobsFromQueue("default").Length, 1);
NUnit.Framework.Assert.IsTrue(client.GetJobsFromQueue("default")[0].GetJobFile().
EndsWith("/job.xml"));
JobQueueInfo qi = client.GetQueueInfo("default");
NUnit.Framework.Assert.AreEqual(qi.GetQueueName(), "default");
NUnit.Framework.Assert.AreEqual(qi.GetQueueState(), "running");
QueueAclsInfo[] aai = client.GetQueueAclsForCurrentUser();
NUnit.Framework.Assert.AreEqual(aai.Length, 2);
NUnit.Framework.Assert.AreEqual(aai[0].GetQueueName(), "root");
NUnit.Framework.Assert.AreEqual(aai[1].GetQueueName(), "default");
// test token
Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier> token = client.
GetDelegationToken(new Text(UserGroupInformation.GetCurrentUser().GetShortUserName
()));
NUnit.Framework.Assert.AreEqual(token.GetKind().ToString(), "RM_DELEGATION_TOKEN"
);
// test JobClient
// The following asserts read JobStatus twice and ensure the returned
// JobStatus objects correspond to the same Job.
NUnit.Framework.Assert.AreEqual("Expected matching JobIDs", jobId, ((JobID)client
.GetJob(jobId).GetJobStatus().GetJobID()));
NUnit.Framework.Assert.AreEqual("Expected matching startTimes", rj.GetJobStatus()
.GetStartTime(), client.GetJob(jobId).GetJobStatus().GetStartTime());
}
finally
{
if (fileSys != null)
{
fileSys.Delete(testDir, true);
}
if (mr != null)
{
mr.Stop();
}
}
}