/// <exception cref="System.Exception"/>
private void TestCachingAtLevel(int level)
{
string namenode = null;
MiniDFSCluster dfs = null;
MiniMRCluster mr = null;
FileSystem fileSys = null;
string testName = "TestMultiLevelCaching";
try
{
int taskTrackers = 1;
// generate the racks
// use rack1 for data node
string rack1 = GetRack(0, level);
// use rack2 for task tracker
string rack2 = GetRack(1, level);
Configuration conf = new Configuration();
// Run a datanode on host1 under /a/b/c/..../d1/e1/f1
dfs = new MiniDFSCluster.Builder(conf).Racks(new string[] { rack1 }).Hosts(new string
[] { "host1.com" }).Build();
dfs.WaitActive();
fileSys = dfs.GetFileSystem();
if (!fileSys.Mkdirs(inDir))
{
throw new IOException("Mkdirs failed to create " + inDir.ToString());
}
UtilsForTests.WriteFile(dfs.GetNameNode(), conf, new Path(inDir + "/file"), (short
)1);
namenode = (dfs.GetFileSystem()).GetUri().GetHost() + ":" + (dfs.GetFileSystem())
.GetUri().GetPort();
// Run a job with the (only)tasktracker on host2 under diff topology
// e.g /a/b/c/..../d2/e2/f2.
JobConf jc = new JobConf();
// cache-level = level (unshared levels) + 1(topmost shared node i.e /a)
// + 1 (for host)
jc.SetInt(JTConfig.JtTaskcacheLevels, level + 2);
mr = new MiniMRCluster(taskTrackers, namenode, 1, new string[] { rack2 }, new string
[] { "host2.com" }, jc);
/* The job is configured with 1 map for one (non-splittable) file.
* Since the datanode is running under different subtree, there is no
* node-level data locality but there should be topological locality.
*/
LaunchJobAndTestCounters(testName, mr, fileSys, inDir, outputPath, 1, 1, 0, 0);
mr.Shutdown();
}
finally
{
if (null != fileSys)
{
// inDir, outputPath only exist if fileSys is valid.
fileSys.Delete(inDir, true);
fileSys.Delete(outputPath, true);
}
if (dfs != null)
{
dfs.Shutdown();
}
}
}
// Run a job that will be failed and wait until it completes
/// <exception cref="System.IO.IOException"/>
public static RunningJob RunJobFail(JobConf conf, Path inDir, Path outDir)
{
conf.SetJobName("test-job-fail");
conf.SetMapperClass(typeof(UtilsForTests.FailMapper));
conf.SetReducerClass(typeof(IdentityReducer));
conf.SetMaxMapAttempts(1);
RunningJob job = UtilsForTests.RunJob(conf, inDir, outDir);
long sleepCount = 0;
while (!job.IsComplete())
{
try
{
if (sleepCount > 300)
{
// 30 seconds
throw new IOException("Job didn't finish in 30 seconds");
}
Sharpen.Thread.Sleep(100);
sleepCount++;
}
catch (Exception)
{
break;
}
}
return(job);
}
/// <summary>Reads the output file into a string</summary>
/// <param name="conf"/>
/// <returns/>
/// <exception cref="System.IO.IOException"/>
public virtual string ReadOutputFile(Configuration conf)
{
FileSystem localFs = FileSystem.GetLocal(conf);
Path file = new Path(outputDir, "part-00000");
return(UtilsForTests.SlurpHadoop(file, localFs));
}
public virtual void TestFormatWithCustomSeparator()
{
JobConf job = new JobConf();
string separator = "\u0001";
job.Set("mapreduce.output.textoutputformat.separator", separator);
job.Set(JobContext.TaskAttemptId, attempt);
FileOutputFormat.SetOutputPath(job, workDir.GetParent().GetParent());
FileOutputFormat.SetWorkOutputPath(job, workDir);
FileSystem fs = workDir.GetFileSystem(job);
if (!fs.Mkdirs(workDir))
{
NUnit.Framework.Assert.Fail("Failed to create output directory");
}
string file = "test_custom.txt";
// A reporter that does nothing
Reporter reporter = Reporter.Null;
TextOutputFormat <object, object> theOutputFormat = new TextOutputFormat <object, object
>();
RecordWriter <object, object> theRecordWriter = theOutputFormat.GetRecordWriter(localFs
, job, file, reporter);
Org.Apache.Hadoop.IO.Text key1 = new Org.Apache.Hadoop.IO.Text("key1");
Org.Apache.Hadoop.IO.Text key2 = new Org.Apache.Hadoop.IO.Text("key2");
Org.Apache.Hadoop.IO.Text val1 = new Org.Apache.Hadoop.IO.Text("val1");
Org.Apache.Hadoop.IO.Text val2 = new Org.Apache.Hadoop.IO.Text("val2");
NullWritable nullWritable = NullWritable.Get();
try
{
theRecordWriter.Write(key1, val1);
theRecordWriter.Write(null, nullWritable);
theRecordWriter.Write(null, val1);
theRecordWriter.Write(nullWritable, val2);
theRecordWriter.Write(key2, nullWritable);
theRecordWriter.Write(key1, null);
theRecordWriter.Write(null, null);
theRecordWriter.Write(key2, val2);
}
finally
{
theRecordWriter.Close(reporter);
}
FilePath expectedFile = new FilePath(new Path(workDir, file).ToString());
StringBuilder expectedOutput = new StringBuilder();
expectedOutput.Append(key1).Append(separator).Append(val1).Append("\n");
expectedOutput.Append(val1).Append("\n");
expectedOutput.Append(val2).Append("\n");
expectedOutput.Append(key2).Append("\n");
expectedOutput.Append(key1).Append("\n");
expectedOutput.Append(key2).Append(separator).Append(val2).Append("\n");
string output = UtilsForTests.Slurp(expectedFile);
NUnit.Framework.Assert.AreEqual(expectedOutput.ToString(), output);
}
public virtual void TestMRTimelineEventHandling()
{
Configuration conf = new YarnConfiguration();
conf.SetBoolean(YarnConfiguration.TimelineServiceEnabled, true);
conf.SetBoolean(MRJobConfig.MapreduceJobEmitTimelineData, true);
MiniMRYarnCluster cluster = null;
try
{
cluster = new MiniMRYarnCluster(typeof(TestJobHistoryEventHandler).Name, 1);
cluster.Init(conf);
cluster.Start();
conf.Set(YarnConfiguration.TimelineServiceWebappAddress, MiniYARNCluster.GetHostname
() + ":" + cluster.GetApplicationHistoryServer().GetPort());
TimelineStore ts = cluster.GetApplicationHistoryServer().GetTimelineStore();
Path inDir = new Path("input");
Path outDir = new Path("output");
RunningJob job = UtilsForTests.RunJobSucceed(new JobConf(conf), inDir, outDir);
NUnit.Framework.Assert.AreEqual(JobStatus.Succeeded, job.GetJobStatus().GetState(
).GetValue());
TimelineEntities entities = ts.GetEntities("MAPREDUCE_JOB", null, null, null, null
, null, null, null, null, null);
NUnit.Framework.Assert.AreEqual(1, entities.GetEntities().Count);
TimelineEntity tEntity = entities.GetEntities()[0];
NUnit.Framework.Assert.AreEqual(job.GetID().ToString(), tEntity.GetEntityId());
NUnit.Framework.Assert.AreEqual("MAPREDUCE_JOB", tEntity.GetEntityType());
NUnit.Framework.Assert.AreEqual(EventType.AmStarted.ToString(), tEntity.GetEvents
()[tEntity.GetEvents().Count - 1].GetEventType());
NUnit.Framework.Assert.AreEqual(EventType.JobFinished.ToString(), tEntity.GetEvents
()[0].GetEventType());
job = UtilsForTests.RunJobFail(new JobConf(conf), inDir, outDir);
NUnit.Framework.Assert.AreEqual(JobStatus.Failed, job.GetJobStatus().GetState().GetValue
());
entities = ts.GetEntities("MAPREDUCE_JOB", null, null, null, null, null, null, null
, null, null);
NUnit.Framework.Assert.AreEqual(2, entities.GetEntities().Count);
tEntity = entities.GetEntities()[0];
NUnit.Framework.Assert.AreEqual(job.GetID().ToString(), tEntity.GetEntityId());
NUnit.Framework.Assert.AreEqual("MAPREDUCE_JOB", tEntity.GetEntityType());
NUnit.Framework.Assert.AreEqual(EventType.AmStarted.ToString(), tEntity.GetEvents
()[tEntity.GetEvents().Count - 1].GetEventType());
NUnit.Framework.Assert.AreEqual(EventType.JobFailed.ToString(), tEntity.GetEvents
()[0].GetEventType());
}
finally
{
if (cluster != null)
{
cluster.Stop();
}
}
}
/// <exception cref="System.Exception"/>
public virtual void TestMR()
{
System.Console.Out.WriteLine(LaunchWordCount(this.CreateJobConf(), "a b c d e f g h"
, 1, 1));
bool keepTrying = true;
for (int tries = 0; tries < 30 && keepTrying; tries++)
{
Sharpen.Thread.Sleep(50);
keepTrying = !(NotificationTestCase.NotificationServlet.counter == 2);
}
NUnit.Framework.Assert.AreEqual(2, NotificationTestCase.NotificationServlet.counter
);
NUnit.Framework.Assert.AreEqual(0, NotificationTestCase.NotificationServlet.failureCounter
);
Path inDir = new Path("notificationjob/input");
Path outDir = new Path("notificationjob/output");
// Hack for local FS that does not have the concept of a 'mounting point'
if (IsLocalFS())
{
string localPathRoot = Runtime.GetProperty("test.build.data", "/tmp").ToString().
Replace(' ', '+');
inDir = new Path(localPathRoot, inDir);
outDir = new Path(localPathRoot, outDir);
}
// run a job with KILLED status
System.Console.Out.WriteLine(UtilsForTests.RunJobKill(this.CreateJobConf(), inDir
, outDir).GetID());
keepTrying = true;
for (int tries_1 = 0; tries_1 < 30 && keepTrying; tries_1++)
{
Sharpen.Thread.Sleep(50);
keepTrying = !(NotificationTestCase.NotificationServlet.counter == 4);
}
NUnit.Framework.Assert.AreEqual(4, NotificationTestCase.NotificationServlet.counter
);
NUnit.Framework.Assert.AreEqual(0, NotificationTestCase.NotificationServlet.failureCounter
);
// run a job with FAILED status
System.Console.Out.WriteLine(UtilsForTests.RunJobFail(this.CreateJobConf(), inDir
, outDir).GetID());
keepTrying = true;
for (int tries_2 = 0; tries_2 < 30 && keepTrying; tries_2++)
{
Sharpen.Thread.Sleep(50);
keepTrying = !(NotificationTestCase.NotificationServlet.counter == 6);
}
NUnit.Framework.Assert.AreEqual(6, NotificationTestCase.NotificationServlet.counter
);
NUnit.Framework.Assert.AreEqual(0, NotificationTestCase.NotificationServlet.failureCounter
);
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestCommitFail()
{
Path inDir = new Path(rootDir, "./input");
Path outDir = new Path(rootDir, "./output");
JobConf jobConf = CreateJobConf();
jobConf.SetMaxMapAttempts(1);
jobConf.SetOutputCommitter(typeof(TestTaskCommit.CommitterWithCommitFail));
RunningJob rJob = UtilsForTests.RunJob(jobConf, inDir, outDir, 1, 0);
rJob.WaitForCompletion();
NUnit.Framework.Assert.AreEqual(JobStatus.Failed, rJob.GetJobState());
}
public virtual void TestReporterProgressForMapOnlyJob()
{
Path test = new Path(testRootTempDir, "testReporterProgressForMapOnlyJob");
JobConf conf = new JobConf();
conf.SetMapperClass(typeof(TestReporter.ProgressTesterMapper));
conf.SetMapOutputKeyClass(typeof(Org.Apache.Hadoop.IO.Text));
// fail early
conf.SetMaxMapAttempts(1);
conf.SetMaxReduceAttempts(0);
RunningJob job = UtilsForTests.RunJob(conf, new Path(test, "in"), new Path(test,
"out"), 1, 0, Input);
job.WaitForCompletion();
NUnit.Framework.Assert.IsTrue("Job failed", job.IsSuccessful());
}
// Run a job that will be killed and wait until it completes
/// <exception cref="System.IO.IOException"/>
public static RunningJob RunJobKill(JobConf conf, Path inDir, Path outDir)
{
conf.SetJobName("test-job-kill");
conf.SetMapperClass(typeof(UtilsForTests.KillMapper));
conf.SetReducerClass(typeof(IdentityReducer));
RunningJob job = UtilsForTests.RunJob(conf, inDir, outDir);
long sleepCount = 0;
while (job.GetJobState() != JobStatus.Running)
{
try
{
if (sleepCount > 300)
{
// 30 seconds
throw new IOException("Job didn't finish in 30 seconds");
}
Sharpen.Thread.Sleep(100);
sleepCount++;
}
catch (Exception)
{
break;
}
}
job.KillJob();
sleepCount = 0;
while (job.CleanupProgress() == 0.0f)
{
try
{
if (sleepCount > 2000)
{
// 20 seconds
throw new IOException("Job cleanup didn't start in 20 seconds");
}
Sharpen.Thread.Sleep(10);
sleepCount++;
}
catch (Exception)
{
break;
}
}
return(job);
}
// run a job which gets stuck in mapper and kill it.
/// <exception cref="System.IO.IOException"/>
private void TestKilledJob(string fileName, Type committer, string[] exclude)
{
JobConf jc = mr.CreateJobConf();
Path outDir = GetNewOutputDir();
ConfigureJob(jc, "kill job with abort()", 1, 0, outDir);
// set the job to wait for long
jc.SetMapperClass(typeof(UtilsForTests.KillMapper));
jc.SetOutputCommitter(committer);
JobClient jobClient = new JobClient(jc);
RunningJob job = jobClient.SubmitJob(jc);
JobID id = job.GetID();
Counters counters = job.GetCounters();
// wait for the map to be launched
while (true)
{
if (counters.GetCounter(JobCounter.TotalLaunchedMaps) == 1)
{
break;
}
Log.Info("Waiting for a map task to be launched");
UtilsForTests.WaitFor(100);
counters = job.GetCounters();
}
job.KillJob();
// kill the job
job.WaitForCompletion();
// wait for the job to complete
NUnit.Framework.Assert.AreEqual("Job was not killed", JobStatus.Killed, job.GetJobState
());
if (fileName != null)
{
Path testFile = new Path(outDir, fileName);
NUnit.Framework.Assert.IsTrue("File " + testFile + " missing for job " + id, fileSys
.Exists(testFile));
}
// check if the files from the missing set exists
foreach (string ex in exclude)
{
Path file = new Path(outDir, ex);
NUnit.Framework.Assert.IsFalse("File " + file + " should not be present for killed job "
+ id, fileSys.Exists(file));
}
}
/// <exception cref="System.Exception"/>
public virtual void TestCommitter()
{
JobConf job = new JobConf();
SetConfForFileOutputCommitter(job);
JobContext jContext = new JobContextImpl(job, ((JobID)taskID.GetJobID()));
TaskAttemptContext tContext = new TaskAttemptContextImpl(job, taskID);
FileOutputCommitter committer = new FileOutputCommitter();
FileOutputFormat.SetWorkOutputPath(job, committer.GetTaskAttemptPath(tContext));
committer.SetupJob(jContext);
committer.SetupTask(tContext);
string file = "test.txt";
// A reporter that does nothing
Reporter reporter = Reporter.Null;
// write output
FileSystem localFs = FileSystem.GetLocal(job);
TextOutputFormat theOutputFormat = new TextOutputFormat();
RecordWriter theRecordWriter = theOutputFormat.GetRecordWriter(localFs, job, file
, reporter);
WriteOutput(theRecordWriter, reporter);
// do commit
committer.CommitTask(tContext);
committer.CommitJob(jContext);
// validate output
FilePath expectedFile = new FilePath(new Path(outDir, file).ToString());
StringBuilder expectedOutput = new StringBuilder();
expectedOutput.Append(key1).Append('\t').Append(val1).Append("\n");
expectedOutput.Append(val1).Append("\n");
expectedOutput.Append(val2).Append("\n");
expectedOutput.Append(key2).Append("\n");
expectedOutput.Append(key1).Append("\n");
expectedOutput.Append(key2).Append('\t').Append(val2).Append("\n");
string output = UtilsForTests.Slurp(expectedFile);
NUnit.Framework.Assert.AreEqual(output, expectedOutput.ToString());
FileUtil.FullyDelete(new FilePath(outDir.ToString()));
}
/// <exception cref="System.Exception"/>
public virtual void TestFormat()
{
JobConf job = new JobConf();
job.Set(JobContext.TaskAttemptId, attempt);
FileOutputFormat.SetOutputPath(job, workDir.GetParent().GetParent());
FileOutputFormat.SetWorkOutputPath(job, workDir);
FileSystem fs = workDir.GetFileSystem(job);
if (!fs.Mkdirs(workDir))
{
Fail("Failed to create output directory");
}
//System.out.printf("workdir: %s\n", workDir.toString());
TestMultipleTextOutputFormat.Test1(job);
TestMultipleTextOutputFormat.Test2(job);
string file_11 = "1-part-00000";
FilePath expectedFile_11 = new FilePath(new Path(workDir, file_11).ToString());
//System.out.printf("expectedFile_11: %s\n", new Path(workDir, file_11).toString());
StringBuilder expectedOutput = new StringBuilder();
for (int i = 10; i < 20; i++)
{
expectedOutput.Append(string.Empty + i).Append('\t').Append(string.Empty + i).Append
("\n");
}
string output = UtilsForTests.Slurp(expectedFile_11);
//System.out.printf("File_2 output: %s\n", output);
NUnit.Framework.Assert.AreEqual(output, expectedOutput.ToString());
string file_12 = "2-part-00000";
FilePath expectedFile_12 = new FilePath(new Path(workDir, file_12).ToString());
//System.out.printf("expectedFile_12: %s\n", new Path(workDir, file_12).toString());
expectedOutput = new StringBuilder();
for (int i_1 = 20; i_1 < 30; i_1++)
{
expectedOutput.Append(string.Empty + i_1).Append('\t').Append(string.Empty + i_1)
.Append("\n");
}
output = UtilsForTests.Slurp(expectedFile_12);
//System.out.printf("File_2 output: %s\n", output);
NUnit.Framework.Assert.AreEqual(output, expectedOutput.ToString());
string file_13 = "3-part-00000";
FilePath expectedFile_13 = new FilePath(new Path(workDir, file_13).ToString());
//System.out.printf("expectedFile_13: %s\n", new Path(workDir, file_13).toString());
expectedOutput = new StringBuilder();
for (int i_2 = 30; i_2 < 40; i_2++)
{
expectedOutput.Append(string.Empty + i_2).Append('\t').Append(string.Empty + i_2)
.Append("\n");
}
output = UtilsForTests.Slurp(expectedFile_13);
//System.out.printf("File_2 output: %s\n", output);
NUnit.Framework.Assert.AreEqual(output, expectedOutput.ToString());
string file_2 = "2/3";
FilePath expectedFile_2 = new FilePath(new Path(workDir, file_2).ToString());
//System.out.printf("expectedFile_2: %s\n", new Path(workDir, file_2).toString());
expectedOutput = new StringBuilder();
for (int i_3 = 10; i_3 < 40; i_3++)
{
expectedOutput.Append(string.Empty + i_3).Append('\t').Append(string.Empty + i_3)
.Append("\n");
}
output = UtilsForTests.Slurp(expectedFile_2);
//System.out.printf("File_2 output: %s\n", output);
NUnit.Framework.Assert.AreEqual(output, expectedOutput.ToString());
}