// run a job with 1 map and let it run to completion
/// <exception cref="System.IO.IOException"/>
private void TestSuccessfulJob(string filename, Type committer, string[] exclude)
{
JobConf jc = mr.CreateJobConf();
Path outDir = GetNewOutputDir();
ConfigureJob(jc, "job with cleanup()", 1, 0, outDir);
jc.SetOutputCommitter(committer);
JobClient jobClient = new JobClient(jc);
RunningJob job = jobClient.SubmitJob(jc);
JobID id = job.GetID();
job.WaitForCompletion();
Log.Info("Job finished : " + job.IsComplete());
Path testFile = new Path(outDir, filename);
NUnit.Framework.Assert.IsTrue("Done file \"" + testFile + "\" missing for job " +
id, fileSys.Exists(testFile));
// check if the files from the missing set exists
foreach (string ex in exclude)
{
Path file = new Path(outDir, ex);
NUnit.Framework.Assert.IsFalse("File " + file + " should not be present for successful job "
+ id, fileSys.Exists(file));
}
}
// run a job for which all the attempts simply fail.
/// <exception cref="System.IO.IOException"/>
private void TestFailedJob(string fileName, Type committer, string[] exclude)
{
JobConf jc = mr.CreateJobConf();
Path outDir = GetNewOutputDir();
ConfigureJob(jc, "fail job with abort()", 1, 0, outDir);
jc.SetMaxMapAttempts(1);
// set the job to fail
jc.SetMapperClass(typeof(UtilsForTests.FailMapper));
jc.SetOutputCommitter(committer);
JobClient jobClient = new JobClient(jc);
RunningJob job = jobClient.SubmitJob(jc);
JobID id = job.GetID();
job.WaitForCompletion();
NUnit.Framework.Assert.AreEqual("Job did not fail", JobStatus.Failed, job.GetJobState
());
if (fileName != null)
{
Path testFile = new Path(outDir, fileName);
NUnit.Framework.Assert.IsTrue("File " + testFile + " missing for failed job " + id
, fileSys.Exists(testFile));
}
// check if the files from the missing set exists
foreach (string ex in exclude)
{
Path file = new Path(outDir, ex);
NUnit.Framework.Assert.IsFalse("File " + file + " should not be present for failed job "
+ id, fileSys.Exists(file));
}
}
/// <exception cref="System.Exception"/>
private void MrRun()
{
FileSystem fs = FileSystem.Get(GetJobConf());
Path inputDir = new Path("input");
fs.Mkdirs(inputDir);
TextWriter writer = new OutputStreamWriter(fs.Create(new Path(inputDir, "data.txt"
)));
writer.Write("hello");
writer.Close();
Path outputDir = new Path("output", "output");
JobConf jobConf = new JobConf(GetJobConf());
jobConf.SetInt("mapred.map.tasks", 1);
jobConf.SetInt("mapred.map.max.attempts", 1);
jobConf.SetInt("mapred.reduce.max.attempts", 1);
jobConf.Set("mapred.input.dir", inputDir.ToString());
jobConf.Set("mapred.output.dir", outputDir.ToString());
JobClient jobClient = new JobClient(jobConf);
RunningJob runJob = jobClient.SubmitJob(jobConf);
runJob.WaitForCompletion();
NUnit.Framework.Assert.IsTrue(runJob.IsComplete());
NUnit.Framework.Assert.IsTrue(runJob.IsSuccessful());
}
// runs a sample job as a user (ugi)
/// <exception cref="System.Exception"/>
internal virtual void RunJobAsUser(JobConf job, UserGroupInformation ugi)
{
RunningJob rj = ugi.DoAs(new _PrivilegedExceptionAction_66(job));
rj.WaitForCompletion();
NUnit.Framework.Assert.AreEqual("SUCCEEDED", JobStatus.GetJobRunState(rj.GetJobState
()));
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestCommitFail()
{
Path inDir = new Path(rootDir, "./input");
Path outDir = new Path(rootDir, "./output");
JobConf jobConf = CreateJobConf();
jobConf.SetMaxMapAttempts(1);
jobConf.SetOutputCommitter(typeof(TestTaskCommit.CommitterWithCommitFail));
RunningJob rJob = UtilsForTests.RunJob(jobConf, inDir, outDir, 1, 0);
rJob.WaitForCompletion();
NUnit.Framework.Assert.AreEqual(JobStatus.Failed, rJob.GetJobState());
}
public virtual void TestReporterProgressForMapOnlyJob()
{
Path test = new Path(testRootTempDir, "testReporterProgressForMapOnlyJob");
JobConf conf = new JobConf();
conf.SetMapperClass(typeof(TestReporter.ProgressTesterMapper));
conf.SetMapOutputKeyClass(typeof(Org.Apache.Hadoop.IO.Text));
// fail early
conf.SetMaxMapAttempts(1);
conf.SetMaxReduceAttempts(0);
RunningJob job = UtilsForTests.RunJob(conf, new Path(test, "in"), new Path(test,
"out"), 1, 0, Input);
job.WaitForCompletion();
NUnit.Framework.Assert.IsTrue("Job failed", job.IsSuccessful());
}
// run a job which gets stuck in mapper and kill it.
/// <exception cref="System.IO.IOException"/>
private void TestKilledJob(string fileName, Type committer, string[] exclude)
{
JobConf jc = mr.CreateJobConf();
Path outDir = GetNewOutputDir();
ConfigureJob(jc, "kill job with abort()", 1, 0, outDir);
// set the job to wait for long
jc.SetMapperClass(typeof(UtilsForTests.KillMapper));
jc.SetOutputCommitter(committer);
JobClient jobClient = new JobClient(jc);
RunningJob job = jobClient.SubmitJob(jc);
JobID id = job.GetID();
Counters counters = job.GetCounters();
// wait for the map to be launched
while (true)
{
if (counters.GetCounter(JobCounter.TotalLaunchedMaps) == 1)
{
break;
}
Log.Info("Waiting for a map task to be launched");
UtilsForTests.WaitFor(100);
counters = job.GetCounters();
}
job.KillJob();
// kill the job
job.WaitForCompletion();
// wait for the job to complete
NUnit.Framework.Assert.AreEqual("Job was not killed", JobStatus.Killed, job.GetJobState
());
if (fileName != null)
{
Path testFile = new Path(outDir, fileName);
NUnit.Framework.Assert.IsTrue("File " + testFile + " missing for job " + id, fileSys
.Exists(testFile));
}
// check if the files from the missing set exists
foreach (string ex in exclude)
{
Path file = new Path(outDir, ex);
NUnit.Framework.Assert.IsFalse("File " + file + " should not be present for killed job "
+ id, fileSys.Exists(file));
}
}
// set up heap options, target value for memory loader and the output
// directory before running the job
/// <exception cref="System.IO.IOException"/>
private static RunningJob RunHeapUsageTestJob(JobConf conf, Path testRootDir, string
heapOptions, long targetMapValue, long targetReduceValue, FileSystem fs, JobClient
client, Path inDir)
{
// define a job
JobConf jobConf = new JobConf(conf);
// configure the jobs
jobConf.SetNumMapTasks(1);
jobConf.SetNumReduceTasks(1);
jobConf.SetMapperClass(typeof(TestJobCounters.MemoryLoaderMapper));
jobConf.SetReducerClass(typeof(TestJobCounters.MemoryLoaderReducer));
jobConf.SetInputFormat(typeof(TextInputFormat));
jobConf.SetOutputKeyClass(typeof(LongWritable));
jobConf.SetOutputValueClass(typeof(Org.Apache.Hadoop.IO.Text));
jobConf.SetMaxMapAttempts(1);
jobConf.SetMaxReduceAttempts(1);
jobConf.Set(JobConf.MapredTaskJavaOpts, heapOptions);
// set the targets
jobConf.SetLong(TestJobCounters.MemoryLoaderMapper.TargetValue, targetMapValue);
jobConf.SetLong(TestJobCounters.MemoryLoaderReducer.TargetValue, targetReduceValue
);
// set the input directory for the job
FileInputFormat.SetInputPaths(jobConf, inDir);
// define job output folder
Path outDir = new Path(testRootDir, "out");
fs.Delete(outDir, true);
FileOutputFormat.SetOutputPath(jobConf, outDir);
// run the job
RunningJob job = client.SubmitJob(jobConf);
job.WaitForCompletion();
JobID jobID = job.GetID();
NUnit.Framework.Assert.IsTrue("Job " + jobID + " failed!", job.IsSuccessful());
return(job);
}
/// <exception cref="System.Exception"/>
private void EncryptedShuffleWithCerts(bool useClientCerts)
{
try
{
Configuration conf = new Configuration();
string keystoresDir = new FilePath(Basedir).GetAbsolutePath();
string sslConfsDir = KeyStoreTestUtil.GetClasspathDir(typeof(TestEncryptedShuffle
));
KeyStoreTestUtil.SetupSSLConfig(keystoresDir, sslConfsDir, conf, useClientCerts);
conf.SetBoolean(MRConfig.ShuffleSslEnabledKey, true);
StartCluster(conf);
FileSystem fs = FileSystem.Get(GetJobConf());
Path inputDir = new Path("input");
fs.Mkdirs(inputDir);
TextWriter writer = new OutputStreamWriter(fs.Create(new Path(inputDir, "data.txt"
)));
writer.Write("hello");
writer.Close();
Path outputDir = new Path("output", "output");
JobConf jobConf = new JobConf(GetJobConf());
jobConf.SetInt("mapred.map.tasks", 1);
jobConf.SetInt("mapred.map.max.attempts", 1);
jobConf.SetInt("mapred.reduce.max.attempts", 1);
jobConf.Set("mapred.input.dir", inputDir.ToString());
jobConf.Set("mapred.output.dir", outputDir.ToString());
JobClient jobClient = new JobClient(jobConf);
RunningJob runJob = jobClient.SubmitJob(jobConf);
runJob.WaitForCompletion();
NUnit.Framework.Assert.IsTrue(runJob.IsComplete());
NUnit.Framework.Assert.IsTrue(runJob.IsSuccessful());
}
finally
{
StopCluster();
}
}
public virtual void TestCombiner()
{
if (!new FilePath(TestRootDir).Mkdirs())
{
throw new RuntimeException("Could not create test dir: " + TestRootDir);
}
FilePath @in = new FilePath(TestRootDir, "input");
if ([email protected]())
{
throw new RuntimeException("Could not create test dir: " + @in);
}
FilePath @out = new FilePath(TestRootDir, "output");
PrintWriter pw = new PrintWriter(new FileWriter(new FilePath(@in, "data.txt")));
pw.WriteLine("A|a,1");
pw.WriteLine("A|b,2");
pw.WriteLine("B|a,3");
pw.WriteLine("B|b,4");
pw.WriteLine("B|c,5");
pw.Close();
JobConf job = new JobConf();
job.Set("mapreduce.framework.name", "local");
TextInputFormat.SetInputPaths(job, new Path(@in.GetPath()));
TextOutputFormat.SetOutputPath(job, new Path(@out.GetPath()));
job.SetMapperClass(typeof(TestOldCombinerGrouping.Map));
job.SetReducerClass(typeof(TestOldCombinerGrouping.Reduce));
job.SetInputFormat(typeof(TextInputFormat));
job.SetMapOutputKeyClass(typeof(Text));
job.SetMapOutputValueClass(typeof(LongWritable));
job.SetOutputFormat(typeof(TextOutputFormat));
job.SetOutputValueGroupingComparator(typeof(TestOldCombinerGrouping.GroupComparator
));
job.SetCombinerClass(typeof(TestOldCombinerGrouping.Combiner));
job.SetCombinerKeyGroupingComparator(typeof(TestOldCombinerGrouping.GroupComparator
));
job.SetInt("min.num.spills.for.combine", 0);
JobClient client = new JobClient(job);
RunningJob runningJob = client.SubmitJob(job);
runningJob.WaitForCompletion();
if (runningJob.IsSuccessful())
{
Counters counters = runningJob.GetCounters();
long combinerInputRecords = counters.GetGroup("org.apache.hadoop.mapreduce.TaskCounter"
).GetCounter("COMBINE_INPUT_RECORDS");
long combinerOutputRecords = counters.GetGroup("org.apache.hadoop.mapreduce.TaskCounter"
).GetCounter("COMBINE_OUTPUT_RECORDS");
NUnit.Framework.Assert.IsTrue(combinerInputRecords > 0);
NUnit.Framework.Assert.IsTrue(combinerInputRecords > combinerOutputRecords);
BufferedReader br = new BufferedReader(new FileReader(new FilePath(@out, "part-00000"
)));
ICollection <string> output = new HashSet <string>();
string line = br.ReadLine();
NUnit.Framework.Assert.IsNotNull(line);
output.AddItem(Sharpen.Runtime.Substring(line, 0, 1) + Sharpen.Runtime.Substring(
line, 4, 5));
line = br.ReadLine();
NUnit.Framework.Assert.IsNotNull(line);
output.AddItem(Sharpen.Runtime.Substring(line, 0, 1) + Sharpen.Runtime.Substring(
line, 4, 5));
line = br.ReadLine();
NUnit.Framework.Assert.IsNull(line);
br.Close();
ICollection <string> expected = new HashSet <string>();
expected.AddItem("A2");
expected.AddItem("B5");
NUnit.Framework.Assert.AreEqual(expected, output);
}
else
{
NUnit.Framework.Assert.Fail("Job failed");
}
}