/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
/// <exception cref="System.TypeLoadException"/>
private Job SubmitAndValidateJob(Configuration conf, int numMaps, int numReds)
{
Job job = MapReduceTestUtil.CreateJob(conf, inDir, outDir, numMaps, numReds);
job.SetJobSetupCleanupNeeded(false);
job.SetOutputFormatClass(typeof(TestNoJobSetupCleanup.MyOutputFormat));
job.WaitForCompletion(true);
NUnit.Framework.Assert.IsTrue(job.IsSuccessful());
NUnit.Framework.Assert.IsTrue(job.GetTaskReports(TaskType.JobSetup).Length == 0);
NUnit.Framework.Assert.IsTrue(job.GetTaskReports(TaskType.JobCleanup).Length == 0
);
NUnit.Framework.Assert.IsTrue(job.GetTaskReports(TaskType.Map).Length == numMaps);
NUnit.Framework.Assert.IsTrue(job.GetTaskReports(TaskType.Reduce).Length == numReds
);
FileSystem fs = FileSystem.Get(conf);
NUnit.Framework.Assert.IsTrue("Job output directory doesn't exit!", fs.Exists(outDir
));
// job commit done only in cleanup
// therefore output should still be in temp location
string tempWorkingPathStr = outDir + Path.Separator + "_temporary" + Path.Separator
+ "0";
Path tempWorkingPath = new Path(tempWorkingPathStr);
FileStatus[] list = fs.ListStatus(tempWorkingPath, new TestNoJobSetupCleanup.OutputFilter
());
int numPartFiles = numReds == 0 ? numMaps : numReds;
NUnit.Framework.Assert.IsTrue("Number of part-files is " + list.Length + " and not "
+ numPartFiles, list.Length == numPartFiles);
return(job);
}
/// <summary>test submit task from file</summary>
/// <exception cref="System.Exception"/>
private void TestSubmit(Configuration conf)
{
CLI jc = CreateJobClient();
Job job = MapReduceTestUtil.CreateJob(conf, GetInputDir(), GetOutputDir(), 1, 1,
"ping");
job.SetJobName("mr");
job.SetPriority(JobPriority.Normal);
FilePath fcon = FilePath.CreateTempFile("config", ".xml");
FileSystem localFs = FileSystem.GetLocal(conf);
string fconUri = new Path(fcon.GetAbsolutePath()).MakeQualified(localFs.GetUri(),
localFs.GetWorkingDirectory()).ToUri().ToString();
job.GetConfiguration().WriteXml(new FileOutputStream(fcon));
ByteArrayOutputStream @out = new ByteArrayOutputStream();
// bad parameters
int exitCode = RunTool(conf, jc, new string[] { "-submit" }, @out);
NUnit.Framework.Assert.AreEqual("Exit code", -1, exitCode);
exitCode = RunTool(conf, jc, new string[] { "-submit", fconUri }, @out);
NUnit.Framework.Assert.AreEqual("Exit code", 0, exitCode);
string answer = Sharpen.Runtime.GetStringForBytes(@out.ToByteArray());
// in console was written
NUnit.Framework.Assert.IsTrue(answer.Contains("Created job "));
}
public virtual void TestContextStatus()
{
Path test = new Path(testRootTempDir, "testContextStatus");
// test with 1 map and 0 reducers
// test with custom task status
int numMaps = 1;
Job job = MapReduceTestUtil.CreateJob(CreateJobConf(), new Path(test, "in"), new
Path(test, "out"), numMaps, 0);
job.SetMapperClass(typeof(TestTaskContext.MyMapper));
job.WaitForCompletion(true);
NUnit.Framework.Assert.IsTrue("Job failed", job.IsSuccessful());
TaskReport[] reports = job.GetTaskReports(TaskType.Map);
NUnit.Framework.Assert.AreEqual(numMaps, reports.Length);
NUnit.Framework.Assert.AreEqual(myStatus, reports[0].GetState());
// test with 1 map and 1 reducer
// test with default task status
int numReduces = 1;
job = MapReduceTestUtil.CreateJob(CreateJobConf(), new Path(test, "in"), new Path
(test, "out"), numMaps, numReduces);
job.SetMapperClass(typeof(MapReduceTestUtil.DataCopyMapper));
job.SetReducerClass(typeof(MapReduceTestUtil.DataCopyReducer));
job.SetMapOutputKeyClass(typeof(Text));
job.SetMapOutputValueClass(typeof(Text));
job.SetOutputKeyClass(typeof(Text));
job.SetOutputValueClass(typeof(Text));
// fail early
job.SetMaxMapAttempts(1);
job.SetMaxReduceAttempts(0);
// run the job and wait for completion
job.WaitForCompletion(true);
NUnit.Framework.Assert.IsTrue("Job failed", job.IsSuccessful());
}
/// <exception cref="System.Exception"/>
private Job RunJob(Configuration conf)
{
string input = "hello1\nhello2\nhello3\n";
Job job = MapReduceTestUtil.CreateJob(conf, GetInputDir(), GetOutputDir(), 1, 1,
input);
job.SetJobName("mr");
job.SetPriority(JobPriority.Normal);
job.WaitForCompletion(true);
return(job);
}
/// <summary>Tests new MapReduce map task's context.getProgress() method.</summary>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
/// <exception cref="System.TypeLoadException"/>
public virtual void TestMapContextProgress()
{
int numMaps = 1;
Path test = new Path(testRootTempDir, "testMapContextProgress");
Job job = MapReduceTestUtil.CreateJob(CreateJobConf(), new Path(test, "in"), new
Path(test, "out"), numMaps, 0, Input);
job.SetMapperClass(typeof(TestTaskContext.ProgressCheckerMapper));
job.SetMapOutputKeyClass(typeof(Text));
// fail early
job.SetMaxMapAttempts(1);
job.WaitForCompletion(true);
NUnit.Framework.Assert.IsTrue("Job failed", job.IsSuccessful());
}
/// <exception cref="System.Exception"/>
private Job RunJobInBackGround(Configuration conf)
{
string input = "hello1\nhello2\nhello3\n";
Job job = MapReduceTestUtil.CreateJob(conf, GetInputDir(), GetOutputDir(), 1, 1,
input);
job.SetJobName("mr");
job.SetPriority(JobPriority.Normal);
job.Submit();
int i = 0;
while (i++ < 200 && job.GetJobID() == null)
{
Log.Info("waiting for jobId...");
Sharpen.Thread.Sleep(100);
}
return(job);
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
/// <exception cref="System.TypeLoadException"/>
private Job SubmitAndValidateJob(JobConf conf, int numMaps, int numReds, bool oldConfigs
)
{
conf.SetBoolean(OldConfigs, oldConfigs);
if (oldConfigs)
{
conf.Set(JobConf.MapredTaskJavaOpts, TaskOptsVal);
}
else
{
conf.Set(JobConf.MapredMapTaskJavaOpts, MapOptsVal);
conf.Set(JobConf.MapredReduceTaskJavaOpts, ReduceOptsVal);
}
conf.Set(JobConf.MapredMapTaskLogLevel, Level.Off.ToString());
conf.Set(JobConf.MapredReduceTaskLogLevel, Level.Off.ToString());
Job job = MapReduceTestUtil.CreateJob(conf, inDir, outDir, numMaps, numReds);
job.SetMapperClass(typeof(TestChild.MyMapper));
job.SetReducerClass(typeof(TestChild.MyReducer));
NUnit.Framework.Assert.IsFalse("Job already has a job tracker connection, before it's submitted"
, job.IsConnected());
job.Submit();
NUnit.Framework.Assert.IsTrue("Job doesn't have a job tracker connection, even though it's been submitted"
, job.IsConnected());
job.WaitForCompletion(true);
NUnit.Framework.Assert.IsTrue(job.IsSuccessful());
// Check output directory
FileSystem fs = FileSystem.Get(conf);
NUnit.Framework.Assert.IsTrue("Job output directory doesn't exit!", fs.Exists(outDir
));
FileStatus[] list = fs.ListStatus(outDir, new TestChild.OutputFilter());
int numPartFiles = numReds == 0 ? numMaps : numReds;
NUnit.Framework.Assert.IsTrue("Number of part-files is " + list.Length + " and not "
+ numPartFiles, list.Length == numPartFiles);
return(job);
}
/// <exception cref="System.Exception"/>
public virtual void TestJobSubmissionSpecsAndFiles()
{
Configuration conf = CreateJobConf();
Job job = MapReduceTestUtil.CreateJob(conf, GetInputDir(), GetOutputDir(), 1, 1);
job.SetOutputFormatClass(typeof(TestMRJobClient.BadOutputFormat));
try
{
job.Submit();
Fail("Should've thrown an exception while checking output specs.");
}
catch (Exception e)
{
NUnit.Framework.Assert.IsTrue(e is IOException);
}
Cluster cluster = new Cluster(conf);
Path jobStagingArea = JobSubmissionFiles.GetStagingDir(cluster, job.GetConfiguration
());
Path submitJobDir = new Path(jobStagingArea, "JobId");
Path submitJobFile = JobSubmissionFiles.GetJobConfPath(submitJobDir);
NUnit.Framework.Assert.IsFalse("Shouldn't have created a job file if job specs failed."
, FileSystem.Get(conf).Exists(submitJobFile));
}