/// <exception cref="System.IO.IOException"/>
public static void VerifyCounters(RunningJob runningJob, int expected)
{
NUnit.Framework.Assert.AreEqual(expected, runningJob.GetCounters().GetCounter(TestUserDefinedCounters.EnumCounter
.MapRecords));
NUnit.Framework.Assert.AreEqual(expected, runningJob.GetCounters().GetGroup("StringCounter"
).GetCounter("MapRecords"));
}
public virtual void TestOldCounterC()
{
JobConf conf = CreateConfiguration();
CreateWordsFile(inFiles[3], conf);
CreateWordsFile(inFiles[4], conf);
long inputSize = 0;
inputSize += GetFileSize(inFiles[0]);
inputSize += GetFileSize(inFiles[1]);
inputSize += GetFileSize(inFiles[2]);
inputSize += GetFileSize(inFiles[3]);
inputSize += GetFileSize(inFiles[4]);
conf.SetNumMapTasks(4);
conf.SetInt(JobContext.IoSortFactor, 3);
FileInputFormat.SetInputPaths(conf, InDir);
FileOutputFormat.SetOutputPath(conf, new Path(OutDir, "outputO2"));
RunningJob myJob = JobClient.RunJob(conf);
Counters c1 = myJob.GetCounters();
// As above, each map spills 2^14 records, so 5 maps spill 81920
// 1st merge: read + write = 6 * 8192
// final merge: unmerged = 2 * 8192
// Total reduce: 45056
// 5 files, 5120 = 5 * 1024 rec/file = 15360 input records
// 4 records/line = 102400 output records
ValidateCounters(c1, 122880, 25600, 102400);
ValidateFileCounters(c1, inputSize, 0, 0, 0);
}
public virtual void TestOldCounterB()
{
JobConf conf = CreateConfiguration();
CreateWordsFile(inFiles[3], conf);
RemoveWordsFile(inFiles[4], conf);
long inputSize = 0;
inputSize += GetFileSize(inFiles[0]);
inputSize += GetFileSize(inFiles[1]);
inputSize += GetFileSize(inFiles[2]);
inputSize += GetFileSize(inFiles[3]);
conf.SetNumMapTasks(4);
conf.SetInt(JobContext.IoSortFactor, 2);
FileInputFormat.SetInputPaths(conf, InDir);
FileOutputFormat.SetOutputPath(conf, new Path(OutDir, "outputO1"));
RunningJob myJob = JobClient.RunJob(conf);
Counters c1 = myJob.GetCounters();
// As above, each map spills 2^14 records, so 4 maps spill 2^16 records
// In the reduce, there are two intermediate merges before the reduce.
// 1st merge: read + write = 8192 * 4
// 2nd merge: read + write = 8192 * 4
// final merge: 0
// Total reduce: 32768
// Total: map + reduce = 2^16 + 2^15 = 98304
// 4 files, 5120 = 5 * 1024 rec/file = 15360 input records
// 4 records/line = 81920 output records
ValidateCounters(c1, 98304, 20480, 81920);
ValidateFileCounters(c1, inputSize, 0, 0, 0);
}
/// <exception cref="System.Exception"/>
public static Counters RunJob(JobConf conf)
{
conf.SetMapperClass(typeof(TestReduceFetchFromPartialMem.MapMB));
conf.SetReducerClass(typeof(TestReduceFetchFromPartialMem.MBValidate));
conf.SetOutputKeyClass(typeof(Org.Apache.Hadoop.IO.Text));
conf.SetOutputValueClass(typeof(Org.Apache.Hadoop.IO.Text));
conf.SetNumReduceTasks(1);
conf.SetInputFormat(typeof(TestReduceFetchFromPartialMem.FakeIF));
conf.SetNumTasksToExecutePerJvm(1);
conf.SetInt(JobContext.MapMaxAttempts, 0);
conf.SetInt(JobContext.ReduceMaxAttempts, 0);
FileInputFormat.SetInputPaths(conf, new Path("/in"));
Path outp = new Path("/out");
FileOutputFormat.SetOutputPath(conf, outp);
RunningJob job = null;
try
{
job = JobClient.RunJob(conf);
NUnit.Framework.Assert.IsTrue(job.IsSuccessful());
}
finally
{
FileSystem fs = dfsCluster.GetFileSystem();
if (fs.Exists(outp))
{
fs.Delete(outp, true);
}
}
return(job.GetCounters());
}
// run a job which gets stuck in mapper and kill it.
/// <exception cref="System.IO.IOException"/>
private void TestKilledJob(string fileName, Type committer, string[] exclude)
{
JobConf jc = mr.CreateJobConf();
Path outDir = GetNewOutputDir();
ConfigureJob(jc, "kill job with abort()", 1, 0, outDir);
// set the job to wait for long
jc.SetMapperClass(typeof(UtilsForTests.KillMapper));
jc.SetOutputCommitter(committer);
JobClient jobClient = new JobClient(jc);
RunningJob job = jobClient.SubmitJob(jc);
JobID id = job.GetID();
Counters counters = job.GetCounters();
// wait for the map to be launched
while (true)
{
if (counters.GetCounter(JobCounter.TotalLaunchedMaps) == 1)
{
break;
}
Log.Info("Waiting for a map task to be launched");
UtilsForTests.WaitFor(100);
counters = job.GetCounters();
}
job.KillJob();
// kill the job
job.WaitForCompletion();
// wait for the job to complete
NUnit.Framework.Assert.AreEqual("Job was not killed", JobStatus.Killed, job.GetJobState
());
if (fileName != null)
{
Path testFile = new Path(outDir, fileName);
NUnit.Framework.Assert.IsTrue("File " + testFile + " missing for job " + id, fileSys
.Exists(testFile));
}
// check if the files from the missing set exists
foreach (string ex in exclude)
{
Path file = new Path(outDir, ex);
NUnit.Framework.Assert.IsFalse("File " + file + " should not be present for killed job "
+ id, fileSys.Exists(file));
}
}
public virtual void TestOldCounterA()
{
JobConf conf = CreateConfiguration();
conf.SetNumMapTasks(3);
conf.SetInt(JobContext.IoSortFactor, 2);
RemoveWordsFile(inFiles[3], conf);
RemoveWordsFile(inFiles[4], conf);
long inputSize = 0;
inputSize += GetFileSize(inFiles[0]);
inputSize += GetFileSize(inFiles[1]);
inputSize += GetFileSize(inFiles[2]);
FileInputFormat.SetInputPaths(conf, InDir);
FileOutputFormat.SetOutputPath(conf, new Path(OutDir, "outputO0"));
RunningJob myJob = JobClient.RunJob(conf);
Counters c1 = myJob.GetCounters();
// Each record requires 16 bytes of metadata, 16 bytes per serialized rec
// (vint word len + word + IntWritable) = (1 + 11 + 4)
// (2^20 buf * .5 spill pcnt) / 32 bytes/record = 2^14 recs per spill
// Each file contains 5 replicas of 4096 words, so the first spill will
// contain 4 (2^14 rec / 2^12 rec/replica) replicas, the second just one.
// Each map spills twice, emitting 4096 records per spill from the
// combiner per spill. The merge adds an additional 8192 records, as
// there are too few spills to combine (2 < 3)
// Each map spills 2^14 records, so maps spill 49152 records, combined.
// The combiner has emitted 24576 records to the reducer; these are all
// fetched straight to memory from the map side. The intermediate merge
// adds 8192 records per segment read; again, there are too few spills to
// combine, so all Total spilled records in the reduce
// is 8192 records / map * 3 maps = 24576.
// Total: map + reduce = 49152 + 24576 = 73728
// 3 files, 5120 = 5 * 1024 rec/file = 15360 input records
// 4 records/line = 61440 output records
ValidateCounters(c1, 73728, 15360, 61440);
ValidateFileCounters(c1, inputSize, 0, 0, 0);
ValidateOldFileCounters(c1, inputSize, 61928, 0, 0);
}
/// <summary>Launches a MR job and tests the job counters against the expected values.
/// </summary>
/// <param name="testName">The name for the job</param>
/// <param name="mr">The MR cluster</param>
/// <param name="fileSys">The FileSystem</param>
/// <param name="in">Input path</param>
/// <param name="out">Output path</param>
/// <param name="numMaps">Number of maps</param>
/// <param name="otherLocalMaps">Expected value of other local maps</param>
/// <param name="datalocalMaps">Expected value of data(node) local maps</param>
/// <param name="racklocalMaps">Expected value of rack local maps</param>
/// <exception cref="System.IO.IOException"/>
internal static void LaunchJobAndTestCounters(string jobName, MiniMRCluster mr, FileSystem
fileSys, Path @in, Path @out, int numMaps, int otherLocalMaps, int dataLocalMaps
, int rackLocalMaps)
{
JobConf jobConf = mr.CreateJobConf();
if (fileSys.Exists(@out))
{
fileSys.Delete(@out, true);
}
RunningJob job = LaunchJob(jobConf, @in, @out, numMaps, jobName);
Counters counters = job.GetCounters();
NUnit.Framework.Assert.AreEqual("Number of local maps", counters.GetCounter(JobCounter
.OtherLocalMaps), otherLocalMaps);
NUnit.Framework.Assert.AreEqual("Number of Data-local maps", counters.GetCounter(
JobCounter.DataLocalMaps), dataLocalMaps);
NUnit.Framework.Assert.AreEqual("Number of Rack-local maps", counters.GetCounter(
JobCounter.RackLocalMaps), rackLocalMaps);
mr.WaitUntilIdle();
mr.Shutdown();
}
public virtual void TestOldCounterD()
{
JobConf conf = CreateConfiguration();
conf.SetNumMapTasks(3);
conf.SetInt(JobContext.IoSortFactor, 2);
conf.SetNumReduceTasks(0);
RemoveWordsFile(inFiles[3], conf);
RemoveWordsFile(inFiles[4], conf);
long inputSize = 0;
inputSize += GetFileSize(inFiles[0]);
inputSize += GetFileSize(inFiles[1]);
inputSize += GetFileSize(inFiles[2]);
FileInputFormat.SetInputPaths(conf, InDir);
FileOutputFormat.SetOutputPath(conf, new Path(OutDir, "outputO3"));
RunningJob myJob = JobClient.RunJob(conf);
Counters c1 = myJob.GetCounters();
// No Reduces. Will go through the direct output collector. Spills=0
ValidateCounters(c1, 0, 15360, 61440);
ValidateFileCounters(c1, inputSize, 0, -1, -1);
}
/// <exception cref="System.IO.IOException"/>
internal static void RunProgram(MiniMRCluster mr, MiniDFSCluster dfs, Path program
, Path inputPath, Path outputPath, int numMaps, int numReduces, string[] expectedResults
, JobConf conf)
{
Path wordExec = new Path("testing/bin/application");
JobConf job = null;
if (conf == null)
{
job = mr.CreateJobConf();
}
else
{
job = new JobConf(conf);
}
job.SetNumMapTasks(numMaps);
job.SetNumReduceTasks(numReduces);
{
FileSystem fs = dfs.GetFileSystem();
fs.Delete(wordExec.GetParent(), true);
fs.CopyFromLocalFile(program, wordExec);
Submitter.SetExecutable(job, fs.MakeQualified(wordExec).ToString());
Submitter.SetIsJavaRecordReader(job, true);
Submitter.SetIsJavaRecordWriter(job, true);
FileInputFormat.SetInputPaths(job, inputPath);
FileOutputFormat.SetOutputPath(job, outputPath);
RunningJob rJob = null;
if (numReduces == 0)
{
rJob = Submitter.JobSubmit(job);
while (!rJob.IsComplete())
{
try
{
Sharpen.Thread.Sleep(1000);
}
catch (Exception ie)
{
throw new RuntimeException(ie);
}
}
}
else
{
rJob = Submitter.RunJob(job);
}
NUnit.Framework.Assert.IsTrue("pipes job failed", rJob.IsSuccessful());
Counters counters = rJob.GetCounters();
Counters.Group wordCountCounters = counters.GetGroup("WORDCOUNT");
int numCounters = 0;
foreach (Counters.Counter c in wordCountCounters)
{
System.Console.Out.WriteLine(c);
++numCounters;
}
NUnit.Framework.Assert.IsTrue("No counters found!", (numCounters > 0));
}
IList <string> results = new AList <string>();
foreach (Path p in FileUtil.Stat2Paths(dfs.GetFileSystem().ListStatus(outputPath,
new Utils.OutputFileUtils.OutputFilesFilter())))
{
results.AddItem(MapReduceTestUtil.ReadOutput(p, job));
}
NUnit.Framework.Assert.AreEqual("number of reduces is wrong", expectedResults.Length
, results.Count);
for (int i = 0; i < results.Count; i++)
{
NUnit.Framework.Assert.AreEqual("pipes program " + program + " output " + i + " wrong"
, expectedResults[i], results[i]);
}
}
/// <exception cref="System.Exception"/>
private void ValidateOutput(JobConf conf, RunningJob runningJob, IList <string> mapperBadRecords
, IList <string> redBadRecords)
{
Log.Info(runningJob.GetCounters().ToString());
NUnit.Framework.Assert.IsTrue(runningJob.IsSuccessful());
//validate counters
Counters counters = runningJob.GetCounters();
NUnit.Framework.Assert.AreEqual(counters.FindCounter(TaskCounter.MapSkippedRecords
).GetCounter(), mapperBadRecords.Count);
int mapRecs = input.Count - mapperBadRecords.Count;
NUnit.Framework.Assert.AreEqual(counters.FindCounter(TaskCounter.MapInputRecords)
.GetCounter(), mapRecs);
NUnit.Framework.Assert.AreEqual(counters.FindCounter(TaskCounter.MapOutputRecords
).GetCounter(), mapRecs);
int redRecs = mapRecs - redBadRecords.Count;
NUnit.Framework.Assert.AreEqual(counters.FindCounter(TaskCounter.ReduceSkippedRecords
).GetCounter(), redBadRecords.Count);
NUnit.Framework.Assert.AreEqual(counters.FindCounter(TaskCounter.ReduceSkippedGroups
).GetCounter(), redBadRecords.Count);
NUnit.Framework.Assert.AreEqual(counters.FindCounter(TaskCounter.ReduceInputGroups
).GetCounter(), redRecs);
NUnit.Framework.Assert.AreEqual(counters.FindCounter(TaskCounter.ReduceInputRecords
).GetCounter(), redRecs);
NUnit.Framework.Assert.AreEqual(counters.FindCounter(TaskCounter.ReduceOutputRecords
).GetCounter(), redRecs);
//validate skipped records
Path skipDir = SkipBadRecords.GetSkipOutputPath(conf);
NUnit.Framework.Assert.IsNotNull(skipDir);
Path[] skips = FileUtil.Stat2Paths(GetFileSystem().ListStatus(skipDir));
IList <string> mapSkipped = new AList <string>();
IList <string> redSkipped = new AList <string>();
foreach (Path skipPath in skips)
{
Log.Info("skipPath: " + skipPath);
SequenceFile.Reader reader = new SequenceFile.Reader(GetFileSystem(), skipPath, conf
);
object key = ReflectionUtils.NewInstance(reader.GetKeyClass(), conf);
object value = ReflectionUtils.NewInstance(reader.GetValueClass(), conf);
key = reader.Next(key);
while (key != null)
{
value = reader.GetCurrentValue(value);
Log.Debug("key:" + key + " value:" + value.ToString());
if (skipPath.GetName().Contains("_r_"))
{
redSkipped.AddItem(value.ToString());
}
else
{
mapSkipped.AddItem(value.ToString());
}
key = reader.Next(key);
}
reader.Close();
}
NUnit.Framework.Assert.IsTrue(mapSkipped.ContainsAll(mapperBadRecords));
NUnit.Framework.Assert.IsTrue(redSkipped.ContainsAll(redBadRecords));
Path[] outputFiles = FileUtil.Stat2Paths(GetFileSystem().ListStatus(GetOutputDir(
), new Utils.OutputFileUtils.OutputFilesFilter()));
IList <string> mapperOutput = GetProcessed(input, mapperBadRecords);
Log.Debug("mapperOutput " + mapperOutput.Count);
IList <string> reducerOutput = GetProcessed(mapperOutput, redBadRecords);
Log.Debug("reducerOutput " + reducerOutput.Count);
if (outputFiles.Length > 0)
{
InputStream @is = GetFileSystem().Open(outputFiles[0]);
BufferedReader reader = new BufferedReader(new InputStreamReader(@is));
string line = reader.ReadLine();
int counter = 0;
while (line != null)
{
counter++;
StringTokenizer tokeniz = new StringTokenizer(line, "\t");
string key = tokeniz.NextToken();
string value = tokeniz.NextToken();
Log.Debug("Output: key:" + key + " value:" + value);
NUnit.Framework.Assert.IsTrue(value.Contains("hello"));
NUnit.Framework.Assert.IsTrue(reducerOutput.Contains(value));
line = reader.ReadLine();
}
reader.Close();
NUnit.Framework.Assert.AreEqual(reducerOutput.Count, counter);
}
}
public virtual void TestCombiner()
{
if (!new FilePath(TestRootDir).Mkdirs())
{
throw new RuntimeException("Could not create test dir: " + TestRootDir);
}
FilePath @in = new FilePath(TestRootDir, "input");
if ([email protected]())
{
throw new RuntimeException("Could not create test dir: " + @in);
}
FilePath @out = new FilePath(TestRootDir, "output");
PrintWriter pw = new PrintWriter(new FileWriter(new FilePath(@in, "data.txt")));
pw.WriteLine("A|a,1");
pw.WriteLine("A|b,2");
pw.WriteLine("B|a,3");
pw.WriteLine("B|b,4");
pw.WriteLine("B|c,5");
pw.Close();
JobConf job = new JobConf();
job.Set("mapreduce.framework.name", "local");
TextInputFormat.SetInputPaths(job, new Path(@in.GetPath()));
TextOutputFormat.SetOutputPath(job, new Path(@out.GetPath()));
job.SetMapperClass(typeof(TestOldCombinerGrouping.Map));
job.SetReducerClass(typeof(TestOldCombinerGrouping.Reduce));
job.SetInputFormat(typeof(TextInputFormat));
job.SetMapOutputKeyClass(typeof(Text));
job.SetMapOutputValueClass(typeof(LongWritable));
job.SetOutputFormat(typeof(TextOutputFormat));
job.SetOutputValueGroupingComparator(typeof(TestOldCombinerGrouping.GroupComparator
));
job.SetCombinerClass(typeof(TestOldCombinerGrouping.Combiner));
job.SetCombinerKeyGroupingComparator(typeof(TestOldCombinerGrouping.GroupComparator
));
job.SetInt("min.num.spills.for.combine", 0);
JobClient client = new JobClient(job);
RunningJob runningJob = client.SubmitJob(job);
runningJob.WaitForCompletion();
if (runningJob.IsSuccessful())
{
Counters counters = runningJob.GetCounters();
long combinerInputRecords = counters.GetGroup("org.apache.hadoop.mapreduce.TaskCounter"
).GetCounter("COMBINE_INPUT_RECORDS");
long combinerOutputRecords = counters.GetGroup("org.apache.hadoop.mapreduce.TaskCounter"
).GetCounter("COMBINE_OUTPUT_RECORDS");
NUnit.Framework.Assert.IsTrue(combinerInputRecords > 0);
NUnit.Framework.Assert.IsTrue(combinerInputRecords > combinerOutputRecords);
BufferedReader br = new BufferedReader(new FileReader(new FilePath(@out, "part-00000"
)));
ICollection <string> output = new HashSet <string>();
string line = br.ReadLine();
NUnit.Framework.Assert.IsNotNull(line);
output.AddItem(Sharpen.Runtime.Substring(line, 0, 1) + Sharpen.Runtime.Substring(
line, 4, 5));
line = br.ReadLine();
NUnit.Framework.Assert.IsNotNull(line);
output.AddItem(Sharpen.Runtime.Substring(line, 0, 1) + Sharpen.Runtime.Substring(
line, 4, 5));
line = br.ReadLine();
NUnit.Framework.Assert.IsNull(line);
br.Close();
ICollection <string> expected = new HashSet <string>();
expected.AddItem("A2");
expected.AddItem("B5");
NUnit.Framework.Assert.AreEqual(expected, output);
}
else
{
NUnit.Framework.Assert.Fail("Job failed");
}
}
/// <exception cref="System.Exception"/>
protected internal virtual void _testMOWithJavaSerialization(bool withCounters)
{
Path inDir = GetDir(InDir);
Path outDir = GetDir(OutDir);
JobConf conf = CreateJobConf();
FileSystem fs = FileSystem.Get(conf);
DataOutputStream file = fs.Create(new Path(inDir, "part-0"));
file.WriteBytes("a\nb\n\nc\nd\ne");
file.Close();
fs.Delete(inDir, true);
fs.Delete(outDir, true);
file = fs.Create(new Path(inDir, "part-1"));
file.WriteBytes("a\nb\n\nc\nd\ne");
file.Close();
conf.SetJobName("mo");
conf.Set("io.serializations", "org.apache.hadoop.io.serializer.JavaSerialization,"
+ "org.apache.hadoop.io.serializer.WritableSerialization");
conf.SetInputFormat(typeof(TextInputFormat));
conf.SetMapOutputKeyClass(typeof(long));
conf.SetMapOutputValueClass(typeof(string));
conf.SetOutputKeyComparatorClass(typeof(JavaSerializationComparator));
conf.SetOutputKeyClass(typeof(long));
conf.SetOutputValueClass(typeof(string));
conf.SetOutputFormat(typeof(TextOutputFormat));
MultipleOutputs.AddNamedOutput(conf, "text", typeof(TextOutputFormat), typeof(long
), typeof(string));
MultipleOutputs.SetCountersEnabled(conf, withCounters);
conf.SetMapperClass(typeof(TestMultipleOutputs.MOJavaSerDeMap));
conf.SetReducerClass(typeof(TestMultipleOutputs.MOJavaSerDeReduce));
FileInputFormat.SetInputPaths(conf, inDir);
FileOutputFormat.SetOutputPath(conf, outDir);
JobClient jc = new JobClient(conf);
RunningJob job = jc.SubmitJob(conf);
while (!job.IsComplete())
{
Sharpen.Thread.Sleep(100);
}
// assert number of named output part files
int namedOutputCount = 0;
FileStatus[] statuses = fs.ListStatus(outDir);
foreach (FileStatus status in statuses)
{
if (status.GetPath().GetName().Equals("text-m-00000") || status.GetPath().GetName
().Equals("text-r-00000"))
{
namedOutputCount++;
}
}
NUnit.Framework.Assert.AreEqual(2, namedOutputCount);
// assert TextOutputFormat files correctness
BufferedReader reader = new BufferedReader(new InputStreamReader(fs.Open(new Path
(FileOutputFormat.GetOutputPath(conf), "text-r-00000"))));
int count = 0;
string line = reader.ReadLine();
while (line != null)
{
NUnit.Framework.Assert.IsTrue(line.EndsWith("text"));
line = reader.ReadLine();
count++;
}
reader.Close();
NUnit.Framework.Assert.IsFalse(count == 0);
Counters.Group counters = job.GetCounters().GetGroup(typeof(MultipleOutputs).FullName
);
if (!withCounters)
{
NUnit.Framework.Assert.AreEqual(0, counters.Size());
}
else
{
NUnit.Framework.Assert.AreEqual(1, counters.Size());
NUnit.Framework.Assert.AreEqual(2, counters.GetCounter("text"));
}
}
/// <exception cref="System.Exception"/>
protected internal virtual void _testMultipleOutputs(bool withCounters)
{
Path inDir = GetDir(InDir);
Path outDir = GetDir(OutDir);
JobConf conf = CreateJobConf();
FileSystem fs = FileSystem.Get(conf);
DataOutputStream file = fs.Create(new Path(inDir, "part-0"));
file.WriteBytes("a\nb\n\nc\nd\ne");
file.Close();
file = fs.Create(new Path(inDir, "part-1"));
file.WriteBytes("a\nb\n\nc\nd\ne");
file.Close();
conf.SetJobName("mo");
conf.SetInputFormat(typeof(TextInputFormat));
conf.SetOutputKeyClass(typeof(LongWritable));
conf.SetOutputValueClass(typeof(Text));
conf.SetMapOutputKeyClass(typeof(LongWritable));
conf.SetMapOutputValueClass(typeof(Text));
conf.SetOutputFormat(typeof(TextOutputFormat));
MultipleOutputs.AddNamedOutput(conf, "text", typeof(TextOutputFormat), typeof(LongWritable
), typeof(Text));
MultipleOutputs.AddMultiNamedOutput(conf, "sequence", typeof(SequenceFileOutputFormat
), typeof(LongWritable), typeof(Text));
MultipleOutputs.SetCountersEnabled(conf, withCounters);
conf.SetMapperClass(typeof(TestMultipleOutputs.MOMap));
conf.SetReducerClass(typeof(TestMultipleOutputs.MOReduce));
FileInputFormat.SetInputPaths(conf, inDir);
FileOutputFormat.SetOutputPath(conf, outDir);
JobClient jc = new JobClient(conf);
RunningJob job = jc.SubmitJob(conf);
while (!job.IsComplete())
{
Sharpen.Thread.Sleep(100);
}
// assert number of named output part files
int namedOutputCount = 0;
FileStatus[] statuses = fs.ListStatus(outDir);
foreach (FileStatus status in statuses)
{
if (status.GetPath().GetName().Equals("text-m-00000") || status.GetPath().GetName
().Equals("text-m-00001") || status.GetPath().GetName().Equals("text-r-00000") ||
status.GetPath().GetName().Equals("sequence_A-m-00000") || status.GetPath().GetName
().Equals("sequence_A-m-00001") || status.GetPath().GetName().Equals("sequence_B-m-00000"
) || status.GetPath().GetName().Equals("sequence_B-m-00001") || status.GetPath()
.GetName().Equals("sequence_B-r-00000") || status.GetPath().GetName().Equals("sequence_C-r-00000"
))
{
namedOutputCount++;
}
}
NUnit.Framework.Assert.AreEqual(9, namedOutputCount);
// assert TextOutputFormat files correctness
BufferedReader reader = new BufferedReader(new InputStreamReader(fs.Open(new Path
(FileOutputFormat.GetOutputPath(conf), "text-r-00000"))));
int count = 0;
string line = reader.ReadLine();
while (line != null)
{
NUnit.Framework.Assert.IsTrue(line.EndsWith("text"));
line = reader.ReadLine();
count++;
}
reader.Close();
NUnit.Framework.Assert.IsFalse(count == 0);
// assert SequenceOutputFormat files correctness
SequenceFile.Reader seqReader = new SequenceFile.Reader(fs, new Path(FileOutputFormat
.GetOutputPath(conf), "sequence_B-r-00000"), conf);
NUnit.Framework.Assert.AreEqual(typeof(LongWritable), seqReader.GetKeyClass());
NUnit.Framework.Assert.AreEqual(typeof(Text), seqReader.GetValueClass());
count = 0;
LongWritable key = new LongWritable();
Text value = new Text();
while (seqReader.Next(key, value))
{
NUnit.Framework.Assert.AreEqual("sequence", value.ToString());
count++;
}
seqReader.Close();
NUnit.Framework.Assert.IsFalse(count == 0);
Counters.Group counters = job.GetCounters().GetGroup(typeof(MultipleOutputs).FullName
);
if (!withCounters)
{
NUnit.Framework.Assert.AreEqual(0, counters.Size());
}
else
{
NUnit.Framework.Assert.AreEqual(4, counters.Size());
NUnit.Framework.Assert.AreEqual(4, counters.GetCounter("text"));
NUnit.Framework.Assert.AreEqual(2, counters.GetCounter("sequence_A"));
NUnit.Framework.Assert.AreEqual(4, counters.GetCounter("sequence_B"));
NUnit.Framework.Assert.AreEqual(2, counters.GetCounter("sequence_C"));
}
}