/// <summary>Tests Reducer throwing exception.</summary>
/// <exception cref="System.Exception"/>
public virtual void TestReducerFail()
{
Configuration conf = CreateJobConf();
Job job = MapReduceTestUtil.CreateJob(conf, inDir, outDir, 1, 1, input);
job.SetJobName("chain");
ChainMapper.AddMapper(job, typeof(Mapper), typeof(LongWritable), typeof(Text), typeof(
LongWritable), typeof(Text), null);
ChainReducer.SetReducer(job, typeof(TestChainErrors.FailReduce), typeof(LongWritable
), typeof(Text), typeof(LongWritable), typeof(Text), null);
ChainReducer.AddMapper(job, typeof(Mapper), typeof(LongWritable), typeof(Text), typeof(
LongWritable), typeof(Text), null);
job.WaitForCompletion(true);
NUnit.Framework.Assert.IsTrue("Job Not failed", !job.IsSuccessful());
}
/// <summary>Tests one of the maps consuming output.</summary>
/// <exception cref="System.Exception"/>
public virtual void TestChainMapNoOuptut()
{
Configuration conf = CreateJobConf();
string expectedOutput = string.Empty;
Job job = MapReduceTestUtil.CreateJob(conf, inDir, outDir, 1, 0, input);
job.SetJobName("chain");
ChainMapper.AddMapper(job, typeof(TestChainErrors.ConsumeMap), typeof(IntWritable
), typeof(Text), typeof(LongWritable), typeof(Text), null);
ChainMapper.AddMapper(job, typeof(Mapper), typeof(LongWritable), typeof(Text), typeof(
LongWritable), typeof(Text), null);
job.WaitForCompletion(true);
NUnit.Framework.Assert.IsTrue("Job failed", job.IsSuccessful());
NUnit.Framework.Assert.AreEqual("Outputs doesn't match", expectedOutput, MapReduceTestUtil
.ReadOutput(outDir, conf));
}
// test chain mapper and reducer by adding single mapper and reducer to chain
/// <exception cref="System.Exception"/>
public virtual void TestNoChain()
{
Path inDir = new Path(localPathRoot, "testing/chain/input");
Path outDir = new Path(localPathRoot, "testing/chain/output");
string input = "a\nb\na\n";
string expectedOutput = "a\t2\nb\t1\n";
Configuration conf = CreateJobConf();
Job job = MapReduceTestUtil.CreateJob(conf, inDir, outDir, 1, 1, input);
job.SetJobName("chain");
ChainMapper.AddMapper(job, typeof(TokenCounterMapper), typeof(object), typeof(Text
), typeof(Text), typeof(IntWritable), null);
ChainReducer.SetReducer(job, typeof(IntSumReducer), typeof(Text), typeof(IntWritable
), typeof(Text), typeof(IntWritable), null);
job.WaitForCompletion(true);
NUnit.Framework.Assert.IsTrue("Job failed", job.IsSuccessful());
NUnit.Framework.Assert.AreEqual("Outputs doesn't match", expectedOutput, MapReduceTestUtil
.ReadOutput(outDir, conf));
}
/// <summary>Tests errors during submission.</summary>
/// <exception cref="System.Exception"/>
public virtual void TestChainSubmission()
{
Configuration conf = CreateJobConf();
Job job = MapReduceTestUtil.CreateJob(conf, inDir, outDir, 0, 0, input);
job.SetJobName("chain");
Exception th = null;
// output key,value classes of first map are not same as that of second map
try
{
ChainMapper.AddMapper(job, typeof(Mapper), typeof(LongWritable), typeof(Text), typeof(
IntWritable), typeof(Text), null);
ChainMapper.AddMapper(job, typeof(Mapper), typeof(LongWritable), typeof(Text), typeof(
LongWritable), typeof(Text), null);
}
catch (ArgumentException iae)
{
th = iae;
}
NUnit.Framework.Assert.IsTrue(th != null);
th = null;
// output key,value classes of reducer are not
// same as that of mapper in the chain
try
{
ChainReducer.SetReducer(job, typeof(Reducer), typeof(LongWritable), typeof(Text),
typeof(IntWritable), typeof(Text), null);
ChainMapper.AddMapper(job, typeof(Mapper), typeof(LongWritable), typeof(Text), typeof(
LongWritable), typeof(Text), null);
}
catch (ArgumentException iae)
{
th = iae;
}
NUnit.Framework.Assert.IsTrue(th != null);
}
/// <exception cref="System.Exception"/>
public virtual void TestChain()
{
Path inDir = new Path(localPathRoot, "testing/chain/input");
Path outDir = new Path(localPathRoot, "testing/chain/output");
string input = "1\n2\n";
string expectedOutput = "0\t1ABCRDEF\n2\t2ABCRDEF\n";
Configuration conf = CreateJobConf();
CleanFlags(conf);
conf.Set("a", "X");
Job job = MapReduceTestUtil.CreateJob(conf, inDir, outDir, 1, 1, input);
job.SetJobName("chain");
Configuration mapAConf = new Configuration(false);
mapAConf.Set("a", "A");
ChainMapper.AddMapper(job, typeof(TestMapReduceChain.AMap), typeof(LongWritable),
typeof(Text), typeof(LongWritable), typeof(Text), mapAConf);
ChainMapper.AddMapper(job, typeof(TestMapReduceChain.BMap), typeof(LongWritable),
typeof(Text), typeof(LongWritable), typeof(Text), null);
ChainMapper.AddMapper(job, typeof(TestMapReduceChain.CMap), typeof(LongWritable),
typeof(Text), typeof(LongWritable), typeof(Text), null);
Configuration reduceConf = new Configuration(false);
reduceConf.Set("a", "C");
ChainReducer.SetReducer(job, typeof(TestMapReduceChain.RReduce), typeof(LongWritable
), typeof(Text), typeof(LongWritable), typeof(Text), reduceConf);
ChainReducer.AddMapper(job, typeof(TestMapReduceChain.DMap), typeof(LongWritable)
, typeof(Text), typeof(LongWritable), typeof(Text), null);
Configuration mapEConf = new Configuration(false);
mapEConf.Set("a", "E");
ChainReducer.AddMapper(job, typeof(TestMapReduceChain.EMap), typeof(LongWritable)
, typeof(Text), typeof(LongWritable), typeof(Text), mapEConf);
ChainReducer.AddMapper(job, typeof(TestMapReduceChain.FMap), typeof(LongWritable)
, typeof(Text), typeof(LongWritable), typeof(Text), null);
job.WaitForCompletion(true);
NUnit.Framework.Assert.IsTrue("Job failed", job.IsSuccessful());
string str = "flag not set";
NUnit.Framework.Assert.IsTrue(str, GetFlag(conf, "map.setup.A"));
NUnit.Framework.Assert.IsTrue(str, GetFlag(conf, "map.setup.B"));
NUnit.Framework.Assert.IsTrue(str, GetFlag(conf, "map.setup.C"));
NUnit.Framework.Assert.IsTrue(str, GetFlag(conf, "reduce.setup.R"));
NUnit.Framework.Assert.IsTrue(str, GetFlag(conf, "map.setup.D"));
NUnit.Framework.Assert.IsTrue(str, GetFlag(conf, "map.setup.E"));
NUnit.Framework.Assert.IsTrue(str, GetFlag(conf, "map.setup.F"));
NUnit.Framework.Assert.IsTrue(str, GetFlag(conf, "map.A.value.1"));
NUnit.Framework.Assert.IsTrue(str, GetFlag(conf, "map.A.value.2"));
NUnit.Framework.Assert.IsTrue(str, GetFlag(conf, "map.B.value.1A"));
NUnit.Framework.Assert.IsTrue(str, GetFlag(conf, "map.B.value.2A"));
NUnit.Framework.Assert.IsTrue(str, GetFlag(conf, "map.C.value.1AB"));
NUnit.Framework.Assert.IsTrue(str, GetFlag(conf, "map.C.value.2AB"));
NUnit.Framework.Assert.IsTrue(str, GetFlag(conf, "reduce.R.value.1ABC"));
NUnit.Framework.Assert.IsTrue(str, GetFlag(conf, "reduce.R.value.2ABC"));
NUnit.Framework.Assert.IsTrue(str, GetFlag(conf, "map.D.value.1ABCR"));
NUnit.Framework.Assert.IsTrue(str, GetFlag(conf, "map.D.value.2ABCR"));
NUnit.Framework.Assert.IsTrue(str, GetFlag(conf, "map.E.value.1ABCRD"));
NUnit.Framework.Assert.IsTrue(str, GetFlag(conf, "map.E.value.2ABCRD"));
NUnit.Framework.Assert.IsTrue(str, GetFlag(conf, "map.F.value.1ABCRDE"));
NUnit.Framework.Assert.IsTrue(str, GetFlag(conf, "map.F.value.2ABCRDE"));
NUnit.Framework.Assert.IsTrue(GetFlag(conf, "map.cleanup.A"));
NUnit.Framework.Assert.IsTrue(GetFlag(conf, "map.cleanup.B"));
NUnit.Framework.Assert.IsTrue(GetFlag(conf, "map.cleanup.C"));
NUnit.Framework.Assert.IsTrue(GetFlag(conf, "reduce.cleanup.R"));
NUnit.Framework.Assert.IsTrue(GetFlag(conf, "map.cleanup.D"));
NUnit.Framework.Assert.IsTrue(GetFlag(conf, "map.cleanup.E"));
NUnit.Framework.Assert.IsTrue(GetFlag(conf, "map.cleanup.F"));
NUnit.Framework.Assert.AreEqual("Outputs doesn't match", expectedOutput, MapReduceTestUtil
.ReadOutput(outDir, conf));
}