/// <summary>Tests Reducer throwing exception.</summary> /// <exception cref="System.Exception"/> public virtual void TestReducerFail() { Configuration conf = CreateJobConf(); Job job = MapReduceTestUtil.CreateJob(conf, inDir, outDir, 1, 1, input); job.SetJobName("chain"); ChainMapper.AddMapper(job, typeof(Mapper), typeof(LongWritable), typeof(Text), typeof( LongWritable), typeof(Text), null); ChainReducer.SetReducer(job, typeof(TestChainErrors.FailReduce), typeof(LongWritable ), typeof(Text), typeof(LongWritable), typeof(Text), null); ChainReducer.AddMapper(job, typeof(Mapper), typeof(LongWritable), typeof(Text), typeof( LongWritable), typeof(Text), null); job.WaitForCompletion(true); NUnit.Framework.Assert.IsTrue("Job Not failed", !job.IsSuccessful()); }
/// <summary>Tests reducer consuming output.</summary> /// <exception cref="System.Exception"/> public virtual void TestChainReduceNoOuptut() { Configuration conf = CreateJobConf(); string expectedOutput = string.Empty; Job job = MapReduceTestUtil.CreateJob(conf, inDir, outDir, 1, 1, input); job.SetJobName("chain"); ChainMapper.AddMapper(job, typeof(Mapper), typeof(IntWritable), typeof(Text), typeof( LongWritable), typeof(Text), null); ChainReducer.SetReducer(job, typeof(TestChainErrors.ConsumeReduce), typeof(LongWritable ), typeof(Text), typeof(LongWritable), typeof(Text), null); ChainReducer.AddMapper(job, typeof(Mapper), typeof(LongWritable), typeof(Text), typeof( LongWritable), typeof(Text), null); job.WaitForCompletion(true); NUnit.Framework.Assert.IsTrue("Job failed", job.IsSuccessful()); NUnit.Framework.Assert.AreEqual("Outputs doesn't match", expectedOutput, MapReduceTestUtil .ReadOutput(outDir, conf)); }
// test chain mapper and reducer by adding single mapper and reducer to chain /// <exception cref="System.Exception"/> public virtual void TestNoChain() { Path inDir = new Path(localPathRoot, "testing/chain/input"); Path outDir = new Path(localPathRoot, "testing/chain/output"); string input = "a\nb\na\n"; string expectedOutput = "a\t2\nb\t1\n"; Configuration conf = CreateJobConf(); Job job = MapReduceTestUtil.CreateJob(conf, inDir, outDir, 1, 1, input); job.SetJobName("chain"); ChainMapper.AddMapper(job, typeof(TokenCounterMapper), typeof(object), typeof(Text ), typeof(Text), typeof(IntWritable), null); ChainReducer.SetReducer(job, typeof(IntSumReducer), typeof(Text), typeof(IntWritable ), typeof(Text), typeof(IntWritable), null); job.WaitForCompletion(true); NUnit.Framework.Assert.IsTrue("Job failed", job.IsSuccessful()); NUnit.Framework.Assert.AreEqual("Outputs doesn't match", expectedOutput, MapReduceTestUtil .ReadOutput(outDir, conf)); }
/// <summary>Tests errors during submission.</summary> /// <exception cref="System.Exception"/> public virtual void TestChainSubmission() { Configuration conf = CreateJobConf(); Job job = MapReduceTestUtil.CreateJob(conf, inDir, outDir, 0, 0, input); job.SetJobName("chain"); Exception th = null; // output key,value classes of first map are not same as that of second map try { ChainMapper.AddMapper(job, typeof(Mapper), typeof(LongWritable), typeof(Text), typeof( IntWritable), typeof(Text), null); ChainMapper.AddMapper(job, typeof(Mapper), typeof(LongWritable), typeof(Text), typeof( LongWritable), typeof(Text), null); } catch (ArgumentException iae) { th = iae; } NUnit.Framework.Assert.IsTrue(th != null); th = null; // output key,value classes of reducer are not // same as that of mapper in the chain try { ChainReducer.SetReducer(job, typeof(Reducer), typeof(LongWritable), typeof(Text), typeof(IntWritable), typeof(Text), null); ChainMapper.AddMapper(job, typeof(Mapper), typeof(LongWritable), typeof(Text), typeof( LongWritable), typeof(Text), null); } catch (ArgumentException iae) { th = iae; } NUnit.Framework.Assert.IsTrue(th != null); }
/// <exception cref="System.Exception"/> public virtual void TestChain() { Path inDir = new Path(localPathRoot, "testing/chain/input"); Path outDir = new Path(localPathRoot, "testing/chain/output"); string input = "1\n2\n"; string expectedOutput = "0\t1ABCRDEF\n2\t2ABCRDEF\n"; Configuration conf = CreateJobConf(); CleanFlags(conf); conf.Set("a", "X"); Job job = MapReduceTestUtil.CreateJob(conf, inDir, outDir, 1, 1, input); job.SetJobName("chain"); Configuration mapAConf = new Configuration(false); mapAConf.Set("a", "A"); ChainMapper.AddMapper(job, typeof(TestMapReduceChain.AMap), typeof(LongWritable), typeof(Text), typeof(LongWritable), typeof(Text), mapAConf); ChainMapper.AddMapper(job, typeof(TestMapReduceChain.BMap), typeof(LongWritable), typeof(Text), typeof(LongWritable), typeof(Text), null); ChainMapper.AddMapper(job, typeof(TestMapReduceChain.CMap), typeof(LongWritable), typeof(Text), typeof(LongWritable), typeof(Text), null); Configuration reduceConf = new Configuration(false); reduceConf.Set("a", "C"); ChainReducer.SetReducer(job, typeof(TestMapReduceChain.RReduce), typeof(LongWritable ), typeof(Text), typeof(LongWritable), typeof(Text), reduceConf); ChainReducer.AddMapper(job, typeof(TestMapReduceChain.DMap), typeof(LongWritable) , typeof(Text), typeof(LongWritable), typeof(Text), null); Configuration mapEConf = new Configuration(false); mapEConf.Set("a", "E"); ChainReducer.AddMapper(job, typeof(TestMapReduceChain.EMap), typeof(LongWritable) , typeof(Text), typeof(LongWritable), typeof(Text), mapEConf); ChainReducer.AddMapper(job, typeof(TestMapReduceChain.FMap), typeof(LongWritable) , typeof(Text), typeof(LongWritable), typeof(Text), null); job.WaitForCompletion(true); NUnit.Framework.Assert.IsTrue("Job failed", job.IsSuccessful()); string str = "flag not set"; NUnit.Framework.Assert.IsTrue(str, GetFlag(conf, "map.setup.A")); NUnit.Framework.Assert.IsTrue(str, GetFlag(conf, "map.setup.B")); NUnit.Framework.Assert.IsTrue(str, GetFlag(conf, "map.setup.C")); NUnit.Framework.Assert.IsTrue(str, GetFlag(conf, "reduce.setup.R")); NUnit.Framework.Assert.IsTrue(str, GetFlag(conf, "map.setup.D")); NUnit.Framework.Assert.IsTrue(str, GetFlag(conf, "map.setup.E")); NUnit.Framework.Assert.IsTrue(str, GetFlag(conf, "map.setup.F")); NUnit.Framework.Assert.IsTrue(str, GetFlag(conf, "map.A.value.1")); NUnit.Framework.Assert.IsTrue(str, GetFlag(conf, "map.A.value.2")); NUnit.Framework.Assert.IsTrue(str, GetFlag(conf, "map.B.value.1A")); NUnit.Framework.Assert.IsTrue(str, GetFlag(conf, "map.B.value.2A")); NUnit.Framework.Assert.IsTrue(str, GetFlag(conf, "map.C.value.1AB")); NUnit.Framework.Assert.IsTrue(str, GetFlag(conf, "map.C.value.2AB")); NUnit.Framework.Assert.IsTrue(str, GetFlag(conf, "reduce.R.value.1ABC")); NUnit.Framework.Assert.IsTrue(str, GetFlag(conf, "reduce.R.value.2ABC")); NUnit.Framework.Assert.IsTrue(str, GetFlag(conf, "map.D.value.1ABCR")); NUnit.Framework.Assert.IsTrue(str, GetFlag(conf, "map.D.value.2ABCR")); NUnit.Framework.Assert.IsTrue(str, GetFlag(conf, "map.E.value.1ABCRD")); NUnit.Framework.Assert.IsTrue(str, GetFlag(conf, "map.E.value.2ABCRD")); NUnit.Framework.Assert.IsTrue(str, GetFlag(conf, "map.F.value.1ABCRDE")); NUnit.Framework.Assert.IsTrue(str, GetFlag(conf, "map.F.value.2ABCRDE")); NUnit.Framework.Assert.IsTrue(GetFlag(conf, "map.cleanup.A")); NUnit.Framework.Assert.IsTrue(GetFlag(conf, "map.cleanup.B")); NUnit.Framework.Assert.IsTrue(GetFlag(conf, "map.cleanup.C")); NUnit.Framework.Assert.IsTrue(GetFlag(conf, "reduce.cleanup.R")); NUnit.Framework.Assert.IsTrue(GetFlag(conf, "map.cleanup.D")); NUnit.Framework.Assert.IsTrue(GetFlag(conf, "map.cleanup.E")); NUnit.Framework.Assert.IsTrue(GetFlag(conf, "map.cleanup.F")); NUnit.Framework.Assert.AreEqual("Outputs doesn't match", expectedOutput, MapReduceTestUtil .ReadOutput(outDir, conf)); }