Exemplo n.º 1
0
        /// <summary>Tests Reducer throwing exception.</summary>
        /// <exception cref="System.Exception"/>
        public virtual void TestReducerFail()
        {
            Configuration conf = CreateJobConf();
            Job           job  = MapReduceTestUtil.CreateJob(conf, inDir, outDir, 1, 1, input);

            job.SetJobName("chain");
            ChainMapper.AddMapper(job, typeof(Mapper), typeof(LongWritable), typeof(Text), typeof(
                                      LongWritable), typeof(Text), null);
            ChainReducer.SetReducer(job, typeof(TestChainErrors.FailReduce), typeof(LongWritable
                                                                                    ), typeof(Text), typeof(LongWritable), typeof(Text), null);
            ChainReducer.AddMapper(job, typeof(Mapper), typeof(LongWritable), typeof(Text), typeof(
                                       LongWritable), typeof(Text), null);
            job.WaitForCompletion(true);
            NUnit.Framework.Assert.IsTrue("Job Not failed", !job.IsSuccessful());
        }
Exemplo n.º 2
0
        /// <summary>Tests one of the maps consuming output.</summary>
        /// <exception cref="System.Exception"/>
        public virtual void TestChainMapNoOuptut()
        {
            Configuration conf           = CreateJobConf();
            string        expectedOutput = string.Empty;
            Job           job            = MapReduceTestUtil.CreateJob(conf, inDir, outDir, 1, 0, input);

            job.SetJobName("chain");
            ChainMapper.AddMapper(job, typeof(TestChainErrors.ConsumeMap), typeof(IntWritable
                                                                                  ), typeof(Text), typeof(LongWritable), typeof(Text), null);
            ChainMapper.AddMapper(job, typeof(Mapper), typeof(LongWritable), typeof(Text), typeof(
                                      LongWritable), typeof(Text), null);
            job.WaitForCompletion(true);
            NUnit.Framework.Assert.IsTrue("Job failed", job.IsSuccessful());
            NUnit.Framework.Assert.AreEqual("Outputs doesn't match", expectedOutput, MapReduceTestUtil
                                            .ReadOutput(outDir, conf));
        }
Exemplo n.º 3
0
        // test chain mapper and reducer by adding single mapper and reducer to chain
        /// <exception cref="System.Exception"/>
        public virtual void TestNoChain()
        {
            Path          inDir          = new Path(localPathRoot, "testing/chain/input");
            Path          outDir         = new Path(localPathRoot, "testing/chain/output");
            string        input          = "a\nb\na\n";
            string        expectedOutput = "a\t2\nb\t1\n";
            Configuration conf           = CreateJobConf();
            Job           job            = MapReduceTestUtil.CreateJob(conf, inDir, outDir, 1, 1, input);

            job.SetJobName("chain");
            ChainMapper.AddMapper(job, typeof(TokenCounterMapper), typeof(object), typeof(Text
                                                                                          ), typeof(Text), typeof(IntWritable), null);
            ChainReducer.SetReducer(job, typeof(IntSumReducer), typeof(Text), typeof(IntWritable
                                                                                     ), typeof(Text), typeof(IntWritable), null);
            job.WaitForCompletion(true);
            NUnit.Framework.Assert.IsTrue("Job failed", job.IsSuccessful());
            NUnit.Framework.Assert.AreEqual("Outputs doesn't match", expectedOutput, MapReduceTestUtil
                                            .ReadOutput(outDir, conf));
        }
Exemplo n.º 4
0
        /// <summary>Tests errors during submission.</summary>
        /// <exception cref="System.Exception"/>
        public virtual void TestChainSubmission()
        {
            Configuration conf = CreateJobConf();
            Job           job  = MapReduceTestUtil.CreateJob(conf, inDir, outDir, 0, 0, input);

            job.SetJobName("chain");
            Exception th = null;

            // output key,value classes of first map are not same as that of second map
            try
            {
                ChainMapper.AddMapper(job, typeof(Mapper), typeof(LongWritable), typeof(Text), typeof(
                                          IntWritable), typeof(Text), null);
                ChainMapper.AddMapper(job, typeof(Mapper), typeof(LongWritable), typeof(Text), typeof(
                                          LongWritable), typeof(Text), null);
            }
            catch (ArgumentException iae)
            {
                th = iae;
            }
            NUnit.Framework.Assert.IsTrue(th != null);
            th = null;
            // output key,value classes of reducer are not
            // same as that of mapper in the chain
            try
            {
                ChainReducer.SetReducer(job, typeof(Reducer), typeof(LongWritable), typeof(Text),
                                        typeof(IntWritable), typeof(Text), null);
                ChainMapper.AddMapper(job, typeof(Mapper), typeof(LongWritable), typeof(Text), typeof(
                                          LongWritable), typeof(Text), null);
            }
            catch (ArgumentException iae)
            {
                th = iae;
            }
            NUnit.Framework.Assert.IsTrue(th != null);
        }
Exemplo n.º 5
0
        /// <exception cref="System.Exception"/>
        public virtual void TestChain()
        {
            Path          inDir          = new Path(localPathRoot, "testing/chain/input");
            Path          outDir         = new Path(localPathRoot, "testing/chain/output");
            string        input          = "1\n2\n";
            string        expectedOutput = "0\t1ABCRDEF\n2\t2ABCRDEF\n";
            Configuration conf           = CreateJobConf();

            CleanFlags(conf);
            conf.Set("a", "X");
            Job job = MapReduceTestUtil.CreateJob(conf, inDir, outDir, 1, 1, input);

            job.SetJobName("chain");
            Configuration mapAConf = new Configuration(false);

            mapAConf.Set("a", "A");
            ChainMapper.AddMapper(job, typeof(TestMapReduceChain.AMap), typeof(LongWritable),
                                  typeof(Text), typeof(LongWritable), typeof(Text), mapAConf);
            ChainMapper.AddMapper(job, typeof(TestMapReduceChain.BMap), typeof(LongWritable),
                                  typeof(Text), typeof(LongWritable), typeof(Text), null);
            ChainMapper.AddMapper(job, typeof(TestMapReduceChain.CMap), typeof(LongWritable),
                                  typeof(Text), typeof(LongWritable), typeof(Text), null);
            Configuration reduceConf = new Configuration(false);

            reduceConf.Set("a", "C");
            ChainReducer.SetReducer(job, typeof(TestMapReduceChain.RReduce), typeof(LongWritable
                                                                                    ), typeof(Text), typeof(LongWritable), typeof(Text), reduceConf);
            ChainReducer.AddMapper(job, typeof(TestMapReduceChain.DMap), typeof(LongWritable)
                                   , typeof(Text), typeof(LongWritable), typeof(Text), null);
            Configuration mapEConf = new Configuration(false);

            mapEConf.Set("a", "E");
            ChainReducer.AddMapper(job, typeof(TestMapReduceChain.EMap), typeof(LongWritable)
                                   , typeof(Text), typeof(LongWritable), typeof(Text), mapEConf);
            ChainReducer.AddMapper(job, typeof(TestMapReduceChain.FMap), typeof(LongWritable)
                                   , typeof(Text), typeof(LongWritable), typeof(Text), null);
            job.WaitForCompletion(true);
            NUnit.Framework.Assert.IsTrue("Job failed", job.IsSuccessful());
            string str = "flag not set";

            NUnit.Framework.Assert.IsTrue(str, GetFlag(conf, "map.setup.A"));
            NUnit.Framework.Assert.IsTrue(str, GetFlag(conf, "map.setup.B"));
            NUnit.Framework.Assert.IsTrue(str, GetFlag(conf, "map.setup.C"));
            NUnit.Framework.Assert.IsTrue(str, GetFlag(conf, "reduce.setup.R"));
            NUnit.Framework.Assert.IsTrue(str, GetFlag(conf, "map.setup.D"));
            NUnit.Framework.Assert.IsTrue(str, GetFlag(conf, "map.setup.E"));
            NUnit.Framework.Assert.IsTrue(str, GetFlag(conf, "map.setup.F"));
            NUnit.Framework.Assert.IsTrue(str, GetFlag(conf, "map.A.value.1"));
            NUnit.Framework.Assert.IsTrue(str, GetFlag(conf, "map.A.value.2"));
            NUnit.Framework.Assert.IsTrue(str, GetFlag(conf, "map.B.value.1A"));
            NUnit.Framework.Assert.IsTrue(str, GetFlag(conf, "map.B.value.2A"));
            NUnit.Framework.Assert.IsTrue(str, GetFlag(conf, "map.C.value.1AB"));
            NUnit.Framework.Assert.IsTrue(str, GetFlag(conf, "map.C.value.2AB"));
            NUnit.Framework.Assert.IsTrue(str, GetFlag(conf, "reduce.R.value.1ABC"));
            NUnit.Framework.Assert.IsTrue(str, GetFlag(conf, "reduce.R.value.2ABC"));
            NUnit.Framework.Assert.IsTrue(str, GetFlag(conf, "map.D.value.1ABCR"));
            NUnit.Framework.Assert.IsTrue(str, GetFlag(conf, "map.D.value.2ABCR"));
            NUnit.Framework.Assert.IsTrue(str, GetFlag(conf, "map.E.value.1ABCRD"));
            NUnit.Framework.Assert.IsTrue(str, GetFlag(conf, "map.E.value.2ABCRD"));
            NUnit.Framework.Assert.IsTrue(str, GetFlag(conf, "map.F.value.1ABCRDE"));
            NUnit.Framework.Assert.IsTrue(str, GetFlag(conf, "map.F.value.2ABCRDE"));
            NUnit.Framework.Assert.IsTrue(GetFlag(conf, "map.cleanup.A"));
            NUnit.Framework.Assert.IsTrue(GetFlag(conf, "map.cleanup.B"));
            NUnit.Framework.Assert.IsTrue(GetFlag(conf, "map.cleanup.C"));
            NUnit.Framework.Assert.IsTrue(GetFlag(conf, "reduce.cleanup.R"));
            NUnit.Framework.Assert.IsTrue(GetFlag(conf, "map.cleanup.D"));
            NUnit.Framework.Assert.IsTrue(GetFlag(conf, "map.cleanup.E"));
            NUnit.Framework.Assert.IsTrue(GetFlag(conf, "map.cleanup.F"));
            NUnit.Framework.Assert.AreEqual("Outputs doesn't match", expectedOutput, MapReduceTestUtil
                                            .ReadOutput(outDir, conf));
        }