Exemplo n.º 1
0
        /// <exception cref="System.Exception"/>
        public static void Launch()
        {
            Configuration conf            = new Configuration();
            FileSystem    fs              = FileSystem.Get(conf);
            int           numOfInputLines = 10;
            Path          outDir          = new Path(testDir, "output_for_field_selection_test");
            Path          inDir           = new Path(testDir, "input_for_field_selection_test");
            StringBuilder inputData       = new StringBuilder();
            StringBuilder expectedOutput  = new StringBuilder();

            ConstructInputOutputData(inputData, expectedOutput, numOfInputLines);
            conf.Set(FieldSelectionHelper.DataFieldSeperator, "-");
            conf.Set(FieldSelectionHelper.MapOutputKeyValueSpec, "6,5,1-3:0-");
            conf.Set(FieldSelectionHelper.ReduceOutputKeyValueSpec, ":4,3,2,1,0,0-");
            Job job = MapReduceTestUtil.CreateJob(conf, inDir, outDir, 1, 1, inputData.ToString
                                                      ());

            job.SetMapperClass(typeof(FieldSelectionMapper));
            job.SetReducerClass(typeof(FieldSelectionReducer));
            job.SetOutputKeyClass(typeof(Org.Apache.Hadoop.IO.Text));
            job.SetOutputValueClass(typeof(Org.Apache.Hadoop.IO.Text));
            job.SetNumReduceTasks(1);
            job.WaitForCompletion(true);
            NUnit.Framework.Assert.IsTrue("Job Failed!", job.IsSuccessful());
            //
            // Finally, we compare the reconstructed answer key with the
            // original one.  Remember, we need to ignore zero-count items
            // in the original key.
            //
            string outdata = MapReduceTestUtil.ReadOutput(outDir, conf);

            NUnit.Framework.Assert.AreEqual("Outputs doesnt match.", expectedOutput.ToString(
                                                ), outdata);
            fs.Delete(outDir, true);
        }
Exemplo n.º 2
0
        /// <exception cref="System.Exception"/>
        public static void Launch()
        {
            JobConf    conf            = new JobConf(typeof(Org.Apache.Hadoop.Mapred.TestFieldSelection));
            FileSystem fs              = FileSystem.Get(conf);
            int        numOfInputLines = 10;
            Path       OutputDir       = new Path("build/test/output_for_field_selection_test");
            Path       InputDir        = new Path("build/test/input_for_field_selection_test");
            string     inputFile       = "input.txt";

            fs.Delete(InputDir, true);
            fs.Mkdirs(InputDir);
            fs.Delete(OutputDir, true);
            StringBuilder inputData      = new StringBuilder();
            StringBuilder expectedOutput = new StringBuilder();

            TestMRFieldSelection.ConstructInputOutputData(inputData, expectedOutput, numOfInputLines
                                                          );
            FSDataOutputStream fileOut = fs.Create(new Path(InputDir, inputFile));

            fileOut.Write(Sharpen.Runtime.GetBytesForString(inputData.ToString(), "utf-8"));
            fileOut.Close();
            System.Console.Out.WriteLine("inputData:");
            System.Console.Out.WriteLine(inputData.ToString());
            JobConf job = new JobConf(conf, typeof(Org.Apache.Hadoop.Mapred.TestFieldSelection
                                                   ));

            FileInputFormat.SetInputPaths(job, InputDir);
            job.SetInputFormat(typeof(TextInputFormat));
            job.SetMapperClass(typeof(FieldSelectionMapReduce));
            job.SetReducerClass(typeof(FieldSelectionMapReduce));
            FileOutputFormat.SetOutputPath(job, OutputDir);
            job.SetOutputKeyClass(typeof(Org.Apache.Hadoop.IO.Text));
            job.SetOutputValueClass(typeof(Org.Apache.Hadoop.IO.Text));
            job.SetOutputFormat(typeof(TextOutputFormat));
            job.SetNumReduceTasks(1);
            job.Set(FieldSelectionHelper.DataFieldSeperator, "-");
            job.Set(FieldSelectionHelper.MapOutputKeyValueSpec, "6,5,1-3:0-");
            job.Set(FieldSelectionHelper.ReduceOutputKeyValueSpec, ":4,3,2,1,0,0-");
            JobClient.RunJob(job);
            //
            // Finally, we compare the reconstructed answer key with the
            // original one.  Remember, we need to ignore zero-count items
            // in the original key.
            //
            bool   success = true;
            Path   outPath = new Path(OutputDir, "part-00000");
            string outdata = MapReduceTestUtil.ReadOutput(outPath, job);

            NUnit.Framework.Assert.AreEqual(expectedOutput.ToString(), outdata);
            fs.Delete(OutputDir, true);
            fs.Delete(InputDir, true);
        }
Exemplo n.º 3
0
        /// <summary>Tests one of the maps consuming output.</summary>
        /// <exception cref="System.Exception"/>
        public virtual void TestChainMapNoOuptut()
        {
            Configuration conf           = CreateJobConf();
            string        expectedOutput = string.Empty;
            Job           job            = MapReduceTestUtil.CreateJob(conf, inDir, outDir, 1, 0, input);

            job.SetJobName("chain");
            ChainMapper.AddMapper(job, typeof(TestChainErrors.ConsumeMap), typeof(IntWritable
                                                                                  ), typeof(Text), typeof(LongWritable), typeof(Text), null);
            ChainMapper.AddMapper(job, typeof(Mapper), typeof(LongWritable), typeof(Text), typeof(
                                      LongWritable), typeof(Text), null);
            job.WaitForCompletion(true);
            NUnit.Framework.Assert.IsTrue("Job failed", job.IsSuccessful());
            NUnit.Framework.Assert.AreEqual("Outputs doesn't match", expectedOutput, MapReduceTestUtil
                                            .ReadOutput(outDir, conf));
        }
Exemplo n.º 4
0
        // test chain mapper and reducer by adding single mapper and reducer to chain
        /// <exception cref="System.Exception"/>
        public virtual void TestNoChain()
        {
            Path          inDir          = new Path(localPathRoot, "testing/chain/input");
            Path          outDir         = new Path(localPathRoot, "testing/chain/output");
            string        input          = "a\nb\na\n";
            string        expectedOutput = "a\t2\nb\t1\n";
            Configuration conf           = CreateJobConf();
            Job           job            = MapReduceTestUtil.CreateJob(conf, inDir, outDir, 1, 1, input);

            job.SetJobName("chain");
            ChainMapper.AddMapper(job, typeof(TokenCounterMapper), typeof(object), typeof(Text
                                                                                          ), typeof(Text), typeof(IntWritable), null);
            ChainReducer.SetReducer(job, typeof(IntSumReducer), typeof(Text), typeof(IntWritable
                                                                                     ), typeof(Text), typeof(IntWritable), null);
            job.WaitForCompletion(true);
            NUnit.Framework.Assert.IsTrue("Job failed", job.IsSuccessful());
            NUnit.Framework.Assert.AreEqual("Outputs doesn't match", expectedOutput, MapReduceTestUtil
                                            .ReadOutput(outDir, conf));
        }
Exemplo n.º 5
0
        /// <exception cref="System.IO.IOException"/>
        public static TestJobSysDirWithDFS.TestResult LaunchWordCount(JobConf conf, Path
                                                                      inDir, Path outDir, string input, int numMaps, int numReduces, string sysDir)
        {
            FileSystem inFs  = inDir.GetFileSystem(conf);
            FileSystem outFs = outDir.GetFileSystem(conf);

            outFs.Delete(outDir, true);
            if (!inFs.Mkdirs(inDir))
            {
                throw new IOException("Mkdirs failed to create " + inDir.ToString());
            }
            {
                DataOutputStream file = inFs.Create(new Path(inDir, "part-0"));
                file.WriteBytes(input);
                file.Close();
            }
            conf.SetJobName("wordcount");
            conf.SetInputFormat(typeof(TextInputFormat));
            // the keys are words (strings)
            conf.SetOutputKeyClass(typeof(Text));
            // the values are counts (ints)
            conf.SetOutputValueClass(typeof(IntWritable));
            conf.SetMapperClass(typeof(WordCount.MapClass));
            conf.SetCombinerClass(typeof(WordCount.Reduce));
            conf.SetReducerClass(typeof(WordCount.Reduce));
            FileInputFormat.SetInputPaths(conf, inDir);
            FileOutputFormat.SetOutputPath(conf, outDir);
            conf.SetNumMapTasks(numMaps);
            conf.SetNumReduceTasks(numReduces);
            conf.Set(JTConfig.JtSystemDir, "/tmp/subru/mapred/system");
            JobClient  jobClient = new JobClient(conf);
            RunningJob job       = JobClient.RunJob(conf);

            // Checking that the Job Client system dir is not used
            NUnit.Framework.Assert.IsFalse(FileSystem.Get(conf).Exists(new Path(conf.Get(JTConfig
                                                                                         .JtSystemDir))));
            // Check if the Job Tracker system dir is propogated to client
            NUnit.Framework.Assert.IsFalse(sysDir.Contains("/tmp/subru/mapred/system"));
            NUnit.Framework.Assert.IsTrue(sysDir.Contains("custom"));
            return(new TestJobSysDirWithDFS.TestResult(job, MapReduceTestUtil.ReadOutput(outDir
                                                                                         , conf)));
        }
Exemplo n.º 6
0
        /// <exception cref="System.IO.IOException"/>
        private string LaunchWordCount(JobConf conf, string input, int numMaps, int numReduces
                                       )
        {
            Path inDir  = new Path("testing/wc/input");
            Path outDir = new Path("testing/wc/output");

            // Hack for local FS that does not have the concept of a 'mounting point'
            if (IsLocalFS())
            {
                string localPathRoot = Runtime.GetProperty("test.build.data", "/tmp").ToString().
                                       Replace(' ', '+');
                inDir  = new Path(localPathRoot, inDir);
                outDir = new Path(localPathRoot, outDir);
            }
            FileSystem fs = FileSystem.Get(conf);

            fs.Delete(outDir, true);
            if (!fs.Mkdirs(inDir))
            {
                throw new IOException("Mkdirs failed to create " + inDir.ToString());
            }
            {
                DataOutputStream file = fs.Create(new Path(inDir, "part-0"));
                file.WriteBytes(input);
                file.Close();
            }
            conf.SetJobName("wordcount");
            conf.SetInputFormat(typeof(TextInputFormat));
            // the keys are words (strings)
            conf.SetOutputKeyClass(typeof(Text));
            // the values are counts (ints)
            conf.SetOutputValueClass(typeof(IntWritable));
            conf.SetMapperClass(typeof(WordCount.MapClass));
            conf.SetCombinerClass(typeof(WordCount.Reduce));
            conf.SetReducerClass(typeof(WordCount.Reduce));
            FileInputFormat.SetInputPaths(conf, inDir);
            FileOutputFormat.SetOutputPath(conf, outDir);
            conf.SetNumMapTasks(numMaps);
            conf.SetNumReduceTasks(numReduces);
            JobClient.RunJob(conf);
            return(MapReduceTestUtil.ReadOutput(outDir, conf));
        }
Exemplo n.º 7
0
        /// <exception cref="System.IO.IOException"/>
        internal static void RunProgram(MiniMRCluster mr, MiniDFSCluster dfs, Path program
                                        , Path inputPath, Path outputPath, int numMaps, int numReduces, string[] expectedResults
                                        , JobConf conf)
        {
            Path    wordExec = new Path("testing/bin/application");
            JobConf job      = null;

            if (conf == null)
            {
                job = mr.CreateJobConf();
            }
            else
            {
                job = new JobConf(conf);
            }
            job.SetNumMapTasks(numMaps);
            job.SetNumReduceTasks(numReduces);
            {
                FileSystem fs = dfs.GetFileSystem();
                fs.Delete(wordExec.GetParent(), true);
                fs.CopyFromLocalFile(program, wordExec);
                Submitter.SetExecutable(job, fs.MakeQualified(wordExec).ToString());
                Submitter.SetIsJavaRecordReader(job, true);
                Submitter.SetIsJavaRecordWriter(job, true);
                FileInputFormat.SetInputPaths(job, inputPath);
                FileOutputFormat.SetOutputPath(job, outputPath);
                RunningJob rJob = null;
                if (numReduces == 0)
                {
                    rJob = Submitter.JobSubmit(job);
                    while (!rJob.IsComplete())
                    {
                        try
                        {
                            Sharpen.Thread.Sleep(1000);
                        }
                        catch (Exception ie)
                        {
                            throw new RuntimeException(ie);
                        }
                    }
                }
                else
                {
                    rJob = Submitter.RunJob(job);
                }
                NUnit.Framework.Assert.IsTrue("pipes job failed", rJob.IsSuccessful());
                Counters       counters          = rJob.GetCounters();
                Counters.Group wordCountCounters = counters.GetGroup("WORDCOUNT");
                int            numCounters       = 0;
                foreach (Counters.Counter c in wordCountCounters)
                {
                    System.Console.Out.WriteLine(c);
                    ++numCounters;
                }
                NUnit.Framework.Assert.IsTrue("No counters found!", (numCounters > 0));
            }
            IList <string> results = new AList <string>();

            foreach (Path p in FileUtil.Stat2Paths(dfs.GetFileSystem().ListStatus(outputPath,
                                                                                  new Utils.OutputFileUtils.OutputFilesFilter())))
            {
                results.AddItem(MapReduceTestUtil.ReadOutput(p, job));
            }
            NUnit.Framework.Assert.AreEqual("number of reduces is wrong", expectedResults.Length
                                            , results.Count);
            for (int i = 0; i < results.Count; i++)
            {
                NUnit.Framework.Assert.AreEqual("pipes program " + program + " output " + i + " wrong"
                                                , expectedResults[i], results[i]);
            }
        }
Exemplo n.º 8
0
        /// <exception cref="System.Exception"/>
        public static void Launch()
        {
            JobConf conf = new JobConf(typeof(Org.Apache.Hadoop.Mapred.Lib.Aggregate.TestAggregates
                                              ));
            FileSystem fs = FileSystem.Get(conf);
            int        numOfInputLines = 20;
            Path       OutputDir       = new Path("build/test/output_for_aggregates_test");
            Path       InputDir        = new Path("build/test/input_for_aggregates_test");
            string     inputFile       = "input.txt";

            fs.Delete(InputDir, true);
            fs.Mkdirs(InputDir);
            fs.Delete(OutputDir, true);
            StringBuilder inputData      = new StringBuilder();
            StringBuilder expectedOutput = new StringBuilder();

            expectedOutput.Append("max\t19\n");
            expectedOutput.Append("min\t1\n");
            FSDataOutputStream fileOut = fs.Create(new Path(InputDir, inputFile));

            for (int i = 1; i < numOfInputLines; i++)
            {
                expectedOutput.Append("count_").Append(idFormat.Format(i));
                expectedOutput.Append("\t").Append(i).Append("\n");
                inputData.Append(idFormat.Format(i));
                for (int j = 1; j < i; j++)
                {
                    inputData.Append(" ").Append(idFormat.Format(i));
                }
                inputData.Append("\n");
            }
            expectedOutput.Append("value_as_string_max\t9\n");
            expectedOutput.Append("value_as_string_min\t1\n");
            expectedOutput.Append("uniq_count\t15\n");
            fileOut.Write(Sharpen.Runtime.GetBytesForString(inputData.ToString(), "utf-8"));
            fileOut.Close();
            System.Console.Out.WriteLine("inputData:");
            System.Console.Out.WriteLine(inputData.ToString());
            JobConf job = new JobConf(conf, typeof(Org.Apache.Hadoop.Mapred.Lib.Aggregate.TestAggregates
                                                   ));

            FileInputFormat.SetInputPaths(job, InputDir);
            job.SetInputFormat(typeof(TextInputFormat));
            FileOutputFormat.SetOutputPath(job, OutputDir);
            job.SetOutputFormat(typeof(TextOutputFormat));
            job.SetMapOutputKeyClass(typeof(Org.Apache.Hadoop.IO.Text));
            job.SetMapOutputValueClass(typeof(Org.Apache.Hadoop.IO.Text));
            job.SetOutputKeyClass(typeof(Org.Apache.Hadoop.IO.Text));
            job.SetOutputValueClass(typeof(Org.Apache.Hadoop.IO.Text));
            job.SetNumReduceTasks(1);
            job.SetMapperClass(typeof(ValueAggregatorMapper));
            job.SetReducerClass(typeof(ValueAggregatorReducer));
            job.SetCombinerClass(typeof(ValueAggregatorCombiner));
            job.SetInt("aggregator.descriptor.num", 1);
            job.Set("aggregator.descriptor.0", "UserDefined,org.apache.hadoop.mapred.lib.aggregate.AggregatorTests"
                    );
            job.SetLong("aggregate.max.num.unique.values", 14);
            JobClient.RunJob(job);
            //
            // Finally, we compare the reconstructed answer key with the
            // original one.  Remember, we need to ignore zero-count items
            // in the original key.
            //
            bool   success = true;
            Path   outPath = new Path(OutputDir, "part-00000");
            string outdata = MapReduceTestUtil.ReadOutput(outPath, job);

            System.Console.Out.WriteLine("full out data:");
            System.Console.Out.WriteLine(outdata.ToString());
            outdata = Sharpen.Runtime.Substring(outdata, 0, expectedOutput.ToString().Length);
            NUnit.Framework.Assert.AreEqual(expectedOutput.ToString(), outdata);
            //fs.delete(OUTPUT_DIR);
            fs.Delete(InputDir, true);
        }
Exemplo n.º 9
0
        /// <exception cref="System.Exception"/>
        public virtual void TestChain()
        {
            Path          inDir          = new Path(localPathRoot, "testing/chain/input");
            Path          outDir         = new Path(localPathRoot, "testing/chain/output");
            string        input          = "1\n2\n";
            string        expectedOutput = "0\t1ABCRDEF\n2\t2ABCRDEF\n";
            Configuration conf           = CreateJobConf();

            CleanFlags(conf);
            conf.Set("a", "X");
            Job job = MapReduceTestUtil.CreateJob(conf, inDir, outDir, 1, 1, input);

            job.SetJobName("chain");
            Configuration mapAConf = new Configuration(false);

            mapAConf.Set("a", "A");
            ChainMapper.AddMapper(job, typeof(TestMapReduceChain.AMap), typeof(LongWritable),
                                  typeof(Text), typeof(LongWritable), typeof(Text), mapAConf);
            ChainMapper.AddMapper(job, typeof(TestMapReduceChain.BMap), typeof(LongWritable),
                                  typeof(Text), typeof(LongWritable), typeof(Text), null);
            ChainMapper.AddMapper(job, typeof(TestMapReduceChain.CMap), typeof(LongWritable),
                                  typeof(Text), typeof(LongWritable), typeof(Text), null);
            Configuration reduceConf = new Configuration(false);

            reduceConf.Set("a", "C");
            ChainReducer.SetReducer(job, typeof(TestMapReduceChain.RReduce), typeof(LongWritable
                                                                                    ), typeof(Text), typeof(LongWritable), typeof(Text), reduceConf);
            ChainReducer.AddMapper(job, typeof(TestMapReduceChain.DMap), typeof(LongWritable)
                                   , typeof(Text), typeof(LongWritable), typeof(Text), null);
            Configuration mapEConf = new Configuration(false);

            mapEConf.Set("a", "E");
            ChainReducer.AddMapper(job, typeof(TestMapReduceChain.EMap), typeof(LongWritable)
                                   , typeof(Text), typeof(LongWritable), typeof(Text), mapEConf);
            ChainReducer.AddMapper(job, typeof(TestMapReduceChain.FMap), typeof(LongWritable)
                                   , typeof(Text), typeof(LongWritable), typeof(Text), null);
            job.WaitForCompletion(true);
            NUnit.Framework.Assert.IsTrue("Job failed", job.IsSuccessful());
            string str = "flag not set";

            NUnit.Framework.Assert.IsTrue(str, GetFlag(conf, "map.setup.A"));
            NUnit.Framework.Assert.IsTrue(str, GetFlag(conf, "map.setup.B"));
            NUnit.Framework.Assert.IsTrue(str, GetFlag(conf, "map.setup.C"));
            NUnit.Framework.Assert.IsTrue(str, GetFlag(conf, "reduce.setup.R"));
            NUnit.Framework.Assert.IsTrue(str, GetFlag(conf, "map.setup.D"));
            NUnit.Framework.Assert.IsTrue(str, GetFlag(conf, "map.setup.E"));
            NUnit.Framework.Assert.IsTrue(str, GetFlag(conf, "map.setup.F"));
            NUnit.Framework.Assert.IsTrue(str, GetFlag(conf, "map.A.value.1"));
            NUnit.Framework.Assert.IsTrue(str, GetFlag(conf, "map.A.value.2"));
            NUnit.Framework.Assert.IsTrue(str, GetFlag(conf, "map.B.value.1A"));
            NUnit.Framework.Assert.IsTrue(str, GetFlag(conf, "map.B.value.2A"));
            NUnit.Framework.Assert.IsTrue(str, GetFlag(conf, "map.C.value.1AB"));
            NUnit.Framework.Assert.IsTrue(str, GetFlag(conf, "map.C.value.2AB"));
            NUnit.Framework.Assert.IsTrue(str, GetFlag(conf, "reduce.R.value.1ABC"));
            NUnit.Framework.Assert.IsTrue(str, GetFlag(conf, "reduce.R.value.2ABC"));
            NUnit.Framework.Assert.IsTrue(str, GetFlag(conf, "map.D.value.1ABCR"));
            NUnit.Framework.Assert.IsTrue(str, GetFlag(conf, "map.D.value.2ABCR"));
            NUnit.Framework.Assert.IsTrue(str, GetFlag(conf, "map.E.value.1ABCRD"));
            NUnit.Framework.Assert.IsTrue(str, GetFlag(conf, "map.E.value.2ABCRD"));
            NUnit.Framework.Assert.IsTrue(str, GetFlag(conf, "map.F.value.1ABCRDE"));
            NUnit.Framework.Assert.IsTrue(str, GetFlag(conf, "map.F.value.2ABCRDE"));
            NUnit.Framework.Assert.IsTrue(GetFlag(conf, "map.cleanup.A"));
            NUnit.Framework.Assert.IsTrue(GetFlag(conf, "map.cleanup.B"));
            NUnit.Framework.Assert.IsTrue(GetFlag(conf, "map.cleanup.C"));
            NUnit.Framework.Assert.IsTrue(GetFlag(conf, "reduce.cleanup.R"));
            NUnit.Framework.Assert.IsTrue(GetFlag(conf, "map.cleanup.D"));
            NUnit.Framework.Assert.IsTrue(GetFlag(conf, "map.cleanup.E"));
            NUnit.Framework.Assert.IsTrue(GetFlag(conf, "map.cleanup.F"));
            NUnit.Framework.Assert.AreEqual("Outputs doesn't match", expectedOutput, MapReduceTestUtil
                                            .ReadOutput(outDir, conf));
        }
Exemplo n.º 10
0
        /// <exception cref="System.Exception"/>
        public static void Launch()
        {
            Configuration conf            = new Configuration();
            FileSystem    fs              = FileSystem.Get(conf);
            int           numOfInputLines = 20;
            Path          OutputDir       = new Path("build/test/output_for_aggregates_test");
            Path          InputDir        = new Path("build/test/input_for_aggregates_test");
            string        inputFile       = "input.txt";

            fs.Delete(InputDir, true);
            fs.Mkdirs(InputDir);
            fs.Delete(OutputDir, true);
            StringBuilder inputData      = new StringBuilder();
            StringBuilder expectedOutput = new StringBuilder();

            expectedOutput.Append("max\t19\n");
            expectedOutput.Append("min\t1\n");
            FSDataOutputStream fileOut = fs.Create(new Path(InputDir, inputFile));

            for (int i = 1; i < numOfInputLines; i++)
            {
                expectedOutput.Append("count_").Append(idFormat.Format(i));
                expectedOutput.Append("\t").Append(i).Append("\n");
                inputData.Append(idFormat.Format(i));
                for (int j = 1; j < i; j++)
                {
                    inputData.Append(" ").Append(idFormat.Format(i));
                }
                inputData.Append("\n");
            }
            expectedOutput.Append("value_as_string_max\t9\n");
            expectedOutput.Append("value_as_string_min\t1\n");
            expectedOutput.Append("uniq_count\t15\n");
            fileOut.Write(Sharpen.Runtime.GetBytesForString(inputData.ToString(), "utf-8"));
            fileOut.Close();
            System.Console.Out.WriteLine("inputData:");
            System.Console.Out.WriteLine(inputData.ToString());
            conf.SetInt(ValueAggregatorJobBase.DescriptorNum, 1);
            conf.Set(ValueAggregatorJobBase.Descriptor + ".0", "UserDefined,org.apache.hadoop.mapreduce.lib.aggregate.AggregatorTests"
                     );
            conf.SetLong(UniqValueCount.MaxNumUniqueValues, 14);
            Job job = Job.GetInstance(conf);

            FileInputFormat.SetInputPaths(job, InputDir);
            job.SetInputFormatClass(typeof(TextInputFormat));
            FileOutputFormat.SetOutputPath(job, OutputDir);
            job.SetOutputFormatClass(typeof(TextOutputFormat));
            job.SetMapOutputKeyClass(typeof(Org.Apache.Hadoop.IO.Text));
            job.SetMapOutputValueClass(typeof(Org.Apache.Hadoop.IO.Text));
            job.SetOutputKeyClass(typeof(Org.Apache.Hadoop.IO.Text));
            job.SetOutputValueClass(typeof(Org.Apache.Hadoop.IO.Text));
            job.SetNumReduceTasks(1);
            job.SetMapperClass(typeof(ValueAggregatorMapper));
            job.SetReducerClass(typeof(ValueAggregatorReducer));
            job.SetCombinerClass(typeof(ValueAggregatorCombiner));
            job.WaitForCompletion(true);
            NUnit.Framework.Assert.IsTrue(job.IsSuccessful());
            //
            // Finally, we compare the reconstructed answer key with the
            // original one.  Remember, we need to ignore zero-count items
            // in the original key.
            //
            string outdata = MapReduceTestUtil.ReadOutput(OutputDir, conf);

            System.Console.Out.WriteLine("full out data:");
            System.Console.Out.WriteLine(outdata.ToString());
            outdata = Sharpen.Runtime.Substring(outdata, 0, expectedOutput.ToString().Length);
            NUnit.Framework.Assert.AreEqual(expectedOutput.ToString(), outdata);
            fs.Delete(OutputDir, true);
            fs.Delete(InputDir, true);
        }