public virtual void TestSubmitter() { JobConf conf = new JobConf(); FilePath[] psw = CleanTokenPasswordFile(); Runtime.SetProperty("test.build.data", "target/tmp/build/TEST_SUBMITTER_MAPPER/data" ); conf.Set("hadoop.log.dir", "target/tmp"); // prepare configuration Submitter.SetIsJavaMapper(conf, false); Submitter.SetIsJavaReducer(conf, false); Submitter.SetKeepCommandFile(conf, false); Submitter.SetIsJavaRecordReader(conf, false); Submitter.SetIsJavaRecordWriter(conf, false); PipesPartitioner <IntWritable, Text> partitioner = new PipesPartitioner <IntWritable , Text>(); partitioner.Configure(conf); Submitter.SetJavaPartitioner(conf, partitioner.GetType()); NUnit.Framework.Assert.AreEqual(typeof(PipesPartitioner), (Submitter.GetJavaPartitioner (conf))); // test going to call main method with System.exit(). Change Security SecurityManager securityManager = Runtime.GetSecurityManager(); // store System.out TextWriter oldps = System.Console.Out; ByteArrayOutputStream @out = new ByteArrayOutputStream(); ExitUtil.DisableSystemExit(); // test without parameters try { Runtime.SetOut(new TextWriter(@out)); Submitter.Main(new string[0]); NUnit.Framework.Assert.Fail(); } catch (ExitUtil.ExitException) { // System.exit prohibited! output message test NUnit.Framework.Assert.IsTrue(@out.ToString().Contains(string.Empty)); NUnit.Framework.Assert.IsTrue(@out.ToString().Contains("bin/hadoop pipes")); NUnit.Framework.Assert.IsTrue(@out.ToString().Contains("[-input <path>] // Input directory" )); NUnit.Framework.Assert.IsTrue(@out.ToString().Contains("[-output <path>] // Output directory" )); NUnit.Framework.Assert.IsTrue(@out.ToString().Contains("[-jar <jar file> // jar filename" )); NUnit.Framework.Assert.IsTrue(@out.ToString().Contains("[-inputformat <class>] // InputFormat class" )); NUnit.Framework.Assert.IsTrue(@out.ToString().Contains("[-map <class>] // Java Map class" )); NUnit.Framework.Assert.IsTrue(@out.ToString().Contains("[-partitioner <class>] // Java Partitioner" )); NUnit.Framework.Assert.IsTrue(@out.ToString().Contains("[-reduce <class>] // Java Reduce class" )); NUnit.Framework.Assert.IsTrue(@out.ToString().Contains("[-writer <class>] // Java RecordWriter" )); NUnit.Framework.Assert.IsTrue(@out.ToString().Contains("[-program <executable>] // executable URI" )); NUnit.Framework.Assert.IsTrue(@out.ToString().Contains("[-reduces <num>] // number of reduces" )); NUnit.Framework.Assert.IsTrue(@out.ToString().Contains("[-lazyOutput <true/false>] // createOutputLazily" )); NUnit.Framework.Assert.IsTrue(@out.ToString().Contains("-conf <configuration file> specify an application configuration file" )); NUnit.Framework.Assert.IsTrue(@out.ToString().Contains("-D <property=value> use value for given property" )); NUnit.Framework.Assert.IsTrue(@out.ToString().Contains("-fs <local|namenode:port> specify a namenode" )); NUnit.Framework.Assert.IsTrue(@out.ToString().Contains("-jt <local|resourcemanager:port> specify a ResourceManager" )); NUnit.Framework.Assert.IsTrue(@out.ToString().Contains("-files <comma separated list of files> specify comma separated files to be copied to the map reduce cluster" )); NUnit.Framework.Assert.IsTrue(@out.ToString().Contains("-libjars <comma separated list of jars> specify comma separated jar files to include in the classpath." )); NUnit.Framework.Assert.IsTrue(@out.ToString().Contains("-archives <comma separated list of archives> specify comma separated archives to be unarchived on the compute machines." )); } finally { Runtime.SetOut(oldps); // restore Runtime.SetSecurityManager(securityManager); if (psw != null) { // remove password files foreach (FilePath file in psw) { file.DeleteOnExit(); } } } // test call Submitter form command line try { FilePath fCommand = GetFileCommand(null); string[] args = new string[22]; FilePath input = new FilePath(workSpace + FilePath.separator + "input"); if (!input.Exists()) { NUnit.Framework.Assert.IsTrue(input.CreateNewFile()); } FilePath outPut = new FilePath(workSpace + FilePath.separator + "output"); FileUtil.FullyDelete(outPut); args[0] = "-input"; args[1] = input.GetAbsolutePath(); // "input"; args[2] = "-output"; args[3] = outPut.GetAbsolutePath(); // "output"; args[4] = "-inputformat"; args[5] = "org.apache.hadoop.mapred.TextInputFormat"; args[6] = "-map"; args[7] = "org.apache.hadoop.mapred.lib.IdentityMapper"; args[8] = "-partitioner"; args[9] = "org.apache.hadoop.mapred.pipes.PipesPartitioner"; args[10] = "-reduce"; args[11] = "org.apache.hadoop.mapred.lib.IdentityReducer"; args[12] = "-writer"; args[13] = "org.apache.hadoop.mapred.TextOutputFormat"; args[14] = "-program"; args[15] = fCommand.GetAbsolutePath(); // "program"; args[16] = "-reduces"; args[17] = "2"; args[18] = "-lazyOutput"; args[19] = "lazyOutput"; args[20] = "-jobconf"; args[21] = "mapreduce.pipes.isjavarecordwriter=false,mapreduce.pipes.isjavarecordreader=false"; Submitter.Main(args); NUnit.Framework.Assert.Fail(); } catch (ExitUtil.ExitException e) { // status should be 0 NUnit.Framework.Assert.AreEqual(e.status, 0); } finally { Runtime.SetOut(oldps); Runtime.SetSecurityManager(securityManager); } }
/// <exception cref="System.IO.IOException"/> internal static void RunProgram(MiniMRCluster mr, MiniDFSCluster dfs, Path program , Path inputPath, Path outputPath, int numMaps, int numReduces, string[] expectedResults , JobConf conf) { Path wordExec = new Path("testing/bin/application"); JobConf job = null; if (conf == null) { job = mr.CreateJobConf(); } else { job = new JobConf(conf); } job.SetNumMapTasks(numMaps); job.SetNumReduceTasks(numReduces); { FileSystem fs = dfs.GetFileSystem(); fs.Delete(wordExec.GetParent(), true); fs.CopyFromLocalFile(program, wordExec); Submitter.SetExecutable(job, fs.MakeQualified(wordExec).ToString()); Submitter.SetIsJavaRecordReader(job, true); Submitter.SetIsJavaRecordWriter(job, true); FileInputFormat.SetInputPaths(job, inputPath); FileOutputFormat.SetOutputPath(job, outputPath); RunningJob rJob = null; if (numReduces == 0) { rJob = Submitter.JobSubmit(job); while (!rJob.IsComplete()) { try { Sharpen.Thread.Sleep(1000); } catch (Exception ie) { throw new RuntimeException(ie); } } } else { rJob = Submitter.RunJob(job); } NUnit.Framework.Assert.IsTrue("pipes job failed", rJob.IsSuccessful()); Counters counters = rJob.GetCounters(); Counters.Group wordCountCounters = counters.GetGroup("WORDCOUNT"); int numCounters = 0; foreach (Counters.Counter c in wordCountCounters) { System.Console.Out.WriteLine(c); ++numCounters; } NUnit.Framework.Assert.IsTrue("No counters found!", (numCounters > 0)); } IList <string> results = new AList <string>(); foreach (Path p in FileUtil.Stat2Paths(dfs.GetFileSystem().ListStatus(outputPath, new Utils.OutputFileUtils.OutputFilesFilter()))) { results.AddItem(MapReduceTestUtil.ReadOutput(p, job)); } NUnit.Framework.Assert.AreEqual("number of reduces is wrong", expectedResults.Length , results.Count); for (int i = 0; i < results.Count; i++) { NUnit.Framework.Assert.AreEqual("pipes program " + program + " output " + i + " wrong" , expectedResults[i], results[i]); } }