public override Counters.Counter GetCounter(string group, string name) { Counters.Counter counter = null; if (this.counters != null) { counter = this.counters.FindCounter(group, name); if (counter == null) { Counters.Group grp = this.counters.AddGroup(group, group); counter = grp.AddCounter(name, name, 10); } } return(counter); }
/// <exception cref="System.Exception"/> protected internal virtual void _testMOWithJavaSerialization(bool withCounters) { Path inDir = GetDir(InDir); Path outDir = GetDir(OutDir); JobConf conf = CreateJobConf(); FileSystem fs = FileSystem.Get(conf); DataOutputStream file = fs.Create(new Path(inDir, "part-0")); file.WriteBytes("a\nb\n\nc\nd\ne"); file.Close(); fs.Delete(inDir, true); fs.Delete(outDir, true); file = fs.Create(new Path(inDir, "part-1")); file.WriteBytes("a\nb\n\nc\nd\ne"); file.Close(); conf.SetJobName("mo"); conf.Set("io.serializations", "org.apache.hadoop.io.serializer.JavaSerialization," + "org.apache.hadoop.io.serializer.WritableSerialization"); conf.SetInputFormat(typeof(TextInputFormat)); conf.SetMapOutputKeyClass(typeof(long)); conf.SetMapOutputValueClass(typeof(string)); conf.SetOutputKeyComparatorClass(typeof(JavaSerializationComparator)); conf.SetOutputKeyClass(typeof(long)); conf.SetOutputValueClass(typeof(string)); conf.SetOutputFormat(typeof(TextOutputFormat)); MultipleOutputs.AddNamedOutput(conf, "text", typeof(TextOutputFormat), typeof(long ), typeof(string)); MultipleOutputs.SetCountersEnabled(conf, withCounters); conf.SetMapperClass(typeof(TestMultipleOutputs.MOJavaSerDeMap)); conf.SetReducerClass(typeof(TestMultipleOutputs.MOJavaSerDeReduce)); FileInputFormat.SetInputPaths(conf, inDir); FileOutputFormat.SetOutputPath(conf, outDir); JobClient jc = new JobClient(conf); RunningJob job = jc.SubmitJob(conf); while (!job.IsComplete()) { Sharpen.Thread.Sleep(100); } // assert number of named output part files int namedOutputCount = 0; FileStatus[] statuses = fs.ListStatus(outDir); foreach (FileStatus status in statuses) { if (status.GetPath().GetName().Equals("text-m-00000") || status.GetPath().GetName ().Equals("text-r-00000")) { namedOutputCount++; } } NUnit.Framework.Assert.AreEqual(2, namedOutputCount); // assert TextOutputFormat files correctness BufferedReader reader = new BufferedReader(new InputStreamReader(fs.Open(new Path (FileOutputFormat.GetOutputPath(conf), "text-r-00000")))); int count = 0; string line = reader.ReadLine(); while (line != null) { NUnit.Framework.Assert.IsTrue(line.EndsWith("text")); line = reader.ReadLine(); count++; } reader.Close(); NUnit.Framework.Assert.IsFalse(count == 0); Counters.Group counters = job.GetCounters().GetGroup(typeof(MultipleOutputs).FullName ); if (!withCounters) { NUnit.Framework.Assert.AreEqual(0, counters.Size()); } else { NUnit.Framework.Assert.AreEqual(1, counters.Size()); NUnit.Framework.Assert.AreEqual(2, counters.GetCounter("text")); } }
/// <exception cref="System.IO.IOException"/> internal static void RunProgram(MiniMRCluster mr, MiniDFSCluster dfs, Path program , Path inputPath, Path outputPath, int numMaps, int numReduces, string[] expectedResults , JobConf conf) { Path wordExec = new Path("testing/bin/application"); JobConf job = null; if (conf == null) { job = mr.CreateJobConf(); } else { job = new JobConf(conf); } job.SetNumMapTasks(numMaps); job.SetNumReduceTasks(numReduces); { FileSystem fs = dfs.GetFileSystem(); fs.Delete(wordExec.GetParent(), true); fs.CopyFromLocalFile(program, wordExec); Submitter.SetExecutable(job, fs.MakeQualified(wordExec).ToString()); Submitter.SetIsJavaRecordReader(job, true); Submitter.SetIsJavaRecordWriter(job, true); FileInputFormat.SetInputPaths(job, inputPath); FileOutputFormat.SetOutputPath(job, outputPath); RunningJob rJob = null; if (numReduces == 0) { rJob = Submitter.JobSubmit(job); while (!rJob.IsComplete()) { try { Sharpen.Thread.Sleep(1000); } catch (Exception ie) { throw new RuntimeException(ie); } } } else { rJob = Submitter.RunJob(job); } NUnit.Framework.Assert.IsTrue("pipes job failed", rJob.IsSuccessful()); Counters counters = rJob.GetCounters(); Counters.Group wordCountCounters = counters.GetGroup("WORDCOUNT"); int numCounters = 0; foreach (Counters.Counter c in wordCountCounters) { System.Console.Out.WriteLine(c); ++numCounters; } NUnit.Framework.Assert.IsTrue("No counters found!", (numCounters > 0)); } IList <string> results = new AList <string>(); foreach (Path p in FileUtil.Stat2Paths(dfs.GetFileSystem().ListStatus(outputPath, new Utils.OutputFileUtils.OutputFilesFilter()))) { results.AddItem(MapReduceTestUtil.ReadOutput(p, job)); } NUnit.Framework.Assert.AreEqual("number of reduces is wrong", expectedResults.Length , results.Count); for (int i = 0; i < results.Count; i++) { NUnit.Framework.Assert.AreEqual("pipes program " + program + " output " + i + " wrong" , expectedResults[i], results[i]); } }
/// <exception cref="System.Exception"/> protected internal virtual void _testMultipleOutputs(bool withCounters) { Path inDir = GetDir(InDir); Path outDir = GetDir(OutDir); JobConf conf = CreateJobConf(); FileSystem fs = FileSystem.Get(conf); DataOutputStream file = fs.Create(new Path(inDir, "part-0")); file.WriteBytes("a\nb\n\nc\nd\ne"); file.Close(); file = fs.Create(new Path(inDir, "part-1")); file.WriteBytes("a\nb\n\nc\nd\ne"); file.Close(); conf.SetJobName("mo"); conf.SetInputFormat(typeof(TextInputFormat)); conf.SetOutputKeyClass(typeof(LongWritable)); conf.SetOutputValueClass(typeof(Text)); conf.SetMapOutputKeyClass(typeof(LongWritable)); conf.SetMapOutputValueClass(typeof(Text)); conf.SetOutputFormat(typeof(TextOutputFormat)); MultipleOutputs.AddNamedOutput(conf, "text", typeof(TextOutputFormat), typeof(LongWritable ), typeof(Text)); MultipleOutputs.AddMultiNamedOutput(conf, "sequence", typeof(SequenceFileOutputFormat ), typeof(LongWritable), typeof(Text)); MultipleOutputs.SetCountersEnabled(conf, withCounters); conf.SetMapperClass(typeof(TestMultipleOutputs.MOMap)); conf.SetReducerClass(typeof(TestMultipleOutputs.MOReduce)); FileInputFormat.SetInputPaths(conf, inDir); FileOutputFormat.SetOutputPath(conf, outDir); JobClient jc = new JobClient(conf); RunningJob job = jc.SubmitJob(conf); while (!job.IsComplete()) { Sharpen.Thread.Sleep(100); } // assert number of named output part files int namedOutputCount = 0; FileStatus[] statuses = fs.ListStatus(outDir); foreach (FileStatus status in statuses) { if (status.GetPath().GetName().Equals("text-m-00000") || status.GetPath().GetName ().Equals("text-m-00001") || status.GetPath().GetName().Equals("text-r-00000") || status.GetPath().GetName().Equals("sequence_A-m-00000") || status.GetPath().GetName ().Equals("sequence_A-m-00001") || status.GetPath().GetName().Equals("sequence_B-m-00000" ) || status.GetPath().GetName().Equals("sequence_B-m-00001") || status.GetPath() .GetName().Equals("sequence_B-r-00000") || status.GetPath().GetName().Equals("sequence_C-r-00000" )) { namedOutputCount++; } } NUnit.Framework.Assert.AreEqual(9, namedOutputCount); // assert TextOutputFormat files correctness BufferedReader reader = new BufferedReader(new InputStreamReader(fs.Open(new Path (FileOutputFormat.GetOutputPath(conf), "text-r-00000")))); int count = 0; string line = reader.ReadLine(); while (line != null) { NUnit.Framework.Assert.IsTrue(line.EndsWith("text")); line = reader.ReadLine(); count++; } reader.Close(); NUnit.Framework.Assert.IsFalse(count == 0); // assert SequenceOutputFormat files correctness SequenceFile.Reader seqReader = new SequenceFile.Reader(fs, new Path(FileOutputFormat .GetOutputPath(conf), "sequence_B-r-00000"), conf); NUnit.Framework.Assert.AreEqual(typeof(LongWritable), seqReader.GetKeyClass()); NUnit.Framework.Assert.AreEqual(typeof(Text), seqReader.GetValueClass()); count = 0; LongWritable key = new LongWritable(); Text value = new Text(); while (seqReader.Next(key, value)) { NUnit.Framework.Assert.AreEqual("sequence", value.ToString()); count++; } seqReader.Close(); NUnit.Framework.Assert.IsFalse(count == 0); Counters.Group counters = job.GetCounters().GetGroup(typeof(MultipleOutputs).FullName ); if (!withCounters) { NUnit.Framework.Assert.AreEqual(0, counters.Size()); } else { NUnit.Framework.Assert.AreEqual(4, counters.Size()); NUnit.Framework.Assert.AreEqual(4, counters.GetCounter("text")); NUnit.Framework.Assert.AreEqual(2, counters.GetCounter("sequence_A")); NUnit.Framework.Assert.AreEqual(4, counters.GetCounter("sequence_B")); NUnit.Framework.Assert.AreEqual(2, counters.GetCounter("sequence_C")); } }