private void PrintCounters(StringBuilder buff, Counters totalCounters, Counters mapCounters , Counters reduceCounters) { // Killed jobs might not have counters if (totalCounters == null) { return; } buff.Append("\nCounters: \n\n"); buff.Append(string.Format("|%1$-30s|%2$-30s|%3$-10s|%4$-10s|%5$-10s|", "Group Name" , "Counter name", "Map Value", "Reduce Value", "Total Value")); buff.Append("\n------------------------------------------" + "---------------------------------------------" ); foreach (string groupName in totalCounters.GetGroupNames()) { CounterGroup totalGroup = totalCounters.GetGroup(groupName); CounterGroup mapGroup = mapCounters.GetGroup(groupName); CounterGroup reduceGroup = reduceCounters.GetGroup(groupName); Format @decimal = new DecimalFormat(); IEnumerator <Counter> ctrItr = totalGroup.GetEnumerator(); while (ctrItr.HasNext()) { Counter counter = ctrItr.Next(); string name = counter.GetName(); string mapValue = @decimal.Format(mapGroup.FindCounter(name).GetValue()); string reduceValue = @decimal.Format(reduceGroup.FindCounter(name).GetValue()); string totalValue = @decimal.Format(counter.GetValue()); buff.Append(string.Format("%n|%1$-30s|%2$-30s|%3$-10s|%4$-10s|%5$-10s", totalGroup .GetDisplayName(), counter.GetDisplayName(), mapValue, reduceValue, totalValue)); } } }
public CounterGroupInfo(string name, CounterGroup group, CounterGroup mg, CounterGroup rg) { this.counterGroupName = name; this.counter = new AList <CounterInfo>(); foreach (Counter c in group) { Counter mc = mg == null ? null : mg.FindCounter(c.GetName()); Counter rc = rg == null ? null : rg.FindCounter(c.GetName()); CounterInfo cinfo = new CounterInfo(c, mc, rc); this.counter.AddItem(cinfo); } }
/// <exception cref="System.Exception"/> protected internal virtual void _testMOWithJavaSerialization(bool withCounters) { string input = "a\nb\nc\nd\ne\nc\nd\ne"; Configuration conf = CreateJobConf(); conf.Set("io.serializations", "org.apache.hadoop.io.serializer.JavaSerialization," + "org.apache.hadoop.io.serializer.WritableSerialization"); Job job = MapReduceTestUtil.CreateJob(conf, InDir, OutDir, 2, 1, input); job.SetJobName("mo"); MultipleOutputs.AddNamedOutput(job, Text, typeof(TextOutputFormat), typeof(long), typeof(string)); MultipleOutputs.SetCountersEnabled(job, withCounters); job.SetSortComparatorClass(typeof(JavaSerializationComparator)); job.SetMapOutputKeyClass(typeof(long)); job.SetMapOutputValueClass(typeof(string)); job.SetOutputKeyClass(typeof(long)); job.SetOutputValueClass(typeof(string)); job.SetMapperClass(typeof(TestMRMultipleOutputs.MOJavaSerDeMap)); job.SetReducerClass(typeof(TestMRMultipleOutputs.MOJavaSerDeReduce)); job.WaitForCompletion(true); // assert number of named output part files int namedOutputCount = 0; int valueBasedOutputCount = 0; FileSystem fs = OutDir.GetFileSystem(conf); FileStatus[] statuses = fs.ListStatus(OutDir); foreach (FileStatus status in statuses) { string fileName = status.GetPath().GetName(); if (fileName.Equals("text-m-00000") || fileName.Equals("text-m-00001") || fileName .Equals("text-r-00000")) { namedOutputCount++; } else { if (fileName.Equals("a-r-00000") || fileName.Equals("b-r-00000") || fileName.Equals ("c-r-00000") || fileName.Equals("d-r-00000") || fileName.Equals("e-r-00000")) { valueBasedOutputCount++; } } } NUnit.Framework.Assert.AreEqual(3, namedOutputCount); NUnit.Framework.Assert.AreEqual(5, valueBasedOutputCount); // assert TextOutputFormat files correctness BufferedReader reader = new BufferedReader(new InputStreamReader(fs.Open(new Path (FileOutputFormat.GetOutputPath(job), "text-r-00000")))); int count = 0; string line = reader.ReadLine(); while (line != null) { NUnit.Framework.Assert.IsTrue(line.EndsWith(Text)); line = reader.ReadLine(); count++; } reader.Close(); NUnit.Framework.Assert.IsFalse(count == 0); if (withCounters) { CounterGroup counters = job.GetCounters().GetGroup(typeof(MultipleOutputs).FullName ); NUnit.Framework.Assert.AreEqual(6, counters.Size()); NUnit.Framework.Assert.AreEqual(4, counters.FindCounter(Text).GetValue()); NUnit.Framework.Assert.AreEqual(2, counters.FindCounter("a").GetValue()); NUnit.Framework.Assert.AreEqual(2, counters.FindCounter("b").GetValue()); NUnit.Framework.Assert.AreEqual(4, counters.FindCounter("c").GetValue()); NUnit.Framework.Assert.AreEqual(4, counters.FindCounter("d").GetValue()); NUnit.Framework.Assert.AreEqual(4, counters.FindCounter("e").GetValue()); } }
/// <exception cref="System.Exception"/> protected internal virtual void _testMultipleOutputs(bool withCounters) { string input = "a\nb\nc\nd\ne\nc\nd\ne"; Configuration conf = CreateJobConf(); Job job = MapReduceTestUtil.CreateJob(conf, InDir, OutDir, 2, 1, input); job.SetJobName("mo"); MultipleOutputs.AddNamedOutput(job, Text, typeof(TextOutputFormat), typeof(LongWritable ), typeof(Text)); MultipleOutputs.AddNamedOutput(job, Sequence, typeof(SequenceFileOutputFormat), typeof( IntWritable), typeof(Text)); MultipleOutputs.SetCountersEnabled(job, withCounters); job.SetMapperClass(typeof(TestMRMultipleOutputs.MOMap)); job.SetReducerClass(typeof(TestMRMultipleOutputs.MOReduce)); job.WaitForCompletion(true); // assert number of named output part files int namedOutputCount = 0; int valueBasedOutputCount = 0; FileSystem fs = OutDir.GetFileSystem(conf); FileStatus[] statuses = fs.ListStatus(OutDir); foreach (FileStatus status in statuses) { string fileName = status.GetPath().GetName(); if (fileName.Equals("text-m-00000") || fileName.Equals("text-m-00001") || fileName .Equals("text-r-00000") || fileName.Equals("sequence_A-m-00000") || fileName.Equals ("sequence_A-m-00001") || fileName.Equals("sequence_B-m-00000") || fileName.Equals ("sequence_B-m-00001") || fileName.Equals("sequence_B-r-00000") || fileName.Equals ("sequence_C-r-00000")) { namedOutputCount++; } else { if (fileName.Equals("a-r-00000") || fileName.Equals("b-r-00000") || fileName.Equals ("c-r-00000") || fileName.Equals("d-r-00000") || fileName.Equals("e-r-00000")) { valueBasedOutputCount++; } } } NUnit.Framework.Assert.AreEqual(9, namedOutputCount); NUnit.Framework.Assert.AreEqual(5, valueBasedOutputCount); // assert TextOutputFormat files correctness BufferedReader reader = new BufferedReader(new InputStreamReader(fs.Open(new Path (FileOutputFormat.GetOutputPath(job), "text-r-00000")))); int count = 0; string line = reader.ReadLine(); while (line != null) { NUnit.Framework.Assert.IsTrue(line.EndsWith(Text)); line = reader.ReadLine(); count++; } reader.Close(); NUnit.Framework.Assert.IsFalse(count == 0); // assert SequenceOutputFormat files correctness SequenceFile.Reader seqReader = new SequenceFile.Reader(fs, new Path(FileOutputFormat .GetOutputPath(job), "sequence_B-r-00000"), conf); NUnit.Framework.Assert.AreEqual(typeof(IntWritable), seqReader.GetKeyClass()); NUnit.Framework.Assert.AreEqual(typeof(Text), seqReader.GetValueClass()); count = 0; IntWritable key = new IntWritable(); Text value = new Text(); while (seqReader.Next(key, value)) { NUnit.Framework.Assert.AreEqual(Sequence, value.ToString()); count++; } seqReader.Close(); NUnit.Framework.Assert.IsFalse(count == 0); if (withCounters) { CounterGroup counters = job.GetCounters().GetGroup(typeof(MultipleOutputs).FullName ); NUnit.Framework.Assert.AreEqual(9, counters.Size()); NUnit.Framework.Assert.AreEqual(4, counters.FindCounter(Text).GetValue()); NUnit.Framework.Assert.AreEqual(2, counters.FindCounter(Sequence + "_A").GetValue ()); NUnit.Framework.Assert.AreEqual(4, counters.FindCounter(Sequence + "_B").GetValue ()); NUnit.Framework.Assert.AreEqual(2, counters.FindCounter(Sequence + "_C").GetValue ()); NUnit.Framework.Assert.AreEqual(2, counters.FindCounter("a").GetValue()); NUnit.Framework.Assert.AreEqual(2, counters.FindCounter("b").GetValue()); NUnit.Framework.Assert.AreEqual(4, counters.FindCounter("c").GetValue()); NUnit.Framework.Assert.AreEqual(4, counters.FindCounter("d").GetValue()); NUnit.Framework.Assert.AreEqual(4, counters.FindCounter("e").GetValue()); } }
protected override void Render(HtmlBlock.Block html) { if (job == null) { html.P().("Sorry, no counters for nonexistent", $(AMParams.JobId, "job")).(); return; } if (!$(AMParams.TaskId).IsEmpty() && task == null) { html.P().("Sorry, no counters for nonexistent", $(AMParams.TaskId, "task")).(); return; } if (total == null || total.GetGroupNames() == null || total.CountCounters() == 0) { string type = $(AMParams.TaskId); if (type == null || type.IsEmpty()) { type = $(AMParams.JobId, "the job"); } html.P().("Sorry it looks like ", type, " has no counters.").(); return; } string urlBase; string urlId; if (task != null) { urlBase = "singletaskcounter"; urlId = MRApps.ToString(task.GetID()); } else { urlBase = "singlejobcounter"; urlId = MRApps.ToString(job.GetID()); } int numGroups = 0; Hamlet.TBODY <Hamlet.TABLE <Hamlet.DIV <Org.Apache.Hadoop.Yarn.Webapp.Hamlet.Hamlet> > > tbody = html.Div(JQueryUI.InfoWrap).Table("#counters").Thead().Tr().Th(".group.ui-state-default" , "Counter Group").Th(".ui-state-default", "Counters").().().Tbody(); foreach (CounterGroup g in total) { CounterGroup mg = map == null ? null : map.GetGroup(g.GetName()); CounterGroup rg = reduce == null ? null : reduce.GetGroup(g.GetName()); ++numGroups; // This is mostly for demonstration :) Typically we'd introduced // a CounterGroup block to reduce the verbosity. OTOH, this // serves as an indicator of where we're in the tag hierarchy. Hamlet.TR <Hamlet.THEAD <Hamlet.TABLE <Hamlet.TD <Hamlet.TR <Hamlet.TBODY <Hamlet.TABLE <Hamlet.DIV <Org.Apache.Hadoop.Yarn.Webapp.Hamlet.Hamlet> > > > > > > > groupHeadRow = tbody .Tr().Th().$title(g.GetName()).$class("ui-state-default").(FixGroupDisplayName(g .GetDisplayName())).().Td().$class(JQueryUI.CTable).Table(".dt-counters").$id(job .GetID() + "." + g.GetName()).Thead().Tr().Th(".name", "Name"); if (map != null) { groupHeadRow.Th("Map").Th("Reduce"); } // Ditto Hamlet.TBODY <Hamlet.TABLE <Hamlet.TD <Hamlet.TR <Hamlet.TBODY <Hamlet.TABLE <Hamlet.DIV <Org.Apache.Hadoop.Yarn.Webapp.Hamlet.Hamlet> > > > > > > group = groupHeadRow.Th(map == null ? "Value" : "Total").().().Tbody(); foreach (Counter counter in g) { // Ditto Hamlet.TR <Hamlet.TBODY <Hamlet.TABLE <Hamlet.TD <Hamlet.TR <Hamlet.TBODY <Hamlet.TABLE <Hamlet.DIV <Org.Apache.Hadoop.Yarn.Webapp.Hamlet.Hamlet> > > > > > > > groupRow = group .Tr(); if (task == null && mg == null && rg == null) { groupRow.Td().$title(counter.GetName()).(counter.GetDisplayName()).(); } else { groupRow.Td().$title(counter.GetName()).A(Url(urlBase, urlId, g.GetName(), counter .GetName()), counter.GetDisplayName()).(); } if (map != null) { Counter mc = mg == null ? null : mg.FindCounter(counter.GetName()); Counter rc = rg == null ? null : rg.FindCounter(counter.GetName()); groupRow.Td(mc == null ? "0" : string.Format("%,d", mc.GetValue())).Td(rc == null ? "0" : string.Format("%,d", rc.GetValue())); } groupRow.Td(string.Format("%,d", counter.GetValue())).(); } group.().().().(); } tbody.().().(); }
private void PopulateMembers(AppContext ctx) { JobId jobID = null; TaskId taskID = null; string tid = $(AMParams.TaskId); if (!tid.IsEmpty()) { taskID = MRApps.ToTaskID(tid); jobID = taskID.GetJobId(); } else { string jid = $(AMParams.JobId); if (!jid.IsEmpty()) { jobID = MRApps.ToJobID(jid); } } if (jobID == null) { return; } job = ctx.GetJob(jobID); if (job == null) { return; } if (taskID != null) { task = job.GetTask(taskID); if (task == null) { return; } foreach (KeyValuePair <TaskAttemptId, TaskAttempt> entry in task.GetAttempts()) { long value = 0; Counters counters = entry.Value.GetCounters(); CounterGroup group = (counters != null) ? counters.GetGroup($(AMParams.CounterGroup )) : null; if (group != null) { Counter c = group.FindCounter($(AMParams.CounterName)); if (c != null) { value = c.GetValue(); } } values[MRApps.ToString(entry.Key)] = value; } return; } // Get all types of counters IDictionary <TaskId, Task> tasks = job.GetTasks(); foreach (KeyValuePair <TaskId, Task> entry_1 in tasks) { long value = 0; Counters counters = entry_1.Value.GetCounters(); CounterGroup group = (counters != null) ? counters.GetGroup($(AMParams.CounterGroup )) : null; if (group != null) { Counter c = group.FindCounter($(AMParams.CounterName)); if (c != null) { value = c.GetValue(); } } values[MRApps.ToString(entry_1.Key)] = value; } }