private void PrintCounters(StringBuilder buff, Counters totalCounters, Counters mapCounters
, Counters reduceCounters)
{
// Killed jobs might not have counters
if (totalCounters == null)
{
return;
}
buff.Append("\nCounters: \n\n");
buff.Append(string.Format("|%1$-30s|%2$-30s|%3$-10s|%4$-10s|%5$-10s|", "Group Name"
, "Counter name", "Map Value", "Reduce Value", "Total Value"));
buff.Append("\n------------------------------------------" + "---------------------------------------------"
);
foreach (string groupName in totalCounters.GetGroupNames())
{
CounterGroup totalGroup = totalCounters.GetGroup(groupName);
CounterGroup mapGroup = mapCounters.GetGroup(groupName);
CounterGroup reduceGroup = reduceCounters.GetGroup(groupName);
Format @decimal = new DecimalFormat();
IEnumerator <Counter> ctrItr = totalGroup.GetEnumerator();
while (ctrItr.HasNext())
{
Counter counter = ctrItr.Next();
string name = counter.GetName();
string mapValue = @decimal.Format(mapGroup.FindCounter(name).GetValue());
string reduceValue = @decimal.Format(reduceGroup.FindCounter(name).GetValue());
string totalValue = @decimal.Format(counter.GetValue());
buff.Append(string.Format("%n|%1$-30s|%2$-30s|%3$-10s|%4$-10s|%5$-10s", totalGroup
.GetDisplayName(), counter.GetDisplayName(), mapValue, reduceValue, totalValue));
}
}
}
public CounterGroupInfo(string name, CounterGroup group, CounterGroup mg, CounterGroup
rg)
{
this.counterGroupName = name;
this.counter = new AList <CounterInfo>();
foreach (Counter c in group)
{
Counter mc = mg == null ? null : mg.FindCounter(c.GetName());
Counter rc = rg == null ? null : rg.FindCounter(c.GetName());
CounterInfo cinfo = new CounterInfo(c, mc, rc);
this.counter.AddItem(cinfo);
}
}
/// <exception cref="System.Exception"/>
protected internal virtual void _testMOWithJavaSerialization(bool withCounters)
{
string input = "a\nb\nc\nd\ne\nc\nd\ne";
Configuration conf = CreateJobConf();
conf.Set("io.serializations", "org.apache.hadoop.io.serializer.JavaSerialization,"
+ "org.apache.hadoop.io.serializer.WritableSerialization");
Job job = MapReduceTestUtil.CreateJob(conf, InDir, OutDir, 2, 1, input);
job.SetJobName("mo");
MultipleOutputs.AddNamedOutput(job, Text, typeof(TextOutputFormat), typeof(long),
typeof(string));
MultipleOutputs.SetCountersEnabled(job, withCounters);
job.SetSortComparatorClass(typeof(JavaSerializationComparator));
job.SetMapOutputKeyClass(typeof(long));
job.SetMapOutputValueClass(typeof(string));
job.SetOutputKeyClass(typeof(long));
job.SetOutputValueClass(typeof(string));
job.SetMapperClass(typeof(TestMRMultipleOutputs.MOJavaSerDeMap));
job.SetReducerClass(typeof(TestMRMultipleOutputs.MOJavaSerDeReduce));
job.WaitForCompletion(true);
// assert number of named output part files
int namedOutputCount = 0;
int valueBasedOutputCount = 0;
FileSystem fs = OutDir.GetFileSystem(conf);
FileStatus[] statuses = fs.ListStatus(OutDir);
foreach (FileStatus status in statuses)
{
string fileName = status.GetPath().GetName();
if (fileName.Equals("text-m-00000") || fileName.Equals("text-m-00001") || fileName
.Equals("text-r-00000"))
{
namedOutputCount++;
}
else
{
if (fileName.Equals("a-r-00000") || fileName.Equals("b-r-00000") || fileName.Equals
("c-r-00000") || fileName.Equals("d-r-00000") || fileName.Equals("e-r-00000"))
{
valueBasedOutputCount++;
}
}
}
NUnit.Framework.Assert.AreEqual(3, namedOutputCount);
NUnit.Framework.Assert.AreEqual(5, valueBasedOutputCount);
// assert TextOutputFormat files correctness
BufferedReader reader = new BufferedReader(new InputStreamReader(fs.Open(new Path
(FileOutputFormat.GetOutputPath(job), "text-r-00000"))));
int count = 0;
string line = reader.ReadLine();
while (line != null)
{
NUnit.Framework.Assert.IsTrue(line.EndsWith(Text));
line = reader.ReadLine();
count++;
}
reader.Close();
NUnit.Framework.Assert.IsFalse(count == 0);
if (withCounters)
{
CounterGroup counters = job.GetCounters().GetGroup(typeof(MultipleOutputs).FullName
);
NUnit.Framework.Assert.AreEqual(6, counters.Size());
NUnit.Framework.Assert.AreEqual(4, counters.FindCounter(Text).GetValue());
NUnit.Framework.Assert.AreEqual(2, counters.FindCounter("a").GetValue());
NUnit.Framework.Assert.AreEqual(2, counters.FindCounter("b").GetValue());
NUnit.Framework.Assert.AreEqual(4, counters.FindCounter("c").GetValue());
NUnit.Framework.Assert.AreEqual(4, counters.FindCounter("d").GetValue());
NUnit.Framework.Assert.AreEqual(4, counters.FindCounter("e").GetValue());
}
}
/// <exception cref="System.Exception"/>
protected internal virtual void _testMultipleOutputs(bool withCounters)
{
string input = "a\nb\nc\nd\ne\nc\nd\ne";
Configuration conf = CreateJobConf();
Job job = MapReduceTestUtil.CreateJob(conf, InDir, OutDir, 2, 1, input);
job.SetJobName("mo");
MultipleOutputs.AddNamedOutput(job, Text, typeof(TextOutputFormat), typeof(LongWritable
), typeof(Text));
MultipleOutputs.AddNamedOutput(job, Sequence, typeof(SequenceFileOutputFormat), typeof(
IntWritable), typeof(Text));
MultipleOutputs.SetCountersEnabled(job, withCounters);
job.SetMapperClass(typeof(TestMRMultipleOutputs.MOMap));
job.SetReducerClass(typeof(TestMRMultipleOutputs.MOReduce));
job.WaitForCompletion(true);
// assert number of named output part files
int namedOutputCount = 0;
int valueBasedOutputCount = 0;
FileSystem fs = OutDir.GetFileSystem(conf);
FileStatus[] statuses = fs.ListStatus(OutDir);
foreach (FileStatus status in statuses)
{
string fileName = status.GetPath().GetName();
if (fileName.Equals("text-m-00000") || fileName.Equals("text-m-00001") || fileName
.Equals("text-r-00000") || fileName.Equals("sequence_A-m-00000") || fileName.Equals
("sequence_A-m-00001") || fileName.Equals("sequence_B-m-00000") || fileName.Equals
("sequence_B-m-00001") || fileName.Equals("sequence_B-r-00000") || fileName.Equals
("sequence_C-r-00000"))
{
namedOutputCount++;
}
else
{
if (fileName.Equals("a-r-00000") || fileName.Equals("b-r-00000") || fileName.Equals
("c-r-00000") || fileName.Equals("d-r-00000") || fileName.Equals("e-r-00000"))
{
valueBasedOutputCount++;
}
}
}
NUnit.Framework.Assert.AreEqual(9, namedOutputCount);
NUnit.Framework.Assert.AreEqual(5, valueBasedOutputCount);
// assert TextOutputFormat files correctness
BufferedReader reader = new BufferedReader(new InputStreamReader(fs.Open(new Path
(FileOutputFormat.GetOutputPath(job), "text-r-00000"))));
int count = 0;
string line = reader.ReadLine();
while (line != null)
{
NUnit.Framework.Assert.IsTrue(line.EndsWith(Text));
line = reader.ReadLine();
count++;
}
reader.Close();
NUnit.Framework.Assert.IsFalse(count == 0);
// assert SequenceOutputFormat files correctness
SequenceFile.Reader seqReader = new SequenceFile.Reader(fs, new Path(FileOutputFormat
.GetOutputPath(job), "sequence_B-r-00000"), conf);
NUnit.Framework.Assert.AreEqual(typeof(IntWritable), seqReader.GetKeyClass());
NUnit.Framework.Assert.AreEqual(typeof(Text), seqReader.GetValueClass());
count = 0;
IntWritable key = new IntWritable();
Text value = new Text();
while (seqReader.Next(key, value))
{
NUnit.Framework.Assert.AreEqual(Sequence, value.ToString());
count++;
}
seqReader.Close();
NUnit.Framework.Assert.IsFalse(count == 0);
if (withCounters)
{
CounterGroup counters = job.GetCounters().GetGroup(typeof(MultipleOutputs).FullName
);
NUnit.Framework.Assert.AreEqual(9, counters.Size());
NUnit.Framework.Assert.AreEqual(4, counters.FindCounter(Text).GetValue());
NUnit.Framework.Assert.AreEqual(2, counters.FindCounter(Sequence + "_A").GetValue
());
NUnit.Framework.Assert.AreEqual(4, counters.FindCounter(Sequence + "_B").GetValue
());
NUnit.Framework.Assert.AreEqual(2, counters.FindCounter(Sequence + "_C").GetValue
());
NUnit.Framework.Assert.AreEqual(2, counters.FindCounter("a").GetValue());
NUnit.Framework.Assert.AreEqual(2, counters.FindCounter("b").GetValue());
NUnit.Framework.Assert.AreEqual(4, counters.FindCounter("c").GetValue());
NUnit.Framework.Assert.AreEqual(4, counters.FindCounter("d").GetValue());
NUnit.Framework.Assert.AreEqual(4, counters.FindCounter("e").GetValue());
}
}
protected override void Render(HtmlBlock.Block html)
{
if (job == null)
{
html.P().("Sorry, no counters for nonexistent", $(AMParams.JobId, "job")).();
return;
}
if (!$(AMParams.TaskId).IsEmpty() && task == null)
{
html.P().("Sorry, no counters for nonexistent", $(AMParams.TaskId, "task")).();
return;
}
if (total == null || total.GetGroupNames() == null || total.CountCounters() == 0)
{
string type = $(AMParams.TaskId);
if (type == null || type.IsEmpty())
{
type = $(AMParams.JobId, "the job");
}
html.P().("Sorry it looks like ", type, " has no counters.").();
return;
}
string urlBase;
string urlId;
if (task != null)
{
urlBase = "singletaskcounter";
urlId = MRApps.ToString(task.GetID());
}
else
{
urlBase = "singlejobcounter";
urlId = MRApps.ToString(job.GetID());
}
int numGroups = 0;
Hamlet.TBODY <Hamlet.TABLE <Hamlet.DIV <Org.Apache.Hadoop.Yarn.Webapp.Hamlet.Hamlet>
> > tbody = html.Div(JQueryUI.InfoWrap).Table("#counters").Thead().Tr().Th(".group.ui-state-default"
, "Counter Group").Th(".ui-state-default", "Counters").().().Tbody();
foreach (CounterGroup g in total)
{
CounterGroup mg = map == null ? null : map.GetGroup(g.GetName());
CounterGroup rg = reduce == null ? null : reduce.GetGroup(g.GetName());
++numGroups;
// This is mostly for demonstration :) Typically we'd introduced
// a CounterGroup block to reduce the verbosity. OTOH, this
// serves as an indicator of where we're in the tag hierarchy.
Hamlet.TR <Hamlet.THEAD <Hamlet.TABLE <Hamlet.TD <Hamlet.TR <Hamlet.TBODY <Hamlet.TABLE
<Hamlet.DIV <Org.Apache.Hadoop.Yarn.Webapp.Hamlet.Hamlet> > > > > > > > groupHeadRow = tbody
.Tr().Th().$title(g.GetName()).$class("ui-state-default").(FixGroupDisplayName(g
.GetDisplayName())).().Td().$class(JQueryUI.CTable).Table(".dt-counters").$id(job
.GetID() + "." + g.GetName()).Thead().Tr().Th(".name", "Name");
if (map != null)
{
groupHeadRow.Th("Map").Th("Reduce");
}
// Ditto
Hamlet.TBODY <Hamlet.TABLE <Hamlet.TD <Hamlet.TR <Hamlet.TBODY <Hamlet.TABLE <Hamlet.DIV
<Org.Apache.Hadoop.Yarn.Webapp.Hamlet.Hamlet> > > > > > > group = groupHeadRow.Th(map
== null ? "Value" : "Total").().().Tbody();
foreach (Counter counter in g)
{
// Ditto
Hamlet.TR <Hamlet.TBODY <Hamlet.TABLE <Hamlet.TD <Hamlet.TR <Hamlet.TBODY <Hamlet.TABLE
<Hamlet.DIV <Org.Apache.Hadoop.Yarn.Webapp.Hamlet.Hamlet> > > > > > > > groupRow = group
.Tr();
if (task == null && mg == null && rg == null)
{
groupRow.Td().$title(counter.GetName()).(counter.GetDisplayName()).();
}
else
{
groupRow.Td().$title(counter.GetName()).A(Url(urlBase, urlId, g.GetName(), counter
.GetName()), counter.GetDisplayName()).();
}
if (map != null)
{
Counter mc = mg == null ? null : mg.FindCounter(counter.GetName());
Counter rc = rg == null ? null : rg.FindCounter(counter.GetName());
groupRow.Td(mc == null ? "0" : string.Format("%,d", mc.GetValue())).Td(rc == null
? "0" : string.Format("%,d", rc.GetValue()));
}
groupRow.Td(string.Format("%,d", counter.GetValue())).();
}
group.().().().();
}
tbody.().().();
}
private void PopulateMembers(AppContext ctx)
{
JobId jobID = null;
TaskId taskID = null;
string tid = $(AMParams.TaskId);
if (!tid.IsEmpty())
{
taskID = MRApps.ToTaskID(tid);
jobID = taskID.GetJobId();
}
else
{
string jid = $(AMParams.JobId);
if (!jid.IsEmpty())
{
jobID = MRApps.ToJobID(jid);
}
}
if (jobID == null)
{
return;
}
job = ctx.GetJob(jobID);
if (job == null)
{
return;
}
if (taskID != null)
{
task = job.GetTask(taskID);
if (task == null)
{
return;
}
foreach (KeyValuePair <TaskAttemptId, TaskAttempt> entry in task.GetAttempts())
{
long value = 0;
Counters counters = entry.Value.GetCounters();
CounterGroup group = (counters != null) ? counters.GetGroup($(AMParams.CounterGroup
)) : null;
if (group != null)
{
Counter c = group.FindCounter($(AMParams.CounterName));
if (c != null)
{
value = c.GetValue();
}
}
values[MRApps.ToString(entry.Key)] = value;
}
return;
}
// Get all types of counters
IDictionary <TaskId, Task> tasks = job.GetTasks();
foreach (KeyValuePair <TaskId, Task> entry_1 in tasks)
{
long value = 0;
Counters counters = entry_1.Value.GetCounters();
CounterGroup group = (counters != null) ? counters.GetGroup($(AMParams.CounterGroup
)) : null;
if (group != null)
{
Counter c = group.FindCounter($(AMParams.CounterName));
if (c != null)
{
value = c.GetValue();
}
}
values[MRApps.ToString(entry_1.Key)] = value;
}
}
