public virtual void TestCountersIncrement()
{
Counters fCounters = new Counters();
Counter fCounter = fCounters.FindCounter(FrameworkCounter);
fCounter.SetValue(100);
Counter gCounter = fCounters.FindCounter("test", "foo");
gCounter.SetValue(200);
Counters counters = new Counters();
counters.IncrAllCounters(fCounters);
Counter counter;
foreach (CounterGroup cg in fCounters)
{
CounterGroup group = counters.GetGroup(cg.GetName());
if (group.GetName().Equals("test"))
{
counter = counters.FindCounter("test", "foo");
NUnit.Framework.Assert.AreEqual(200, counter.GetValue());
}
else
{
counter = counters.FindCounter(FrameworkCounter);
NUnit.Framework.Assert.AreEqual(100, counter.GetValue());
}
}
}
private void TestMaxCounters(Counters counters)
{
Log.Info("counters max=" + Limits.GetCountersMax());
for (int i = 0; i < Limits.GetCountersMax(); ++i)
{
counters.FindCounter("test", "test" + i);
}
SetExpected(counters);
ShouldThrow(typeof(LimitExceededException), new _Runnable_109(counters));
CheckExpected(counters);
}
public static Counters FromYarn(Counters yCntrs)
{
if (yCntrs == null)
{
return(null);
}
Counters counters = new Counters();
foreach (CounterGroup yGrp in yCntrs.GetAllCounterGroups().Values)
{
counters.AddGroup(yGrp.GetName(), yGrp.GetDisplayName());
foreach (Counter yCntr in yGrp.GetAllCounters().Values)
{
Counter c = counters.FindCounter(yGrp.GetName(), yCntr.GetName());
// if c can be found, or it will be skipped.
if (c != null)
{
c.SetValue(yCntr.GetValue());
}
}
}
return(counters);
}
public virtual void TestCombiner()
{
if (!new FilePath(TestRootDir).Mkdirs())
{
throw new RuntimeException("Could not create test dir: " + TestRootDir);
}
FilePath @in = new FilePath(TestRootDir, "input");
if ([email protected]())
{
throw new RuntimeException("Could not create test dir: " + @in);
}
FilePath @out = new FilePath(TestRootDir, "output");
PrintWriter pw = new PrintWriter(new FileWriter(new FilePath(@in, "data.txt")));
pw.WriteLine("A|a,1");
pw.WriteLine("A|b,2");
pw.WriteLine("B|a,3");
pw.WriteLine("B|b,4");
pw.WriteLine("B|c,5");
pw.Close();
JobConf conf = new JobConf();
conf.Set("mapreduce.framework.name", "local");
Job job = new Job(conf);
TextInputFormat.SetInputPaths(job, new Path(@in.GetPath()));
TextOutputFormat.SetOutputPath(job, new Path(@out.GetPath()));
job.SetMapperClass(typeof(TestNewCombinerGrouping.Map));
job.SetReducerClass(typeof(TestNewCombinerGrouping.Reduce));
job.SetInputFormatClass(typeof(TextInputFormat));
job.SetMapOutputKeyClass(typeof(Text));
job.SetMapOutputValueClass(typeof(LongWritable));
job.SetOutputFormatClass(typeof(TextOutputFormat));
job.SetGroupingComparatorClass(typeof(TestNewCombinerGrouping.GroupComparator));
job.SetCombinerKeyGroupingComparatorClass(typeof(TestNewCombinerGrouping.GroupComparator
));
job.SetCombinerClass(typeof(TestNewCombinerGrouping.Combiner));
job.GetConfiguration().SetInt("min.num.spills.for.combine", 0);
job.Submit();
job.WaitForCompletion(false);
if (job.IsSuccessful())
{
Counters counters = job.GetCounters();
long combinerInputRecords = counters.FindCounter("org.apache.hadoop.mapreduce.TaskCounter"
, "COMBINE_INPUT_RECORDS").GetValue();
long combinerOutputRecords = counters.FindCounter("org.apache.hadoop.mapreduce.TaskCounter"
, "COMBINE_OUTPUT_RECORDS").GetValue();
NUnit.Framework.Assert.IsTrue(combinerInputRecords > 0);
NUnit.Framework.Assert.IsTrue(combinerInputRecords > combinerOutputRecords);
BufferedReader br = new BufferedReader(new FileReader(new FilePath(@out, "part-r-00000"
)));
ICollection <string> output = new HashSet <string>();
string line = br.ReadLine();
NUnit.Framework.Assert.IsNotNull(line);
output.AddItem(Sharpen.Runtime.Substring(line, 0, 1) + Sharpen.Runtime.Substring(
line, 4, 5));
line = br.ReadLine();
NUnit.Framework.Assert.IsNotNull(line);
output.AddItem(Sharpen.Runtime.Substring(line, 0, 1) + Sharpen.Runtime.Substring(
line, 4, 5));
line = br.ReadLine();
NUnit.Framework.Assert.IsNull(line);
br.Close();
ICollection <string> expected = new HashSet <string>();
expected.AddItem("A2");
expected.AddItem("B5");
NUnit.Framework.Assert.AreEqual(expected, output);
}
else
{
NUnit.Framework.Assert.Fail("Job failed");
}
}