private void CheckExpected(Counters counters)
{
NUnit.Framework.Assert.AreEqual(FrameworkCounterValue, counters.FindCounter(FrameworkCounter
).GetValue());
NUnit.Framework.Assert.AreEqual(FsCounterValue, counters.FindCounter(FsScheme, FsCounter
).GetValue());
}
private void ValidateCounters(Counters counter, long spillRecCnt, long mapInputRecords
, long mapOutputRecords)
{
// Check if the numer of Spilled Records is same as expected
NUnit.Framework.Assert.AreEqual(spillRecCnt, counter.FindCounter(TaskCounter.SpilledRecords
).GetCounter());
NUnit.Framework.Assert.AreEqual(mapInputRecords, counter.FindCounter(TaskCounter.
MapInputRecords).GetCounter());
NUnit.Framework.Assert.AreEqual(mapOutputRecords, counter.FindCounter(TaskCounter
.MapOutputRecords).GetCounter());
}
private void ValidateCounters(Counters counters, long mapInputRecords, long mapOutputRecords
, long reduceInputGroups, long reduceOutputRecords)
{
NUnit.Framework.Assert.AreEqual("MapInputRecords", mapInputRecords, counters.FindCounter
("MyCounterGroup", "MAP_INPUT_RECORDS").GetValue());
NUnit.Framework.Assert.AreEqual("MapOutputRecords", mapOutputRecords, counters.FindCounter
("MyCounterGroup", "MAP_OUTPUT_RECORDS").GetValue());
NUnit.Framework.Assert.AreEqual("ReduceInputGroups", reduceInputGroups, counters.
FindCounter("MyCounterGroup", "REDUCE_INPUT_GROUPS").GetValue());
NUnit.Framework.Assert.AreEqual("ReduceOutputRecords", reduceOutputRecords, counters
.FindCounter("MyCounterGroup", "REDUCE_OUTPUT_RECORDS").GetValue());
}
// some records hit twice
/// <summary>Verify that no segment hits disk.</summary>
/// <exception cref="System.Exception">might be thrown</exception>
public virtual void TestReduceFromMem()
{
int MapTasks = 3;
JobConf job = mrCluster.CreateJobConf();
job.Set(JobContext.ReduceInputBufferPercent, "1.0");
job.Set(JobContext.ShuffleInputBufferPercent, "1.0");
job.SetLong(JobContext.ReduceMemoryTotalBytes, 128 << 20);
job.SetNumMapTasks(MapTasks);
Counters c = RunJob(job);
long spill = c.FindCounter(TaskCounter.SpilledRecords).GetCounter();
long @out = c.FindCounter(TaskCounter.MapOutputRecords).GetCounter();
NUnit.Framework.Assert.AreEqual("Spilled records: " + spill, @out, spill);
}
public virtual void TestCounterValue()
{
Counters counters = new Counters();
int NumberTests = 100;
int NumberInc = 10;
Random rand = new Random();
for (int i = 0; i < NumberTests; i++)
{
long initValue = rand.Next();
long expectedValue = initValue;
Counters.Counter counter = counters.FindCounter("foo", "bar");
counter.SetValue(initValue);
NUnit.Framework.Assert.AreEqual("Counter value is not initialized correctly", expectedValue
, counter.GetValue());
for (int j = 0; j < NumberInc; j++)
{
int incValue = rand.Next();
counter.Increment(incValue);
expectedValue += incValue;
NUnit.Framework.Assert.AreEqual("Counter value is not incremented correctly", expectedValue
, counter.GetValue());
}
expectedValue = rand.Next();
counter.SetValue(expectedValue);
NUnit.Framework.Assert.AreEqual("Counter value is not set correctly", expectedValue
, counter.GetValue());
}
}
public virtual void TestWriteWithLegacyNames()
{
Counters counters = new Counters();
counters.IncrCounter(Task.Counter.MapInputRecords, 1);
counters.IncrCounter(JobInProgress.Counter.DataLocalMaps, 1);
counters.FindCounter("FileSystemCounters", "FILE_BYTES_READ").Increment(1);
CheckLegacyNames(counters);
}
public virtual void TestReadWithLegacyNames()
{
Counters counters = new Counters();
counters.IncrCounter(TaskCounter.MapInputRecords, 1);
counters.IncrCounter(JobCounter.DataLocalMaps, 1);
counters.FindCounter("file", FileSystemCounter.BytesRead).Increment(1);
CheckLegacyNames(counters);
}
public virtual void TestLegacyGetGroupNames()
{
Counters counters = new Counters();
// create 2 filesystem counter groups
counters.FindCounter("fs1", FileSystemCounter.BytesRead).Increment(1);
counters.FindCounter("fs2", FileSystemCounter.BytesRead).Increment(1);
counters.IncrCounter("group1", "counter1", 1);
HashSet <string> groups = new HashSet <string>(((ICollection <string>)counters.GetGroupNames
()));
HashSet <string> expectedGroups = new HashSet <string>();
expectedGroups.AddItem("group1");
expectedGroups.AddItem("FileSystemCounters");
//Legacy Name
expectedGroups.AddItem("org.apache.hadoop.mapreduce.FileSystemCounter");
NUnit.Framework.Assert.AreEqual(expectedGroups, groups);
}
public virtual void TestFileSystemGroupIteratorConcurrency()
{
Counters counters = new Counters();
// create 2 filesystem counter groups
counters.FindCounter("fs1", FileSystemCounter.BytesRead).Increment(1);
counters.FindCounter("fs2", FileSystemCounter.BytesRead).Increment(1);
// Iterate over the counters in this group while updating counters in
// the group
Counters.Group group = counters.GetGroup(typeof(FileSystemCounter).FullName);
IEnumerator <Counters.Counter> iterator = group.GetEnumerator();
counters.FindCounter("fs3", FileSystemCounter.BytesRead).Increment(1);
NUnit.Framework.Assert.IsTrue(iterator.HasNext());
iterator.Next();
counters.FindCounter("fs3", FileSystemCounter.BytesRead).Increment(1);
NUnit.Framework.Assert.IsTrue(iterator.HasNext());
iterator.Next();
}
private void ValidateFileCounters(Counters counter, long fileBytesRead, long fileBytesWritten
, long mapOutputBytes, long mapOutputMaterializedBytes)
{
NUnit.Framework.Assert.IsTrue(counter.FindCounter(FileInputFormatCounter.BytesRead
).GetValue() != 0);
NUnit.Framework.Assert.AreEqual(fileBytesRead, counter.FindCounter(FileInputFormatCounter
.BytesRead).GetValue());
NUnit.Framework.Assert.IsTrue(counter.FindCounter(FileOutputFormatCounter.BytesWritten
).GetValue() != 0);
if (mapOutputBytes >= 0)
{
NUnit.Framework.Assert.IsTrue(counter.FindCounter(TaskCounter.MapOutputBytes).GetValue
() != 0);
}
if (mapOutputMaterializedBytes >= 0)
{
NUnit.Framework.Assert.IsTrue(counter.FindCounter(TaskCounter.MapOutputMaterializedBytes
).GetValue() != 0);
}
}
private void TestMaxCountersLimits(Counters counters)
{
for (int i = 0; i < Counters.MaxCounterLimit; ++i)
{
counters.FindCounter("test", "test" + i);
}
SetExpected(counters);
ShouldThrow(typeof(Counters.CountersExceededException), new _Runnable_281(counters
));
CheckExpected(counters);
}
/// <summary>Verify that at least one segment does not hit disk</summary>
/// <exception cref="System.Exception"/>
public virtual void TestReduceFromPartialMem()
{
int MapTasks = 7;
JobConf job = mrCluster.CreateJobConf();
job.SetNumMapTasks(MapTasks);
job.SetInt(JobContext.ReduceMergeInmemThreshold, 0);
job.Set(JobContext.ReduceInputBufferPercent, "1.0");
job.SetInt(JobContext.ShuffleParallelCopies, 1);
job.SetInt(JobContext.IoSortMb, 10);
job.Set(JobConf.MapredReduceTaskJavaOpts, "-Xmx128m");
job.SetLong(JobContext.ReduceMemoryTotalBytes, 128 << 20);
job.Set(JobContext.ShuffleInputBufferPercent, "0.14");
job.Set(JobContext.ShuffleMergePercent, "1.0");
Counters c = RunJob(job);
long @out = c.FindCounter(TaskCounter.MapOutputRecords).GetCounter();
long spill = c.FindCounter(TaskCounter.SpilledRecords).GetCounter();
NUnit.Framework.Assert.IsTrue("Expected some records not spilled during reduce" +
spill + ")", spill < 2 * @out);
}
private void TestMaxGroupsLimits(Counters counters)
{
for (int i = 0; i < Counters.MaxGroupLimit; ++i)
{
// assuming COUNTERS_MAX > GROUPS_MAX
counters.FindCounter("test" + i, "test");
}
SetExpected(counters);
ShouldThrow(typeof(Counters.CountersExceededException), new _Runnable_295(counters
));
CheckExpected(counters);
}
/// <summary>Verify that all segments are read from disk</summary>
/// <exception cref="System.Exception">might be thrown</exception>
public virtual void TestReduceFromDisk()
{
int MapTasks = 8;
JobConf job = mrCluster.CreateJobConf();
job.Set(JobContext.ReduceInputBufferPercent, "0.0");
job.SetNumMapTasks(MapTasks);
job.Set(JobConf.MapredReduceTaskJavaOpts, "-Xmx128m");
job.SetLong(JobContext.ReduceMemoryTotalBytes, 128 << 20);
job.Set(JobContext.ShuffleInputBufferPercent, "0.05");
job.SetInt(JobContext.IoSortFactor, 2);
job.SetInt(JobContext.ReduceMergeInmemThreshold, 4);
Counters c = RunJob(job);
long spill = c.FindCounter(TaskCounter.SpilledRecords).GetCounter();
long @out = c.FindCounter(TaskCounter.MapOutputRecords).GetCounter();
NUnit.Framework.Assert.IsTrue("Expected all records spilled during reduce (" + spill
+ ")", spill >= 2 * @out);
// all records spill at map, reduce
NUnit.Framework.Assert.IsTrue("Expected intermediate merges (" + spill + ")", spill
>= 2 * @out + (@out / MapTasks));
}
private void CheckLegacyNames(Counters counters)
{
NUnit.Framework.Assert.AreEqual("New name", 1, counters.FindCounter(typeof(TaskCounter
).FullName, "MAP_INPUT_RECORDS").GetValue());
NUnit.Framework.Assert.AreEqual("Legacy name", 1, counters.FindCounter("org.apache.hadoop.mapred.Task$Counter"
, "MAP_INPUT_RECORDS").GetValue());
NUnit.Framework.Assert.AreEqual("Legacy enum", 1, counters.FindCounter(Task.Counter
.MapInputRecords).GetValue());
NUnit.Framework.Assert.AreEqual("New name", 1, counters.FindCounter(typeof(JobCounter
).FullName, "DATA_LOCAL_MAPS").GetValue());
NUnit.Framework.Assert.AreEqual("Legacy name", 1, counters.FindCounter("org.apache.hadoop.mapred.JobInProgress$Counter"
, "DATA_LOCAL_MAPS").GetValue());
NUnit.Framework.Assert.AreEqual("Legacy enum", 1, counters.FindCounter(JobInProgress.Counter
.DataLocalMaps).GetValue());
NUnit.Framework.Assert.AreEqual("New name", 1, counters.FindCounter(typeof(FileSystemCounter
).FullName, "FILE_BYTES_READ").GetValue());
NUnit.Framework.Assert.AreEqual("New name and method", 1, counters.FindCounter("file"
, FileSystemCounter.BytesRead).GetValue());
NUnit.Framework.Assert.AreEqual("Legacy name", 1, counters.FindCounter("FileSystemCounters"
, "FILE_BYTES_READ").GetValue());
}
private void SetExpected(Counters counters)
{
counters.FindCounter(FrameworkCounter).SetValue(FrameworkCounterValue);
counters.FindCounter(FsScheme, FsCounter).SetValue(FsCounterValue);
}
/// <exception cref="System.Exception"/>
private void ValidateOutput(JobConf conf, RunningJob runningJob, IList <string> mapperBadRecords
, IList <string> redBadRecords)
{
Log.Info(runningJob.GetCounters().ToString());
NUnit.Framework.Assert.IsTrue(runningJob.IsSuccessful());
//validate counters
Counters counters = runningJob.GetCounters();
NUnit.Framework.Assert.AreEqual(counters.FindCounter(TaskCounter.MapSkippedRecords
).GetCounter(), mapperBadRecords.Count);
int mapRecs = input.Count - mapperBadRecords.Count;
NUnit.Framework.Assert.AreEqual(counters.FindCounter(TaskCounter.MapInputRecords)
.GetCounter(), mapRecs);
NUnit.Framework.Assert.AreEqual(counters.FindCounter(TaskCounter.MapOutputRecords
).GetCounter(), mapRecs);
int redRecs = mapRecs - redBadRecords.Count;
NUnit.Framework.Assert.AreEqual(counters.FindCounter(TaskCounter.ReduceSkippedRecords
).GetCounter(), redBadRecords.Count);
NUnit.Framework.Assert.AreEqual(counters.FindCounter(TaskCounter.ReduceSkippedGroups
).GetCounter(), redBadRecords.Count);
NUnit.Framework.Assert.AreEqual(counters.FindCounter(TaskCounter.ReduceInputGroups
).GetCounter(), redRecs);
NUnit.Framework.Assert.AreEqual(counters.FindCounter(TaskCounter.ReduceInputRecords
).GetCounter(), redRecs);
NUnit.Framework.Assert.AreEqual(counters.FindCounter(TaskCounter.ReduceOutputRecords
).GetCounter(), redRecs);
//validate skipped records
Path skipDir = SkipBadRecords.GetSkipOutputPath(conf);
NUnit.Framework.Assert.IsNotNull(skipDir);
Path[] skips = FileUtil.Stat2Paths(GetFileSystem().ListStatus(skipDir));
IList <string> mapSkipped = new AList <string>();
IList <string> redSkipped = new AList <string>();
foreach (Path skipPath in skips)
{
Log.Info("skipPath: " + skipPath);
SequenceFile.Reader reader = new SequenceFile.Reader(GetFileSystem(), skipPath, conf
);
object key = ReflectionUtils.NewInstance(reader.GetKeyClass(), conf);
object value = ReflectionUtils.NewInstance(reader.GetValueClass(), conf);
key = reader.Next(key);
while (key != null)
{
value = reader.GetCurrentValue(value);
Log.Debug("key:" + key + " value:" + value.ToString());
if (skipPath.GetName().Contains("_r_"))
{
redSkipped.AddItem(value.ToString());
}
else
{
mapSkipped.AddItem(value.ToString());
}
key = reader.Next(key);
}
reader.Close();
}
NUnit.Framework.Assert.IsTrue(mapSkipped.ContainsAll(mapperBadRecords));
NUnit.Framework.Assert.IsTrue(redSkipped.ContainsAll(redBadRecords));
Path[] outputFiles = FileUtil.Stat2Paths(GetFileSystem().ListStatus(GetOutputDir(
), new Utils.OutputFileUtils.OutputFilesFilter()));
IList <string> mapperOutput = GetProcessed(input, mapperBadRecords);
Log.Debug("mapperOutput " + mapperOutput.Count);
IList <string> reducerOutput = GetProcessed(mapperOutput, redBadRecords);
Log.Debug("reducerOutput " + reducerOutput.Count);
if (outputFiles.Length > 0)
{
InputStream @is = GetFileSystem().Open(outputFiles[0]);
BufferedReader reader = new BufferedReader(new InputStreamReader(@is));
string line = reader.ReadLine();
int counter = 0;
while (line != null)
{
counter++;
StringTokenizer tokeniz = new StringTokenizer(line, "\t");
string key = tokeniz.NextToken();
string value = tokeniz.NextToken();
Log.Debug("Output: key:" + key + " value:" + value);
NUnit.Framework.Assert.IsTrue(value.Contains("hello"));
NUnit.Framework.Assert.IsTrue(reducerOutput.Contains(value));
line = reader.ReadLine();
}
reader.Close();
NUnit.Framework.Assert.AreEqual(reducerOutput.Count, counter);
}
}