/// <exception cref="System.Exception"/>
private void ValidateOutput(JobConf conf, RunningJob runningJob, IList <string> mapperBadRecords
, IList <string> redBadRecords)
{
Log.Info(runningJob.GetCounters().ToString());
NUnit.Framework.Assert.IsTrue(runningJob.IsSuccessful());
//validate counters
Counters counters = runningJob.GetCounters();
NUnit.Framework.Assert.AreEqual(counters.FindCounter(TaskCounter.MapSkippedRecords
).GetCounter(), mapperBadRecords.Count);
int mapRecs = input.Count - mapperBadRecords.Count;
NUnit.Framework.Assert.AreEqual(counters.FindCounter(TaskCounter.MapInputRecords)
.GetCounter(), mapRecs);
NUnit.Framework.Assert.AreEqual(counters.FindCounter(TaskCounter.MapOutputRecords
).GetCounter(), mapRecs);
int redRecs = mapRecs - redBadRecords.Count;
NUnit.Framework.Assert.AreEqual(counters.FindCounter(TaskCounter.ReduceSkippedRecords
).GetCounter(), redBadRecords.Count);
NUnit.Framework.Assert.AreEqual(counters.FindCounter(TaskCounter.ReduceSkippedGroups
).GetCounter(), redBadRecords.Count);
NUnit.Framework.Assert.AreEqual(counters.FindCounter(TaskCounter.ReduceInputGroups
).GetCounter(), redRecs);
NUnit.Framework.Assert.AreEqual(counters.FindCounter(TaskCounter.ReduceInputRecords
).GetCounter(), redRecs);
NUnit.Framework.Assert.AreEqual(counters.FindCounter(TaskCounter.ReduceOutputRecords
).GetCounter(), redRecs);
//validate skipped records
Path skipDir = SkipBadRecords.GetSkipOutputPath(conf);
NUnit.Framework.Assert.IsNotNull(skipDir);
Path[] skips = FileUtil.Stat2Paths(GetFileSystem().ListStatus(skipDir));
IList <string> mapSkipped = new AList <string>();
IList <string> redSkipped = new AList <string>();
foreach (Path skipPath in skips)
{
Log.Info("skipPath: " + skipPath);
SequenceFile.Reader reader = new SequenceFile.Reader(GetFileSystem(), skipPath, conf
);
object key = ReflectionUtils.NewInstance(reader.GetKeyClass(), conf);
object value = ReflectionUtils.NewInstance(reader.GetValueClass(), conf);
key = reader.Next(key);
while (key != null)
{
value = reader.GetCurrentValue(value);
Log.Debug("key:" + key + " value:" + value.ToString());
if (skipPath.GetName().Contains("_r_"))
{
redSkipped.AddItem(value.ToString());
}
else
{
mapSkipped.AddItem(value.ToString());
}
key = reader.Next(key);
}
reader.Close();
}
NUnit.Framework.Assert.IsTrue(mapSkipped.ContainsAll(mapperBadRecords));
NUnit.Framework.Assert.IsTrue(redSkipped.ContainsAll(redBadRecords));
Path[] outputFiles = FileUtil.Stat2Paths(GetFileSystem().ListStatus(GetOutputDir(
), new Utils.OutputFileUtils.OutputFilesFilter()));
IList <string> mapperOutput = GetProcessed(input, mapperBadRecords);
Log.Debug("mapperOutput " + mapperOutput.Count);
IList <string> reducerOutput = GetProcessed(mapperOutput, redBadRecords);
Log.Debug("reducerOutput " + reducerOutput.Count);
if (outputFiles.Length > 0)
{
InputStream @is = GetFileSystem().Open(outputFiles[0]);
BufferedReader reader = new BufferedReader(new InputStreamReader(@is));
string line = reader.ReadLine();
int counter = 0;
while (line != null)
{
counter++;
StringTokenizer tokeniz = new StringTokenizer(line, "\t");
string key = tokeniz.NextToken();
string value = tokeniz.NextToken();
Log.Debug("Output: key:" + key + " value:" + value);
NUnit.Framework.Assert.IsTrue(value.Contains("hello"));
NUnit.Framework.Assert.IsTrue(reducerOutput.Contains(value));
line = reader.ReadLine();
}
reader.Close();
NUnit.Framework.Assert.AreEqual(reducerOutput.Count, counter);
}
}
public virtual void TestCombiner()
{
if (!new FilePath(TestRootDir).Mkdirs())
{
throw new RuntimeException("Could not create test dir: " + TestRootDir);
}
FilePath @in = new FilePath(TestRootDir, "input");
if ([email protected]())
{
throw new RuntimeException("Could not create test dir: " + @in);
}
FilePath @out = new FilePath(TestRootDir, "output");
PrintWriter pw = new PrintWriter(new FileWriter(new FilePath(@in, "data.txt")));
pw.WriteLine("A|a,1");
pw.WriteLine("A|b,2");
pw.WriteLine("B|a,3");
pw.WriteLine("B|b,4");
pw.WriteLine("B|c,5");
pw.Close();
JobConf job = new JobConf();
job.Set("mapreduce.framework.name", "local");
TextInputFormat.SetInputPaths(job, new Path(@in.GetPath()));
TextOutputFormat.SetOutputPath(job, new Path(@out.GetPath()));
job.SetMapperClass(typeof(TestOldCombinerGrouping.Map));
job.SetReducerClass(typeof(TestOldCombinerGrouping.Reduce));
job.SetInputFormat(typeof(TextInputFormat));
job.SetMapOutputKeyClass(typeof(Text));
job.SetMapOutputValueClass(typeof(LongWritable));
job.SetOutputFormat(typeof(TextOutputFormat));
job.SetOutputValueGroupingComparator(typeof(TestOldCombinerGrouping.GroupComparator
));
job.SetCombinerClass(typeof(TestOldCombinerGrouping.Combiner));
job.SetCombinerKeyGroupingComparator(typeof(TestOldCombinerGrouping.GroupComparator
));
job.SetInt("min.num.spills.for.combine", 0);
JobClient client = new JobClient(job);
RunningJob runningJob = client.SubmitJob(job);
runningJob.WaitForCompletion();
if (runningJob.IsSuccessful())
{
Counters counters = runningJob.GetCounters();
long combinerInputRecords = counters.GetGroup("org.apache.hadoop.mapreduce.TaskCounter"
).GetCounter("COMBINE_INPUT_RECORDS");
long combinerOutputRecords = counters.GetGroup("org.apache.hadoop.mapreduce.TaskCounter"
).GetCounter("COMBINE_OUTPUT_RECORDS");
NUnit.Framework.Assert.IsTrue(combinerInputRecords > 0);
NUnit.Framework.Assert.IsTrue(combinerInputRecords > combinerOutputRecords);
BufferedReader br = new BufferedReader(new FileReader(new FilePath(@out, "part-00000"
)));
ICollection <string> output = new HashSet <string>();
string line = br.ReadLine();
NUnit.Framework.Assert.IsNotNull(line);
output.AddItem(Sharpen.Runtime.Substring(line, 0, 1) + Sharpen.Runtime.Substring(
line, 4, 5));
line = br.ReadLine();
NUnit.Framework.Assert.IsNotNull(line);
output.AddItem(Sharpen.Runtime.Substring(line, 0, 1) + Sharpen.Runtime.Substring(
line, 4, 5));
line = br.ReadLine();
NUnit.Framework.Assert.IsNull(line);
br.Close();
ICollection <string> expected = new HashSet <string>();
expected.AddItem("A2");
expected.AddItem("B5");
NUnit.Framework.Assert.AreEqual(expected, output);
}
else
{
NUnit.Framework.Assert.Fail("Job failed");
}
}
public virtual void TestRedirect()
{
Configuration conf = new YarnConfiguration();
conf.Set(MRConfig.FrameworkName, MRConfig.YarnFrameworkName);
conf.Set(YarnConfiguration.RmAddress, Rmaddress);
conf.Set(JHAdminConfig.MrHistoryAddress, Hshostaddress);
// Start the RM.
TestClientRedirect.RMService rmService = new TestClientRedirect.RMService(this, "test"
);
rmService.Init(conf);
rmService.Start();
// Start the AM.
TestClientRedirect.AMService amService = new TestClientRedirect.AMService(this);
amService.Init(conf);
amService.Start(conf);
// Start the HS.
TestClientRedirect.HistoryService historyService = new TestClientRedirect.HistoryService
(this);
historyService.Init(conf);
historyService.Start(conf);
Log.Info("services started");
Cluster cluster = new Cluster(conf);
JobID jobID = new JobID("201103121733", 1);
Counters counters = cluster.GetJob(jobID).GetCounters();
ValidateCounters(counters);
NUnit.Framework.Assert.IsTrue(amContact);
Log.Info("Sleeping for 5 seconds before stop for" + " the client socket to not get EOF immediately.."
);
Sharpen.Thread.Sleep(5000);
//bring down the AM service
amService.Stop();
Log.Info("Sleeping for 5 seconds after stop for" + " the server to exit cleanly.."
);
Sharpen.Thread.Sleep(5000);
amRestarting = true;
// Same client
//results are returned from fake (not started job)
counters = cluster.GetJob(jobID).GetCounters();
NUnit.Framework.Assert.AreEqual(0, counters.CountCounters());
Job job = cluster.GetJob(jobID);
TaskID taskId = new TaskID(jobID, TaskType.Map, 0);
TaskAttemptID tId = new TaskAttemptID(taskId, 0);
//invoke all methods to check that no exception is thrown
job.KillJob();
job.KillTask(tId);
job.FailTask(tId);
job.GetTaskCompletionEvents(0, 100);
job.GetStatus();
job.GetTaskDiagnostics(tId);
job.GetTaskReports(TaskType.Map);
job.GetTrackingURL();
amRestarting = false;
amService = new TestClientRedirect.AMService(this);
amService.Init(conf);
amService.Start(conf);
amContact = false;
//reset
counters = cluster.GetJob(jobID).GetCounters();
ValidateCounters(counters);
NUnit.Framework.Assert.IsTrue(amContact);
// Stop the AM. It is not even restarting. So it should be treated as
// completed.
amService.Stop();
// Same client
counters = cluster.GetJob(jobID).GetCounters();
ValidateCounters(counters);
NUnit.Framework.Assert.IsTrue(hsContact);
rmService.Stop();
historyService.Stop();
}
private void SetExpected(Counters counters)
{
counters.FindCounter(FrameworkCounter).SetValue(FrameworkCounterValue);
counters.FindCounter(FsScheme, FsCounter).SetValue(FsCounterValue);
}
public _Runnable_295(Counters counters)
{
this.counters = counters;
}
/// <summary>Set the task's counters.</summary>
/// <param name="counters"/>
public virtual void SetCounters(Counters counters)
{
this.counters = counters;
}