/// <exception cref="System.Exception"/>
private void ValidateOutput(JobConf conf, RunningJob runningJob, IList <string> mapperBadRecords
, IList <string> redBadRecords)
{
Log.Info(runningJob.GetCounters().ToString());
NUnit.Framework.Assert.IsTrue(runningJob.IsSuccessful());
//validate counters
Counters counters = runningJob.GetCounters();
NUnit.Framework.Assert.AreEqual(counters.FindCounter(TaskCounter.MapSkippedRecords
).GetCounter(), mapperBadRecords.Count);
int mapRecs = input.Count - mapperBadRecords.Count;
NUnit.Framework.Assert.AreEqual(counters.FindCounter(TaskCounter.MapInputRecords)
.GetCounter(), mapRecs);
NUnit.Framework.Assert.AreEqual(counters.FindCounter(TaskCounter.MapOutputRecords
).GetCounter(), mapRecs);
int redRecs = mapRecs - redBadRecords.Count;
NUnit.Framework.Assert.AreEqual(counters.FindCounter(TaskCounter.ReduceSkippedRecords
).GetCounter(), redBadRecords.Count);
NUnit.Framework.Assert.AreEqual(counters.FindCounter(TaskCounter.ReduceSkippedGroups
).GetCounter(), redBadRecords.Count);
NUnit.Framework.Assert.AreEqual(counters.FindCounter(TaskCounter.ReduceInputGroups
).GetCounter(), redRecs);
NUnit.Framework.Assert.AreEqual(counters.FindCounter(TaskCounter.ReduceInputRecords
).GetCounter(), redRecs);
NUnit.Framework.Assert.AreEqual(counters.FindCounter(TaskCounter.ReduceOutputRecords
).GetCounter(), redRecs);
//validate skipped records
Path skipDir = SkipBadRecords.GetSkipOutputPath(conf);
NUnit.Framework.Assert.IsNotNull(skipDir);
Path[] skips = FileUtil.Stat2Paths(GetFileSystem().ListStatus(skipDir));
IList <string> mapSkipped = new AList <string>();
IList <string> redSkipped = new AList <string>();
foreach (Path skipPath in skips)
{
Log.Info("skipPath: " + skipPath);
SequenceFile.Reader reader = new SequenceFile.Reader(GetFileSystem(), skipPath, conf
);
object key = ReflectionUtils.NewInstance(reader.GetKeyClass(), conf);
object value = ReflectionUtils.NewInstance(reader.GetValueClass(), conf);
key = reader.Next(key);
while (key != null)
{
value = reader.GetCurrentValue(value);
Log.Debug("key:" + key + " value:" + value.ToString());
if (skipPath.GetName().Contains("_r_"))
{
redSkipped.AddItem(value.ToString());
}
else
{
mapSkipped.AddItem(value.ToString());
}
key = reader.Next(key);
}
reader.Close();
}
NUnit.Framework.Assert.IsTrue(mapSkipped.ContainsAll(mapperBadRecords));
NUnit.Framework.Assert.IsTrue(redSkipped.ContainsAll(redBadRecords));
Path[] outputFiles = FileUtil.Stat2Paths(GetFileSystem().ListStatus(GetOutputDir(
), new Utils.OutputFileUtils.OutputFilesFilter()));
IList <string> mapperOutput = GetProcessed(input, mapperBadRecords);
Log.Debug("mapperOutput " + mapperOutput.Count);
IList <string> reducerOutput = GetProcessed(mapperOutput, redBadRecords);
Log.Debug("reducerOutput " + reducerOutput.Count);
if (outputFiles.Length > 0)
{
InputStream @is = GetFileSystem().Open(outputFiles[0]);
BufferedReader reader = new BufferedReader(new InputStreamReader(@is));
string line = reader.ReadLine();
int counter = 0;
while (line != null)
{
counter++;
StringTokenizer tokeniz = new StringTokenizer(line, "\t");
string key = tokeniz.NextToken();
string value = tokeniz.NextToken();
Log.Debug("Output: key:" + key + " value:" + value);
NUnit.Framework.Assert.IsTrue(value.Contains("hello"));
NUnit.Framework.Assert.IsTrue(reducerOutput.Contains(value));
line = reader.ReadLine();
}
reader.Close();
NUnit.Framework.Assert.AreEqual(reducerOutput.Count, counter);
}
}
private int VerifyContainerLog(int containerNum, IList <string> expectedContent, bool
count, string expectedWord)
{
FilePath logFolder = new FilePath(yarnCluster.GetNodeManager(0).GetConfig().Get(YarnConfiguration
.NmLogDirs, YarnConfiguration.DefaultNmLogDirs));
FilePath[] listOfFiles = logFolder.ListFiles();
int currentContainerLogFileIndex = -1;
for (int i = listOfFiles.Length - 1; i >= 0; i--)
{
if (listOfFiles[i].ListFiles().Length == containerNum + 1)
{
currentContainerLogFileIndex = i;
break;
}
}
NUnit.Framework.Assert.IsTrue(currentContainerLogFileIndex != -1);
FilePath[] containerFiles = listOfFiles[currentContainerLogFileIndex].ListFiles();
int numOfWords = 0;
for (int i_1 = 0; i_1 < containerFiles.Length; i_1++)
{
foreach (FilePath output in containerFiles[i_1].ListFiles())
{
if (output.GetName().Trim().Contains("stdout"))
{
BufferedReader br = null;
IList <string> stdOutContent = new AList <string>();
try
{
string sCurrentLine;
br = new BufferedReader(new FileReader(output));
int numOfline = 0;
while ((sCurrentLine = br.ReadLine()) != null)
{
if (count)
{
if (sCurrentLine.Contains(expectedWord))
{
numOfWords++;
}
}
else
{
if (output.GetName().Trim().Equals("stdout"))
{
if (!Shell.Windows)
{
NUnit.Framework.Assert.AreEqual("The current is" + sCurrentLine, expectedContent[
numOfline], sCurrentLine.Trim());
numOfline++;
}
else
{
stdOutContent.AddItem(sCurrentLine.Trim());
}
}
}
}
/* By executing bat script using cmd /c,
* it will output all contents from bat script first
* It is hard for us to do check line by line
* Simply check whether output from bat file contains
* all the expected messages
*/
if (Shell.Windows && !count && output.GetName().Trim().Equals("stdout"))
{
NUnit.Framework.Assert.IsTrue(stdOutContent.ContainsAll(expectedContent));
}
}
catch (IOException e)
{
Sharpen.Runtime.PrintStackTrace(e);
}
finally
{
try
{
if (br != null)
{
br.Close();
}
}
catch (IOException ex)
{
Sharpen.Runtime.PrintStackTrace(ex);
}
}
}
}
}
return(numOfWords);
}
