/// <exception cref="System.Exception"/> private void ValidateOutput(JobConf conf, RunningJob runningJob, IList <string> mapperBadRecords , IList <string> redBadRecords) { Log.Info(runningJob.GetCounters().ToString()); NUnit.Framework.Assert.IsTrue(runningJob.IsSuccessful()); //validate counters Counters counters = runningJob.GetCounters(); NUnit.Framework.Assert.AreEqual(counters.FindCounter(TaskCounter.MapSkippedRecords ).GetCounter(), mapperBadRecords.Count); int mapRecs = input.Count - mapperBadRecords.Count; NUnit.Framework.Assert.AreEqual(counters.FindCounter(TaskCounter.MapInputRecords) .GetCounter(), mapRecs); NUnit.Framework.Assert.AreEqual(counters.FindCounter(TaskCounter.MapOutputRecords ).GetCounter(), mapRecs); int redRecs = mapRecs - redBadRecords.Count; NUnit.Framework.Assert.AreEqual(counters.FindCounter(TaskCounter.ReduceSkippedRecords ).GetCounter(), redBadRecords.Count); NUnit.Framework.Assert.AreEqual(counters.FindCounter(TaskCounter.ReduceSkippedGroups ).GetCounter(), redBadRecords.Count); NUnit.Framework.Assert.AreEqual(counters.FindCounter(TaskCounter.ReduceInputGroups ).GetCounter(), redRecs); NUnit.Framework.Assert.AreEqual(counters.FindCounter(TaskCounter.ReduceInputRecords ).GetCounter(), redRecs); NUnit.Framework.Assert.AreEqual(counters.FindCounter(TaskCounter.ReduceOutputRecords ).GetCounter(), redRecs); //validate skipped records Path skipDir = SkipBadRecords.GetSkipOutputPath(conf); NUnit.Framework.Assert.IsNotNull(skipDir); Path[] skips = FileUtil.Stat2Paths(GetFileSystem().ListStatus(skipDir)); IList <string> mapSkipped = new AList <string>(); IList <string> redSkipped = new AList <string>(); foreach (Path skipPath in skips) { Log.Info("skipPath: " + skipPath); SequenceFile.Reader reader = new SequenceFile.Reader(GetFileSystem(), skipPath, conf ); object key = ReflectionUtils.NewInstance(reader.GetKeyClass(), conf); object value = ReflectionUtils.NewInstance(reader.GetValueClass(), conf); key = reader.Next(key); while (key != null) { value = reader.GetCurrentValue(value); Log.Debug("key:" + key + " value:" + value.ToString()); if (skipPath.GetName().Contains("_r_")) { redSkipped.AddItem(value.ToString()); } else { mapSkipped.AddItem(value.ToString()); } key = reader.Next(key); } reader.Close(); } NUnit.Framework.Assert.IsTrue(mapSkipped.ContainsAll(mapperBadRecords)); NUnit.Framework.Assert.IsTrue(redSkipped.ContainsAll(redBadRecords)); Path[] outputFiles = FileUtil.Stat2Paths(GetFileSystem().ListStatus(GetOutputDir( ), new Utils.OutputFileUtils.OutputFilesFilter())); IList <string> mapperOutput = GetProcessed(input, mapperBadRecords); Log.Debug("mapperOutput " + mapperOutput.Count); IList <string> reducerOutput = GetProcessed(mapperOutput, redBadRecords); Log.Debug("reducerOutput " + reducerOutput.Count); if (outputFiles.Length > 0) { InputStream @is = GetFileSystem().Open(outputFiles[0]); BufferedReader reader = new BufferedReader(new InputStreamReader(@is)); string line = reader.ReadLine(); int counter = 0; while (line != null) { counter++; StringTokenizer tokeniz = new StringTokenizer(line, "\t"); string key = tokeniz.NextToken(); string value = tokeniz.NextToken(); Log.Debug("Output: key:" + key + " value:" + value); NUnit.Framework.Assert.IsTrue(value.Contains("hello")); NUnit.Framework.Assert.IsTrue(reducerOutput.Contains(value)); line = reader.ReadLine(); } reader.Close(); NUnit.Framework.Assert.AreEqual(reducerOutput.Count, counter); } }
private int VerifyContainerLog(int containerNum, IList <string> expectedContent, bool count, string expectedWord) { FilePath logFolder = new FilePath(yarnCluster.GetNodeManager(0).GetConfig().Get(YarnConfiguration .NmLogDirs, YarnConfiguration.DefaultNmLogDirs)); FilePath[] listOfFiles = logFolder.ListFiles(); int currentContainerLogFileIndex = -1; for (int i = listOfFiles.Length - 1; i >= 0; i--) { if (listOfFiles[i].ListFiles().Length == containerNum + 1) { currentContainerLogFileIndex = i; break; } } NUnit.Framework.Assert.IsTrue(currentContainerLogFileIndex != -1); FilePath[] containerFiles = listOfFiles[currentContainerLogFileIndex].ListFiles(); int numOfWords = 0; for (int i_1 = 0; i_1 < containerFiles.Length; i_1++) { foreach (FilePath output in containerFiles[i_1].ListFiles()) { if (output.GetName().Trim().Contains("stdout")) { BufferedReader br = null; IList <string> stdOutContent = new AList <string>(); try { string sCurrentLine; br = new BufferedReader(new FileReader(output)); int numOfline = 0; while ((sCurrentLine = br.ReadLine()) != null) { if (count) { if (sCurrentLine.Contains(expectedWord)) { numOfWords++; } } else { if (output.GetName().Trim().Equals("stdout")) { if (!Shell.Windows) { NUnit.Framework.Assert.AreEqual("The current is" + sCurrentLine, expectedContent[ numOfline], sCurrentLine.Trim()); numOfline++; } else { stdOutContent.AddItem(sCurrentLine.Trim()); } } } } /* By executing bat script using cmd /c, * it will output all contents from bat script first * It is hard for us to do check line by line * Simply check whether output from bat file contains * all the expected messages */ if (Shell.Windows && !count && output.GetName().Trim().Equals("stdout")) { NUnit.Framework.Assert.IsTrue(stdOutContent.ContainsAll(expectedContent)); } } catch (IOException e) { Sharpen.Runtime.PrintStackTrace(e); } finally { try { if (br != null) { br.Close(); } } catch (IOException ex) { Sharpen.Runtime.PrintStackTrace(ex); } } } } } return(numOfWords); }