public virtual void TestJobWithNonNormalizedCapabilities() { if (!(new FilePath(MiniMRYarnCluster.Appjar)).Exists()) { Log.Info("MRAppJar " + MiniMRYarnCluster.Appjar + " not found. Not running test." ); return; } JobConf jobConf = new JobConf(mrCluster.GetConfig()); jobConf.SetInt("mapreduce.map.memory.mb", 700); jobConf.SetInt("mapred.reduce.memory.mb", 1500); SleepJob sleepJob = new SleepJob(); sleepJob.SetConf(jobConf); Job job = sleepJob.CreateJob(3, 2, 1000, 1, 500, 1); job.SetJarByClass(typeof(SleepJob)); job.AddFileToClassPath(AppJar); // The AppMaster jar itself. job.Submit(); bool completed = job.WaitForCompletion(true); NUnit.Framework.Assert.IsTrue("Job should be completed", completed); NUnit.Framework.Assert.AreEqual("Job should be finished successfully", JobStatus.State .Succeeded, job.GetJobState()); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> /// <exception cref="System.TypeLoadException"/> private void TestJobClassloader(bool useCustomClasses) { Log.Info("\n\n\nStarting testJobClassloader()" + " useCustomClasses=" + useCustomClasses ); if (!(new FilePath(MiniMRYarnCluster.Appjar)).Exists()) { Log.Info("MRAppJar " + MiniMRYarnCluster.Appjar + " not found. Not running test." ); return; } Configuration sleepConf = new Configuration(mrCluster.GetConfig()); // set master address to local to test that local mode applied iff framework == local sleepConf.Set(MRConfig.MasterAddress, "local"); sleepConf.SetBoolean(MRJobConfig.MapreduceJobClassloader, true); if (useCustomClasses) { // to test AM loading user classes such as output format class, we want // to blacklist them from the system classes (they need to be prepended // as the first match wins) string systemClasses = ApplicationClassLoader.SystemClassesDefault; // exclude the custom classes from system classes systemClasses = "-" + typeof(TestMRJobs.CustomOutputFormat).FullName + ",-" + typeof( TestMRJobs.CustomSpeculator).FullName + "," + systemClasses; sleepConf.Set(MRJobConfig.MapreduceJobClassloaderSystemClasses, systemClasses); } sleepConf.Set(MRJobConfig.IoSortMb, TestIoSortMb); sleepConf.Set(MRJobConfig.MrAmLogLevel, Level.All.ToString()); sleepConf.Set(MRJobConfig.MapLogLevel, Level.All.ToString()); sleepConf.Set(MRJobConfig.ReduceLogLevel, Level.All.ToString()); sleepConf.Set(MRJobConfig.MapJavaOpts, "-verbose:class"); SleepJob sleepJob = new SleepJob(); sleepJob.SetConf(sleepConf); Job job = sleepJob.CreateJob(1, 1, 10, 1, 10, 1); job.SetMapperClass(typeof(TestMRJobs.ConfVerificationMapper)); job.AddFileToClassPath(AppJar); // The AppMaster jar itself. job.SetJarByClass(typeof(SleepJob)); job.SetMaxMapAttempts(1); // speed up failures if (useCustomClasses) { // set custom output format class and speculator class job.SetOutputFormatClass(typeof(TestMRJobs.CustomOutputFormat)); Configuration jobConf = job.GetConfiguration(); jobConf.SetClass(MRJobConfig.MrAmJobSpeculator, typeof(TestMRJobs.CustomSpeculator ), typeof(Speculator)); // speculation needs to be enabled for the speculator to be loaded jobConf.SetBoolean(MRJobConfig.MapSpeculative, true); } job.Submit(); bool succeeded = job.WaitForCompletion(true); NUnit.Framework.Assert.IsTrue("Job status: " + job.GetStatus().GetFailureInfo(), succeeded); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> /// <exception cref="Org.Apache.Avro.AvroRemoteException"/> /// <exception cref="System.TypeLoadException"/> public virtual void TestJobHistoryData() { if (!(new FilePath(MiniMRYarnCluster.Appjar)).Exists()) { Log.Info("MRAppJar " + MiniMRYarnCluster.Appjar + " not found. Not running test." ); return; } SleepJob sleepJob = new SleepJob(); sleepJob.SetConf(mrCluster.GetConfig()); // Job with 3 maps and 2 reduces Job job = sleepJob.CreateJob(3, 2, 1000, 1, 500, 1); job.SetJarByClass(typeof(SleepJob)); job.AddFileToClassPath(AppJar); // The AppMaster jar itself. job.WaitForCompletion(true); Counters counterMR = job.GetCounters(); JobId jobId = TypeConverter.ToYarn(job.GetJobID()); ApplicationId appID = jobId.GetAppId(); int pollElapsed = 0; while (true) { Sharpen.Thread.Sleep(1000); pollElapsed += 1000; if (TerminalRmAppStates.Contains(mrCluster.GetResourceManager().GetRMContext().GetRMApps ()[appID].GetState())) { break; } if (pollElapsed >= 60000) { Log.Warn("application did not reach terminal state within 60 seconds"); break; } } NUnit.Framework.Assert.AreEqual(RMAppState.Finished, mrCluster.GetResourceManager ().GetRMContext().GetRMApps()[appID].GetState()); Counters counterHS = job.GetCounters(); //TODO the Assert below worked. need to check //Should we compare each field or convert to V2 counter and compare Log.Info("CounterHS " + counterHS); Log.Info("CounterMR " + counterMR); NUnit.Framework.Assert.AreEqual(counterHS, counterMR); HSClientProtocol historyClient = InstantiateHistoryProxy(); GetJobReportRequest gjReq = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <GetJobReportRequest >(); gjReq.SetJobId(jobId); JobReport jobReport = historyClient.GetJobReport(gjReq).GetJobReport(); VerifyJobReport(jobReport, jobId); }
/// <exception cref="System.Exception"/> private void TestSleepJobInternal(bool useRemoteJar) { Log.Info("\n\n\nStarting testSleepJob: useRemoteJar=" + useRemoteJar); if (!(new FilePath(MiniMRYarnCluster.Appjar)).Exists()) { Log.Info("MRAppJar " + MiniMRYarnCluster.Appjar + " not found. Not running test." ); return; } Configuration sleepConf = new Configuration(mrCluster.GetConfig()); // set master address to local to test that local mode applied iff framework == local sleepConf.Set(MRConfig.MasterAddress, "local"); SleepJob sleepJob = new SleepJob(); sleepJob.SetConf(sleepConf); // job with 3 maps (10s) and numReduces reduces (5s), 1 "record" each: Job job = sleepJob.CreateJob(3, numSleepReducers, 10000, 1, 5000, 1); job.AddFileToClassPath(AppJar); // The AppMaster jar itself. if (useRemoteJar) { Path localJar = new Path(ClassUtil.FindContainingJar(typeof(SleepJob))); ConfigUtil.AddLink(job.GetConfiguration(), "/jobjars", localFs.MakeQualified(localJar .GetParent()).ToUri()); job.SetJar("viewfs:///jobjars/" + localJar.GetName()); } else { job.SetJarByClass(typeof(SleepJob)); } job.SetMaxMapAttempts(1); // speed up failures job.Submit(); string trackingUrl = job.GetTrackingURL(); string jobId = job.GetJobID().ToString(); bool succeeded = job.WaitForCompletion(true); NUnit.Framework.Assert.IsTrue(succeeded); NUnit.Framework.Assert.AreEqual(JobStatus.State.Succeeded, job.GetJobState()); NUnit.Framework.Assert.IsTrue("Tracking URL was " + trackingUrl + " but didn't Match Job ID " + jobId, trackingUrl.EndsWith(Sharpen.Runtime.Substring(jobId, jobId.LastIndexOf ("_")) + "/")); VerifySleepJobCounters(job); VerifyTaskProgress(job); }
/// <exception cref="System.Exception"/> public Void Run() { SleepJob sleepJob = new SleepJob(); sleepJob.SetConf(TestMRJobs.mrCluster.GetConfig()); Job job = sleepJob.CreateJob(3, 0, 10000, 1, 0, 0); // //Job with reduces // Job job = sleepJob.createJob(3, 2, 10000, 1, 10000, 1); job.AddFileToClassPath(TestMRJobs.AppJar); // The AppMaster jar itself. job.Submit(); string trackingUrl = job.GetTrackingURL(); string jobId = job.GetJobID().ToString(); job.WaitForCompletion(true); NUnit.Framework.Assert.AreEqual(JobStatus.State.Succeeded, job.GetJobState()); NUnit.Framework.Assert.IsTrue("Tracking URL was " + trackingUrl + " but didn't Match Job ID " + jobId, trackingUrl.EndsWith(Sharpen.Runtime.Substring(jobId, jobId.LastIndexOf ("_")) + "/")); return(null); }
/// <exception cref="System.Exception"/> private void TestProfilerInternal(bool useDefault) { if (!(new FilePath(MiniMRYarnCluster.Appjar)).Exists()) { Log.Info("MRAppJar " + MiniMRYarnCluster.Appjar + " not found. Not running test." ); return; } SleepJob sleepJob = new SleepJob(); JobConf sleepConf = new JobConf(mrCluster.GetConfig()); sleepConf.SetProfileEnabled(true); sleepConf.SetProfileTaskRange(true, ProfiledTaskId.ToString()); sleepConf.SetProfileTaskRange(false, ProfiledTaskId.ToString()); if (!useDefault) { // use hprof for map to profile.out sleepConf.Set(MRJobConfig.TaskMapProfileParams, "-agentlib:hprof=cpu=times,heap=sites,force=n,thread=y,verbose=n," + "file=%s"); // use Xprof for reduce to stdout sleepConf.Set(MRJobConfig.TaskReduceProfileParams, "-Xprof"); } sleepJob.SetConf(sleepConf); // 2-map-2-reduce SleepJob Job job = sleepJob.CreateJob(2, 2, 500, 1, 500, 1); job.SetJarByClass(typeof(SleepJob)); job.AddFileToClassPath(AppJar); // The AppMaster jar itself. job.WaitForCompletion(true); JobId jobId = TypeConverter.ToYarn(job.GetJobID()); ApplicationId appID = jobId.GetAppId(); int pollElapsed = 0; while (true) { Sharpen.Thread.Sleep(1000); pollElapsed += 1000; if (TerminalRmAppStates.Contains(mrCluster.GetResourceManager().GetRMContext().GetRMApps ()[appID].GetState())) { break; } if (pollElapsed >= 60000) { Log.Warn("application did not reach terminal state within 60 seconds"); break; } } NUnit.Framework.Assert.AreEqual(RMAppState.Finished, mrCluster.GetResourceManager ().GetRMContext().GetRMApps()[appID].GetState()); // Job finished, verify logs // Configuration nmConf = mrCluster.GetNodeManager(0).GetConfig(); string appIdStr = appID.ToString(); string appIdSuffix = Sharpen.Runtime.Substring(appIdStr, "application_".Length, appIdStr .Length); string containerGlob = "container_" + appIdSuffix + "_*_*"; IDictionary <TaskAttemptID, Path> taLogDirs = new Dictionary <TaskAttemptID, Path>( ); Sharpen.Pattern taskPattern = Sharpen.Pattern.Compile(".*Task:(attempt_" + appIdSuffix + "_[rm]_" + "[0-9]+_[0-9]+).*"); foreach (string logDir in nmConf.GetTrimmedStrings(YarnConfiguration.NmLogDirs)) { // filter out MRAppMaster and create attemptId->logDir map // foreach (FileStatus fileStatus in localFs.GlobStatus(new Path(logDir + Path.Separator + appIdStr + Path.Separator + containerGlob + Path.Separator + TaskLog.LogName. Syslog))) { BufferedReader br = new BufferedReader(new InputStreamReader(localFs.Open(fileStatus .GetPath()))); string line; while ((line = br.ReadLine()) != null) { Matcher m = taskPattern.Matcher(line); if (m.Matches()) { // found Task done message taLogDirs[TaskAttemptID.ForName(m.Group(1))] = fileStatus.GetPath().GetParent(); break; } } br.Close(); } } NUnit.Framework.Assert.AreEqual(4, taLogDirs.Count); // all 4 attempts found foreach (KeyValuePair <TaskAttemptID, Path> dirEntry in taLogDirs) { TaskAttemptID tid = dirEntry.Key; Path profilePath = new Path(dirEntry.Value, TaskLog.LogName.Profile.ToString()); Path stdoutPath = new Path(dirEntry.Value, TaskLog.LogName.Stdout.ToString()); if (useDefault || tid.GetTaskType() == TaskType.Map) { if (tid.GetTaskID().GetId() == ProfiledTaskId) { // verify profile.out BufferedReader br = new BufferedReader(new InputStreamReader(localFs.Open(profilePath ))); string line = br.ReadLine(); NUnit.Framework.Assert.IsTrue("No hprof content found!", line != null && line.StartsWith ("JAVA PROFILE")); br.Close(); NUnit.Framework.Assert.AreEqual(0L, localFs.GetFileStatus(stdoutPath).GetLen()); } else { NUnit.Framework.Assert.IsFalse("hprof file should not exist", localFs.Exists(profilePath )); } } else { NUnit.Framework.Assert.IsFalse("hprof file should not exist", localFs.Exists(profilePath )); if (tid.GetTaskID().GetId() == ProfiledTaskId) { // reducer is profiled with Xprof BufferedReader br = new BufferedReader(new InputStreamReader(localFs.Open(stdoutPath ))); bool flatProfFound = false; string line; while ((line = br.ReadLine()) != null) { if (line.StartsWith("Flat profile")) { flatProfFound = true; break; } } br.Close(); NUnit.Framework.Assert.IsTrue("Xprof flat profile not found!", flatProfFound); } else { NUnit.Framework.Assert.AreEqual(0L, localFs.GetFileStatus(stdoutPath).GetLen()); } } } }
// TODO later: add explicit "isUber()" checks of some sort /// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> /// <exception cref="System.TypeLoadException"/> public virtual void TestContainerRollingLog() { if (!(new FilePath(MiniMRYarnCluster.Appjar)).Exists()) { Log.Info("MRAppJar " + MiniMRYarnCluster.Appjar + " not found. Not running test." ); return; } SleepJob sleepJob = new SleepJob(); JobConf sleepConf = new JobConf(mrCluster.GetConfig()); sleepConf.Set(MRJobConfig.MapLogLevel, Level.All.ToString()); long userLogKb = 4; sleepConf.SetLong(MRJobConfig.TaskUserlogLimit, userLogKb); sleepConf.SetInt(MRJobConfig.TaskLogBackups, 3); sleepConf.Set(MRJobConfig.MrAmLogLevel, Level.All.ToString()); long amLogKb = 7; sleepConf.SetLong(MRJobConfig.MrAmLogKb, amLogKb); sleepConf.SetInt(MRJobConfig.MrAmLogBackups, 7); sleepJob.SetConf(sleepConf); Job job = sleepJob.CreateJob(1, 0, 1L, 100, 0L, 0); job.SetJarByClass(typeof(SleepJob)); job.AddFileToClassPath(AppJar); // The AppMaster jar itself. job.WaitForCompletion(true); JobId jobId = TypeConverter.ToYarn(job.GetJobID()); ApplicationId appID = jobId.GetAppId(); int pollElapsed = 0; while (true) { Sharpen.Thread.Sleep(1000); pollElapsed += 1000; if (TerminalRmAppStates.Contains(mrCluster.GetResourceManager().GetRMContext().GetRMApps ()[appID].GetState())) { break; } if (pollElapsed >= 60000) { Log.Warn("application did not reach terminal state within 60 seconds"); break; } } NUnit.Framework.Assert.AreEqual(RMAppState.Finished, mrCluster.GetResourceManager ().GetRMContext().GetRMApps()[appID].GetState()); // Job finished, verify logs // string appIdStr = appID.ToString(); string appIdSuffix = Sharpen.Runtime.Substring(appIdStr, "application_".Length, appIdStr .Length); string containerGlob = "container_" + appIdSuffix + "_*_*"; string syslogGlob = appIdStr + Path.Separator + containerGlob + Path.Separator + TaskLog.LogName.Syslog; int numAppMasters = 0; int numMapTasks = 0; for (int i = 0; i < NumNodeMgrs; i++) { Configuration nmConf = mrCluster.GetNodeManager(i).GetConfig(); foreach (string logDir in nmConf.GetTrimmedStrings(YarnConfiguration.NmLogDirs)) { Path absSyslogGlob = new Path(logDir + Path.Separator + syslogGlob); Log.Info("Checking for glob: " + absSyslogGlob); FileStatus[] syslogs = localFs.GlobStatus(absSyslogGlob); foreach (FileStatus slog in syslogs) { bool foundAppMaster = job.IsUber(); Path containerPathComponent = slog.GetPath().GetParent(); if (!foundAppMaster) { ContainerId cid = ConverterUtils.ToContainerId(containerPathComponent.GetName()); foundAppMaster = ((cid.GetContainerId() & ContainerId.ContainerIdBitmask) == 1); } FileStatus[] sysSiblings = localFs.GlobStatus(new Path(containerPathComponent, TaskLog.LogName .Syslog + "*")); // sort to ensure for i > 0 sysSiblings[i] == "syslog.i" Arrays.Sort(sysSiblings); if (foundAppMaster) { numAppMasters++; } else { numMapTasks++; } if (foundAppMaster) { NUnit.Framework.Assert.AreSame("Unexpected number of AM sylog* files", sleepConf. GetInt(MRJobConfig.MrAmLogBackups, 0) + 1, sysSiblings.Length); NUnit.Framework.Assert.IsTrue("AM syslog.1 length kb should be >= " + amLogKb, sysSiblings [1].GetLen() >= amLogKb * 1024); } else { NUnit.Framework.Assert.AreSame("Unexpected number of MR task sylog* files", sleepConf .GetInt(MRJobConfig.TaskLogBackups, 0) + 1, sysSiblings.Length); NUnit.Framework.Assert.IsTrue("MR syslog.1 length kb should be >= " + userLogKb, sysSiblings[1].GetLen() >= userLogKb * 1024); } } } } // Make sure we checked non-empty set // NUnit.Framework.Assert.AreEqual("No AppMaster log found!", 1, numAppMasters); if (sleepConf.GetBoolean(MRJobConfig.JobUbertaskEnable, false)) { NUnit.Framework.Assert.AreEqual("MapTask log with uber found!", 0, numMapTasks); } else { NUnit.Framework.Assert.AreEqual("No MapTask log found!", 1, numMapTasks); } }