/// <exception cref="System.IO.IOException"/> public static MiniMRClientCluster Create(Type caller, string identifier, int noOfNMs , Configuration conf) { if (conf == null) { conf = new Configuration(); } FileSystem fs = FileSystem.Get(conf); Path testRootDir = new Path("target", identifier + "-tmpDir").MakeQualified(fs); Path appJar = new Path(testRootDir, "MRAppJar.jar"); // Copy MRAppJar and make it private. Path appMasterJar = new Path(MiniMRYarnCluster.Appjar); fs.CopyFromLocalFile(appMasterJar, appJar); fs.SetPermission(appJar, new FsPermission("744")); Job job = Job.GetInstance(conf); job.AddFileToClassPath(appJar); Path callerJar = new Path(JarFinder.GetJar(caller)); Path remoteCallerJar = new Path(testRootDir, callerJar.GetName()); fs.CopyFromLocalFile(callerJar, remoteCallerJar); fs.SetPermission(remoteCallerJar, new FsPermission("744")); job.AddFileToClassPath(remoteCallerJar); MiniMRYarnCluster miniMRYarnCluster = new MiniMRYarnCluster(identifier, noOfNMs); job.GetConfiguration().Set("minimrclientcluster.caller.name", identifier); job.GetConfiguration().SetInt("minimrclientcluster.nodemanagers.number", noOfNMs); miniMRYarnCluster.Init(job.GetConfiguration()); miniMRYarnCluster.Start(); return(new MiniMRYarnClusterAdapter(miniMRYarnCluster)); }
/// <exception cref="System.Exception"/> public virtual void _testDistributedCache(string jobJarPath) { if (!(new FilePath(MiniMRYarnCluster.Appjar)).Exists()) { Log.Info("MRAppJar " + MiniMRYarnCluster.Appjar + " not found. Not running test." ); return; } // Create a temporary file of length 1. Path first = CreateTempFile("distributed.first", "x"); // Create two jars with a single file inside them. Path second = MakeJar(new Path(TestRootDir, "distributed.second.jar"), 2); Path third = MakeJar(new Path(TestRootDir, "distributed.third.jar"), 3); Path fourth = MakeJar(new Path(TestRootDir, "distributed.fourth.jar"), 4); Job job = Job.GetInstance(mrCluster.GetConfig()); // Set the job jar to a new "dummy" jar so we can check that its extracted // properly job.SetJar(jobJarPath); // Because the job jar is a "dummy" jar, we need to include the jar with // DistributedCacheChecker or it won't be able to find it Path distributedCacheCheckerJar = new Path(JarFinder.GetJar(typeof(TestMRJobs.DistributedCacheChecker ))); job.AddFileToClassPath(distributedCacheCheckerJar.MakeQualified(localFs.GetUri(), distributedCacheCheckerJar.GetParent())); job.SetMapperClass(typeof(TestMRJobs.DistributedCacheChecker)); job.SetOutputFormatClass(typeof(NullOutputFormat)); FileInputFormat.SetInputPaths(job, first); // Creates the Job Configuration job.AddCacheFile(new URI(first.ToUri().ToString() + "#distributed.first.symlink") ); job.AddFileToClassPath(second); // The AppMaster jar itself job.AddFileToClassPath(AppJar.MakeQualified(localFs.GetUri(), AppJar.GetParent()) ); job.AddArchiveToClassPath(third); job.AddCacheArchive(fourth.ToUri()); job.SetMaxMapAttempts(1); // speed up failures job.Submit(); string trackingUrl = job.GetTrackingURL(); string jobId = job.GetJobID().ToString(); NUnit.Framework.Assert.IsTrue(job.WaitForCompletion(false)); NUnit.Framework.Assert.IsTrue("Tracking URL was " + trackingUrl + " but didn't Match Job ID " + jobId, trackingUrl.EndsWith(Sharpen.Runtime.Substring(jobId, jobId.LastIndexOf ("_")) + "/")); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> /// <exception cref="System.TypeLoadException"/> /// <exception cref="Sharpen.URISyntaxException"/> private void TestWithConf(Configuration conf) { // Create a temporary file of length 1. Path first = CreateTempFile("distributed.first", "x"); // Create two jars with a single file inside them. Path second = MakeJar(new Path(TestRootDir, "distributed.second.jar"), 2); Path third = MakeJar(new Path(TestRootDir, "distributed.third.jar"), 3); Path fourth = MakeJar(new Path(TestRootDir, "distributed.fourth.jar"), 4); Job job = Job.GetInstance(conf); job.SetMapperClass(typeof(TestMRWithDistributedCache.DistributedCacheCheckerMapper )); job.SetReducerClass(typeof(TestMRWithDistributedCache.DistributedCacheCheckerReducer )); job.SetOutputFormatClass(typeof(NullOutputFormat)); FileInputFormat.SetInputPaths(job, first); // Creates the Job Configuration job.AddCacheFile(new URI(first.ToUri().ToString() + "#distributed.first.symlink") ); job.AddFileToClassPath(second); job.AddArchiveToClassPath(third); job.AddCacheArchive(fourth.ToUri()); job.SetMaxMapAttempts(1); // speed up failures job.Submit(); NUnit.Framework.Assert.IsTrue(job.WaitForCompletion(false)); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> /// <exception cref="System.TypeLoadException"/> protected internal virtual Job RunFailingMapperJob() { Configuration myConf = new Configuration(mrCluster.GetConfig()); myConf.SetInt(MRJobConfig.NumMaps, 1); myConf.SetInt(MRJobConfig.MapMaxAttempts, 2); //reduce the number of attempts Job job = Job.GetInstance(myConf); job.SetJarByClass(typeof(FailingMapper)); job.SetJobName("failmapper"); job.SetOutputKeyClass(typeof(Text)); job.SetOutputValueClass(typeof(Text)); job.SetInputFormatClass(typeof(RandomTextWriterJob.RandomInputFormat)); job.SetOutputFormatClass(typeof(TextOutputFormat)); job.SetMapperClass(typeof(FailingMapper)); job.SetNumReduceTasks(0); FileOutputFormat.SetOutputPath(job, new Path(OutputRootDir, "failmapper-output")); job.AddFileToClassPath(AppJar); // The AppMaster jar itself. job.Submit(); string trackingUrl = job.GetTrackingURL(); string jobId = job.GetJobID().ToString(); bool succeeded = job.WaitForCompletion(true); NUnit.Framework.Assert.IsFalse(succeeded); NUnit.Framework.Assert.IsTrue("Tracking URL was " + trackingUrl + " but didn't Match Job ID " + jobId, trackingUrl.EndsWith(Sharpen.Runtime.Substring(jobId, jobId.LastIndexOf ("_")) + "/")); return(job); }
public virtual void TestJobWithNonNormalizedCapabilities() { if (!(new FilePath(MiniMRYarnCluster.Appjar)).Exists()) { Log.Info("MRAppJar " + MiniMRYarnCluster.Appjar + " not found. Not running test." ); return; } JobConf jobConf = new JobConf(mrCluster.GetConfig()); jobConf.SetInt("mapreduce.map.memory.mb", 700); jobConf.SetInt("mapred.reduce.memory.mb", 1500); SleepJob sleepJob = new SleepJob(); sleepJob.SetConf(jobConf); Job job = sleepJob.CreateJob(3, 2, 1000, 1, 500, 1); job.SetJarByClass(typeof(SleepJob)); job.AddFileToClassPath(AppJar); // The AppMaster jar itself. job.Submit(); bool completed = job.WaitForCompletion(true); NUnit.Framework.Assert.IsTrue("Job should be completed", completed); NUnit.Framework.Assert.AreEqual("Job should be finished successfully", JobStatus.State .Succeeded, job.GetJobState()); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> /// <exception cref="System.TypeLoadException"/> private void TestJobClassloader(bool useCustomClasses) { Log.Info("\n\n\nStarting testJobClassloader()" + " useCustomClasses=" + useCustomClasses ); if (!(new FilePath(MiniMRYarnCluster.Appjar)).Exists()) { Log.Info("MRAppJar " + MiniMRYarnCluster.Appjar + " not found. Not running test." ); return; } Configuration sleepConf = new Configuration(mrCluster.GetConfig()); // set master address to local to test that local mode applied iff framework == local sleepConf.Set(MRConfig.MasterAddress, "local"); sleepConf.SetBoolean(MRJobConfig.MapreduceJobClassloader, true); if (useCustomClasses) { // to test AM loading user classes such as output format class, we want // to blacklist them from the system classes (they need to be prepended // as the first match wins) string systemClasses = ApplicationClassLoader.SystemClassesDefault; // exclude the custom classes from system classes systemClasses = "-" + typeof(TestMRJobs.CustomOutputFormat).FullName + ",-" + typeof( TestMRJobs.CustomSpeculator).FullName + "," + systemClasses; sleepConf.Set(MRJobConfig.MapreduceJobClassloaderSystemClasses, systemClasses); } sleepConf.Set(MRJobConfig.IoSortMb, TestIoSortMb); sleepConf.Set(MRJobConfig.MrAmLogLevel, Level.All.ToString()); sleepConf.Set(MRJobConfig.MapLogLevel, Level.All.ToString()); sleepConf.Set(MRJobConfig.ReduceLogLevel, Level.All.ToString()); sleepConf.Set(MRJobConfig.MapJavaOpts, "-verbose:class"); SleepJob sleepJob = new SleepJob(); sleepJob.SetConf(sleepConf); Job job = sleepJob.CreateJob(1, 1, 10, 1, 10, 1); job.SetMapperClass(typeof(TestMRJobs.ConfVerificationMapper)); job.AddFileToClassPath(AppJar); // The AppMaster jar itself. job.SetJarByClass(typeof(SleepJob)); job.SetMaxMapAttempts(1); // speed up failures if (useCustomClasses) { // set custom output format class and speculator class job.SetOutputFormatClass(typeof(TestMRJobs.CustomOutputFormat)); Configuration jobConf = job.GetConfiguration(); jobConf.SetClass(MRJobConfig.MrAmJobSpeculator, typeof(TestMRJobs.CustomSpeculator ), typeof(Speculator)); // speculation needs to be enabled for the speculator to be loaded jobConf.SetBoolean(MRJobConfig.MapSpeculative, true); } job.Submit(); bool succeeded = job.WaitForCompletion(true); NUnit.Framework.Assert.IsTrue("Job status: " + job.GetStatus().GetFailureInfo(), succeeded); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> /// <exception cref="Org.Apache.Avro.AvroRemoteException"/> /// <exception cref="System.TypeLoadException"/> public virtual void TestJobHistoryData() { if (!(new FilePath(MiniMRYarnCluster.Appjar)).Exists()) { Log.Info("MRAppJar " + MiniMRYarnCluster.Appjar + " not found. Not running test." ); return; } SleepJob sleepJob = new SleepJob(); sleepJob.SetConf(mrCluster.GetConfig()); // Job with 3 maps and 2 reduces Job job = sleepJob.CreateJob(3, 2, 1000, 1, 500, 1); job.SetJarByClass(typeof(SleepJob)); job.AddFileToClassPath(AppJar); // The AppMaster jar itself. job.WaitForCompletion(true); Counters counterMR = job.GetCounters(); JobId jobId = TypeConverter.ToYarn(job.GetJobID()); ApplicationId appID = jobId.GetAppId(); int pollElapsed = 0; while (true) { Sharpen.Thread.Sleep(1000); pollElapsed += 1000; if (TerminalRmAppStates.Contains(mrCluster.GetResourceManager().GetRMContext().GetRMApps ()[appID].GetState())) { break; } if (pollElapsed >= 60000) { Log.Warn("application did not reach terminal state within 60 seconds"); break; } } NUnit.Framework.Assert.AreEqual(RMAppState.Finished, mrCluster.GetResourceManager ().GetRMContext().GetRMApps()[appID].GetState()); Counters counterHS = job.GetCounters(); //TODO the Assert below worked. need to check //Should we compare each field or convert to V2 counter and compare Log.Info("CounterHS " + counterHS); Log.Info("CounterMR " + counterMR); NUnit.Framework.Assert.AreEqual(counterHS, counterMR); HSClientProtocol historyClient = InstantiateHistoryProxy(); GetJobReportRequest gjReq = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <GetJobReportRequest >(); gjReq.SetJobId(jobId); JobReport jobReport = historyClient.GetJobReport(gjReq).GetJobReport(); VerifyJobReport(jobReport, jobId); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> /// <exception cref="System.TypeLoadException"/> public virtual void TestRandomWriter() { Log.Info("\n\n\nStarting testRandomWriter()."); if (!(new FilePath(MiniMRYarnCluster.Appjar)).Exists()) { Log.Info("MRAppJar " + MiniMRYarnCluster.Appjar + " not found. Not running test." ); return; } RandomTextWriterJob randomWriterJob = new RandomTextWriterJob(); mrCluster.GetConfig().Set(RandomTextWriterJob.TotalBytes, "3072"); mrCluster.GetConfig().Set(RandomTextWriterJob.BytesPerMap, "1024"); Job job = randomWriterJob.CreateJob(mrCluster.GetConfig()); Path outputDir = new Path(OutputRootDir, "random-output"); FileOutputFormat.SetOutputPath(job, outputDir); job.SetSpeculativeExecution(false); job.AddFileToClassPath(AppJar); // The AppMaster jar itself. job.SetJarByClass(typeof(RandomTextWriterJob)); job.SetMaxMapAttempts(1); // speed up failures job.Submit(); string trackingUrl = job.GetTrackingURL(); string jobId = job.GetJobID().ToString(); bool succeeded = job.WaitForCompletion(true); NUnit.Framework.Assert.IsTrue(succeeded); NUnit.Framework.Assert.AreEqual(JobStatus.State.Succeeded, job.GetJobState()); NUnit.Framework.Assert.IsTrue("Tracking URL was " + trackingUrl + " but didn't Match Job ID " + jobId, trackingUrl.EndsWith(Sharpen.Runtime.Substring(jobId, jobId.LastIndexOf ("_")) + "/")); // Make sure there are three files in the output-dir RemoteIterator <FileStatus> iterator = FileContext.GetFileContext(mrCluster.GetConfig ()).ListStatus(outputDir); int count = 0; while (iterator.HasNext()) { FileStatus file = iterator.Next(); if (!file.GetPath().GetName().Equals(FileOutputCommitter.SucceededFileName)) { count++; } } NUnit.Framework.Assert.AreEqual("Number of part files is wrong!", 3, count); VerifyRandomWriterCounters(job); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> /// <exception cref="System.TypeLoadException"/> internal virtual void RunTestTaskEnv(JobConf conf, Path inDir, Path outDir, bool oldConfigs) { string input = "The input"; Configure(conf, inDir, outDir, input, typeof(TestMiniMRChildTask.EnvCheckMapper), typeof(TestMiniMRChildTask.EnvCheckReducer)); // test // - new SET of new var (MY_PATH) // - set of old var (LANG) // - append to an old var from modified env (LD_LIBRARY_PATH) // - append to an old var from tt's env (PATH) // - append to a new var (NEW_PATH) string mapTaskEnvKey = JobConf.MapredMapTaskEnv; string reduceTaskEnvKey = JobConf.MapredMapTaskEnv; string mapTaskJavaOptsKey = JobConf.MapredMapTaskJavaOpts; string reduceTaskJavaOptsKey = JobConf.MapredReduceTaskJavaOpts; string mapTaskJavaOpts = MapOptsVal; string reduceTaskJavaOpts = ReduceOptsVal; conf.SetBoolean(OldConfigs, oldConfigs); if (oldConfigs) { mapTaskEnvKey = reduceTaskEnvKey = JobConf.MapredTaskEnv; mapTaskJavaOptsKey = reduceTaskJavaOptsKey = JobConf.MapredTaskJavaOpts; mapTaskJavaOpts = reduceTaskJavaOpts = TaskOptsVal; } conf.Set(mapTaskEnvKey, Shell.Windows ? "MY_PATH=/tmp,LANG=en_us_8859_1,LD_LIBRARY_PATH=%LD_LIBRARY_PATH%;/tmp," + "PATH=%PATH%;/tmp,NEW_PATH=%NEW_PATH%;/tmp" : "MY_PATH=/tmp,LANG=en_us_8859_1,LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/tmp," + "PATH=$PATH:/tmp,NEW_PATH=$NEW_PATH:/tmp"); conf.Set(reduceTaskEnvKey, Shell.Windows ? "MY_PATH=/tmp,LANG=en_us_8859_1,LD_LIBRARY_PATH=%LD_LIBRARY_PATH%;/tmp," + "PATH=%PATH%;/tmp,NEW_PATH=%NEW_PATH%;/tmp" : "MY_PATH=/tmp,LANG=en_us_8859_1,LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/tmp," + "PATH=$PATH:/tmp,NEW_PATH=$NEW_PATH:/tmp"); conf.Set("path", Runtime.Getenv("PATH")); conf.Set(mapTaskJavaOptsKey, mapTaskJavaOpts); conf.Set(reduceTaskJavaOptsKey, reduceTaskJavaOpts); Job job = Job.GetInstance(conf); job.AddFileToClassPath(AppJar); job.SetJarByClass(typeof(TestMiniMRChildTask)); job.SetMaxMapAttempts(1); // speed up failures job.WaitForCompletion(true); bool succeeded = job.WaitForCompletion(true); NUnit.Framework.Assert.IsTrue("The environment checker job failed.", succeeded); }
/// <exception cref="System.Exception"/> private void TestSleepJobInternal(bool useRemoteJar) { Log.Info("\n\n\nStarting testSleepJob: useRemoteJar=" + useRemoteJar); if (!(new FilePath(MiniMRYarnCluster.Appjar)).Exists()) { Log.Info("MRAppJar " + MiniMRYarnCluster.Appjar + " not found. Not running test." ); return; } Configuration sleepConf = new Configuration(mrCluster.GetConfig()); // set master address to local to test that local mode applied iff framework == local sleepConf.Set(MRConfig.MasterAddress, "local"); SleepJob sleepJob = new SleepJob(); sleepJob.SetConf(sleepConf); // job with 3 maps (10s) and numReduces reduces (5s), 1 "record" each: Job job = sleepJob.CreateJob(3, numSleepReducers, 10000, 1, 5000, 1); job.AddFileToClassPath(AppJar); // The AppMaster jar itself. if (useRemoteJar) { Path localJar = new Path(ClassUtil.FindContainingJar(typeof(SleepJob))); ConfigUtil.AddLink(job.GetConfiguration(), "/jobjars", localFs.MakeQualified(localJar .GetParent()).ToUri()); job.SetJar("viewfs:///jobjars/" + localJar.GetName()); } else { job.SetJarByClass(typeof(SleepJob)); } job.SetMaxMapAttempts(1); // speed up failures job.Submit(); string trackingUrl = job.GetTrackingURL(); string jobId = job.GetJobID().ToString(); bool succeeded = job.WaitForCompletion(true); NUnit.Framework.Assert.IsTrue(succeeded); NUnit.Framework.Assert.AreEqual(JobStatus.State.Succeeded, job.GetJobState()); NUnit.Framework.Assert.IsTrue("Tracking URL was " + trackingUrl + " but didn't Match Job ID " + jobId, trackingUrl.EndsWith(Sharpen.Runtime.Substring(jobId, jobId.LastIndexOf ("_")) + "/")); VerifySleepJobCounters(job); VerifyTaskProgress(job); }
/// <summary>Launch tests</summary> /// <param name="conf">Configuration of the mapreduce job.</param> /// <param name="inDir">input path</param> /// <param name="outDir">output path</param> /// <param name="input">Input text</param> /// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> /// <exception cref="System.TypeLoadException"/> public virtual void LaunchTest(JobConf conf, Path inDir, Path outDir, string input ) { FileSystem outFs = outDir.GetFileSystem(conf); // Launch job with default option for temp dir. // i.e. temp dir is ./tmp Job job = Job.GetInstance(conf); job.AddFileToClassPath(AppJar); job.SetJarByClass(typeof(TestMiniMRChildTask)); job.SetMaxMapAttempts(1); // speed up failures job.WaitForCompletion(true); bool succeeded = job.WaitForCompletion(true); NUnit.Framework.Assert.IsTrue(succeeded); outFs.Delete(outDir, true); }
/// <exception cref="System.Exception"/> public Void Run() { SleepJob sleepJob = new SleepJob(); sleepJob.SetConf(TestMRJobs.mrCluster.GetConfig()); Job job = sleepJob.CreateJob(3, 0, 10000, 1, 0, 0); // //Job with reduces // Job job = sleepJob.createJob(3, 2, 10000, 1, 10000, 1); job.AddFileToClassPath(TestMRJobs.AppJar); // The AppMaster jar itself. job.Submit(); string trackingUrl = job.GetTrackingURL(); string jobId = job.GetJobID().ToString(); job.WaitForCompletion(true); NUnit.Framework.Assert.AreEqual(JobStatus.State.Succeeded, job.GetJobState()); NUnit.Framework.Assert.IsTrue("Tracking URL was " + trackingUrl + " but didn't Match Job ID " + jobId, trackingUrl.EndsWith(Sharpen.Runtime.Substring(jobId, jobId.LastIndexOf ("_")) + "/")); return(null); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.TypeLoadException"/> /// <exception cref="System.Exception"/> private Job RunSpecTest(bool mapspec, bool redspec) { Path first = CreateTempFile("specexec_map_input1", "a\nz"); Path secnd = CreateTempFile("specexec_map_input2", "a\nz"); Configuration conf = mrCluster.GetConfig(); conf.SetBoolean(MRJobConfig.MapSpeculative, mapspec); conf.SetBoolean(MRJobConfig.ReduceSpeculative, redspec); conf.SetClass(MRJobConfig.MrAmTaskEstimator, typeof(TestSpeculativeExecution.TestSpecEstimator ), typeof(TaskRuntimeEstimator)); Job job = Job.GetInstance(conf); job.SetJarByClass(typeof(TestSpeculativeExecution)); job.SetMapperClass(typeof(TestSpeculativeExecution.SpeculativeMapper)); job.SetReducerClass(typeof(TestSpeculativeExecution.SpeculativeReducer)); job.SetOutputKeyClass(typeof(Text)); job.SetOutputValueClass(typeof(IntWritable)); job.SetNumReduceTasks(2); FileInputFormat.SetInputPaths(job, first); FileInputFormat.AddInputPath(job, secnd); FileOutputFormat.SetOutputPath(job, TestOutDir); // Delete output directory if it exists. try { localFs.Delete(TestOutDir, true); } catch (IOException) { } // ignore // Creates the Job Configuration job.AddFileToClassPath(AppJar); // The AppMaster jar itself. job.SetMaxMapAttempts(2); job.Submit(); return(job); }
/// <exception cref="System.Exception"/> private void TestProfilerInternal(bool useDefault) { if (!(new FilePath(MiniMRYarnCluster.Appjar)).Exists()) { Log.Info("MRAppJar " + MiniMRYarnCluster.Appjar + " not found. Not running test." ); return; } SleepJob sleepJob = new SleepJob(); JobConf sleepConf = new JobConf(mrCluster.GetConfig()); sleepConf.SetProfileEnabled(true); sleepConf.SetProfileTaskRange(true, ProfiledTaskId.ToString()); sleepConf.SetProfileTaskRange(false, ProfiledTaskId.ToString()); if (!useDefault) { // use hprof for map to profile.out sleepConf.Set(MRJobConfig.TaskMapProfileParams, "-agentlib:hprof=cpu=times,heap=sites,force=n,thread=y,verbose=n," + "file=%s"); // use Xprof for reduce to stdout sleepConf.Set(MRJobConfig.TaskReduceProfileParams, "-Xprof"); } sleepJob.SetConf(sleepConf); // 2-map-2-reduce SleepJob Job job = sleepJob.CreateJob(2, 2, 500, 1, 500, 1); job.SetJarByClass(typeof(SleepJob)); job.AddFileToClassPath(AppJar); // The AppMaster jar itself. job.WaitForCompletion(true); JobId jobId = TypeConverter.ToYarn(job.GetJobID()); ApplicationId appID = jobId.GetAppId(); int pollElapsed = 0; while (true) { Sharpen.Thread.Sleep(1000); pollElapsed += 1000; if (TerminalRmAppStates.Contains(mrCluster.GetResourceManager().GetRMContext().GetRMApps ()[appID].GetState())) { break; } if (pollElapsed >= 60000) { Log.Warn("application did not reach terminal state within 60 seconds"); break; } } NUnit.Framework.Assert.AreEqual(RMAppState.Finished, mrCluster.GetResourceManager ().GetRMContext().GetRMApps()[appID].GetState()); // Job finished, verify logs // Configuration nmConf = mrCluster.GetNodeManager(0).GetConfig(); string appIdStr = appID.ToString(); string appIdSuffix = Sharpen.Runtime.Substring(appIdStr, "application_".Length, appIdStr .Length); string containerGlob = "container_" + appIdSuffix + "_*_*"; IDictionary <TaskAttemptID, Path> taLogDirs = new Dictionary <TaskAttemptID, Path>( ); Sharpen.Pattern taskPattern = Sharpen.Pattern.Compile(".*Task:(attempt_" + appIdSuffix + "_[rm]_" + "[0-9]+_[0-9]+).*"); foreach (string logDir in nmConf.GetTrimmedStrings(YarnConfiguration.NmLogDirs)) { // filter out MRAppMaster and create attemptId->logDir map // foreach (FileStatus fileStatus in localFs.GlobStatus(new Path(logDir + Path.Separator + appIdStr + Path.Separator + containerGlob + Path.Separator + TaskLog.LogName. Syslog))) { BufferedReader br = new BufferedReader(new InputStreamReader(localFs.Open(fileStatus .GetPath()))); string line; while ((line = br.ReadLine()) != null) { Matcher m = taskPattern.Matcher(line); if (m.Matches()) { // found Task done message taLogDirs[TaskAttemptID.ForName(m.Group(1))] = fileStatus.GetPath().GetParent(); break; } } br.Close(); } } NUnit.Framework.Assert.AreEqual(4, taLogDirs.Count); // all 4 attempts found foreach (KeyValuePair <TaskAttemptID, Path> dirEntry in taLogDirs) { TaskAttemptID tid = dirEntry.Key; Path profilePath = new Path(dirEntry.Value, TaskLog.LogName.Profile.ToString()); Path stdoutPath = new Path(dirEntry.Value, TaskLog.LogName.Stdout.ToString()); if (useDefault || tid.GetTaskType() == TaskType.Map) { if (tid.GetTaskID().GetId() == ProfiledTaskId) { // verify profile.out BufferedReader br = new BufferedReader(new InputStreamReader(localFs.Open(profilePath ))); string line = br.ReadLine(); NUnit.Framework.Assert.IsTrue("No hprof content found!", line != null && line.StartsWith ("JAVA PROFILE")); br.Close(); NUnit.Framework.Assert.AreEqual(0L, localFs.GetFileStatus(stdoutPath).GetLen()); } else { NUnit.Framework.Assert.IsFalse("hprof file should not exist", localFs.Exists(profilePath )); } } else { NUnit.Framework.Assert.IsFalse("hprof file should not exist", localFs.Exists(profilePath )); if (tid.GetTaskID().GetId() == ProfiledTaskId) { // reducer is profiled with Xprof BufferedReader br = new BufferedReader(new InputStreamReader(localFs.Open(stdoutPath ))); bool flatProfFound = false; string line; while ((line = br.ReadLine()) != null) { if (line.StartsWith("Flat profile")) { flatProfFound = true; break; } } br.Close(); NUnit.Framework.Assert.IsTrue("Xprof flat profile not found!", flatProfFound); } else { NUnit.Framework.Assert.AreEqual(0L, localFs.GetFileStatus(stdoutPath).GetLen()); } } } }
// TODO later: add explicit "isUber()" checks of some sort /// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> /// <exception cref="System.TypeLoadException"/> public virtual void TestContainerRollingLog() { if (!(new FilePath(MiniMRYarnCluster.Appjar)).Exists()) { Log.Info("MRAppJar " + MiniMRYarnCluster.Appjar + " not found. Not running test." ); return; } SleepJob sleepJob = new SleepJob(); JobConf sleepConf = new JobConf(mrCluster.GetConfig()); sleepConf.Set(MRJobConfig.MapLogLevel, Level.All.ToString()); long userLogKb = 4; sleepConf.SetLong(MRJobConfig.TaskUserlogLimit, userLogKb); sleepConf.SetInt(MRJobConfig.TaskLogBackups, 3); sleepConf.Set(MRJobConfig.MrAmLogLevel, Level.All.ToString()); long amLogKb = 7; sleepConf.SetLong(MRJobConfig.MrAmLogKb, amLogKb); sleepConf.SetInt(MRJobConfig.MrAmLogBackups, 7); sleepJob.SetConf(sleepConf); Job job = sleepJob.CreateJob(1, 0, 1L, 100, 0L, 0); job.SetJarByClass(typeof(SleepJob)); job.AddFileToClassPath(AppJar); // The AppMaster jar itself. job.WaitForCompletion(true); JobId jobId = TypeConverter.ToYarn(job.GetJobID()); ApplicationId appID = jobId.GetAppId(); int pollElapsed = 0; while (true) { Sharpen.Thread.Sleep(1000); pollElapsed += 1000; if (TerminalRmAppStates.Contains(mrCluster.GetResourceManager().GetRMContext().GetRMApps ()[appID].GetState())) { break; } if (pollElapsed >= 60000) { Log.Warn("application did not reach terminal state within 60 seconds"); break; } } NUnit.Framework.Assert.AreEqual(RMAppState.Finished, mrCluster.GetResourceManager ().GetRMContext().GetRMApps()[appID].GetState()); // Job finished, verify logs // string appIdStr = appID.ToString(); string appIdSuffix = Sharpen.Runtime.Substring(appIdStr, "application_".Length, appIdStr .Length); string containerGlob = "container_" + appIdSuffix + "_*_*"; string syslogGlob = appIdStr + Path.Separator + containerGlob + Path.Separator + TaskLog.LogName.Syslog; int numAppMasters = 0; int numMapTasks = 0; for (int i = 0; i < NumNodeMgrs; i++) { Configuration nmConf = mrCluster.GetNodeManager(i).GetConfig(); foreach (string logDir in nmConf.GetTrimmedStrings(YarnConfiguration.NmLogDirs)) { Path absSyslogGlob = new Path(logDir + Path.Separator + syslogGlob); Log.Info("Checking for glob: " + absSyslogGlob); FileStatus[] syslogs = localFs.GlobStatus(absSyslogGlob); foreach (FileStatus slog in syslogs) { bool foundAppMaster = job.IsUber(); Path containerPathComponent = slog.GetPath().GetParent(); if (!foundAppMaster) { ContainerId cid = ConverterUtils.ToContainerId(containerPathComponent.GetName()); foundAppMaster = ((cid.GetContainerId() & ContainerId.ContainerIdBitmask) == 1); } FileStatus[] sysSiblings = localFs.GlobStatus(new Path(containerPathComponent, TaskLog.LogName .Syslog + "*")); // sort to ensure for i > 0 sysSiblings[i] == "syslog.i" Arrays.Sort(sysSiblings); if (foundAppMaster) { numAppMasters++; } else { numMapTasks++; } if (foundAppMaster) { NUnit.Framework.Assert.AreSame("Unexpected number of AM sylog* files", sleepConf. GetInt(MRJobConfig.MrAmLogBackups, 0) + 1, sysSiblings.Length); NUnit.Framework.Assert.IsTrue("AM syslog.1 length kb should be >= " + amLogKb, sysSiblings [1].GetLen() >= amLogKb * 1024); } else { NUnit.Framework.Assert.AreSame("Unexpected number of MR task sylog* files", sleepConf .GetInt(MRJobConfig.TaskLogBackups, 0) + 1, sysSiblings.Length); NUnit.Framework.Assert.IsTrue("MR syslog.1 length kb should be >= " + userLogKb, sysSiblings[1].GetLen() >= userLogKb * 1024); } } } } // Make sure we checked non-empty set // NUnit.Framework.Assert.AreEqual("No AppMaster log found!", 1, numAppMasters); if (sleepConf.GetBoolean(MRJobConfig.JobUbertaskEnable, false)) { NUnit.Framework.Assert.AreEqual("MapTask log with uber found!", 0, numMapTasks); } else { NUnit.Framework.Assert.AreEqual("No MapTask log found!", 1, numMapTasks); } }