public virtual void TestSetClasspathWithNoUserPrecendence() { Configuration conf = new Configuration(); conf.SetBoolean(MRConfig.MapreduceAppSubmissionCrossPlatform, true); conf.SetBoolean(MRJobConfig.MapreduceJobUserClasspathFirst, false); IDictionary <string, string> env = new Dictionary <string, string>(); try { MRApps.SetClasspath(env, conf); } catch (Exception) { NUnit.Framework.Assert.Fail("Got exception while setting classpath"); } string env_str = env[ApplicationConstants.Environment.Classpath.ToString()]; string expectedClasspath = StringUtils.Join(ApplicationConstants.ClassPathSeparator , Arrays.AsList("job.jar/job.jar", "job.jar/classes/", "job.jar/lib/*", ApplicationConstants.Environment .Pwd.$$() + "/*")); NUnit.Framework.Assert.IsTrue("MAPREDUCE_JOB_USER_CLASSPATH_FIRST false, and job.jar is not in" + " the classpath!", env_str.Contains(expectedClasspath)); NUnit.Framework.Assert.IsFalse("MAPREDUCE_JOB_USER_CLASSPATH_FIRST false, but taking effect!" , env_str.StartsWith(expectedClasspath)); }
/// <exception cref="System.IO.IOException"/> public virtual void TestSetClasspath() { Configuration conf = new Configuration(); conf.SetBoolean(MRConfig.MapreduceAppSubmissionCrossPlatform, true); Job job = Job.GetInstance(conf); IDictionary <string, string> environment = new Dictionary <string, string>(); MRApps.SetClasspath(environment, job.GetConfiguration()); NUnit.Framework.Assert.IsTrue(environment[ApplicationConstants.Environment.Classpath .ToString()].StartsWith(ApplicationConstants.Environment.Pwd.$$() + ApplicationConstants .ClassPathSeparator)); string yarnAppClasspath = job.GetConfiguration().Get(YarnConfiguration.YarnApplicationClasspath , StringUtils.Join(",", YarnConfiguration.DefaultYarnCrossPlatformApplicationClasspath )); if (yarnAppClasspath != null) { yarnAppClasspath = yarnAppClasspath.ReplaceAll(",\\s*", ApplicationConstants.ClassPathSeparator ).Trim(); } NUnit.Framework.Assert.IsTrue(environment[ApplicationConstants.Environment.Classpath .ToString()].Contains(yarnAppClasspath)); string mrAppClasspath = job.GetConfiguration().Get(MRJobConfig.MapreduceApplicationClasspath , MRJobConfig.DefaultMapreduceCrossPlatformApplicationClasspath); if (mrAppClasspath != null) { mrAppClasspath = mrAppClasspath.ReplaceAll(",\\s*", ApplicationConstants.ClassPathSeparator ).Trim(); } NUnit.Framework.Assert.IsTrue(environment[ApplicationConstants.Environment.Classpath .ToString()].Contains(mrAppClasspath)); }
/// <exception cref="System.IO.IOException"/> public virtual void TestSetClasspathWithArchives() { FilePath testTGZ = new FilePath(testWorkDir, "test.tgz"); FileOutputStream @out = new FileOutputStream(testTGZ); @out.Write(0); @out.Close(); Configuration conf = new Configuration(); conf.SetBoolean(MRConfig.MapreduceAppSubmissionCrossPlatform, true); Job job = Job.GetInstance(conf); conf = job.GetConfiguration(); string testTGZQualifiedPath = FileSystem.GetLocal(conf).MakeQualified(new Path(testTGZ .GetAbsolutePath())).ToString(); conf.Set(MRJobConfig.ClasspathArchives, testTGZQualifiedPath); conf.Set(MRJobConfig.CacheArchives, testTGZQualifiedPath + "#testTGZ"); IDictionary <string, string> environment = new Dictionary <string, string>(); MRApps.SetClasspath(environment, conf); NUnit.Framework.Assert.IsTrue(environment[ApplicationConstants.Environment.Classpath .ToString()].StartsWith(ApplicationConstants.Environment.Pwd.$$() + ApplicationConstants .ClassPathSeparator)); NUnit.Framework.Assert.IsTrue(environment[ApplicationConstants.Environment.HadoopClasspath .ToString()].StartsWith(ApplicationConstants.Environment.Pwd.$$() + ApplicationConstants .ClassPathSeparator)); string confClasspath = job.GetConfiguration().Get(YarnConfiguration.YarnApplicationClasspath , StringUtils.Join(",", YarnConfiguration.DefaultYarnCrossPlatformApplicationClasspath )); if (confClasspath != null) { confClasspath = confClasspath.ReplaceAll(",\\s*", ApplicationConstants.ClassPathSeparator ).Trim(); } Log.Info("CLASSPATH: " + environment[ApplicationConstants.Environment.Classpath.ToString ()]); Log.Info("confClasspath: " + confClasspath); NUnit.Framework.Assert.IsTrue(environment[ApplicationConstants.Environment.Classpath .ToString()].Contains(confClasspath)); Log.Info("HADOOP_CLASSPATH: " + environment[ApplicationConstants.Environment.HadoopClasspath .ToString()]); NUnit.Framework.Assert.IsTrue(environment[ApplicationConstants.Environment.Classpath .ToString()].Contains("testTGZ")); NUnit.Framework.Assert.IsTrue(environment[ApplicationConstants.Environment.HadoopClasspath .ToString()].Contains("testTGZ")); }
/// <exception cref="System.IO.IOException"/> public virtual void TestSetClasspathWithFramework() { string FrameworkName = "some-framework-name"; string FrameworkPath = "some-framework-path#" + FrameworkName; Configuration conf = new Configuration(); conf.SetBoolean(MRConfig.MapreduceAppSubmissionCrossPlatform, true); conf.Set(MRJobConfig.MapreduceApplicationFrameworkPath, FrameworkPath); IDictionary <string, string> env = new Dictionary <string, string>(); try { MRApps.SetClasspath(env, conf); NUnit.Framework.Assert.Fail("Failed to catch framework path set without classpath change" ); } catch (ArgumentException e) { NUnit.Framework.Assert.IsTrue("Unexpected IllegalArgumentException", e.Message.Contains ("Could not locate MapReduce framework name '" + FrameworkName + "'")); } env.Clear(); string FrameworkClasspath = FrameworkName + "/*.jar"; conf.Set(MRJobConfig.MapreduceApplicationClasspath, FrameworkClasspath); MRApps.SetClasspath(env, conf); string stdClasspath = StringUtils.Join(ApplicationConstants.ClassPathSeparator, Arrays .AsList("job.jar/job.jar", "job.jar/classes/", "job.jar/lib/*", ApplicationConstants.Environment .Pwd.$$() + "/*")); string expectedClasspath = StringUtils.Join(ApplicationConstants.ClassPathSeparator , Arrays.AsList(ApplicationConstants.Environment.Pwd.$$(), FrameworkClasspath, stdClasspath )); NUnit.Framework.Assert.AreEqual("Incorrect classpath with framework and no user precedence" , expectedClasspath, env[ApplicationConstants.Environment.Classpath.ToString()]); env.Clear(); conf.SetBoolean(MRJobConfig.MapreduceJobUserClasspathFirst, true); MRApps.SetClasspath(env, conf); expectedClasspath = StringUtils.Join(ApplicationConstants.ClassPathSeparator, Arrays .AsList(ApplicationConstants.Environment.Pwd.$$(), stdClasspath, FrameworkClasspath )); NUnit.Framework.Assert.AreEqual("Incorrect classpath with framework and user precedence" , expectedClasspath, env[ApplicationConstants.Environment.Classpath.ToString()]); }
/// <exception cref="System.IO.IOException"/> public virtual void TestSetClasspathWithJobClassloader() { Configuration conf = new Configuration(); conf.SetBoolean(MRConfig.MapreduceAppSubmissionCrossPlatform, true); conf.SetBoolean(MRJobConfig.MapreduceJobClassloader, true); IDictionary <string, string> env = new Dictionary <string, string>(); MRApps.SetClasspath(env, conf); string cp = env[ApplicationConstants.Environment.Classpath.ToString()]; string appCp = env[ApplicationConstants.Environment.AppClasspath.ToString()]; NUnit.Framework.Assert.IsFalse("MAPREDUCE_JOB_CLASSLOADER true, but job.jar is in the" + " classpath!", cp.Contains("jar" + ApplicationConstants.ClassPathSeparator + "job")); NUnit.Framework.Assert.IsFalse("MAPREDUCE_JOB_CLASSLOADER true, but PWD is in the classpath!" , cp.Contains("PWD")); string expectedAppClasspath = StringUtils.Join(ApplicationConstants.ClassPathSeparator , Arrays.AsList(ApplicationConstants.Environment.Pwd.$$(), "job.jar/job.jar", "job.jar/classes/" , "job.jar/lib/*", ApplicationConstants.Environment.Pwd.$$() + "/*")); NUnit.Framework.Assert.AreEqual("MAPREDUCE_JOB_CLASSLOADER true, but job.jar is not in the app" + " classpath!", expectedAppClasspath, appCp); }