public virtual void TestSetClasspathWithNoUserPrecendence()
{
Configuration conf = new Configuration();
conf.SetBoolean(MRConfig.MapreduceAppSubmissionCrossPlatform, true);
conf.SetBoolean(MRJobConfig.MapreduceJobUserClasspathFirst, false);
IDictionary <string, string> env = new Dictionary <string, string>();
try
{
MRApps.SetClasspath(env, conf);
}
catch (Exception)
{
NUnit.Framework.Assert.Fail("Got exception while setting classpath");
}
string env_str = env[ApplicationConstants.Environment.Classpath.ToString()];
string expectedClasspath = StringUtils.Join(ApplicationConstants.ClassPathSeparator
, Arrays.AsList("job.jar/job.jar", "job.jar/classes/", "job.jar/lib/*", ApplicationConstants.Environment
.Pwd.$$() + "/*"));
NUnit.Framework.Assert.IsTrue("MAPREDUCE_JOB_USER_CLASSPATH_FIRST false, and job.jar is not in"
+ " the classpath!", env_str.Contains(expectedClasspath));
NUnit.Framework.Assert.IsFalse("MAPREDUCE_JOB_USER_CLASSPATH_FIRST false, but taking effect!"
, env_str.StartsWith(expectedClasspath));
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestSetClasspath()
{
Configuration conf = new Configuration();
conf.SetBoolean(MRConfig.MapreduceAppSubmissionCrossPlatform, true);
Job job = Job.GetInstance(conf);
IDictionary <string, string> environment = new Dictionary <string, string>();
MRApps.SetClasspath(environment, job.GetConfiguration());
NUnit.Framework.Assert.IsTrue(environment[ApplicationConstants.Environment.Classpath
.ToString()].StartsWith(ApplicationConstants.Environment.Pwd.$$() + ApplicationConstants
.ClassPathSeparator));
string yarnAppClasspath = job.GetConfiguration().Get(YarnConfiguration.YarnApplicationClasspath
, StringUtils.Join(",", YarnConfiguration.DefaultYarnCrossPlatformApplicationClasspath
));
if (yarnAppClasspath != null)
{
yarnAppClasspath = yarnAppClasspath.ReplaceAll(",\\s*", ApplicationConstants.ClassPathSeparator
).Trim();
}
NUnit.Framework.Assert.IsTrue(environment[ApplicationConstants.Environment.Classpath
.ToString()].Contains(yarnAppClasspath));
string mrAppClasspath = job.GetConfiguration().Get(MRJobConfig.MapreduceApplicationClasspath
, MRJobConfig.DefaultMapreduceCrossPlatformApplicationClasspath);
if (mrAppClasspath != null)
{
mrAppClasspath = mrAppClasspath.ReplaceAll(",\\s*", ApplicationConstants.ClassPathSeparator
).Trim();
}
NUnit.Framework.Assert.IsTrue(environment[ApplicationConstants.Environment.Classpath
.ToString()].Contains(mrAppClasspath));
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestSetClasspathWithArchives()
{
FilePath testTGZ = new FilePath(testWorkDir, "test.tgz");
FileOutputStream @out = new FileOutputStream(testTGZ);
@out.Write(0);
@out.Close();
Configuration conf = new Configuration();
conf.SetBoolean(MRConfig.MapreduceAppSubmissionCrossPlatform, true);
Job job = Job.GetInstance(conf);
conf = job.GetConfiguration();
string testTGZQualifiedPath = FileSystem.GetLocal(conf).MakeQualified(new Path(testTGZ
.GetAbsolutePath())).ToString();
conf.Set(MRJobConfig.ClasspathArchives, testTGZQualifiedPath);
conf.Set(MRJobConfig.CacheArchives, testTGZQualifiedPath + "#testTGZ");
IDictionary <string, string> environment = new Dictionary <string, string>();
MRApps.SetClasspath(environment, conf);
NUnit.Framework.Assert.IsTrue(environment[ApplicationConstants.Environment.Classpath
.ToString()].StartsWith(ApplicationConstants.Environment.Pwd.$$() + ApplicationConstants
.ClassPathSeparator));
NUnit.Framework.Assert.IsTrue(environment[ApplicationConstants.Environment.HadoopClasspath
.ToString()].StartsWith(ApplicationConstants.Environment.Pwd.$$() + ApplicationConstants
.ClassPathSeparator));
string confClasspath = job.GetConfiguration().Get(YarnConfiguration.YarnApplicationClasspath
, StringUtils.Join(",", YarnConfiguration.DefaultYarnCrossPlatformApplicationClasspath
));
if (confClasspath != null)
{
confClasspath = confClasspath.ReplaceAll(",\\s*", ApplicationConstants.ClassPathSeparator
).Trim();
}
Log.Info("CLASSPATH: " + environment[ApplicationConstants.Environment.Classpath.ToString
()]);
Log.Info("confClasspath: " + confClasspath);
NUnit.Framework.Assert.IsTrue(environment[ApplicationConstants.Environment.Classpath
.ToString()].Contains(confClasspath));
Log.Info("HADOOP_CLASSPATH: " + environment[ApplicationConstants.Environment.HadoopClasspath
.ToString()]);
NUnit.Framework.Assert.IsTrue(environment[ApplicationConstants.Environment.Classpath
.ToString()].Contains("testTGZ"));
NUnit.Framework.Assert.IsTrue(environment[ApplicationConstants.Environment.HadoopClasspath
.ToString()].Contains("testTGZ"));
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestSetClasspathWithFramework()
{
string FrameworkName = "some-framework-name";
string FrameworkPath = "some-framework-path#" + FrameworkName;
Configuration conf = new Configuration();
conf.SetBoolean(MRConfig.MapreduceAppSubmissionCrossPlatform, true);
conf.Set(MRJobConfig.MapreduceApplicationFrameworkPath, FrameworkPath);
IDictionary <string, string> env = new Dictionary <string, string>();
try
{
MRApps.SetClasspath(env, conf);
NUnit.Framework.Assert.Fail("Failed to catch framework path set without classpath change"
);
}
catch (ArgumentException e)
{
NUnit.Framework.Assert.IsTrue("Unexpected IllegalArgumentException", e.Message.Contains
("Could not locate MapReduce framework name '" + FrameworkName + "'"));
}
env.Clear();
string FrameworkClasspath = FrameworkName + "/*.jar";
conf.Set(MRJobConfig.MapreduceApplicationClasspath, FrameworkClasspath);
MRApps.SetClasspath(env, conf);
string stdClasspath = StringUtils.Join(ApplicationConstants.ClassPathSeparator, Arrays
.AsList("job.jar/job.jar", "job.jar/classes/", "job.jar/lib/*", ApplicationConstants.Environment
.Pwd.$$() + "/*"));
string expectedClasspath = StringUtils.Join(ApplicationConstants.ClassPathSeparator
, Arrays.AsList(ApplicationConstants.Environment.Pwd.$$(), FrameworkClasspath, stdClasspath
));
NUnit.Framework.Assert.AreEqual("Incorrect classpath with framework and no user precedence"
, expectedClasspath, env[ApplicationConstants.Environment.Classpath.ToString()]);
env.Clear();
conf.SetBoolean(MRJobConfig.MapreduceJobUserClasspathFirst, true);
MRApps.SetClasspath(env, conf);
expectedClasspath = StringUtils.Join(ApplicationConstants.ClassPathSeparator, Arrays
.AsList(ApplicationConstants.Environment.Pwd.$$(), stdClasspath, FrameworkClasspath
));
NUnit.Framework.Assert.AreEqual("Incorrect classpath with framework and user precedence"
, expectedClasspath, env[ApplicationConstants.Environment.Classpath.ToString()]);
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestSetClasspathWithJobClassloader()
{
Configuration conf = new Configuration();
conf.SetBoolean(MRConfig.MapreduceAppSubmissionCrossPlatform, true);
conf.SetBoolean(MRJobConfig.MapreduceJobClassloader, true);
IDictionary <string, string> env = new Dictionary <string, string>();
MRApps.SetClasspath(env, conf);
string cp = env[ApplicationConstants.Environment.Classpath.ToString()];
string appCp = env[ApplicationConstants.Environment.AppClasspath.ToString()];
NUnit.Framework.Assert.IsFalse("MAPREDUCE_JOB_CLASSLOADER true, but job.jar is in the"
+ " classpath!", cp.Contains("jar" + ApplicationConstants.ClassPathSeparator +
"job"));
NUnit.Framework.Assert.IsFalse("MAPREDUCE_JOB_CLASSLOADER true, but PWD is in the classpath!"
, cp.Contains("PWD"));
string expectedAppClasspath = StringUtils.Join(ApplicationConstants.ClassPathSeparator
, Arrays.AsList(ApplicationConstants.Environment.Pwd.$$(), "job.jar/job.jar", "job.jar/classes/"
, "job.jar/lib/*", ApplicationConstants.Environment.Pwd.$$() + "/*"));
NUnit.Framework.Assert.AreEqual("MAPREDUCE_JOB_CLASSLOADER true, but job.jar is not in the app"
+ " classpath!", expectedAppClasspath, appCp);
}