// TODO: Remove duplicates.
/// <exception cref="System.IO.IOException"/>
public static void SetClasspath(IDictionary <string, string> environment, Configuration
conf)
{
bool userClassesTakesPrecedence = conf.GetBoolean(MRJobConfig.MapreduceJobUserClasspathFirst
, false);
string classpathEnvVar = conf.GetBoolean(MRJobConfig.MapreduceJobClassloader, false
) ? ApplicationConstants.Environment.AppClasspath.ToString() : ApplicationConstants.Environment
.Classpath.ToString();
string hadoopClasspathEnvVar = ApplicationConstants.Environment.HadoopClasspath.ToString
();
MRApps.AddToEnvironment(environment, classpathEnvVar, CrossPlatformifyMREnv(conf,
ApplicationConstants.Environment.Pwd), conf);
MRApps.AddToEnvironment(environment, hadoopClasspathEnvVar, CrossPlatformifyMREnv
(conf, ApplicationConstants.Environment.Pwd), conf);
if (!userClassesTakesPrecedence)
{
MRApps.SetMRFrameworkClasspath(environment, conf);
}
AddClasspathToEnv(environment, classpathEnvVar, conf);
AddClasspathToEnv(environment, hadoopClasspathEnvVar, conf);
if (userClassesTakesPrecedence)
{
MRApps.SetMRFrameworkClasspath(environment, conf);
}
}
/// <exception cref="System.IO.IOException"/>
public static void AddClasspathToEnv(IDictionary <string, string> environment, string
classpathEnvVar, Configuration conf)
{
MRApps.AddToEnvironment(environment, classpathEnvVar, MRJobConfig.JobJar + Path.Separator
+ MRJobConfig.JobJar, conf);
MRApps.AddToEnvironment(environment, classpathEnvVar, MRJobConfig.JobJar + Path.Separator
+ "classes" + Path.Separator, conf);
MRApps.AddToEnvironment(environment, classpathEnvVar, MRJobConfig.JobJar + Path.Separator
+ "lib" + Path.Separator + "*", conf);
MRApps.AddToEnvironment(environment, classpathEnvVar, CrossPlatformifyMREnv(conf,
ApplicationConstants.Environment.Pwd) + Path.Separator + "*", conf);
// a * in the classpath will only find a .jar, so we need to filter out
// all .jars and add everything else
AddToClasspathIfNotJar(DistributedCache.GetFileClassPaths(conf), DistributedCache
.GetCacheFiles(conf), conf, environment, classpathEnvVar);
AddToClasspathIfNotJar(DistributedCache.GetArchiveClassPaths(conf), DistributedCache
.GetCacheArchives(conf), conf, environment, classpathEnvVar);
}
/// <exception cref="System.IO.IOException"/>
private static void SetMRFrameworkClasspath(IDictionary <string, string> environment
, Configuration conf)
{
// Propagate the system classpath when using the mini cluster
if (conf.GetBoolean(YarnConfiguration.IsMiniYarnCluster, false))
{
MRApps.AddToEnvironment(environment, ApplicationConstants.Environment.Classpath.ToString
(), Runtime.GetProperty("java.class.path"), conf);
}
bool crossPlatform = conf.GetBoolean(MRConfig.MapreduceAppSubmissionCrossPlatform
, MRConfig.DefaultMapreduceAppSubmissionCrossPlatform);
// if the framework is specified then only use the MR classpath
string frameworkName = GetMRFrameworkName(conf);
if (frameworkName == null)
{
// Add standard Hadoop classes
foreach (string c in conf.GetStrings(YarnConfiguration.YarnApplicationClasspath,
crossPlatform ? YarnConfiguration.DefaultYarnCrossPlatformApplicationClasspath :
YarnConfiguration.DefaultYarnApplicationClasspath))
{
MRApps.AddToEnvironment(environment, ApplicationConstants.Environment.Classpath.ToString
(), c.Trim(), conf);
}
}
bool foundFrameworkInClasspath = (frameworkName == null);
foreach (string c_1 in conf.GetStrings(MRJobConfig.MapreduceApplicationClasspath,
crossPlatform ? StringUtils.GetStrings(MRJobConfig.DefaultMapreduceCrossPlatformApplicationClasspath
) : StringUtils.GetStrings(MRJobConfig.DefaultMapreduceApplicationClasspath)))
{
MRApps.AddToEnvironment(environment, ApplicationConstants.Environment.Classpath.ToString
(), c_1.Trim(), conf);
if (!foundFrameworkInClasspath)
{
foundFrameworkInClasspath = c_1.Contains(frameworkName);
}
}
if (!foundFrameworkInClasspath)
{
throw new ArgumentException("Could not locate MapReduce framework name '" + frameworkName
+ "' in " + MRJobConfig.MapreduceApplicationClasspath);
}
}
/// <summary>Add the paths to the classpath if they are not jars</summary>
/// <param name="paths">the paths to add to the classpath</param>
/// <param name="withLinks">the corresponding paths that may have a link name in them
/// </param>
/// <param name="conf">used to resolve the paths</param>
/// <param name="environment">the environment to update CLASSPATH in</param>
/// <exception cref="System.IO.IOException">if there is an error resolving any of the paths.
/// </exception>
private static void AddToClasspathIfNotJar(Path[] paths, URI[] withLinks, Configuration
conf, IDictionary <string, string> environment, string classpathEnvVar)
{
if (paths != null)
{
Dictionary <Path, string> linkLookup = new Dictionary <Path, string>();
if (withLinks != null)
{
foreach (URI u in withLinks)
{
Path p = new Path(u);
FileSystem remoteFS = p.GetFileSystem(conf);
p = remoteFS.ResolvePath(p.MakeQualified(remoteFS.GetUri(), remoteFS.GetWorkingDirectory
()));
string name = (null == u.GetFragment()) ? p.GetName() : u.GetFragment();
if (!StringUtils.ToLowerCase(name).EndsWith(".jar"))
{
linkLookup[p] = name;
}
}
}
foreach (Path p_1 in paths)
{
FileSystem remoteFS = p_1.GetFileSystem(conf);
p_1 = remoteFS.ResolvePath(p_1.MakeQualified(remoteFS.GetUri(), remoteFS.GetWorkingDirectory
()));
string name = linkLookup[p_1];
if (name == null)
{
name = p_1.GetName();
}
if (!StringUtils.ToLowerCase(name).EndsWith(".jar"))
{
MRApps.AddToEnvironment(environment, classpathEnvVar, CrossPlatformifyMREnv(conf,
ApplicationConstants.Environment.Pwd) + Path.Separator + name, conf);
}
}
}
}