// TODO: Remove duplicates. /// <exception cref="System.IO.IOException"/> public static void SetClasspath(IDictionary <string, string> environment, Configuration conf) { bool userClassesTakesPrecedence = conf.GetBoolean(MRJobConfig.MapreduceJobUserClasspathFirst , false); string classpathEnvVar = conf.GetBoolean(MRJobConfig.MapreduceJobClassloader, false ) ? ApplicationConstants.Environment.AppClasspath.ToString() : ApplicationConstants.Environment .Classpath.ToString(); string hadoopClasspathEnvVar = ApplicationConstants.Environment.HadoopClasspath.ToString (); MRApps.AddToEnvironment(environment, classpathEnvVar, CrossPlatformifyMREnv(conf, ApplicationConstants.Environment.Pwd), conf); MRApps.AddToEnvironment(environment, hadoopClasspathEnvVar, CrossPlatformifyMREnv (conf, ApplicationConstants.Environment.Pwd), conf); if (!userClassesTakesPrecedence) { MRApps.SetMRFrameworkClasspath(environment, conf); } AddClasspathToEnv(environment, classpathEnvVar, conf); AddClasspathToEnv(environment, hadoopClasspathEnvVar, conf); if (userClassesTakesPrecedence) { MRApps.SetMRFrameworkClasspath(environment, conf); } }
/// <exception cref="System.IO.IOException"/> public static void AddClasspathToEnv(IDictionary <string, string> environment, string classpathEnvVar, Configuration conf) { MRApps.AddToEnvironment(environment, classpathEnvVar, MRJobConfig.JobJar + Path.Separator + MRJobConfig.JobJar, conf); MRApps.AddToEnvironment(environment, classpathEnvVar, MRJobConfig.JobJar + Path.Separator + "classes" + Path.Separator, conf); MRApps.AddToEnvironment(environment, classpathEnvVar, MRJobConfig.JobJar + Path.Separator + "lib" + Path.Separator + "*", conf); MRApps.AddToEnvironment(environment, classpathEnvVar, CrossPlatformifyMREnv(conf, ApplicationConstants.Environment.Pwd) + Path.Separator + "*", conf); // a * in the classpath will only find a .jar, so we need to filter out // all .jars and add everything else AddToClasspathIfNotJar(DistributedCache.GetFileClassPaths(conf), DistributedCache .GetCacheFiles(conf), conf, environment, classpathEnvVar); AddToClasspathIfNotJar(DistributedCache.GetArchiveClassPaths(conf), DistributedCache .GetCacheArchives(conf), conf, environment, classpathEnvVar); }
/// <exception cref="System.IO.IOException"/> private static void SetMRFrameworkClasspath(IDictionary <string, string> environment , Configuration conf) { // Propagate the system classpath when using the mini cluster if (conf.GetBoolean(YarnConfiguration.IsMiniYarnCluster, false)) { MRApps.AddToEnvironment(environment, ApplicationConstants.Environment.Classpath.ToString (), Runtime.GetProperty("java.class.path"), conf); } bool crossPlatform = conf.GetBoolean(MRConfig.MapreduceAppSubmissionCrossPlatform , MRConfig.DefaultMapreduceAppSubmissionCrossPlatform); // if the framework is specified then only use the MR classpath string frameworkName = GetMRFrameworkName(conf); if (frameworkName == null) { // Add standard Hadoop classes foreach (string c in conf.GetStrings(YarnConfiguration.YarnApplicationClasspath, crossPlatform ? YarnConfiguration.DefaultYarnCrossPlatformApplicationClasspath : YarnConfiguration.DefaultYarnApplicationClasspath)) { MRApps.AddToEnvironment(environment, ApplicationConstants.Environment.Classpath.ToString (), c.Trim(), conf); } } bool foundFrameworkInClasspath = (frameworkName == null); foreach (string c_1 in conf.GetStrings(MRJobConfig.MapreduceApplicationClasspath, crossPlatform ? StringUtils.GetStrings(MRJobConfig.DefaultMapreduceCrossPlatformApplicationClasspath ) : StringUtils.GetStrings(MRJobConfig.DefaultMapreduceApplicationClasspath))) { MRApps.AddToEnvironment(environment, ApplicationConstants.Environment.Classpath.ToString (), c_1.Trim(), conf); if (!foundFrameworkInClasspath) { foundFrameworkInClasspath = c_1.Contains(frameworkName); } } if (!foundFrameworkInClasspath) { throw new ArgumentException("Could not locate MapReduce framework name '" + frameworkName + "' in " + MRJobConfig.MapreduceApplicationClasspath); } }
/// <summary>Add the paths to the classpath if they are not jars</summary> /// <param name="paths">the paths to add to the classpath</param> /// <param name="withLinks">the corresponding paths that may have a link name in them /// </param> /// <param name="conf">used to resolve the paths</param> /// <param name="environment">the environment to update CLASSPATH in</param> /// <exception cref="System.IO.IOException">if there is an error resolving any of the paths. /// </exception> private static void AddToClasspathIfNotJar(Path[] paths, URI[] withLinks, Configuration conf, IDictionary <string, string> environment, string classpathEnvVar) { if (paths != null) { Dictionary <Path, string> linkLookup = new Dictionary <Path, string>(); if (withLinks != null) { foreach (URI u in withLinks) { Path p = new Path(u); FileSystem remoteFS = p.GetFileSystem(conf); p = remoteFS.ResolvePath(p.MakeQualified(remoteFS.GetUri(), remoteFS.GetWorkingDirectory ())); string name = (null == u.GetFragment()) ? p.GetName() : u.GetFragment(); if (!StringUtils.ToLowerCase(name).EndsWith(".jar")) { linkLookup[p] = name; } } } foreach (Path p_1 in paths) { FileSystem remoteFS = p_1.GetFileSystem(conf); p_1 = remoteFS.ResolvePath(p_1.MakeQualified(remoteFS.GetUri(), remoteFS.GetWorkingDirectory ())); string name = linkLookup[p_1]; if (name == null) { name = p_1.GetName(); } if (!StringUtils.ToLowerCase(name).EndsWith(".jar")) { MRApps.AddToEnvironment(environment, classpathEnvVar, CrossPlatformifyMREnv(conf, ApplicationConstants.Environment.Pwd) + Path.Separator + name, conf); } } } }