public virtual void TestAMStandardEnv() { string AdminLibPath = "foo"; string UserLibPath = "bar"; string UserShell = "shell"; JobConf jobConf = new JobConf(); jobConf.Set(MRJobConfig.MrAmAdminUserEnv, "LD_LIBRARY_PATH=" + AdminLibPath); jobConf.Set(MRJobConfig.MrAmEnv, "LD_LIBRARY_PATH=" + UserLibPath); jobConf.Set(MRJobConfig.MapredAdminUserShell, UserShell); YARNRunner yarnRunner = new YARNRunner(jobConf); ApplicationSubmissionContext appSubCtx = BuildSubmitContext(yarnRunner, jobConf); // make sure PWD is first in the lib path ContainerLaunchContext clc = appSubCtx.GetAMContainerSpec(); IDictionary <string, string> env = clc.GetEnvironment(); string libPath = env[ApplicationConstants.Environment.LdLibraryPath.ToString()]; NUnit.Framework.Assert.IsNotNull("LD_LIBRARY_PATH not set", libPath); string cps = jobConf.GetBoolean(MRConfig.MapreduceAppSubmissionCrossPlatform, MRConfig .DefaultMapreduceAppSubmissionCrossPlatform) ? ApplicationConstants.ClassPathSeparator : FilePath.pathSeparator; NUnit.Framework.Assert.AreEqual("Bad AM LD_LIBRARY_PATH setting", MRApps.CrossPlatformifyMREnv (conf, ApplicationConstants.Environment.Pwd) + cps + AdminLibPath + cps + UserLibPath , libPath); // make sure SHELL is set string shell = env[ApplicationConstants.Environment.Shell.ToString()]; NUnit.Framework.Assert.IsNotNull("SHELL not set", shell); NUnit.Framework.Assert.AreEqual("Bad SHELL setting", UserShell, shell); }
/// <exception cref="System.IO.IOException"/> public virtual ApplicationSubmissionContext CreateApplicationSubmissionContext(Configuration jobConf, string jobSubmitDir, Credentials ts) { ApplicationId applicationId = resMgrDelegate.GetApplicationId(); // Setup resource requirements Resource capability = recordFactory.NewRecordInstance <Resource>(); capability.SetMemory(conf.GetInt(MRJobConfig.MrAmVmemMb, MRJobConfig.DefaultMrAmVmemMb )); capability.SetVirtualCores(conf.GetInt(MRJobConfig.MrAmCpuVcores, MRJobConfig.DefaultMrAmCpuVcores )); Log.Debug("AppMaster capability = " + capability); // Setup LocalResources IDictionary <string, LocalResource> localResources = new Dictionary <string, LocalResource >(); Path jobConfPath = new Path(jobSubmitDir, MRJobConfig.JobConfFile); URL yarnUrlForJobSubmitDir = ConverterUtils.GetYarnUrlFromPath(defaultFileContext .GetDefaultFileSystem().ResolvePath(defaultFileContext.MakeQualified(new Path(jobSubmitDir )))); Log.Debug("Creating setup context, jobSubmitDir url is " + yarnUrlForJobSubmitDir ); localResources[MRJobConfig.JobConfFile] = CreateApplicationResource(defaultFileContext , jobConfPath, LocalResourceType.File); if (jobConf.Get(MRJobConfig.Jar) != null) { Path jobJarPath = new Path(jobConf.Get(MRJobConfig.Jar)); LocalResource rc = CreateApplicationResource(FileContext.GetFileContext(jobJarPath .ToUri(), jobConf), jobJarPath, LocalResourceType.Pattern); string pattern = conf.GetPattern(JobContext.JarUnpackPattern, JobConf.UnpackJarPatternDefault ).Pattern(); rc.SetPattern(pattern); localResources[MRJobConfig.JobJar] = rc; } else { // Job jar may be null. For e.g, for pipes, the job jar is the hadoop // mapreduce jar itself which is already on the classpath. Log.Info("Job jar is not present. " + "Not adding any jar to the list of resources." ); } // TODO gross hack foreach (string s in new string[] { MRJobConfig.JobSplit, MRJobConfig.JobSplitMetainfo }) { localResources[MRJobConfig.JobSubmitDir + "/" + s] = CreateApplicationResource(defaultFileContext , new Path(jobSubmitDir, s), LocalResourceType.File); } // Setup security tokens DataOutputBuffer dob = new DataOutputBuffer(); ts.WriteTokenStorageToStream(dob); ByteBuffer securityTokens = ByteBuffer.Wrap(dob.GetData(), 0, dob.GetLength()); // Setup the command to run the AM IList <string> vargs = new AList <string>(8); vargs.AddItem(MRApps.CrossPlatformifyMREnv(jobConf, ApplicationConstants.Environment .JavaHome) + "/bin/java"); Path amTmpDir = new Path(MRApps.CrossPlatformifyMREnv(conf, ApplicationConstants.Environment .Pwd), YarnConfiguration.DefaultContainerTempDir); vargs.AddItem("-Djava.io.tmpdir=" + amTmpDir); MRApps.AddLog4jSystemProperties(null, vargs, conf); // Check for Java Lib Path usage in MAP and REDUCE configs WarnForJavaLibPath(conf.Get(MRJobConfig.MapJavaOpts, string.Empty), "map", MRJobConfig .MapJavaOpts, MRJobConfig.MapEnv); WarnForJavaLibPath(conf.Get(MRJobConfig.MapredMapAdminJavaOpts, string.Empty), "map" , MRJobConfig.MapredMapAdminJavaOpts, MRJobConfig.MapredAdminUserEnv); WarnForJavaLibPath(conf.Get(MRJobConfig.ReduceJavaOpts, string.Empty), "reduce", MRJobConfig.ReduceJavaOpts, MRJobConfig.ReduceEnv); WarnForJavaLibPath(conf.Get(MRJobConfig.MapredReduceAdminJavaOpts, string.Empty), "reduce", MRJobConfig.MapredReduceAdminJavaOpts, MRJobConfig.MapredAdminUserEnv); // Add AM admin command opts before user command opts // so that it can be overridden by user string mrAppMasterAdminOptions = conf.Get(MRJobConfig.MrAmAdminCommandOpts, MRJobConfig .DefaultMrAmAdminCommandOpts); WarnForJavaLibPath(mrAppMasterAdminOptions, "app master", MRJobConfig.MrAmAdminCommandOpts , MRJobConfig.MrAmAdminUserEnv); vargs.AddItem(mrAppMasterAdminOptions); // Add AM user command opts string mrAppMasterUserOptions = conf.Get(MRJobConfig.MrAmCommandOpts, MRJobConfig .DefaultMrAmCommandOpts); WarnForJavaLibPath(mrAppMasterUserOptions, "app master", MRJobConfig.MrAmCommandOpts , MRJobConfig.MrAmEnv); vargs.AddItem(mrAppMasterUserOptions); if (jobConf.GetBoolean(MRJobConfig.MrAmProfile, MRJobConfig.DefaultMrAmProfile)) { string profileParams = jobConf.Get(MRJobConfig.MrAmProfileParams, MRJobConfig.DefaultTaskProfileParams ); if (profileParams != null) { vargs.AddItem(string.Format(profileParams, ApplicationConstants.LogDirExpansionVar + Path.Separator + TaskLog.LogName.Profile)); } } vargs.AddItem(MRJobConfig.ApplicationMasterClass); vargs.AddItem("1>" + ApplicationConstants.LogDirExpansionVar + Path.Separator + ApplicationConstants .Stdout); vargs.AddItem("2>" + ApplicationConstants.LogDirExpansionVar + Path.Separator + ApplicationConstants .Stderr); Vector <string> vargsFinal = new Vector <string>(8); // Final command StringBuilder mergedCommand = new StringBuilder(); foreach (CharSequence str in vargs) { mergedCommand.Append(str).Append(" "); } vargsFinal.AddItem(mergedCommand.ToString()); Log.Debug("Command to launch container for ApplicationMaster is : " + mergedCommand ); // Setup the CLASSPATH in environment // i.e. add { Hadoop jars, job jar, CWD } to classpath. IDictionary <string, string> environment = new Dictionary <string, string>(); MRApps.SetClasspath(environment, conf); // Shell environment[ApplicationConstants.Environment.Shell.ToString()] = conf.Get(MRJobConfig .MapredAdminUserShell, MRJobConfig.DefaultShell); // Add the container working directory at the front of LD_LIBRARY_PATH MRApps.AddToEnvironment(environment, ApplicationConstants.Environment.LdLibraryPath .ToString(), MRApps.CrossPlatformifyMREnv(conf, ApplicationConstants.Environment .Pwd), conf); // Setup the environment variables for Admin first MRApps.SetEnvFromInputString(environment, conf.Get(MRJobConfig.MrAmAdminUserEnv), conf); // Setup the environment variables (LD_LIBRARY_PATH, etc) MRApps.SetEnvFromInputString(environment, conf.Get(MRJobConfig.MrAmEnv), conf); // Parse distributed cache MRApps.SetupDistributedCache(jobConf, localResources); IDictionary <ApplicationAccessType, string> acls = new Dictionary <ApplicationAccessType , string>(2); acls[ApplicationAccessType.ViewApp] = jobConf.Get(MRJobConfig.JobAclViewJob, MRJobConfig .DefaultJobAclViewJob); acls[ApplicationAccessType.ModifyApp] = jobConf.Get(MRJobConfig.JobAclModifyJob, MRJobConfig.DefaultJobAclModifyJob); // Setup ContainerLaunchContext for AM container ContainerLaunchContext amContainer = ContainerLaunchContext.NewInstance(localResources , environment, vargsFinal, null, securityTokens, acls); ICollection <string> tagsFromConf = jobConf.GetTrimmedStringCollection(MRJobConfig .JobTags); // Set up the ApplicationSubmissionContext ApplicationSubmissionContext appContext = recordFactory.NewRecordInstance <ApplicationSubmissionContext >(); appContext.SetApplicationId(applicationId); // ApplicationId appContext.SetQueue(jobConf.Get(JobContext.QueueName, YarnConfiguration.DefaultQueueName )); // Queue name // add reservationID if present ReservationId reservationID = null; try { reservationID = ReservationId.ParseReservationId(jobConf.Get(JobContext.ReservationId )); } catch (FormatException) { // throw exception as reservationid as is invalid string errMsg = "Invalid reservationId: " + jobConf.Get(JobContext.ReservationId) + " specified for the app: " + applicationId; Log.Warn(errMsg); throw new IOException(errMsg); } if (reservationID != null) { appContext.SetReservationID(reservationID); Log.Info("SUBMITTING ApplicationSubmissionContext app:" + applicationId + " to queue:" + appContext.GetQueue() + " with reservationId:" + appContext.GetReservationID( )); } appContext.SetApplicationName(jobConf.Get(JobContext.JobName, YarnConfiguration.DefaultApplicationName )); // Job name appContext.SetCancelTokensWhenComplete(conf.GetBoolean(MRJobConfig.JobCancelDelegationToken , true)); appContext.SetAMContainerSpec(amContainer); // AM Container appContext.SetMaxAppAttempts(conf.GetInt(MRJobConfig.MrAmMaxAttempts, MRJobConfig .DefaultMrAmMaxAttempts)); appContext.SetResource(capability); appContext.SetApplicationType(MRJobConfig.MrApplicationType); if (tagsFromConf != null && !tagsFromConf.IsEmpty()) { appContext.SetApplicationTags(new HashSet <string>(tagsFromConf)); } return(appContext); }
public static IList <string> GetVMCommand(IPEndPoint taskAttemptListenerAddr, Task task, JVMId jvmID) { TaskAttemptID attemptID = task.GetTaskID(); JobConf conf = task.conf; Vector <string> vargs = new Vector <string>(8); vargs.AddItem(MRApps.CrossPlatformifyMREnv(task.conf, ApplicationConstants.Environment .JavaHome) + "/bin/java"); // Add child (task) java-vm options. // // The following symbols if present in mapred.{map|reduce}.child.java.opts // value are replaced: // + @taskid@ is interpolated with value of TaskID. // Other occurrences of @ will not be altered. // // Example with multiple arguments and substitutions, showing // jvm GC logging, and start of a passwordless JVM JMX agent so can // connect with jconsole and the likes to watch child memory, threads // and get thread dumps. // // <property> // <name>mapred.map.child.java.opts</name> // <value>-Xmx 512M -verbose:gc -Xloggc:/tmp/@[email protected] \ // -Dcom.sun.management.jmxremote.authenticate=false \ // -Dcom.sun.management.jmxremote.ssl=false \ // </value> // </property> // // <property> // <name>mapred.reduce.child.java.opts</name> // <value>-Xmx 1024M -verbose:gc -Xloggc:/tmp/@[email protected] \ // -Dcom.sun.management.jmxremote.authenticate=false \ // -Dcom.sun.management.jmxremote.ssl=false \ // </value> // </property> // string javaOpts = GetChildJavaOpts(conf, task.IsMapTask()); javaOpts = javaOpts.Replace("@taskid@", attemptID.ToString()); string[] javaOptsSplit = javaOpts.Split(" "); for (int i = 0; i < javaOptsSplit.Length; i++) { vargs.AddItem(javaOptsSplit[i]); } Path childTmpDir = new Path(MRApps.CrossPlatformifyMREnv(conf, ApplicationConstants.Environment .Pwd), YarnConfiguration.DefaultContainerTempDir); vargs.AddItem("-Djava.io.tmpdir=" + childTmpDir); MRApps.AddLog4jSystemProperties(task, vargs, conf); if (conf.GetProfileEnabled()) { if (conf.GetProfileTaskRange(task.IsMapTask()).IsIncluded(task.GetPartition())) { string profileParams = conf.Get(task.IsMapTask() ? MRJobConfig.TaskMapProfileParams : MRJobConfig.TaskReduceProfileParams, conf.GetProfileParams()); vargs.AddItem(string.Format(profileParams, GetTaskLogFile(TaskLog.LogName.Profile ))); } } // Add main class and its arguments vargs.AddItem(typeof(YarnChild).FullName); // main of Child // pass TaskAttemptListener's address vargs.AddItem(taskAttemptListenerAddr.Address.GetHostAddress()); vargs.AddItem(Sharpen.Extensions.ToString(taskAttemptListenerAddr.Port)); vargs.AddItem(attemptID.ToString()); // pass task identifier // Finally add the jvmID vargs.AddItem(jvmID.GetId().ToString()); vargs.AddItem("1>" + GetTaskLogFile(TaskLog.LogName.Stdout)); vargs.AddItem("2>" + GetTaskLogFile(TaskLog.LogName.Stderr)); // Final commmand StringBuilder mergedCommand = new StringBuilder(); foreach (CharSequence str in vargs) { mergedCommand.Append(str).Append(" "); } Vector <string> vargsFinal = new Vector <string>(1); vargsFinal.AddItem(mergedCommand.ToString()); return(vargsFinal); }