Beispiel #1
0
        /// <exception cref="System.IO.IOException"/>
        public virtual ApplicationSubmissionContext CreateApplicationSubmissionContext(Configuration
                                                                                       jobConf, string jobSubmitDir, Credentials ts)
        {
            ApplicationId applicationId = resMgrDelegate.GetApplicationId();
            // Setup resource requirements
            Resource capability = recordFactory.NewRecordInstance <Resource>();

            capability.SetMemory(conf.GetInt(MRJobConfig.MrAmVmemMb, MRJobConfig.DefaultMrAmVmemMb
                                             ));
            capability.SetVirtualCores(conf.GetInt(MRJobConfig.MrAmCpuVcores, MRJobConfig.DefaultMrAmCpuVcores
                                                   ));
            Log.Debug("AppMaster capability = " + capability);
            // Setup LocalResources
            IDictionary <string, LocalResource> localResources = new Dictionary <string, LocalResource
                                                                                 >();
            Path jobConfPath            = new Path(jobSubmitDir, MRJobConfig.JobConfFile);
            URL  yarnUrlForJobSubmitDir = ConverterUtils.GetYarnUrlFromPath(defaultFileContext
                                                                            .GetDefaultFileSystem().ResolvePath(defaultFileContext.MakeQualified(new Path(jobSubmitDir
                                                                                                                                                          ))));

            Log.Debug("Creating setup context, jobSubmitDir url is " + yarnUrlForJobSubmitDir
                      );
            localResources[MRJobConfig.JobConfFile] = CreateApplicationResource(defaultFileContext
                                                                                , jobConfPath, LocalResourceType.File);
            if (jobConf.Get(MRJobConfig.Jar) != null)
            {
                Path          jobJarPath = new Path(jobConf.Get(MRJobConfig.Jar));
                LocalResource rc         = CreateApplicationResource(FileContext.GetFileContext(jobJarPath
                                                                                                .ToUri(), jobConf), jobJarPath, LocalResourceType.Pattern);
                string pattern = conf.GetPattern(JobContext.JarUnpackPattern, JobConf.UnpackJarPatternDefault
                                                 ).Pattern();
                rc.SetPattern(pattern);
                localResources[MRJobConfig.JobJar] = rc;
            }
            else
            {
                // Job jar may be null. For e.g, for pipes, the job jar is the hadoop
                // mapreduce jar itself which is already on the classpath.
                Log.Info("Job jar is not present. " + "Not adding any jar to the list of resources."
                         );
            }
            // TODO gross hack
            foreach (string s in new string[] { MRJobConfig.JobSplit, MRJobConfig.JobSplitMetainfo })
            {
                localResources[MRJobConfig.JobSubmitDir + "/" + s] = CreateApplicationResource(defaultFileContext
                                                                                               , new Path(jobSubmitDir, s), LocalResourceType.File);
            }
            // Setup security tokens
            DataOutputBuffer dob = new DataOutputBuffer();

            ts.WriteTokenStorageToStream(dob);
            ByteBuffer securityTokens = ByteBuffer.Wrap(dob.GetData(), 0, dob.GetLength());
            // Setup the command to run the AM
            IList <string> vargs = new AList <string>(8);

            vargs.AddItem(MRApps.CrossPlatformifyMREnv(jobConf, ApplicationConstants.Environment
                                                       .JavaHome) + "/bin/java");
            Path amTmpDir = new Path(MRApps.CrossPlatformifyMREnv(conf, ApplicationConstants.Environment
                                                                  .Pwd), YarnConfiguration.DefaultContainerTempDir);

            vargs.AddItem("-Djava.io.tmpdir=" + amTmpDir);
            MRApps.AddLog4jSystemProperties(null, vargs, conf);
            // Check for Java Lib Path usage in MAP and REDUCE configs
            WarnForJavaLibPath(conf.Get(MRJobConfig.MapJavaOpts, string.Empty), "map", MRJobConfig
                               .MapJavaOpts, MRJobConfig.MapEnv);
            WarnForJavaLibPath(conf.Get(MRJobConfig.MapredMapAdminJavaOpts, string.Empty), "map"
                               , MRJobConfig.MapredMapAdminJavaOpts, MRJobConfig.MapredAdminUserEnv);
            WarnForJavaLibPath(conf.Get(MRJobConfig.ReduceJavaOpts, string.Empty), "reduce",
                               MRJobConfig.ReduceJavaOpts, MRJobConfig.ReduceEnv);
            WarnForJavaLibPath(conf.Get(MRJobConfig.MapredReduceAdminJavaOpts, string.Empty),
                               "reduce", MRJobConfig.MapredReduceAdminJavaOpts, MRJobConfig.MapredAdminUserEnv);
            // Add AM admin command opts before user command opts
            // so that it can be overridden by user
            string mrAppMasterAdminOptions = conf.Get(MRJobConfig.MrAmAdminCommandOpts, MRJobConfig
                                                      .DefaultMrAmAdminCommandOpts);

            WarnForJavaLibPath(mrAppMasterAdminOptions, "app master", MRJobConfig.MrAmAdminCommandOpts
                               , MRJobConfig.MrAmAdminUserEnv);
            vargs.AddItem(mrAppMasterAdminOptions);
            // Add AM user command opts
            string mrAppMasterUserOptions = conf.Get(MRJobConfig.MrAmCommandOpts, MRJobConfig
                                                     .DefaultMrAmCommandOpts);

            WarnForJavaLibPath(mrAppMasterUserOptions, "app master", MRJobConfig.MrAmCommandOpts
                               , MRJobConfig.MrAmEnv);
            vargs.AddItem(mrAppMasterUserOptions);
            if (jobConf.GetBoolean(MRJobConfig.MrAmProfile, MRJobConfig.DefaultMrAmProfile))
            {
                string profileParams = jobConf.Get(MRJobConfig.MrAmProfileParams, MRJobConfig.DefaultTaskProfileParams
                                                   );
                if (profileParams != null)
                {
                    vargs.AddItem(string.Format(profileParams, ApplicationConstants.LogDirExpansionVar
                                                + Path.Separator + TaskLog.LogName.Profile));
                }
            }
            vargs.AddItem(MRJobConfig.ApplicationMasterClass);
            vargs.AddItem("1>" + ApplicationConstants.LogDirExpansionVar + Path.Separator + ApplicationConstants
                          .Stdout);
            vargs.AddItem("2>" + ApplicationConstants.LogDirExpansionVar + Path.Separator + ApplicationConstants
                          .Stderr);
            Vector <string> vargsFinal = new Vector <string>(8);
            // Final command
            StringBuilder mergedCommand = new StringBuilder();

            foreach (CharSequence str in vargs)
            {
                mergedCommand.Append(str).Append(" ");
            }
            vargsFinal.AddItem(mergedCommand.ToString());
            Log.Debug("Command to launch container for ApplicationMaster is : " + mergedCommand
                      );
            // Setup the CLASSPATH in environment
            // i.e. add { Hadoop jars, job jar, CWD } to classpath.
            IDictionary <string, string> environment = new Dictionary <string, string>();

            MRApps.SetClasspath(environment, conf);
            // Shell
            environment[ApplicationConstants.Environment.Shell.ToString()] = conf.Get(MRJobConfig
                                                                                      .MapredAdminUserShell, MRJobConfig.DefaultShell);
            // Add the container working directory at the front of LD_LIBRARY_PATH
            MRApps.AddToEnvironment(environment, ApplicationConstants.Environment.LdLibraryPath
                                    .ToString(), MRApps.CrossPlatformifyMREnv(conf, ApplicationConstants.Environment
                                                                              .Pwd), conf);
            // Setup the environment variables for Admin first
            MRApps.SetEnvFromInputString(environment, conf.Get(MRJobConfig.MrAmAdminUserEnv),
                                         conf);
            // Setup the environment variables (LD_LIBRARY_PATH, etc)
            MRApps.SetEnvFromInputString(environment, conf.Get(MRJobConfig.MrAmEnv), conf);
            // Parse distributed cache
            MRApps.SetupDistributedCache(jobConf, localResources);
            IDictionary <ApplicationAccessType, string> acls = new Dictionary <ApplicationAccessType
                                                                               , string>(2);

            acls[ApplicationAccessType.ViewApp] = jobConf.Get(MRJobConfig.JobAclViewJob, MRJobConfig
                                                              .DefaultJobAclViewJob);
            acls[ApplicationAccessType.ModifyApp] = jobConf.Get(MRJobConfig.JobAclModifyJob,
                                                                MRJobConfig.DefaultJobAclModifyJob);
            // Setup ContainerLaunchContext for AM container
            ContainerLaunchContext amContainer = ContainerLaunchContext.NewInstance(localResources
                                                                                    , environment, vargsFinal, null, securityTokens, acls);
            ICollection <string> tagsFromConf = jobConf.GetTrimmedStringCollection(MRJobConfig
                                                                                   .JobTags);
            // Set up the ApplicationSubmissionContext
            ApplicationSubmissionContext appContext = recordFactory.NewRecordInstance <ApplicationSubmissionContext
                                                                                       >();

            appContext.SetApplicationId(applicationId);
            // ApplicationId
            appContext.SetQueue(jobConf.Get(JobContext.QueueName, YarnConfiguration.DefaultQueueName
                                            ));
            // Queue name
            // add reservationID if present
            ReservationId reservationID = null;

            try
            {
                reservationID = ReservationId.ParseReservationId(jobConf.Get(JobContext.ReservationId
                                                                             ));
            }
            catch (FormatException)
            {
                // throw exception as reservationid as is invalid
                string errMsg = "Invalid reservationId: " + jobConf.Get(JobContext.ReservationId)
                                + " specified for the app: " + applicationId;
                Log.Warn(errMsg);
                throw new IOException(errMsg);
            }
            if (reservationID != null)
            {
                appContext.SetReservationID(reservationID);
                Log.Info("SUBMITTING ApplicationSubmissionContext app:" + applicationId + " to queue:"
                         + appContext.GetQueue() + " with reservationId:" + appContext.GetReservationID(
                             ));
            }
            appContext.SetApplicationName(jobConf.Get(JobContext.JobName, YarnConfiguration.DefaultApplicationName
                                                      ));
            // Job name
            appContext.SetCancelTokensWhenComplete(conf.GetBoolean(MRJobConfig.JobCancelDelegationToken
                                                                   , true));
            appContext.SetAMContainerSpec(amContainer);
            // AM Container
            appContext.SetMaxAppAttempts(conf.GetInt(MRJobConfig.MrAmMaxAttempts, MRJobConfig
                                                     .DefaultMrAmMaxAttempts));
            appContext.SetResource(capability);
            appContext.SetApplicationType(MRJobConfig.MrApplicationType);
            if (tagsFromConf != null && !tagsFromConf.IsEmpty())
            {
                appContext.SetApplicationTags(new HashSet <string>(tagsFromConf));
            }
            return(appContext);
        }