Exemple #1
0
        public static ApplicationSubmissionContext NewApplicationSubmissionContext(ApplicationId
                                                                                   applicationId, string applicationName, string queue, Priority priority, ContainerLaunchContext
                                                                                   amContainer, bool isUnmanagedAM, bool cancelTokensWhenComplete, int maxAppAttempts
                                                                                   , Resource resource, string applicationType)
        {
            ApplicationSubmissionContext context = recordFactory.NewRecordInstance <ApplicationSubmissionContext
                                                                                    >();

            context.SetApplicationId(applicationId);
            context.SetApplicationName(applicationName);
            context.SetQueue(queue);
            context.SetPriority(priority);
            context.SetAMContainerSpec(amContainer);
            context.SetUnmanagedAM(isUnmanagedAM);
            context.SetCancelTokensWhenComplete(cancelTokensWhenComplete);
            context.SetMaxAppAttempts(maxAppAttempts);
            context.SetResource(resource);
            context.SetApplicationType(applicationType);
            return(context);
        }
Exemple #2
0
        public virtual void Setup()
        {
            // start minicluster
            conf        = new YarnConfiguration();
            yarnCluster = new MiniYARNCluster(typeof(TestAMRMClient).FullName, nodeCount, 1,
                                              1);
            yarnCluster.Init(conf);
            yarnCluster.Start();
            NUnit.Framework.Assert.IsNotNull(yarnCluster);
            NUnit.Framework.Assert.AreEqual(Service.STATE.Started, yarnCluster.GetServiceState
                                                ());
            // start rm client
            yarnClient = (YarnClientImpl)YarnClient.CreateYarnClient();
            yarnClient.Init(conf);
            yarnClient.Start();
            NUnit.Framework.Assert.IsNotNull(yarnClient);
            NUnit.Framework.Assert.AreEqual(Service.STATE.Started, yarnClient.GetServiceState
                                                ());
            // get node info
            nodeReports = yarnClient.GetNodeReports(NodeState.Running);
            // submit new app
            ApplicationSubmissionContext appContext = yarnClient.CreateApplication().GetApplicationSubmissionContext
                                                          ();
            ApplicationId appId = appContext.GetApplicationId();

            // set the application name
            appContext.SetApplicationName("Test");
            // Set the priority for the application master
            Priority pri = Priority.NewInstance(0);

            appContext.SetPriority(pri);
            // Set the queue to which this application is to be submitted in the RM
            appContext.SetQueue("default");
            // Set up the container launch context for the application master
            ContainerLaunchContext amContainer = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord
                                                 <ContainerLaunchContext>();

            appContext.SetAMContainerSpec(amContainer);
            // unmanaged AM
            appContext.SetUnmanagedAM(true);
            // Create the request to send to the applications manager
            SubmitApplicationRequest appRequest = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord
                                                  <SubmitApplicationRequest>();

            appRequest.SetApplicationSubmissionContext(appContext);
            // Submit the application to the applications manager
            yarnClient.SubmitApplication(appContext);
            // wait for app to start
            int          iterationsLeft = 30;
            RMAppAttempt appAttempt     = null;

            while (iterationsLeft > 0)
            {
                ApplicationReport appReport = yarnClient.GetApplicationReport(appId);
                if (appReport.GetYarnApplicationState() == YarnApplicationState.Accepted)
                {
                    attemptId  = appReport.GetCurrentApplicationAttemptId();
                    appAttempt = yarnCluster.GetResourceManager().GetRMContext().GetRMApps()[attemptId
                                                                                             .GetApplicationId()].GetCurrentAppAttempt();
                    while (true)
                    {
                        if (appAttempt.GetAppAttemptState() == RMAppAttemptState.Launched)
                        {
                            break;
                        }
                    }
                    break;
                }
                Sleep(1000);
                --iterationsLeft;
            }
            if (iterationsLeft == 0)
            {
                NUnit.Framework.Assert.Fail("Application hasn't bee started");
            }
            // Just dig into the ResourceManager and get the AMRMToken just for the sake
            // of testing.
            UserGroupInformation.SetLoginUser(UserGroupInformation.CreateRemoteUser(UserGroupInformation
                                                                                    .GetCurrentUser().GetUserName()));
            UserGroupInformation.GetCurrentUser().AddToken(appAttempt.GetAMRMToken());
            //creating an instance NMTokenCase
            nmTokenCache = new NMTokenCache();
            // start am rm client
            rmClient = (AMRMClientImpl <AMRMClient.ContainerRequest>)AMRMClient.CreateAMRMClient
                       <AMRMClient.ContainerRequest>();
            //setting an instance NMTokenCase
            rmClient.SetNMTokenCache(nmTokenCache);
            rmClient.Init(conf);
            rmClient.Start();
            NUnit.Framework.Assert.IsNotNull(rmClient);
            NUnit.Framework.Assert.AreEqual(Service.STATE.Started, rmClient.GetServiceState()
                                            );
            // start am nm client
            nmClient = (NMClientImpl)NMClient.CreateNMClient();
            //propagating the AMRMClient NMTokenCache instance
            nmClient.SetNMTokenCache(rmClient.GetNMTokenCache());
            nmClient.Init(conf);
            nmClient.Start();
            NUnit.Framework.Assert.IsNotNull(nmClient);
            NUnit.Framework.Assert.AreEqual(Service.STATE.Started, nmClient.GetServiceState()
                                            );
        }
Exemple #3
0
 /// <exception cref="System.IO.IOException"/>
 /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
 public virtual bool Run()
 {
     Log.Info("Starting Client");
     // Connect to ResourceManager
     rmClient.Start();
     try
     {
         // Create launch context for app master
         Log.Info("Setting up application submission context for ASM");
         ApplicationSubmissionContext appContext = rmClient.CreateApplication().GetApplicationSubmissionContext
                                                       ();
         ApplicationId appId = appContext.GetApplicationId();
         // set the application name
         appContext.SetApplicationName(appName);
         // Set the priority for the application master
         Priority pri = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <Priority>();
         pri.SetPriority(amPriority);
         appContext.SetPriority(pri);
         // Set the queue to which this application is to be submitted in the RM
         appContext.SetQueue(amQueue);
         // Set up the container launch context for the application master
         ContainerLaunchContext amContainer = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord
                                              <ContainerLaunchContext>();
         appContext.SetAMContainerSpec(amContainer);
         // unmanaged AM
         appContext.SetUnmanagedAM(true);
         Log.Info("Setting unmanaged AM");
         // Submit the application to the applications manager
         Log.Info("Submitting application to ASM");
         rmClient.SubmitApplication(appContext);
         ApplicationReport appReport = MonitorApplication(appId, EnumSet.Of(YarnApplicationState
                                                                            .Accepted, YarnApplicationState.Killed, YarnApplicationState.Failed, YarnApplicationState
                                                                            .Finished));
         if (appReport.GetYarnApplicationState() == YarnApplicationState.Accepted)
         {
             // Monitor the application attempt to wait for launch state
             ApplicationAttemptReport attemptReport = MonitorCurrentAppAttempt(appId, YarnApplicationAttemptState
                                                                               .Launched);
             ApplicationAttemptId attemptId = attemptReport.GetApplicationAttemptId();
             Log.Info("Launching AM with application attempt id " + attemptId);
             // launch AM
             LaunchAM(attemptId);
             // Monitor the application for end state
             appReport = MonitorApplication(appId, EnumSet.Of(YarnApplicationState.Killed, YarnApplicationState
                                                              .Failed, YarnApplicationState.Finished));
         }
         YarnApplicationState   appState  = appReport.GetYarnApplicationState();
         FinalApplicationStatus appStatus = appReport.GetFinalApplicationStatus();
         Log.Info("App ended with state: " + appReport.GetYarnApplicationState() + " and status: "
                  + appStatus);
         bool success;
         if (YarnApplicationState.Finished == appState && FinalApplicationStatus.Succeeded
             == appStatus)
         {
             Log.Info("Application has completed successfully.");
             success = true;
         }
         else
         {
             Log.Info("Application did finished unsuccessfully." + " YarnState=" + appState.ToString
                          () + ", FinalStatus=" + appStatus.ToString());
             success = false;
         }
         return(success);
     }
     finally
     {
         rmClient.Stop();
     }
 }
Exemple #4
0
        /// <exception cref="System.IO.IOException"/>
        public virtual ApplicationSubmissionContext CreateApplicationSubmissionContext(Configuration
                                                                                       jobConf, string jobSubmitDir, Credentials ts)
        {
            ApplicationId applicationId = resMgrDelegate.GetApplicationId();
            // Setup resource requirements
            Resource capability = recordFactory.NewRecordInstance <Resource>();

            capability.SetMemory(conf.GetInt(MRJobConfig.MrAmVmemMb, MRJobConfig.DefaultMrAmVmemMb
                                             ));
            capability.SetVirtualCores(conf.GetInt(MRJobConfig.MrAmCpuVcores, MRJobConfig.DefaultMrAmCpuVcores
                                                   ));
            Log.Debug("AppMaster capability = " + capability);
            // Setup LocalResources
            IDictionary <string, LocalResource> localResources = new Dictionary <string, LocalResource
                                                                                 >();
            Path jobConfPath            = new Path(jobSubmitDir, MRJobConfig.JobConfFile);
            URL  yarnUrlForJobSubmitDir = ConverterUtils.GetYarnUrlFromPath(defaultFileContext
                                                                            .GetDefaultFileSystem().ResolvePath(defaultFileContext.MakeQualified(new Path(jobSubmitDir
                                                                                                                                                          ))));

            Log.Debug("Creating setup context, jobSubmitDir url is " + yarnUrlForJobSubmitDir
                      );
            localResources[MRJobConfig.JobConfFile] = CreateApplicationResource(defaultFileContext
                                                                                , jobConfPath, LocalResourceType.File);
            if (jobConf.Get(MRJobConfig.Jar) != null)
            {
                Path          jobJarPath = new Path(jobConf.Get(MRJobConfig.Jar));
                LocalResource rc         = CreateApplicationResource(FileContext.GetFileContext(jobJarPath
                                                                                                .ToUri(), jobConf), jobJarPath, LocalResourceType.Pattern);
                string pattern = conf.GetPattern(JobContext.JarUnpackPattern, JobConf.UnpackJarPatternDefault
                                                 ).Pattern();
                rc.SetPattern(pattern);
                localResources[MRJobConfig.JobJar] = rc;
            }
            else
            {
                // Job jar may be null. For e.g, for pipes, the job jar is the hadoop
                // mapreduce jar itself which is already on the classpath.
                Log.Info("Job jar is not present. " + "Not adding any jar to the list of resources."
                         );
            }
            // TODO gross hack
            foreach (string s in new string[] { MRJobConfig.JobSplit, MRJobConfig.JobSplitMetainfo })
            {
                localResources[MRJobConfig.JobSubmitDir + "/" + s] = CreateApplicationResource(defaultFileContext
                                                                                               , new Path(jobSubmitDir, s), LocalResourceType.File);
            }
            // Setup security tokens
            DataOutputBuffer dob = new DataOutputBuffer();

            ts.WriteTokenStorageToStream(dob);
            ByteBuffer securityTokens = ByteBuffer.Wrap(dob.GetData(), 0, dob.GetLength());
            // Setup the command to run the AM
            IList <string> vargs = new AList <string>(8);

            vargs.AddItem(MRApps.CrossPlatformifyMREnv(jobConf, ApplicationConstants.Environment
                                                       .JavaHome) + "/bin/java");
            Path amTmpDir = new Path(MRApps.CrossPlatformifyMREnv(conf, ApplicationConstants.Environment
                                                                  .Pwd), YarnConfiguration.DefaultContainerTempDir);

            vargs.AddItem("-Djava.io.tmpdir=" + amTmpDir);
            MRApps.AddLog4jSystemProperties(null, vargs, conf);
            // Check for Java Lib Path usage in MAP and REDUCE configs
            WarnForJavaLibPath(conf.Get(MRJobConfig.MapJavaOpts, string.Empty), "map", MRJobConfig
                               .MapJavaOpts, MRJobConfig.MapEnv);
            WarnForJavaLibPath(conf.Get(MRJobConfig.MapredMapAdminJavaOpts, string.Empty), "map"
                               , MRJobConfig.MapredMapAdminJavaOpts, MRJobConfig.MapredAdminUserEnv);
            WarnForJavaLibPath(conf.Get(MRJobConfig.ReduceJavaOpts, string.Empty), "reduce",
                               MRJobConfig.ReduceJavaOpts, MRJobConfig.ReduceEnv);
            WarnForJavaLibPath(conf.Get(MRJobConfig.MapredReduceAdminJavaOpts, string.Empty),
                               "reduce", MRJobConfig.MapredReduceAdminJavaOpts, MRJobConfig.MapredAdminUserEnv);
            // Add AM admin command opts before user command opts
            // so that it can be overridden by user
            string mrAppMasterAdminOptions = conf.Get(MRJobConfig.MrAmAdminCommandOpts, MRJobConfig
                                                      .DefaultMrAmAdminCommandOpts);

            WarnForJavaLibPath(mrAppMasterAdminOptions, "app master", MRJobConfig.MrAmAdminCommandOpts
                               , MRJobConfig.MrAmAdminUserEnv);
            vargs.AddItem(mrAppMasterAdminOptions);
            // Add AM user command opts
            string mrAppMasterUserOptions = conf.Get(MRJobConfig.MrAmCommandOpts, MRJobConfig
                                                     .DefaultMrAmCommandOpts);

            WarnForJavaLibPath(mrAppMasterUserOptions, "app master", MRJobConfig.MrAmCommandOpts
                               , MRJobConfig.MrAmEnv);
            vargs.AddItem(mrAppMasterUserOptions);
            if (jobConf.GetBoolean(MRJobConfig.MrAmProfile, MRJobConfig.DefaultMrAmProfile))
            {
                string profileParams = jobConf.Get(MRJobConfig.MrAmProfileParams, MRJobConfig.DefaultTaskProfileParams
                                                   );
                if (profileParams != null)
                {
                    vargs.AddItem(string.Format(profileParams, ApplicationConstants.LogDirExpansionVar
                                                + Path.Separator + TaskLog.LogName.Profile));
                }
            }
            vargs.AddItem(MRJobConfig.ApplicationMasterClass);
            vargs.AddItem("1>" + ApplicationConstants.LogDirExpansionVar + Path.Separator + ApplicationConstants
                          .Stdout);
            vargs.AddItem("2>" + ApplicationConstants.LogDirExpansionVar + Path.Separator + ApplicationConstants
                          .Stderr);
            Vector <string> vargsFinal = new Vector <string>(8);
            // Final command
            StringBuilder mergedCommand = new StringBuilder();

            foreach (CharSequence str in vargs)
            {
                mergedCommand.Append(str).Append(" ");
            }
            vargsFinal.AddItem(mergedCommand.ToString());
            Log.Debug("Command to launch container for ApplicationMaster is : " + mergedCommand
                      );
            // Setup the CLASSPATH in environment
            // i.e. add { Hadoop jars, job jar, CWD } to classpath.
            IDictionary <string, string> environment = new Dictionary <string, string>();

            MRApps.SetClasspath(environment, conf);
            // Shell
            environment[ApplicationConstants.Environment.Shell.ToString()] = conf.Get(MRJobConfig
                                                                                      .MapredAdminUserShell, MRJobConfig.DefaultShell);
            // Add the container working directory at the front of LD_LIBRARY_PATH
            MRApps.AddToEnvironment(environment, ApplicationConstants.Environment.LdLibraryPath
                                    .ToString(), MRApps.CrossPlatformifyMREnv(conf, ApplicationConstants.Environment
                                                                              .Pwd), conf);
            // Setup the environment variables for Admin first
            MRApps.SetEnvFromInputString(environment, conf.Get(MRJobConfig.MrAmAdminUserEnv),
                                         conf);
            // Setup the environment variables (LD_LIBRARY_PATH, etc)
            MRApps.SetEnvFromInputString(environment, conf.Get(MRJobConfig.MrAmEnv), conf);
            // Parse distributed cache
            MRApps.SetupDistributedCache(jobConf, localResources);
            IDictionary <ApplicationAccessType, string> acls = new Dictionary <ApplicationAccessType
                                                                               , string>(2);

            acls[ApplicationAccessType.ViewApp] = jobConf.Get(MRJobConfig.JobAclViewJob, MRJobConfig
                                                              .DefaultJobAclViewJob);
            acls[ApplicationAccessType.ModifyApp] = jobConf.Get(MRJobConfig.JobAclModifyJob,
                                                                MRJobConfig.DefaultJobAclModifyJob);
            // Setup ContainerLaunchContext for AM container
            ContainerLaunchContext amContainer = ContainerLaunchContext.NewInstance(localResources
                                                                                    , environment, vargsFinal, null, securityTokens, acls);
            ICollection <string> tagsFromConf = jobConf.GetTrimmedStringCollection(MRJobConfig
                                                                                   .JobTags);
            // Set up the ApplicationSubmissionContext
            ApplicationSubmissionContext appContext = recordFactory.NewRecordInstance <ApplicationSubmissionContext
                                                                                       >();

            appContext.SetApplicationId(applicationId);
            // ApplicationId
            appContext.SetQueue(jobConf.Get(JobContext.QueueName, YarnConfiguration.DefaultQueueName
                                            ));
            // Queue name
            // add reservationID if present
            ReservationId reservationID = null;

            try
            {
                reservationID = ReservationId.ParseReservationId(jobConf.Get(JobContext.ReservationId
                                                                             ));
            }
            catch (FormatException)
            {
                // throw exception as reservationid as is invalid
                string errMsg = "Invalid reservationId: " + jobConf.Get(JobContext.ReservationId)
                                + " specified for the app: " + applicationId;
                Log.Warn(errMsg);
                throw new IOException(errMsg);
            }
            if (reservationID != null)
            {
                appContext.SetReservationID(reservationID);
                Log.Info("SUBMITTING ApplicationSubmissionContext app:" + applicationId + " to queue:"
                         + appContext.GetQueue() + " with reservationId:" + appContext.GetReservationID(
                             ));
            }
            appContext.SetApplicationName(jobConf.Get(JobContext.JobName, YarnConfiguration.DefaultApplicationName
                                                      ));
            // Job name
            appContext.SetCancelTokensWhenComplete(conf.GetBoolean(MRJobConfig.JobCancelDelegationToken
                                                                   , true));
            appContext.SetAMContainerSpec(amContainer);
            // AM Container
            appContext.SetMaxAppAttempts(conf.GetInt(MRJobConfig.MrAmMaxAttempts, MRJobConfig
                                                     .DefaultMrAmMaxAttempts));
            appContext.SetResource(capability);
            appContext.SetApplicationType(MRJobConfig.MrApplicationType);
            if (tagsFromConf != null && !tagsFromConf.IsEmpty())
            {
                appContext.SetApplicationTags(new HashSet <string>(tagsFromConf));
            }
            return(appContext);
        }
Exemple #5
0
        /// <exception cref="System.Exception"/>
        public virtual RMApp SubmitApp(int masterMemory, string name, string user, IDictionary
                                       <ApplicationAccessType, string> acls, bool unmanaged, string queue, int maxAppAttempts
                                       , Credentials ts, string appType, bool waitForAccepted, bool keepContainers, bool
                                       isAppIdProvided, ApplicationId applicationId, long attemptFailuresValidityInterval
                                       , LogAggregationContext logAggregationContext, bool cancelTokensWhenComplete)
        {
            ApplicationId             appId  = isAppIdProvided ? applicationId : null;
            ApplicationClientProtocol client = GetClientRMService();

            if (!isAppIdProvided)
            {
                GetNewApplicationResponse resp = client.GetNewApplication(Org.Apache.Hadoop.Yarn.Util.Records
                                                                          .NewRecord <GetNewApplicationRequest>());
                appId = resp.GetApplicationId();
            }
            SubmitApplicationRequest req = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <SubmitApplicationRequest
                                                                                          >();
            ApplicationSubmissionContext sub = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <
                ApplicationSubmissionContext>();

            sub.SetKeepContainersAcrossApplicationAttempts(keepContainers);
            sub.SetApplicationId(appId);
            sub.SetApplicationName(name);
            sub.SetMaxAppAttempts(maxAppAttempts);
            if (unmanaged)
            {
                sub.SetUnmanagedAM(true);
            }
            if (queue != null)
            {
                sub.SetQueue(queue);
            }
            sub.SetApplicationType(appType);
            ContainerLaunchContext clc = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <ContainerLaunchContext
                                                                                        >();
            Resource capability = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <Resource>();

            capability.SetMemory(masterMemory);
            sub.SetResource(capability);
            clc.SetApplicationACLs(acls);
            if (ts != null && UserGroupInformation.IsSecurityEnabled())
            {
                DataOutputBuffer dob = new DataOutputBuffer();
                ts.WriteTokenStorageToStream(dob);
                ByteBuffer securityTokens = ByteBuffer.Wrap(dob.GetData(), 0, dob.GetLength());
                clc.SetTokens(securityTokens);
            }
            sub.SetAMContainerSpec(clc);
            sub.SetAttemptFailuresValidityInterval(attemptFailuresValidityInterval);
            if (logAggregationContext != null)
            {
                sub.SetLogAggregationContext(logAggregationContext);
            }
            sub.SetCancelTokensWhenComplete(cancelTokensWhenComplete);
            req.SetApplicationSubmissionContext(sub);
            UserGroupInformation fakeUser = UserGroupInformation.CreateUserForTesting(user, new
                                                                                      string[] { "someGroup" });
            PrivilegedAction <SubmitApplicationResponse> action = new _PrivilegedAction_415().
                                                                  SetClientReq(client, req);

            fakeUser.DoAs(action);
            // make sure app is immediately available after submit
            if (waitForAccepted)
            {
                WaitForState(appId, RMAppState.Accepted);
            }
            RMApp rmApp = GetRMContext().GetRMApps()[appId];

            // unmanaged AM won't go to RMAppAttemptState.SCHEDULED.
            if (waitForAccepted && !unmanaged)
            {
                WaitForState(rmApp.GetCurrentAppAttempt().GetAppAttemptId(), RMAppAttemptState.Scheduled
                             );
            }
            return(rmApp);
        }
Exemple #6
0
        /// <summary>Main run function for the client</summary>
        /// <returns>true if application completed successfully</returns>
        /// <exception cref="System.IO.IOException"/>
        /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
        public virtual bool Run()
        {
            Log.Info("Running Client");
            yarnClient.Start();
            YarnClusterMetrics clusterMetrics = yarnClient.GetYarnClusterMetrics();

            Log.Info("Got Cluster metric info from ASM" + ", numNodeManagers=" + clusterMetrics
                     .GetNumNodeManagers());
            IList <NodeReport> clusterNodeReports = yarnClient.GetNodeReports(NodeState.Running
                                                                              );

            Log.Info("Got Cluster node info from ASM");
            foreach (NodeReport node in clusterNodeReports)
            {
                Log.Info("Got node report from ASM for" + ", nodeId=" + node.GetNodeId() + ", nodeAddress"
                         + node.GetHttpAddress() + ", nodeRackName" + node.GetRackName() + ", nodeNumContainers"
                         + node.GetNumContainers());
            }
            QueueInfo queueInfo = yarnClient.GetQueueInfo(this.amQueue);

            Log.Info("Queue info" + ", queueName=" + queueInfo.GetQueueName() + ", queueCurrentCapacity="
                     + queueInfo.GetCurrentCapacity() + ", queueMaxCapacity=" + queueInfo.GetMaximumCapacity
                         () + ", queueApplicationCount=" + queueInfo.GetApplications().Count + ", queueChildQueueCount="
                     + queueInfo.GetChildQueues().Count);
            IList <QueueUserACLInfo> listAclInfo = yarnClient.GetQueueAclsInfo();

            foreach (QueueUserACLInfo aclInfo in listAclInfo)
            {
                foreach (QueueACL userAcl in aclInfo.GetUserAcls())
                {
                    Log.Info("User ACL Info for Queue" + ", queueName=" + aclInfo.GetQueueName() + ", userAcl="
                             + userAcl.ToString());
                }
            }
            if (domainId != null && domainId.Length > 0 && toCreateDomain)
            {
                PrepareTimelineDomain();
            }
            // Get a new application id
            YarnClientApplication     app         = yarnClient.CreateApplication();
            GetNewApplicationResponse appResponse = app.GetNewApplicationResponse();
            // TODO get min/max resource capabilities from RM and change memory ask if needed
            // If we do not have min/max, we may not be able to correctly request
            // the required resources from the RM for the app master
            // Memory ask has to be a multiple of min and less than max.
            // Dump out information about cluster capability as seen by the resource manager
            int maxMem = appResponse.GetMaximumResourceCapability().GetMemory();

            Log.Info("Max mem capabililty of resources in this cluster " + maxMem);
            // A resource ask cannot exceed the max.
            if (amMemory > maxMem)
            {
                Log.Info("AM memory specified above max threshold of cluster. Using max value." +
                         ", specified=" + amMemory + ", max=" + maxMem);
                amMemory = maxMem;
            }
            int maxVCores = appResponse.GetMaximumResourceCapability().GetVirtualCores();

            Log.Info("Max virtual cores capabililty of resources in this cluster " + maxVCores
                     );
            if (amVCores > maxVCores)
            {
                Log.Info("AM virtual cores specified above max threshold of cluster. " + "Using max value."
                         + ", specified=" + amVCores + ", max=" + maxVCores);
                amVCores = maxVCores;
            }
            // set the application name
            ApplicationSubmissionContext appContext = app.GetApplicationSubmissionContext();
            ApplicationId appId = appContext.GetApplicationId();

            appContext.SetKeepContainersAcrossApplicationAttempts(keepContainers);
            appContext.SetApplicationName(appName);
            if (attemptFailuresValidityInterval >= 0)
            {
                appContext.SetAttemptFailuresValidityInterval(attemptFailuresValidityInterval);
            }
            // set local resources for the application master
            // local files or archives as needed
            // In this scenario, the jar file for the application master is part of the local resources
            IDictionary <string, LocalResource> localResources = new Dictionary <string, LocalResource
                                                                                 >();

            Log.Info("Copy App Master jar from local filesystem and add to local environment"
                     );
            // Copy the application master jar to the filesystem
            // Create a local resource to point to the destination jar path
            FileSystem fs = FileSystem.Get(conf);

            AddToLocalResources(fs, appMasterJar, appMasterJarPath, appId.ToString(), localResources
                                , null);
            // Set the log4j properties if needed
            if (!log4jPropFile.IsEmpty())
            {
                AddToLocalResources(fs, log4jPropFile, log4jPath, appId.ToString(), localResources
                                    , null);
            }
            // The shell script has to be made available on the final container(s)
            // where it will be executed.
            // To do this, we need to first copy into the filesystem that is visible
            // to the yarn framework.
            // We do not need to set this as a local resource for the application
            // master as the application master does not need it.
            string hdfsShellScriptLocation  = string.Empty;
            long   hdfsShellScriptLen       = 0;
            long   hdfsShellScriptTimestamp = 0;

            if (!shellScriptPath.IsEmpty())
            {
                Path   shellSrc        = new Path(shellScriptPath);
                string shellPathSuffix = appName + "/" + appId.ToString() + "/" + ScriptPath;
                Path   shellDst        = new Path(fs.GetHomeDirectory(), shellPathSuffix);
                fs.CopyFromLocalFile(false, true, shellSrc, shellDst);
                hdfsShellScriptLocation = shellDst.ToUri().ToString();
                FileStatus shellFileStatus = fs.GetFileStatus(shellDst);
                hdfsShellScriptLen       = shellFileStatus.GetLen();
                hdfsShellScriptTimestamp = shellFileStatus.GetModificationTime();
            }
            if (!shellCommand.IsEmpty())
            {
                AddToLocalResources(fs, null, shellCommandPath, appId.ToString(), localResources,
                                    shellCommand);
            }
            if (shellArgs.Length > 0)
            {
                AddToLocalResources(fs, null, shellArgsPath, appId.ToString(), localResources, StringUtils
                                    .Join(shellArgs, " "));
            }
            // Set the necessary security tokens as needed
            //amContainer.setContainerTokens(containerToken);
            // Set the env variables to be setup in the env where the application master will be run
            Log.Info("Set the environment for the application master");
            IDictionary <string, string> env = new Dictionary <string, string>();

            // put location of shell script into env
            // using the env info, the application master will create the correct local resource for the
            // eventual containers that will be launched to execute the shell scripts
            env[DSConstants.Distributedshellscriptlocation]  = hdfsShellScriptLocation;
            env[DSConstants.Distributedshellscripttimestamp] = System.Convert.ToString(hdfsShellScriptTimestamp
                                                                                       );
            env[DSConstants.Distributedshellscriptlen] = System.Convert.ToString(hdfsShellScriptLen
                                                                                 );
            if (domainId != null && domainId.Length > 0)
            {
                env[DSConstants.Distributedshelltimelinedomain] = domainId;
            }
            // Add AppMaster.jar location to classpath
            // At some point we should not be required to add
            // the hadoop specific classpaths to the env.
            // It should be provided out of the box.
            // For now setting all required classpaths including
            // the classpath to "." for the application jar
            StringBuilder classPathEnv = new StringBuilder(ApplicationConstants.Environment.Classpath
                                                           .$$()).Append(ApplicationConstants.ClassPathSeparator).Append("./*");

            foreach (string c in conf.GetStrings(YarnConfiguration.YarnApplicationClasspath,
                                                 YarnConfiguration.DefaultYarnCrossPlatformApplicationClasspath))
            {
                classPathEnv.Append(ApplicationConstants.ClassPathSeparator);
                classPathEnv.Append(c.Trim());
            }
            classPathEnv.Append(ApplicationConstants.ClassPathSeparator).Append("./log4j.properties"
                                                                                );
            // add the runtime classpath needed for tests to work
            if (conf.GetBoolean(YarnConfiguration.IsMiniYarnCluster, false))
            {
                classPathEnv.Append(':');
                classPathEnv.Append(Runtime.GetProperty("java.class.path"));
            }
            env["CLASSPATH"] = classPathEnv.ToString();
            // Set the necessary command to execute the application master
            Vector <CharSequence> vargs = new Vector <CharSequence>(30);

            // Set java executable command
            Log.Info("Setting up app master command");
            vargs.AddItem(ApplicationConstants.Environment.JavaHome.$$() + "/bin/java");
            // Set Xmx based on am memory size
            vargs.AddItem("-Xmx" + amMemory + "m");
            // Set class name
            vargs.AddItem(appMasterMainClass);
            // Set params for Application Master
            vargs.AddItem("--container_memory " + containerMemory.ToString());
            vargs.AddItem("--container_vcores " + containerVirtualCores.ToString());
            vargs.AddItem("--num_containers " + numContainers.ToString());
            if (null != nodeLabelExpression)
            {
                appContext.SetNodeLabelExpression(nodeLabelExpression);
            }
            vargs.AddItem("--priority " + shellCmdPriority.ToString());
            foreach (KeyValuePair <string, string> entry in shellEnv)
            {
                vargs.AddItem("--shell_env " + entry.Key + "=" + entry.Value);
            }
            if (debugFlag)
            {
                vargs.AddItem("--debug");
            }
            vargs.AddItem("1>" + ApplicationConstants.LogDirExpansionVar + "/AppMaster.stdout"
                          );
            vargs.AddItem("2>" + ApplicationConstants.LogDirExpansionVar + "/AppMaster.stderr"
                          );
            // Get final commmand
            StringBuilder command = new StringBuilder();

            foreach (CharSequence str in vargs)
            {
                command.Append(str).Append(" ");
            }
            Log.Info("Completed setting up app master command " + command.ToString());
            IList <string> commands = new AList <string>();

            commands.AddItem(command.ToString());
            // Set up the container launch context for the application master
            ContainerLaunchContext amContainer = ContainerLaunchContext.NewInstance(localResources
                                                                                    , env, commands, null, null, null);
            // Set up resource type requirements
            // For now, both memory and vcores are supported, so we set memory and
            // vcores requirements
            Resource capability = Resource.NewInstance(amMemory, amVCores);

            appContext.SetResource(capability);
            // Service data is a binary blob that can be passed to the application
            // Not needed in this scenario
            // amContainer.setServiceData(serviceData);
            // Setup security tokens
            if (UserGroupInformation.IsSecurityEnabled())
            {
                // Note: Credentials class is marked as LimitedPrivate for HDFS and MapReduce
                Credentials credentials  = new Credentials();
                string      tokenRenewer = conf.Get(YarnConfiguration.RmPrincipal);
                if (tokenRenewer == null || tokenRenewer.Length == 0)
                {
                    throw new IOException("Can't get Master Kerberos principal for the RM to use as renewer"
                                          );
                }
                // For now, only getting tokens for the default file-system.
                Org.Apache.Hadoop.Security.Token.Token <object>[] tokens = fs.AddDelegationTokens(
                    tokenRenewer, credentials);
                if (tokens != null)
                {
                    foreach (Org.Apache.Hadoop.Security.Token.Token <object> token in tokens)
                    {
                        Log.Info("Got dt for " + fs.GetUri() + "; " + token);
                    }
                }
                DataOutputBuffer dob = new DataOutputBuffer();
                credentials.WriteTokenStorageToStream(dob);
                ByteBuffer fsTokens = ByteBuffer.Wrap(dob.GetData(), 0, dob.GetLength());
                amContainer.SetTokens(fsTokens);
            }
            appContext.SetAMContainerSpec(amContainer);
            // Set the priority for the application master
            // TODO - what is the range for priority? how to decide?
            Priority pri = Priority.NewInstance(amPriority);

            appContext.SetPriority(pri);
            // Set the queue to which this application is to be submitted in the RM
            appContext.SetQueue(amQueue);
            // Submit the application to the applications manager
            // SubmitApplicationResponse submitResp = applicationsManager.submitApplication(appRequest);
            // Ignore the response as either a valid response object is returned on success
            // or an exception thrown to denote some form of a failure
            Log.Info("Submitting application to ASM");
            yarnClient.SubmitApplication(appContext);
            // TODO
            // Try submitting the same request again
            // app submission failure?
            // Monitor the application
            return(MonitorApplication(appId));
        }