Example #1
0
        /// <exception cref="System.Exception"/>
        public virtual RMApp SubmitApp(int masterMemory, string name, string user, IDictionary
                                       <ApplicationAccessType, string> acls, bool unmanaged, string queue, int maxAppAttempts
                                       , Credentials ts, string appType, bool waitForAccepted, bool keepContainers, bool
                                       isAppIdProvided, ApplicationId applicationId, long attemptFailuresValidityInterval
                                       , LogAggregationContext logAggregationContext, bool cancelTokensWhenComplete)
        {
            ApplicationId             appId  = isAppIdProvided ? applicationId : null;
            ApplicationClientProtocol client = GetClientRMService();

            if (!isAppIdProvided)
            {
                GetNewApplicationResponse resp = client.GetNewApplication(Org.Apache.Hadoop.Yarn.Util.Records
                                                                          .NewRecord <GetNewApplicationRequest>());
                appId = resp.GetApplicationId();
            }
            SubmitApplicationRequest req = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <SubmitApplicationRequest
                                                                                          >();
            ApplicationSubmissionContext sub = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <
                ApplicationSubmissionContext>();

            sub.SetKeepContainersAcrossApplicationAttempts(keepContainers);
            sub.SetApplicationId(appId);
            sub.SetApplicationName(name);
            sub.SetMaxAppAttempts(maxAppAttempts);
            if (unmanaged)
            {
                sub.SetUnmanagedAM(true);
            }
            if (queue != null)
            {
                sub.SetQueue(queue);
            }
            sub.SetApplicationType(appType);
            ContainerLaunchContext clc = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <ContainerLaunchContext
                                                                                        >();
            Resource capability = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <Resource>();

            capability.SetMemory(masterMemory);
            sub.SetResource(capability);
            clc.SetApplicationACLs(acls);
            if (ts != null && UserGroupInformation.IsSecurityEnabled())
            {
                DataOutputBuffer dob = new DataOutputBuffer();
                ts.WriteTokenStorageToStream(dob);
                ByteBuffer securityTokens = ByteBuffer.Wrap(dob.GetData(), 0, dob.GetLength());
                clc.SetTokens(securityTokens);
            }
            sub.SetAMContainerSpec(clc);
            sub.SetAttemptFailuresValidityInterval(attemptFailuresValidityInterval);
            if (logAggregationContext != null)
            {
                sub.SetLogAggregationContext(logAggregationContext);
            }
            sub.SetCancelTokensWhenComplete(cancelTokensWhenComplete);
            req.SetApplicationSubmissionContext(sub);
            UserGroupInformation fakeUser = UserGroupInformation.CreateUserForTesting(user, new
                                                                                      string[] { "someGroup" });
            PrivilegedAction <SubmitApplicationResponse> action = new _PrivilegedAction_415().
                                                                  SetClientReq(client, req);

            fakeUser.DoAs(action);
            // make sure app is immediately available after submit
            if (waitForAccepted)
            {
                WaitForState(appId, RMAppState.Accepted);
            }
            RMApp rmApp = GetRMContext().GetRMApps()[appId];

            // unmanaged AM won't go to RMAppAttemptState.SCHEDULED.
            if (waitForAccepted && !unmanaged)
            {
                WaitForState(rmApp.GetCurrentAppAttempt().GetAppAttemptId(), RMAppAttemptState.Scheduled
                             );
            }
            return(rmApp);
        }
Example #2
0
        /// <summary>Main run function for the client</summary>
        /// <returns>true if application completed successfully</returns>
        /// <exception cref="System.IO.IOException"/>
        /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
        public virtual bool Run()
        {
            Log.Info("Running Client");
            yarnClient.Start();
            YarnClusterMetrics clusterMetrics = yarnClient.GetYarnClusterMetrics();

            Log.Info("Got Cluster metric info from ASM" + ", numNodeManagers=" + clusterMetrics
                     .GetNumNodeManagers());
            IList <NodeReport> clusterNodeReports = yarnClient.GetNodeReports(NodeState.Running
                                                                              );

            Log.Info("Got Cluster node info from ASM");
            foreach (NodeReport node in clusterNodeReports)
            {
                Log.Info("Got node report from ASM for" + ", nodeId=" + node.GetNodeId() + ", nodeAddress"
                         + node.GetHttpAddress() + ", nodeRackName" + node.GetRackName() + ", nodeNumContainers"
                         + node.GetNumContainers());
            }
            QueueInfo queueInfo = yarnClient.GetQueueInfo(this.amQueue);

            Log.Info("Queue info" + ", queueName=" + queueInfo.GetQueueName() + ", queueCurrentCapacity="
                     + queueInfo.GetCurrentCapacity() + ", queueMaxCapacity=" + queueInfo.GetMaximumCapacity
                         () + ", queueApplicationCount=" + queueInfo.GetApplications().Count + ", queueChildQueueCount="
                     + queueInfo.GetChildQueues().Count);
            IList <QueueUserACLInfo> listAclInfo = yarnClient.GetQueueAclsInfo();

            foreach (QueueUserACLInfo aclInfo in listAclInfo)
            {
                foreach (QueueACL userAcl in aclInfo.GetUserAcls())
                {
                    Log.Info("User ACL Info for Queue" + ", queueName=" + aclInfo.GetQueueName() + ", userAcl="
                             + userAcl.ToString());
                }
            }
            if (domainId != null && domainId.Length > 0 && toCreateDomain)
            {
                PrepareTimelineDomain();
            }
            // Get a new application id
            YarnClientApplication     app         = yarnClient.CreateApplication();
            GetNewApplicationResponse appResponse = app.GetNewApplicationResponse();
            // TODO get min/max resource capabilities from RM and change memory ask if needed
            // If we do not have min/max, we may not be able to correctly request
            // the required resources from the RM for the app master
            // Memory ask has to be a multiple of min and less than max.
            // Dump out information about cluster capability as seen by the resource manager
            int maxMem = appResponse.GetMaximumResourceCapability().GetMemory();

            Log.Info("Max mem capabililty of resources in this cluster " + maxMem);
            // A resource ask cannot exceed the max.
            if (amMemory > maxMem)
            {
                Log.Info("AM memory specified above max threshold of cluster. Using max value." +
                         ", specified=" + amMemory + ", max=" + maxMem);
                amMemory = maxMem;
            }
            int maxVCores = appResponse.GetMaximumResourceCapability().GetVirtualCores();

            Log.Info("Max virtual cores capabililty of resources in this cluster " + maxVCores
                     );
            if (amVCores > maxVCores)
            {
                Log.Info("AM virtual cores specified above max threshold of cluster. " + "Using max value."
                         + ", specified=" + amVCores + ", max=" + maxVCores);
                amVCores = maxVCores;
            }
            // set the application name
            ApplicationSubmissionContext appContext = app.GetApplicationSubmissionContext();
            ApplicationId appId = appContext.GetApplicationId();

            appContext.SetKeepContainersAcrossApplicationAttempts(keepContainers);
            appContext.SetApplicationName(appName);
            if (attemptFailuresValidityInterval >= 0)
            {
                appContext.SetAttemptFailuresValidityInterval(attemptFailuresValidityInterval);
            }
            // set local resources for the application master
            // local files or archives as needed
            // In this scenario, the jar file for the application master is part of the local resources
            IDictionary <string, LocalResource> localResources = new Dictionary <string, LocalResource
                                                                                 >();

            Log.Info("Copy App Master jar from local filesystem and add to local environment"
                     );
            // Copy the application master jar to the filesystem
            // Create a local resource to point to the destination jar path
            FileSystem fs = FileSystem.Get(conf);

            AddToLocalResources(fs, appMasterJar, appMasterJarPath, appId.ToString(), localResources
                                , null);
            // Set the log4j properties if needed
            if (!log4jPropFile.IsEmpty())
            {
                AddToLocalResources(fs, log4jPropFile, log4jPath, appId.ToString(), localResources
                                    , null);
            }
            // The shell script has to be made available on the final container(s)
            // where it will be executed.
            // To do this, we need to first copy into the filesystem that is visible
            // to the yarn framework.
            // We do not need to set this as a local resource for the application
            // master as the application master does not need it.
            string hdfsShellScriptLocation  = string.Empty;
            long   hdfsShellScriptLen       = 0;
            long   hdfsShellScriptTimestamp = 0;

            if (!shellScriptPath.IsEmpty())
            {
                Path   shellSrc        = new Path(shellScriptPath);
                string shellPathSuffix = appName + "/" + appId.ToString() + "/" + ScriptPath;
                Path   shellDst        = new Path(fs.GetHomeDirectory(), shellPathSuffix);
                fs.CopyFromLocalFile(false, true, shellSrc, shellDst);
                hdfsShellScriptLocation = shellDst.ToUri().ToString();
                FileStatus shellFileStatus = fs.GetFileStatus(shellDst);
                hdfsShellScriptLen       = shellFileStatus.GetLen();
                hdfsShellScriptTimestamp = shellFileStatus.GetModificationTime();
            }
            if (!shellCommand.IsEmpty())
            {
                AddToLocalResources(fs, null, shellCommandPath, appId.ToString(), localResources,
                                    shellCommand);
            }
            if (shellArgs.Length > 0)
            {
                AddToLocalResources(fs, null, shellArgsPath, appId.ToString(), localResources, StringUtils
                                    .Join(shellArgs, " "));
            }
            // Set the necessary security tokens as needed
            //amContainer.setContainerTokens(containerToken);
            // Set the env variables to be setup in the env where the application master will be run
            Log.Info("Set the environment for the application master");
            IDictionary <string, string> env = new Dictionary <string, string>();

            // put location of shell script into env
            // using the env info, the application master will create the correct local resource for the
            // eventual containers that will be launched to execute the shell scripts
            env[DSConstants.Distributedshellscriptlocation]  = hdfsShellScriptLocation;
            env[DSConstants.Distributedshellscripttimestamp] = System.Convert.ToString(hdfsShellScriptTimestamp
                                                                                       );
            env[DSConstants.Distributedshellscriptlen] = System.Convert.ToString(hdfsShellScriptLen
                                                                                 );
            if (domainId != null && domainId.Length > 0)
            {
                env[DSConstants.Distributedshelltimelinedomain] = domainId;
            }
            // Add AppMaster.jar location to classpath
            // At some point we should not be required to add
            // the hadoop specific classpaths to the env.
            // It should be provided out of the box.
            // For now setting all required classpaths including
            // the classpath to "." for the application jar
            StringBuilder classPathEnv = new StringBuilder(ApplicationConstants.Environment.Classpath
                                                           .$$()).Append(ApplicationConstants.ClassPathSeparator).Append("./*");

            foreach (string c in conf.GetStrings(YarnConfiguration.YarnApplicationClasspath,
                                                 YarnConfiguration.DefaultYarnCrossPlatformApplicationClasspath))
            {
                classPathEnv.Append(ApplicationConstants.ClassPathSeparator);
                classPathEnv.Append(c.Trim());
            }
            classPathEnv.Append(ApplicationConstants.ClassPathSeparator).Append("./log4j.properties"
                                                                                );
            // add the runtime classpath needed for tests to work
            if (conf.GetBoolean(YarnConfiguration.IsMiniYarnCluster, false))
            {
                classPathEnv.Append(':');
                classPathEnv.Append(Runtime.GetProperty("java.class.path"));
            }
            env["CLASSPATH"] = classPathEnv.ToString();
            // Set the necessary command to execute the application master
            Vector <CharSequence> vargs = new Vector <CharSequence>(30);

            // Set java executable command
            Log.Info("Setting up app master command");
            vargs.AddItem(ApplicationConstants.Environment.JavaHome.$$() + "/bin/java");
            // Set Xmx based on am memory size
            vargs.AddItem("-Xmx" + amMemory + "m");
            // Set class name
            vargs.AddItem(appMasterMainClass);
            // Set params for Application Master
            vargs.AddItem("--container_memory " + containerMemory.ToString());
            vargs.AddItem("--container_vcores " + containerVirtualCores.ToString());
            vargs.AddItem("--num_containers " + numContainers.ToString());
            if (null != nodeLabelExpression)
            {
                appContext.SetNodeLabelExpression(nodeLabelExpression);
            }
            vargs.AddItem("--priority " + shellCmdPriority.ToString());
            foreach (KeyValuePair <string, string> entry in shellEnv)
            {
                vargs.AddItem("--shell_env " + entry.Key + "=" + entry.Value);
            }
            if (debugFlag)
            {
                vargs.AddItem("--debug");
            }
            vargs.AddItem("1>" + ApplicationConstants.LogDirExpansionVar + "/AppMaster.stdout"
                          );
            vargs.AddItem("2>" + ApplicationConstants.LogDirExpansionVar + "/AppMaster.stderr"
                          );
            // Get final commmand
            StringBuilder command = new StringBuilder();

            foreach (CharSequence str in vargs)
            {
                command.Append(str).Append(" ");
            }
            Log.Info("Completed setting up app master command " + command.ToString());
            IList <string> commands = new AList <string>();

            commands.AddItem(command.ToString());
            // Set up the container launch context for the application master
            ContainerLaunchContext amContainer = ContainerLaunchContext.NewInstance(localResources
                                                                                    , env, commands, null, null, null);
            // Set up resource type requirements
            // For now, both memory and vcores are supported, so we set memory and
            // vcores requirements
            Resource capability = Resource.NewInstance(amMemory, amVCores);

            appContext.SetResource(capability);
            // Service data is a binary blob that can be passed to the application
            // Not needed in this scenario
            // amContainer.setServiceData(serviceData);
            // Setup security tokens
            if (UserGroupInformation.IsSecurityEnabled())
            {
                // Note: Credentials class is marked as LimitedPrivate for HDFS and MapReduce
                Credentials credentials  = new Credentials();
                string      tokenRenewer = conf.Get(YarnConfiguration.RmPrincipal);
                if (tokenRenewer == null || tokenRenewer.Length == 0)
                {
                    throw new IOException("Can't get Master Kerberos principal for the RM to use as renewer"
                                          );
                }
                // For now, only getting tokens for the default file-system.
                Org.Apache.Hadoop.Security.Token.Token <object>[] tokens = fs.AddDelegationTokens(
                    tokenRenewer, credentials);
                if (tokens != null)
                {
                    foreach (Org.Apache.Hadoop.Security.Token.Token <object> token in tokens)
                    {
                        Log.Info("Got dt for " + fs.GetUri() + "; " + token);
                    }
                }
                DataOutputBuffer dob = new DataOutputBuffer();
                credentials.WriteTokenStorageToStream(dob);
                ByteBuffer fsTokens = ByteBuffer.Wrap(dob.GetData(), 0, dob.GetLength());
                amContainer.SetTokens(fsTokens);
            }
            appContext.SetAMContainerSpec(amContainer);
            // Set the priority for the application master
            // TODO - what is the range for priority? how to decide?
            Priority pri = Priority.NewInstance(amPriority);

            appContext.SetPriority(pri);
            // Set the queue to which this application is to be submitted in the RM
            appContext.SetQueue(amQueue);
            // Submit the application to the applications manager
            // SubmitApplicationResponse submitResp = applicationsManager.submitApplication(appRequest);
            // Ignore the response as either a valid response object is returned on success
            // or an exception thrown to denote some form of a failure
            Log.Info("Submitting application to ASM");
            yarnClient.SubmitApplication(appContext);
            // TODO
            // Try submitting the same request again
            // app submission failure?
            // Monitor the application
            return(MonitorApplication(appId));
        }