/// <exception cref="System.Exception"/> private ApplicationId SubmitAppAndGetAppId(string submitter, string queueName, bool setupACLs) { GetNewApplicationRequest newAppRequest = GetNewApplicationRequest.NewInstance(); ApplicationClientProtocol submitterClient = GetRMClientForUser(submitter); ApplicationId applicationId = submitterClient.GetNewApplication(newAppRequest).GetApplicationId (); Resource resource = BuilderUtils.NewResource(1024, 1); IDictionary <ApplicationAccessType, string> acls = CreateACLs(submitter, setupACLs ); ContainerLaunchContext amContainerSpec = ContainerLaunchContext.NewInstance(null, null, null, null, null, acls); ApplicationSubmissionContext appSubmissionContext = ApplicationSubmissionContext. NewInstance(applicationId, "applicationName", queueName, null, amContainerSpec, false, true, 1, resource, "applicationType"); appSubmissionContext.SetApplicationId(applicationId); appSubmissionContext.SetQueue(queueName); SubmitApplicationRequest submitRequest = SubmitApplicationRequest.NewInstance(appSubmissionContext ); submitterClient.SubmitApplication(submitRequest); resourceManager.WaitForState(applicationId, RMAppState.Accepted); return(applicationId); }
public virtual void TestApplicationRecovery() { YarnConfiguration conf = new YarnConfiguration(); conf.SetBoolean(YarnConfiguration.NmRecoveryEnabled, true); conf.Set(YarnConfiguration.NmAddress, "localhost:1234"); conf.SetBoolean(YarnConfiguration.YarnAclEnable, true); conf.Set(YarnConfiguration.YarnAdminAcl, "yarn_admin_user"); NMStateStoreService stateStore = new NMMemoryStateStoreService(); stateStore.Init(conf); stateStore.Start(); Context context = new NodeManager.NMContext(new NMContainerTokenSecretManager(conf ), new NMTokenSecretManagerInNM(), null, new ApplicationACLsManager(conf), stateStore ); ContainerManagerImpl cm = CreateContainerManager(context); cm.Init(conf); cm.Start(); // simulate registration with RM MasterKey masterKey = new MasterKeyPBImpl(); masterKey.SetKeyId(123); masterKey.SetBytes(ByteBuffer.Wrap(new byte[] { 123 })); context.GetContainerTokenSecretManager().SetMasterKey(masterKey); context.GetNMTokenSecretManager().SetMasterKey(masterKey); // add an application by starting a container string appUser = "******"; string modUser = "******"; string viewUser = "******"; string enemyUser = "******"; ApplicationId appId = ApplicationId.NewInstance(0, 1); ApplicationAttemptId attemptId = ApplicationAttemptId.NewInstance(appId, 1); ContainerId cid = ContainerId.NewContainerId(attemptId, 1); IDictionary <string, LocalResource> localResources = Collections.EmptyMap(); IDictionary <string, string> containerEnv = Sharpen.Collections.EmptyMap(); IList <string> containerCmds = Sharpen.Collections.EmptyList(); IDictionary <string, ByteBuffer> serviceData = Sharpen.Collections.EmptyMap(); Credentials containerCreds = new Credentials(); DataOutputBuffer dob = new DataOutputBuffer(); containerCreds.WriteTokenStorageToStream(dob); ByteBuffer containerTokens = ByteBuffer.Wrap(dob.GetData(), 0, dob.GetLength()); IDictionary <ApplicationAccessType, string> acls = new Dictionary <ApplicationAccessType , string>(); acls[ApplicationAccessType.ModifyApp] = modUser; acls[ApplicationAccessType.ViewApp] = viewUser; ContainerLaunchContext clc = ContainerLaunchContext.NewInstance(localResources, containerEnv , containerCmds, serviceData, containerTokens, acls); // create the logAggregationContext LogAggregationContext logAggregationContext = LogAggregationContext.NewInstance("includePattern" , "excludePattern", "includePatternInRollingAggregation", "excludePatternInRollingAggregation" ); StartContainersResponse startResponse = StartContainer(context, cm, cid, clc, logAggregationContext ); NUnit.Framework.Assert.IsTrue(startResponse.GetFailedRequests().IsEmpty()); NUnit.Framework.Assert.AreEqual(1, context.GetApplications().Count); Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application app = context.GetApplications()[appId]; NUnit.Framework.Assert.IsNotNull(app); WaitForAppState(app, ApplicationState.Initing); NUnit.Framework.Assert.IsTrue(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation .CreateRemoteUser(modUser), ApplicationAccessType.ModifyApp, appUser, appId)); NUnit.Framework.Assert.IsFalse(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation .CreateRemoteUser(viewUser), ApplicationAccessType.ModifyApp, appUser, appId)); NUnit.Framework.Assert.IsTrue(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation .CreateRemoteUser(viewUser), ApplicationAccessType.ViewApp, appUser, appId)); NUnit.Framework.Assert.IsFalse(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation .CreateRemoteUser(enemyUser), ApplicationAccessType.ViewApp, appUser, appId)); // reset container manager and verify app recovered with proper acls cm.Stop(); context = new NodeManager.NMContext(new NMContainerTokenSecretManager(conf), new NMTokenSecretManagerInNM(), null, new ApplicationACLsManager(conf), stateStore); cm = CreateContainerManager(context); cm.Init(conf); cm.Start(); NUnit.Framework.Assert.AreEqual(1, context.GetApplications().Count); app = context.GetApplications()[appId]; NUnit.Framework.Assert.IsNotNull(app); // check whether LogAggregationContext is recovered correctly LogAggregationContext recovered = ((ApplicationImpl)app).GetLogAggregationContext (); NUnit.Framework.Assert.IsNotNull(recovered); NUnit.Framework.Assert.AreEqual(logAggregationContext.GetIncludePattern(), recovered .GetIncludePattern()); NUnit.Framework.Assert.AreEqual(logAggregationContext.GetExcludePattern(), recovered .GetExcludePattern()); NUnit.Framework.Assert.AreEqual(logAggregationContext.GetRolledLogsIncludePattern (), recovered.GetRolledLogsIncludePattern()); NUnit.Framework.Assert.AreEqual(logAggregationContext.GetRolledLogsExcludePattern (), recovered.GetRolledLogsExcludePattern()); WaitForAppState(app, ApplicationState.Initing); NUnit.Framework.Assert.IsTrue(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation .CreateRemoteUser(modUser), ApplicationAccessType.ModifyApp, appUser, appId)); NUnit.Framework.Assert.IsFalse(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation .CreateRemoteUser(viewUser), ApplicationAccessType.ModifyApp, appUser, appId)); NUnit.Framework.Assert.IsTrue(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation .CreateRemoteUser(viewUser), ApplicationAccessType.ViewApp, appUser, appId)); NUnit.Framework.Assert.IsFalse(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation .CreateRemoteUser(enemyUser), ApplicationAccessType.ViewApp, appUser, appId)); // simulate application completion IList <ApplicationId> finishedApps = new AList <ApplicationId>(); finishedApps.AddItem(appId); cm.Handle(new CMgrCompletedAppsEvent(finishedApps, CMgrCompletedAppsEvent.Reason. ByResourcemanager)); WaitForAppState(app, ApplicationState.ApplicationResourcesCleaningup); // restart and verify app is marked for finishing cm.Stop(); context = new NodeManager.NMContext(new NMContainerTokenSecretManager(conf), new NMTokenSecretManagerInNM(), null, new ApplicationACLsManager(conf), stateStore); cm = CreateContainerManager(context); cm.Init(conf); cm.Start(); NUnit.Framework.Assert.AreEqual(1, context.GetApplications().Count); app = context.GetApplications()[appId]; NUnit.Framework.Assert.IsNotNull(app); WaitForAppState(app, ApplicationState.ApplicationResourcesCleaningup); NUnit.Framework.Assert.IsTrue(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation .CreateRemoteUser(modUser), ApplicationAccessType.ModifyApp, appUser, appId)); NUnit.Framework.Assert.IsFalse(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation .CreateRemoteUser(viewUser), ApplicationAccessType.ModifyApp, appUser, appId)); NUnit.Framework.Assert.IsTrue(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation .CreateRemoteUser(viewUser), ApplicationAccessType.ViewApp, appUser, appId)); NUnit.Framework.Assert.IsFalse(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation .CreateRemoteUser(enemyUser), ApplicationAccessType.ViewApp, appUser, appId)); // simulate log aggregation completion app.Handle(new ApplicationEvent(app.GetAppId(), ApplicationEventType.ApplicationResourcesCleanedup )); NUnit.Framework.Assert.AreEqual(app.GetApplicationState(), ApplicationState.Finished ); app.Handle(new ApplicationEvent(app.GetAppId(), ApplicationEventType.ApplicationLogHandlingFinished )); // restart and verify app is no longer present after recovery cm.Stop(); context = new NodeManager.NMContext(new NMContainerTokenSecretManager(conf), new NMTokenSecretManagerInNM(), null, new ApplicationACLsManager(conf), stateStore); cm = CreateContainerManager(context); cm.Init(conf); cm.Start(); NUnit.Framework.Assert.IsTrue(context.GetApplications().IsEmpty()); cm.Stop(); }
/// <exception cref="System.IO.IOException"/> public virtual ApplicationSubmissionContext CreateApplicationSubmissionContext(Configuration jobConf, string jobSubmitDir, Credentials ts) { ApplicationId applicationId = resMgrDelegate.GetApplicationId(); // Setup resource requirements Resource capability = recordFactory.NewRecordInstance <Resource>(); capability.SetMemory(conf.GetInt(MRJobConfig.MrAmVmemMb, MRJobConfig.DefaultMrAmVmemMb )); capability.SetVirtualCores(conf.GetInt(MRJobConfig.MrAmCpuVcores, MRJobConfig.DefaultMrAmCpuVcores )); Log.Debug("AppMaster capability = " + capability); // Setup LocalResources IDictionary <string, LocalResource> localResources = new Dictionary <string, LocalResource >(); Path jobConfPath = new Path(jobSubmitDir, MRJobConfig.JobConfFile); URL yarnUrlForJobSubmitDir = ConverterUtils.GetYarnUrlFromPath(defaultFileContext .GetDefaultFileSystem().ResolvePath(defaultFileContext.MakeQualified(new Path(jobSubmitDir )))); Log.Debug("Creating setup context, jobSubmitDir url is " + yarnUrlForJobSubmitDir ); localResources[MRJobConfig.JobConfFile] = CreateApplicationResource(defaultFileContext , jobConfPath, LocalResourceType.File); if (jobConf.Get(MRJobConfig.Jar) != null) { Path jobJarPath = new Path(jobConf.Get(MRJobConfig.Jar)); LocalResource rc = CreateApplicationResource(FileContext.GetFileContext(jobJarPath .ToUri(), jobConf), jobJarPath, LocalResourceType.Pattern); string pattern = conf.GetPattern(JobContext.JarUnpackPattern, JobConf.UnpackJarPatternDefault ).Pattern(); rc.SetPattern(pattern); localResources[MRJobConfig.JobJar] = rc; } else { // Job jar may be null. For e.g, for pipes, the job jar is the hadoop // mapreduce jar itself which is already on the classpath. Log.Info("Job jar is not present. " + "Not adding any jar to the list of resources." ); } // TODO gross hack foreach (string s in new string[] { MRJobConfig.JobSplit, MRJobConfig.JobSplitMetainfo }) { localResources[MRJobConfig.JobSubmitDir + "/" + s] = CreateApplicationResource(defaultFileContext , new Path(jobSubmitDir, s), LocalResourceType.File); } // Setup security tokens DataOutputBuffer dob = new DataOutputBuffer(); ts.WriteTokenStorageToStream(dob); ByteBuffer securityTokens = ByteBuffer.Wrap(dob.GetData(), 0, dob.GetLength()); // Setup the command to run the AM IList <string> vargs = new AList <string>(8); vargs.AddItem(MRApps.CrossPlatformifyMREnv(jobConf, ApplicationConstants.Environment .JavaHome) + "/bin/java"); Path amTmpDir = new Path(MRApps.CrossPlatformifyMREnv(conf, ApplicationConstants.Environment .Pwd), YarnConfiguration.DefaultContainerTempDir); vargs.AddItem("-Djava.io.tmpdir=" + amTmpDir); MRApps.AddLog4jSystemProperties(null, vargs, conf); // Check for Java Lib Path usage in MAP and REDUCE configs WarnForJavaLibPath(conf.Get(MRJobConfig.MapJavaOpts, string.Empty), "map", MRJobConfig .MapJavaOpts, MRJobConfig.MapEnv); WarnForJavaLibPath(conf.Get(MRJobConfig.MapredMapAdminJavaOpts, string.Empty), "map" , MRJobConfig.MapredMapAdminJavaOpts, MRJobConfig.MapredAdminUserEnv); WarnForJavaLibPath(conf.Get(MRJobConfig.ReduceJavaOpts, string.Empty), "reduce", MRJobConfig.ReduceJavaOpts, MRJobConfig.ReduceEnv); WarnForJavaLibPath(conf.Get(MRJobConfig.MapredReduceAdminJavaOpts, string.Empty), "reduce", MRJobConfig.MapredReduceAdminJavaOpts, MRJobConfig.MapredAdminUserEnv); // Add AM admin command opts before user command opts // so that it can be overridden by user string mrAppMasterAdminOptions = conf.Get(MRJobConfig.MrAmAdminCommandOpts, MRJobConfig .DefaultMrAmAdminCommandOpts); WarnForJavaLibPath(mrAppMasterAdminOptions, "app master", MRJobConfig.MrAmAdminCommandOpts , MRJobConfig.MrAmAdminUserEnv); vargs.AddItem(mrAppMasterAdminOptions); // Add AM user command opts string mrAppMasterUserOptions = conf.Get(MRJobConfig.MrAmCommandOpts, MRJobConfig .DefaultMrAmCommandOpts); WarnForJavaLibPath(mrAppMasterUserOptions, "app master", MRJobConfig.MrAmCommandOpts , MRJobConfig.MrAmEnv); vargs.AddItem(mrAppMasterUserOptions); if (jobConf.GetBoolean(MRJobConfig.MrAmProfile, MRJobConfig.DefaultMrAmProfile)) { string profileParams = jobConf.Get(MRJobConfig.MrAmProfileParams, MRJobConfig.DefaultTaskProfileParams ); if (profileParams != null) { vargs.AddItem(string.Format(profileParams, ApplicationConstants.LogDirExpansionVar + Path.Separator + TaskLog.LogName.Profile)); } } vargs.AddItem(MRJobConfig.ApplicationMasterClass); vargs.AddItem("1>" + ApplicationConstants.LogDirExpansionVar + Path.Separator + ApplicationConstants .Stdout); vargs.AddItem("2>" + ApplicationConstants.LogDirExpansionVar + Path.Separator + ApplicationConstants .Stderr); Vector <string> vargsFinal = new Vector <string>(8); // Final command StringBuilder mergedCommand = new StringBuilder(); foreach (CharSequence str in vargs) { mergedCommand.Append(str).Append(" "); } vargsFinal.AddItem(mergedCommand.ToString()); Log.Debug("Command to launch container for ApplicationMaster is : " + mergedCommand ); // Setup the CLASSPATH in environment // i.e. add { Hadoop jars, job jar, CWD } to classpath. IDictionary <string, string> environment = new Dictionary <string, string>(); MRApps.SetClasspath(environment, conf); // Shell environment[ApplicationConstants.Environment.Shell.ToString()] = conf.Get(MRJobConfig .MapredAdminUserShell, MRJobConfig.DefaultShell); // Add the container working directory at the front of LD_LIBRARY_PATH MRApps.AddToEnvironment(environment, ApplicationConstants.Environment.LdLibraryPath .ToString(), MRApps.CrossPlatformifyMREnv(conf, ApplicationConstants.Environment .Pwd), conf); // Setup the environment variables for Admin first MRApps.SetEnvFromInputString(environment, conf.Get(MRJobConfig.MrAmAdminUserEnv), conf); // Setup the environment variables (LD_LIBRARY_PATH, etc) MRApps.SetEnvFromInputString(environment, conf.Get(MRJobConfig.MrAmEnv), conf); // Parse distributed cache MRApps.SetupDistributedCache(jobConf, localResources); IDictionary <ApplicationAccessType, string> acls = new Dictionary <ApplicationAccessType , string>(2); acls[ApplicationAccessType.ViewApp] = jobConf.Get(MRJobConfig.JobAclViewJob, MRJobConfig .DefaultJobAclViewJob); acls[ApplicationAccessType.ModifyApp] = jobConf.Get(MRJobConfig.JobAclModifyJob, MRJobConfig.DefaultJobAclModifyJob); // Setup ContainerLaunchContext for AM container ContainerLaunchContext amContainer = ContainerLaunchContext.NewInstance(localResources , environment, vargsFinal, null, securityTokens, acls); ICollection <string> tagsFromConf = jobConf.GetTrimmedStringCollection(MRJobConfig .JobTags); // Set up the ApplicationSubmissionContext ApplicationSubmissionContext appContext = recordFactory.NewRecordInstance <ApplicationSubmissionContext >(); appContext.SetApplicationId(applicationId); // ApplicationId appContext.SetQueue(jobConf.Get(JobContext.QueueName, YarnConfiguration.DefaultQueueName )); // Queue name // add reservationID if present ReservationId reservationID = null; try { reservationID = ReservationId.ParseReservationId(jobConf.Get(JobContext.ReservationId )); } catch (FormatException) { // throw exception as reservationid as is invalid string errMsg = "Invalid reservationId: " + jobConf.Get(JobContext.ReservationId) + " specified for the app: " + applicationId; Log.Warn(errMsg); throw new IOException(errMsg); } if (reservationID != null) { appContext.SetReservationID(reservationID); Log.Info("SUBMITTING ApplicationSubmissionContext app:" + applicationId + " to queue:" + appContext.GetQueue() + " with reservationId:" + appContext.GetReservationID( )); } appContext.SetApplicationName(jobConf.Get(JobContext.JobName, YarnConfiguration.DefaultApplicationName )); // Job name appContext.SetCancelTokensWhenComplete(conf.GetBoolean(MRJobConfig.JobCancelDelegationToken , true)); appContext.SetAMContainerSpec(amContainer); // AM Container appContext.SetMaxAppAttempts(conf.GetInt(MRJobConfig.MrAmMaxAttempts, MRJobConfig .DefaultMrAmMaxAttempts)); appContext.SetResource(capability); appContext.SetApplicationType(MRJobConfig.MrApplicationType); if (tagsFromConf != null && !tagsFromConf.IsEmpty()) { appContext.SetApplicationTags(new HashSet <string>(tagsFromConf)); } return(appContext); }
/// <summary>Main run function for the client</summary> /// <returns>true if application completed successfully</returns> /// <exception cref="System.IO.IOException"/> /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> public virtual bool Run() { Log.Info("Running Client"); yarnClient.Start(); YarnClusterMetrics clusterMetrics = yarnClient.GetYarnClusterMetrics(); Log.Info("Got Cluster metric info from ASM" + ", numNodeManagers=" + clusterMetrics .GetNumNodeManagers()); IList <NodeReport> clusterNodeReports = yarnClient.GetNodeReports(NodeState.Running ); Log.Info("Got Cluster node info from ASM"); foreach (NodeReport node in clusterNodeReports) { Log.Info("Got node report from ASM for" + ", nodeId=" + node.GetNodeId() + ", nodeAddress" + node.GetHttpAddress() + ", nodeRackName" + node.GetRackName() + ", nodeNumContainers" + node.GetNumContainers()); } QueueInfo queueInfo = yarnClient.GetQueueInfo(this.amQueue); Log.Info("Queue info" + ", queueName=" + queueInfo.GetQueueName() + ", queueCurrentCapacity=" + queueInfo.GetCurrentCapacity() + ", queueMaxCapacity=" + queueInfo.GetMaximumCapacity () + ", queueApplicationCount=" + queueInfo.GetApplications().Count + ", queueChildQueueCount=" + queueInfo.GetChildQueues().Count); IList <QueueUserACLInfo> listAclInfo = yarnClient.GetQueueAclsInfo(); foreach (QueueUserACLInfo aclInfo in listAclInfo) { foreach (QueueACL userAcl in aclInfo.GetUserAcls()) { Log.Info("User ACL Info for Queue" + ", queueName=" + aclInfo.GetQueueName() + ", userAcl=" + userAcl.ToString()); } } if (domainId != null && domainId.Length > 0 && toCreateDomain) { PrepareTimelineDomain(); } // Get a new application id YarnClientApplication app = yarnClient.CreateApplication(); GetNewApplicationResponse appResponse = app.GetNewApplicationResponse(); // TODO get min/max resource capabilities from RM and change memory ask if needed // If we do not have min/max, we may not be able to correctly request // the required resources from the RM for the app master // Memory ask has to be a multiple of min and less than max. // Dump out information about cluster capability as seen by the resource manager int maxMem = appResponse.GetMaximumResourceCapability().GetMemory(); Log.Info("Max mem capabililty of resources in this cluster " + maxMem); // A resource ask cannot exceed the max. if (amMemory > maxMem) { Log.Info("AM memory specified above max threshold of cluster. Using max value." + ", specified=" + amMemory + ", max=" + maxMem); amMemory = maxMem; } int maxVCores = appResponse.GetMaximumResourceCapability().GetVirtualCores(); Log.Info("Max virtual cores capabililty of resources in this cluster " + maxVCores ); if (amVCores > maxVCores) { Log.Info("AM virtual cores specified above max threshold of cluster. " + "Using max value." + ", specified=" + amVCores + ", max=" + maxVCores); amVCores = maxVCores; } // set the application name ApplicationSubmissionContext appContext = app.GetApplicationSubmissionContext(); ApplicationId appId = appContext.GetApplicationId(); appContext.SetKeepContainersAcrossApplicationAttempts(keepContainers); appContext.SetApplicationName(appName); if (attemptFailuresValidityInterval >= 0) { appContext.SetAttemptFailuresValidityInterval(attemptFailuresValidityInterval); } // set local resources for the application master // local files or archives as needed // In this scenario, the jar file for the application master is part of the local resources IDictionary <string, LocalResource> localResources = new Dictionary <string, LocalResource >(); Log.Info("Copy App Master jar from local filesystem and add to local environment" ); // Copy the application master jar to the filesystem // Create a local resource to point to the destination jar path FileSystem fs = FileSystem.Get(conf); AddToLocalResources(fs, appMasterJar, appMasterJarPath, appId.ToString(), localResources , null); // Set the log4j properties if needed if (!log4jPropFile.IsEmpty()) { AddToLocalResources(fs, log4jPropFile, log4jPath, appId.ToString(), localResources , null); } // The shell script has to be made available on the final container(s) // where it will be executed. // To do this, we need to first copy into the filesystem that is visible // to the yarn framework. // We do not need to set this as a local resource for the application // master as the application master does not need it. string hdfsShellScriptLocation = string.Empty; long hdfsShellScriptLen = 0; long hdfsShellScriptTimestamp = 0; if (!shellScriptPath.IsEmpty()) { Path shellSrc = new Path(shellScriptPath); string shellPathSuffix = appName + "/" + appId.ToString() + "/" + ScriptPath; Path shellDst = new Path(fs.GetHomeDirectory(), shellPathSuffix); fs.CopyFromLocalFile(false, true, shellSrc, shellDst); hdfsShellScriptLocation = shellDst.ToUri().ToString(); FileStatus shellFileStatus = fs.GetFileStatus(shellDst); hdfsShellScriptLen = shellFileStatus.GetLen(); hdfsShellScriptTimestamp = shellFileStatus.GetModificationTime(); } if (!shellCommand.IsEmpty()) { AddToLocalResources(fs, null, shellCommandPath, appId.ToString(), localResources, shellCommand); } if (shellArgs.Length > 0) { AddToLocalResources(fs, null, shellArgsPath, appId.ToString(), localResources, StringUtils .Join(shellArgs, " ")); } // Set the necessary security tokens as needed //amContainer.setContainerTokens(containerToken); // Set the env variables to be setup in the env where the application master will be run Log.Info("Set the environment for the application master"); IDictionary <string, string> env = new Dictionary <string, string>(); // put location of shell script into env // using the env info, the application master will create the correct local resource for the // eventual containers that will be launched to execute the shell scripts env[DSConstants.Distributedshellscriptlocation] = hdfsShellScriptLocation; env[DSConstants.Distributedshellscripttimestamp] = System.Convert.ToString(hdfsShellScriptTimestamp ); env[DSConstants.Distributedshellscriptlen] = System.Convert.ToString(hdfsShellScriptLen ); if (domainId != null && domainId.Length > 0) { env[DSConstants.Distributedshelltimelinedomain] = domainId; } // Add AppMaster.jar location to classpath // At some point we should not be required to add // the hadoop specific classpaths to the env. // It should be provided out of the box. // For now setting all required classpaths including // the classpath to "." for the application jar StringBuilder classPathEnv = new StringBuilder(ApplicationConstants.Environment.Classpath .$$()).Append(ApplicationConstants.ClassPathSeparator).Append("./*"); foreach (string c in conf.GetStrings(YarnConfiguration.YarnApplicationClasspath, YarnConfiguration.DefaultYarnCrossPlatformApplicationClasspath)) { classPathEnv.Append(ApplicationConstants.ClassPathSeparator); classPathEnv.Append(c.Trim()); } classPathEnv.Append(ApplicationConstants.ClassPathSeparator).Append("./log4j.properties" ); // add the runtime classpath needed for tests to work if (conf.GetBoolean(YarnConfiguration.IsMiniYarnCluster, false)) { classPathEnv.Append(':'); classPathEnv.Append(Runtime.GetProperty("java.class.path")); } env["CLASSPATH"] = classPathEnv.ToString(); // Set the necessary command to execute the application master Vector <CharSequence> vargs = new Vector <CharSequence>(30); // Set java executable command Log.Info("Setting up app master command"); vargs.AddItem(ApplicationConstants.Environment.JavaHome.$$() + "/bin/java"); // Set Xmx based on am memory size vargs.AddItem("-Xmx" + amMemory + "m"); // Set class name vargs.AddItem(appMasterMainClass); // Set params for Application Master vargs.AddItem("--container_memory " + containerMemory.ToString()); vargs.AddItem("--container_vcores " + containerVirtualCores.ToString()); vargs.AddItem("--num_containers " + numContainers.ToString()); if (null != nodeLabelExpression) { appContext.SetNodeLabelExpression(nodeLabelExpression); } vargs.AddItem("--priority " + shellCmdPriority.ToString()); foreach (KeyValuePair <string, string> entry in shellEnv) { vargs.AddItem("--shell_env " + entry.Key + "=" + entry.Value); } if (debugFlag) { vargs.AddItem("--debug"); } vargs.AddItem("1>" + ApplicationConstants.LogDirExpansionVar + "/AppMaster.stdout" ); vargs.AddItem("2>" + ApplicationConstants.LogDirExpansionVar + "/AppMaster.stderr" ); // Get final commmand StringBuilder command = new StringBuilder(); foreach (CharSequence str in vargs) { command.Append(str).Append(" "); } Log.Info("Completed setting up app master command " + command.ToString()); IList <string> commands = new AList <string>(); commands.AddItem(command.ToString()); // Set up the container launch context for the application master ContainerLaunchContext amContainer = ContainerLaunchContext.NewInstance(localResources , env, commands, null, null, null); // Set up resource type requirements // For now, both memory and vcores are supported, so we set memory and // vcores requirements Resource capability = Resource.NewInstance(amMemory, amVCores); appContext.SetResource(capability); // Service data is a binary blob that can be passed to the application // Not needed in this scenario // amContainer.setServiceData(serviceData); // Setup security tokens if (UserGroupInformation.IsSecurityEnabled()) { // Note: Credentials class is marked as LimitedPrivate for HDFS and MapReduce Credentials credentials = new Credentials(); string tokenRenewer = conf.Get(YarnConfiguration.RmPrincipal); if (tokenRenewer == null || tokenRenewer.Length == 0) { throw new IOException("Can't get Master Kerberos principal for the RM to use as renewer" ); } // For now, only getting tokens for the default file-system. Org.Apache.Hadoop.Security.Token.Token <object>[] tokens = fs.AddDelegationTokens( tokenRenewer, credentials); if (tokens != null) { foreach (Org.Apache.Hadoop.Security.Token.Token <object> token in tokens) { Log.Info("Got dt for " + fs.GetUri() + "; " + token); } } DataOutputBuffer dob = new DataOutputBuffer(); credentials.WriteTokenStorageToStream(dob); ByteBuffer fsTokens = ByteBuffer.Wrap(dob.GetData(), 0, dob.GetLength()); amContainer.SetTokens(fsTokens); } appContext.SetAMContainerSpec(amContainer); // Set the priority for the application master // TODO - what is the range for priority? how to decide? Priority pri = Priority.NewInstance(amPriority); appContext.SetPriority(pri); // Set the queue to which this application is to be submitted in the RM appContext.SetQueue(amQueue); // Submit the application to the applications manager // SubmitApplicationResponse submitResp = applicationsManager.submitApplication(appRequest); // Ignore the response as either a valid response object is returned on success // or an exception thrown to denote some form of a failure Log.Info("Submitting application to ASM"); yarnClient.SubmitApplication(appContext); // TODO // Try submitting the same request again // app submission failure? // Monitor the application return(MonitorApplication(appId)); }