/// <exception cref="System.Exception"/> internal virtual Org.Apache.Hadoop.Security.Token.Token <object>[] AddTokensWithCreds (FileSystem fs, Credentials creds) { Credentials savedCreds; savedCreds = new Credentials(creds); Org.Apache.Hadoop.Security.Token.Token <object>[] tokens = fs.AddDelegationTokens( "me", creds); // test that we got the token we wanted, and that creds were modified Assert.Equal(1, tokens.Length); Assert.Equal(fs.GetCanonicalServiceName(), tokens[0].GetService ().ToString()); Assert.True(creds.GetAllTokens().Contains(tokens[0])); Assert.True(creds.GetAllTokens().ContainsAll(savedCreds.GetAllTokens ())); Assert.Equal(savedCreds.NumberOfTokens() + 1, creds.NumberOfTokens ()); // shouldn't get any new tokens since already in creds savedCreds = new Credentials(creds); Org.Apache.Hadoop.Security.Token.Token <object>[] tokenRefetch = fs.AddDelegationTokens ("me", creds); Assert.Equal(0, tokenRefetch.Length); Assert.True(creds.GetAllTokens().ContainsAll(savedCreds.GetAllTokens ())); Assert.Equal(savedCreds.NumberOfTokens(), creds.NumberOfTokens ()); return(tokens); }
public virtual void TestGetDelegationTokens() { Org.Apache.Hadoop.Security.Token.Token <object>[] delTokens = fsView.AddDelegationTokens ("sanjay", new Credentials()); Assert.Equal(GetExpectedDelegationTokenCount(), delTokens.Length ); }
/// <exception cref="System.IO.IOException"/> private Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier> GetDelegationToken (FileSystem fs, string renewer) { Org.Apache.Hadoop.Security.Token.Token <object>[] tokens = fs.AddDelegationTokens( renewer, null); NUnit.Framework.Assert.AreEqual(1, tokens.Length); return((Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier>)tokens[ 0]); }
/// <summary>get delegation token for a specific FS</summary> /// <param name="fs"/> /// <param name="credentials"/> /// <param name="p"/> /// <param name="conf"/> /// <exception cref="System.IO.IOException"/> internal static void ObtainTokensForNamenodesInternal(FileSystem fs, Credentials credentials, Configuration conf) { string delegTokenRenewer = Master.GetMasterPrincipal(conf); if (delegTokenRenewer == null || delegTokenRenewer.Length == 0) { throw new IOException("Can't get Master Kerberos principal for use as renewer"); } MergeBinaryTokens(credentials, conf); Org.Apache.Hadoop.Security.Token.Token <object>[] tokens = fs.AddDelegationTokens( delegTokenRenewer, credentials); if (tokens != null) { foreach (Org.Apache.Hadoop.Security.Token.Token <object> token in tokens) { Log.Info("Got dt for " + fs.GetUri() + "; " + token); } } }
/// <exception cref="System.Exception"/> private void TestDelegationTokenWithFS(Type fileSystemClass) { CreateHttpFSServer(); Configuration conf = new Configuration(); conf.Set("fs.webhdfs.impl", fileSystemClass.FullName); conf.Set("fs.hdfs.impl.disable.cache", "true"); URI uri = new URI("webhdfs://" + TestJettyHelper.GetJettyURL().ToURI().GetAuthority ()); FileSystem fs = FileSystem.Get(uri, conf); Org.Apache.Hadoop.Security.Token.Token <object>[] tokens = fs.AddDelegationTokens( "foo", null); fs.Close(); NUnit.Framework.Assert.AreEqual(1, tokens.Length); fs = FileSystem.Get(uri, conf); ((DelegationTokenRenewer.Renewable)fs).SetDelegationToken(tokens[0]); fs.ListStatus(new Path("/")); fs.Close(); }
/// <exception cref="System.Exception"/> public object Run() { if (print) { DelegationTokenIdentifier id = new DelegationTokenSecretManager(0, 0, 0, 0, null) .CreateIdentifier(); foreach (Org.Apache.Hadoop.Security.Token.Token <object> token in DelegationTokenFetcher .ReadTokens(tokenFile, conf)) { DataInputStream @in = new DataInputStream(new ByteArrayInputStream(token.GetIdentifier ())); id.ReadFields(@in); System.Console.Out.WriteLine("Token (" + id + ") for " + token.GetService()); } return(null); } if (renew) { foreach (Org.Apache.Hadoop.Security.Token.Token <object> token in DelegationTokenFetcher .ReadTokens(tokenFile, conf)) { if (token.IsManaged()) { long result = token.Renew(conf); if (DelegationTokenFetcher.Log.IsDebugEnabled()) { DelegationTokenFetcher.Log.Debug("Renewed token for " + token.GetService() + " until: " + Sharpen.Extensions.CreateDate(result)); } } } } else { if (cancel) { foreach (Org.Apache.Hadoop.Security.Token.Token <object> token in DelegationTokenFetcher .ReadTokens(tokenFile, conf)) { if (token.IsManaged()) { token.Cancel(conf); if (DelegationTokenFetcher.Log.IsDebugEnabled()) { DelegationTokenFetcher.Log.Debug("Cancelled token for " + token.GetService()); } } } } else { // otherwise we are fetching if (webUrl != null) { Credentials creds = DelegationTokenFetcher.GetDTfromRemote(connectionFactory, new URI(webUrl), renewer, null); creds.WriteTokenStorageFile(tokenFile, conf); foreach (Org.Apache.Hadoop.Security.Token.Token <object> token in creds.GetAllTokens ()) { System.Console.Out.WriteLine("Fetched token via " + webUrl + " for " + token.GetService () + " into " + tokenFile); } } else { FileSystem fs = FileSystem.Get(conf); Credentials cred = new Credentials(); Org.Apache.Hadoop.Security.Token.Token <object>[] tokens = fs.AddDelegationTokens( renewer, cred); cred.WriteTokenStorageFile(tokenFile, conf); foreach (Org.Apache.Hadoop.Security.Token.Token <object> token in tokens) { System.Console.Out.WriteLine("Fetched token for " + token.GetService() + " into " + tokenFile); } } } } return(null); }
/// <exception cref="System.IO.IOException"/> public override IList <Org.Apache.Hadoop.Security.Token.Token <object> > GetDelegationTokens (string renewer) { //AbstractFileSystem return(Arrays.AsList(fsImpl.AddDelegationTokens(renewer, null))); }
/// <summary>Main run function for the client</summary> /// <returns>true if application completed successfully</returns> /// <exception cref="System.IO.IOException"/> /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> public virtual bool Run() { Log.Info("Running Client"); yarnClient.Start(); YarnClusterMetrics clusterMetrics = yarnClient.GetYarnClusterMetrics(); Log.Info("Got Cluster metric info from ASM" + ", numNodeManagers=" + clusterMetrics .GetNumNodeManagers()); IList <NodeReport> clusterNodeReports = yarnClient.GetNodeReports(NodeState.Running ); Log.Info("Got Cluster node info from ASM"); foreach (NodeReport node in clusterNodeReports) { Log.Info("Got node report from ASM for" + ", nodeId=" + node.GetNodeId() + ", nodeAddress" + node.GetHttpAddress() + ", nodeRackName" + node.GetRackName() + ", nodeNumContainers" + node.GetNumContainers()); } QueueInfo queueInfo = yarnClient.GetQueueInfo(this.amQueue); Log.Info("Queue info" + ", queueName=" + queueInfo.GetQueueName() + ", queueCurrentCapacity=" + queueInfo.GetCurrentCapacity() + ", queueMaxCapacity=" + queueInfo.GetMaximumCapacity () + ", queueApplicationCount=" + queueInfo.GetApplications().Count + ", queueChildQueueCount=" + queueInfo.GetChildQueues().Count); IList <QueueUserACLInfo> listAclInfo = yarnClient.GetQueueAclsInfo(); foreach (QueueUserACLInfo aclInfo in listAclInfo) { foreach (QueueACL userAcl in aclInfo.GetUserAcls()) { Log.Info("User ACL Info for Queue" + ", queueName=" + aclInfo.GetQueueName() + ", userAcl=" + userAcl.ToString()); } } if (domainId != null && domainId.Length > 0 && toCreateDomain) { PrepareTimelineDomain(); } // Get a new application id YarnClientApplication app = yarnClient.CreateApplication(); GetNewApplicationResponse appResponse = app.GetNewApplicationResponse(); // TODO get min/max resource capabilities from RM and change memory ask if needed // If we do not have min/max, we may not be able to correctly request // the required resources from the RM for the app master // Memory ask has to be a multiple of min and less than max. // Dump out information about cluster capability as seen by the resource manager int maxMem = appResponse.GetMaximumResourceCapability().GetMemory(); Log.Info("Max mem capabililty of resources in this cluster " + maxMem); // A resource ask cannot exceed the max. if (amMemory > maxMem) { Log.Info("AM memory specified above max threshold of cluster. Using max value." + ", specified=" + amMemory + ", max=" + maxMem); amMemory = maxMem; } int maxVCores = appResponse.GetMaximumResourceCapability().GetVirtualCores(); Log.Info("Max virtual cores capabililty of resources in this cluster " + maxVCores ); if (amVCores > maxVCores) { Log.Info("AM virtual cores specified above max threshold of cluster. " + "Using max value." + ", specified=" + amVCores + ", max=" + maxVCores); amVCores = maxVCores; } // set the application name ApplicationSubmissionContext appContext = app.GetApplicationSubmissionContext(); ApplicationId appId = appContext.GetApplicationId(); appContext.SetKeepContainersAcrossApplicationAttempts(keepContainers); appContext.SetApplicationName(appName); if (attemptFailuresValidityInterval >= 0) { appContext.SetAttemptFailuresValidityInterval(attemptFailuresValidityInterval); } // set local resources for the application master // local files or archives as needed // In this scenario, the jar file for the application master is part of the local resources IDictionary <string, LocalResource> localResources = new Dictionary <string, LocalResource >(); Log.Info("Copy App Master jar from local filesystem and add to local environment" ); // Copy the application master jar to the filesystem // Create a local resource to point to the destination jar path FileSystem fs = FileSystem.Get(conf); AddToLocalResources(fs, appMasterJar, appMasterJarPath, appId.ToString(), localResources , null); // Set the log4j properties if needed if (!log4jPropFile.IsEmpty()) { AddToLocalResources(fs, log4jPropFile, log4jPath, appId.ToString(), localResources , null); } // The shell script has to be made available on the final container(s) // where it will be executed. // To do this, we need to first copy into the filesystem that is visible // to the yarn framework. // We do not need to set this as a local resource for the application // master as the application master does not need it. string hdfsShellScriptLocation = string.Empty; long hdfsShellScriptLen = 0; long hdfsShellScriptTimestamp = 0; if (!shellScriptPath.IsEmpty()) { Path shellSrc = new Path(shellScriptPath); string shellPathSuffix = appName + "/" + appId.ToString() + "/" + ScriptPath; Path shellDst = new Path(fs.GetHomeDirectory(), shellPathSuffix); fs.CopyFromLocalFile(false, true, shellSrc, shellDst); hdfsShellScriptLocation = shellDst.ToUri().ToString(); FileStatus shellFileStatus = fs.GetFileStatus(shellDst); hdfsShellScriptLen = shellFileStatus.GetLen(); hdfsShellScriptTimestamp = shellFileStatus.GetModificationTime(); } if (!shellCommand.IsEmpty()) { AddToLocalResources(fs, null, shellCommandPath, appId.ToString(), localResources, shellCommand); } if (shellArgs.Length > 0) { AddToLocalResources(fs, null, shellArgsPath, appId.ToString(), localResources, StringUtils .Join(shellArgs, " ")); } // Set the necessary security tokens as needed //amContainer.setContainerTokens(containerToken); // Set the env variables to be setup in the env where the application master will be run Log.Info("Set the environment for the application master"); IDictionary <string, string> env = new Dictionary <string, string>(); // put location of shell script into env // using the env info, the application master will create the correct local resource for the // eventual containers that will be launched to execute the shell scripts env[DSConstants.Distributedshellscriptlocation] = hdfsShellScriptLocation; env[DSConstants.Distributedshellscripttimestamp] = System.Convert.ToString(hdfsShellScriptTimestamp ); env[DSConstants.Distributedshellscriptlen] = System.Convert.ToString(hdfsShellScriptLen ); if (domainId != null && domainId.Length > 0) { env[DSConstants.Distributedshelltimelinedomain] = domainId; } // Add AppMaster.jar location to classpath // At some point we should not be required to add // the hadoop specific classpaths to the env. // It should be provided out of the box. // For now setting all required classpaths including // the classpath to "." for the application jar StringBuilder classPathEnv = new StringBuilder(ApplicationConstants.Environment.Classpath .$$()).Append(ApplicationConstants.ClassPathSeparator).Append("./*"); foreach (string c in conf.GetStrings(YarnConfiguration.YarnApplicationClasspath, YarnConfiguration.DefaultYarnCrossPlatformApplicationClasspath)) { classPathEnv.Append(ApplicationConstants.ClassPathSeparator); classPathEnv.Append(c.Trim()); } classPathEnv.Append(ApplicationConstants.ClassPathSeparator).Append("./log4j.properties" ); // add the runtime classpath needed for tests to work if (conf.GetBoolean(YarnConfiguration.IsMiniYarnCluster, false)) { classPathEnv.Append(':'); classPathEnv.Append(Runtime.GetProperty("java.class.path")); } env["CLASSPATH"] = classPathEnv.ToString(); // Set the necessary command to execute the application master Vector <CharSequence> vargs = new Vector <CharSequence>(30); // Set java executable command Log.Info("Setting up app master command"); vargs.AddItem(ApplicationConstants.Environment.JavaHome.$$() + "/bin/java"); // Set Xmx based on am memory size vargs.AddItem("-Xmx" + amMemory + "m"); // Set class name vargs.AddItem(appMasterMainClass); // Set params for Application Master vargs.AddItem("--container_memory " + containerMemory.ToString()); vargs.AddItem("--container_vcores " + containerVirtualCores.ToString()); vargs.AddItem("--num_containers " + numContainers.ToString()); if (null != nodeLabelExpression) { appContext.SetNodeLabelExpression(nodeLabelExpression); } vargs.AddItem("--priority " + shellCmdPriority.ToString()); foreach (KeyValuePair <string, string> entry in shellEnv) { vargs.AddItem("--shell_env " + entry.Key + "=" + entry.Value); } if (debugFlag) { vargs.AddItem("--debug"); } vargs.AddItem("1>" + ApplicationConstants.LogDirExpansionVar + "/AppMaster.stdout" ); vargs.AddItem("2>" + ApplicationConstants.LogDirExpansionVar + "/AppMaster.stderr" ); // Get final commmand StringBuilder command = new StringBuilder(); foreach (CharSequence str in vargs) { command.Append(str).Append(" "); } Log.Info("Completed setting up app master command " + command.ToString()); IList <string> commands = new AList <string>(); commands.AddItem(command.ToString()); // Set up the container launch context for the application master ContainerLaunchContext amContainer = ContainerLaunchContext.NewInstance(localResources , env, commands, null, null, null); // Set up resource type requirements // For now, both memory and vcores are supported, so we set memory and // vcores requirements Resource capability = Resource.NewInstance(amMemory, amVCores); appContext.SetResource(capability); // Service data is a binary blob that can be passed to the application // Not needed in this scenario // amContainer.setServiceData(serviceData); // Setup security tokens if (UserGroupInformation.IsSecurityEnabled()) { // Note: Credentials class is marked as LimitedPrivate for HDFS and MapReduce Credentials credentials = new Credentials(); string tokenRenewer = conf.Get(YarnConfiguration.RmPrincipal); if (tokenRenewer == null || tokenRenewer.Length == 0) { throw new IOException("Can't get Master Kerberos principal for the RM to use as renewer" ); } // For now, only getting tokens for the default file-system. Org.Apache.Hadoop.Security.Token.Token <object>[] tokens = fs.AddDelegationTokens( tokenRenewer, credentials); if (tokens != null) { foreach (Org.Apache.Hadoop.Security.Token.Token <object> token in tokens) { Log.Info("Got dt for " + fs.GetUri() + "; " + token); } } DataOutputBuffer dob = new DataOutputBuffer(); credentials.WriteTokenStorageToStream(dob); ByteBuffer fsTokens = ByteBuffer.Wrap(dob.GetData(), 0, dob.GetLength()); amContainer.SetTokens(fsTokens); } appContext.SetAMContainerSpec(amContainer); // Set the priority for the application master // TODO - what is the range for priority? how to decide? Priority pri = Priority.NewInstance(amPriority); appContext.SetPriority(pri); // Set the queue to which this application is to be submitted in the RM appContext.SetQueue(amQueue); // Submit the application to the applications manager // SubmitApplicationResponse submitResp = applicationsManager.submitApplication(appRequest); // Ignore the response as either a valid response object is returned on success // or an exception thrown to denote some form of a failure Log.Info("Submitting application to ASM"); yarnClient.SubmitApplication(appContext); // TODO // Try submitting the same request again // app submission failure? // Monitor the application return(MonitorApplication(appId)); }