public static ApplicationAttemptId GetMockApplicationAttemptId(int appId, int attemptId ) { ApplicationId applicationId = BuilderUtils.NewApplicationId(0l, appId); ApplicationAttemptId applicationAttemptId = Org.Mockito.Mockito.Mock <ApplicationAttemptId >(); Org.Mockito.Mockito.When(applicationAttemptId.GetApplicationId()).ThenReturn(applicationId ); Org.Mockito.Mockito.When(applicationAttemptId.GetAttemptId()).ThenReturn(attemptId ); return(applicationAttemptId); }
/// <exception cref="System.IO.IOException"/> public static Path GetPreviousJobHistoryPath(Configuration conf, ApplicationAttemptId applicationAttemptId) { string jobId = TypeConverter.FromYarn(applicationAttemptId.GetApplicationId()).ToString (); string jobhistoryDir = JobHistoryUtils.GetConfiguredHistoryStagingDirPrefix(conf, jobId); Path histDirPath = FileContext.GetFileContext(conf).MakeQualified(new Path(jobhistoryDir )); FileContext fc = FileContext.GetFileContext(histDirPath.ToUri(), conf); return(fc.MakeQualified(JobHistoryUtils.GetStagingJobHistoryFile(histDirPath, jobId , (applicationAttemptId.GetAttemptId() - 1)))); }
internal static ContainerId GetMockContainer(long id) { ApplicationId appId = Org.Mockito.Mockito.Mock <ApplicationId>(); Org.Mockito.Mockito.When(appId.GetClusterTimestamp()).ThenReturn(314159265L); Org.Mockito.Mockito.When(appId.GetId()).ThenReturn(3); ApplicationAttemptId appAttemptId = Org.Mockito.Mockito.Mock <ApplicationAttemptId >(); Org.Mockito.Mockito.When(appAttemptId.GetApplicationId()).ThenReturn(appId); Org.Mockito.Mockito.When(appAttemptId.GetAttemptId()).ThenReturn(0); ContainerId container = Org.Mockito.Mockito.Mock <ContainerId>(); Org.Mockito.Mockito.When(container.GetContainerId()).ThenReturn(id); Org.Mockito.Mockito.When(container.GetApplicationAttemptId()).ThenReturn(appAttemptId ); return(container); }
/// <exception cref="System.IO.IOException"/> private static void ConfigureTask(JobConf job, Task task, Credentials credentials , Org.Apache.Hadoop.Security.Token.Token <JobTokenIdentifier> jt) { job.SetCredentials(credentials); ApplicationAttemptId appAttemptId = ConverterUtils.ToContainerId(Runtime.Getenv(ApplicationConstants.Environment .ContainerId.ToString())).GetApplicationAttemptId(); Log.Debug("APPLICATION_ATTEMPT_ID: " + appAttemptId); // Set it in conf, so as to be able to be used the the OutputCommitter. job.SetInt(MRJobConfig.ApplicationAttemptId, appAttemptId.GetAttemptId()); // set tcp nodelay job.SetBoolean("ipc.client.tcpnodelay", true); job.SetClass(MRConfig.TaskLocalOutputClass, typeof(YarnOutputFiles), typeof(MapOutputFile )); // set the jobToken and shuffle secrets into task task.SetJobTokenSecret(JobTokenSecretManager.CreateSecretKey(jt.GetPassword())); byte[] shuffleSecret = TokenCache.GetShuffleSecretKey(credentials); if (shuffleSecret == null) { Log.Warn("Shuffle secret missing from task credentials." + " Using job token secret as shuffle secret." ); shuffleSecret = jt.GetPassword(); } task.SetShuffleSecret(JobTokenSecretManager.CreateSecretKey(shuffleSecret)); // setup the child's MRConfig.LOCAL_DIR. ConfigureLocalDirs(task, job); // setup the child's attempt directories // Do the task-type specific localization task.LocalizeConfiguration(job); // Set up the DistributedCache related configs MRApps.SetupDistributedCacheLocal(job); // Overwrite the localized task jobconf which is linked to in the current // work-dir. Path localTaskFile = new Path(MRJobConfig.JobConfFile); WriteLocalJobFile(localTaskFile, job); task.SetJobFile(localTaskFile.ToString()); task.SetConf(job); }
/// <exception cref="System.IO.IOException"/> private ApplicationAttemptHistoryData GetLastAttempt(ApplicationId appId) { IDictionary <ApplicationAttemptId, ApplicationAttemptHistoryData> attempts = historyStore .GetApplicationAttempts(appId); ApplicationAttemptId prevMaxAttemptId = null; foreach (ApplicationAttemptId attemptId in attempts.Keys) { if (prevMaxAttemptId == null) { prevMaxAttemptId = attemptId; } else { if (prevMaxAttemptId.GetAttemptId() < attemptId.GetAttemptId()) { prevMaxAttemptId = attemptId; } } } return(attempts[prevMaxAttemptId]); }
/// <exception cref="System.Exception"/> private void TestNMTokens(Configuration conf) { NMTokenSecretManagerInRM nmTokenSecretManagerRM = yarnCluster.GetResourceManager( ).GetRMContext().GetNMTokenSecretManager(); NMTokenSecretManagerInNM nmTokenSecretManagerNM = yarnCluster.GetNodeManager(0).GetNMContext ().GetNMTokenSecretManager(); RMContainerTokenSecretManager containerTokenSecretManager = yarnCluster.GetResourceManager ().GetRMContext().GetContainerTokenSecretManager(); NodeManager nm = yarnCluster.GetNodeManager(0); WaitForNMToReceiveNMTokenKey(nmTokenSecretManagerNM, nm); // Both id should be equal. NUnit.Framework.Assert.AreEqual(nmTokenSecretManagerNM.GetCurrentKey().GetKeyId() , nmTokenSecretManagerRM.GetCurrentKey().GetKeyId()); /* * Below cases should be tested. * 1) If Invalid NMToken is used then it should be rejected. * 2) If valid NMToken but belonging to another Node is used then that * too should be rejected. * 3) NMToken for say appAttempt-1 is used for starting/stopping/retrieving * status for container with containerId for say appAttempt-2 should * be rejected. * 4) After start container call is successful nmtoken should have been * saved in NMTokenSecretManagerInNM. * 5) If start container call was successful (no matter if container is * still running or not), appAttempt->NMToken should be present in * NMTokenSecretManagerInNM's cache. Any future getContainerStatus call * for containerId belonging to that application attempt using * applicationAttempt's older nmToken should not get any invalid * nmToken error. (This can be best tested if we roll over NMToken * master key twice). */ YarnRPC rpc = YarnRPC.Create(conf); string user = "******"; Resource r = Resource.NewInstance(1024, 1); ApplicationId appId = ApplicationId.NewInstance(1, 1); ApplicationAttemptId validAppAttemptId = ApplicationAttemptId.NewInstance(appId, 1); ContainerId validContainerId = ContainerId.NewContainerId(validAppAttemptId, 0); NodeId validNode = yarnCluster.GetNodeManager(0).GetNMContext().GetNodeId(); NodeId invalidNode = NodeId.NewInstance("InvalidHost", 1234); Token validNMToken = nmTokenSecretManagerRM.CreateNMToken(validAppAttemptId, validNode , user); Token validContainerToken = containerTokenSecretManager.CreateContainerToken(validContainerId , validNode, user, r, Priority.NewInstance(10), 1234); ContainerTokenIdentifier identifier = BuilderUtils.NewContainerTokenIdentifier(validContainerToken ); NUnit.Framework.Assert.AreEqual(Priority.NewInstance(10), identifier.GetPriority( )); NUnit.Framework.Assert.AreEqual(1234, identifier.GetCreationTime()); StringBuilder sb; // testInvalidNMToken ... creating NMToken using different secret manager. NMTokenSecretManagerInRM tempManager = new NMTokenSecretManagerInRM(conf); tempManager.RollMasterKey(); do { tempManager.RollMasterKey(); tempManager.ActivateNextMasterKey(); }while (tempManager.GetCurrentKey().GetKeyId() == nmTokenSecretManagerRM.GetCurrentKey ().GetKeyId()); // Making sure key id is different. // Testing that NM rejects the requests when we don't send any token. if (UserGroupInformation.IsSecurityEnabled()) { sb = new StringBuilder("Client cannot authenticate via:[TOKEN]"); } else { sb = new StringBuilder("SIMPLE authentication is not enabled. Available:[TOKEN]" ); } string errorMsg = TestStartContainer(rpc, validAppAttemptId, validNode, validContainerToken , null, true); NUnit.Framework.Assert.IsTrue(errorMsg.Contains(sb.ToString())); Token invalidNMToken = tempManager.CreateNMToken(validAppAttemptId, validNode, user ); sb = new StringBuilder("Given NMToken for application : "); sb.Append(validAppAttemptId.ToString()).Append(" seems to have been generated illegally." ); NUnit.Framework.Assert.IsTrue(sb.ToString().Contains(TestStartContainer(rpc, validAppAttemptId , validNode, validContainerToken, invalidNMToken, true))); // valid NMToken but belonging to other node invalidNMToken = nmTokenSecretManagerRM.CreateNMToken(validAppAttemptId, invalidNode , user); sb = new StringBuilder("Given NMToken for application : "); sb.Append(validAppAttemptId).Append(" is not valid for current node manager.expected : " ).Append(validNode.ToString()).Append(" found : ").Append(invalidNode.ToString() ); NUnit.Framework.Assert.IsTrue(sb.ToString().Contains(TestStartContainer(rpc, validAppAttemptId , validNode, validContainerToken, invalidNMToken, true))); // using correct tokens. nmtoken for app attempt should get saved. conf.SetInt(YarnConfiguration.RmContainerAllocExpiryIntervalMs, 4 * 60 * 1000); validContainerToken = containerTokenSecretManager.CreateContainerToken(validContainerId , validNode, user, r, Priority.NewInstance(0), 0); NUnit.Framework.Assert.IsTrue(TestStartContainer(rpc, validAppAttemptId, validNode , validContainerToken, validNMToken, false).IsEmpty()); NUnit.Framework.Assert.IsTrue(nmTokenSecretManagerNM.IsAppAttemptNMTokenKeyPresent (validAppAttemptId)); // using a new compatible version nmtoken, expect container can be started // successfully. ApplicationAttemptId validAppAttemptId2 = ApplicationAttemptId.NewInstance(appId, 2); ContainerId validContainerId2 = ContainerId.NewContainerId(validAppAttemptId2, 0); Token validContainerToken2 = containerTokenSecretManager.CreateContainerToken(validContainerId2 , validNode, user, r, Priority.NewInstance(0), 0); Token validNMToken2 = nmTokenSecretManagerRM.CreateNMToken(validAppAttemptId2, validNode , user); // First, get a new NMTokenIdentifier. NMTokenIdentifier newIdentifier = new NMTokenIdentifier(); byte[] tokenIdentifierContent = ((byte[])validNMToken2.GetIdentifier().Array()); DataInputBuffer dib = new DataInputBuffer(); dib.Reset(tokenIdentifierContent, tokenIdentifierContent.Length); newIdentifier.ReadFields(dib); // Then, generate a new version NMTokenIdentifier (NMTokenIdentifierNewForTest) // with additional field of message. NMTokenIdentifierNewForTest newVersionIdentifier = new NMTokenIdentifierNewForTest (newIdentifier, "message"); // check new version NMTokenIdentifier has correct info. NUnit.Framework.Assert.AreEqual("The ApplicationAttemptId is changed after set to " + "newVersionIdentifier", validAppAttemptId2.GetAttemptId(), newVersionIdentifier .GetApplicationAttemptId().GetAttemptId()); NUnit.Framework.Assert.AreEqual("The message is changed after set to newVersionIdentifier" , "message", newVersionIdentifier.GetMessage()); NUnit.Framework.Assert.AreEqual("The NodeId is changed after set to newVersionIdentifier" , validNode, newVersionIdentifier.GetNodeId()); // create new Token based on new version NMTokenIdentifier. Token newVersionedNMToken = BaseNMTokenSecretManager.NewInstance(nmTokenSecretManagerRM .RetrievePassword(newVersionIdentifier), newVersionIdentifier); // Verify startContainer is successful and no exception is thrown. NUnit.Framework.Assert.IsTrue(TestStartContainer(rpc, validAppAttemptId2, validNode , validContainerToken2, newVersionedNMToken, false).IsEmpty()); NUnit.Framework.Assert.IsTrue(nmTokenSecretManagerNM.IsAppAttemptNMTokenKeyPresent (validAppAttemptId2)); //Now lets wait till container finishes and is removed from node manager. WaitForContainerToFinishOnNM(validContainerId); sb = new StringBuilder("Attempt to relaunch the same container with id "); sb.Append(validContainerId); NUnit.Framework.Assert.IsTrue(TestStartContainer(rpc, validAppAttemptId, validNode , validContainerToken, validNMToken, true).Contains(sb.ToString())); // Container is removed from node manager's memory by this time. // trying to stop the container. It should not throw any exception. TestStopContainer(rpc, validAppAttemptId, validNode, validContainerId, validNMToken , false); // Rolling over master key twice so that we can check whether older keys // are used for authentication. RollNMTokenMasterKey(nmTokenSecretManagerRM, nmTokenSecretManagerNM); // Key rolled over once.. rolling over again RollNMTokenMasterKey(nmTokenSecretManagerRM, nmTokenSecretManagerNM); // trying get container status. Now saved nmToken should be used for // authentication... It should complain saying container was recently // stopped. sb = new StringBuilder("Container "); sb.Append(validContainerId); sb.Append(" was recently stopped on node manager"); NUnit.Framework.Assert.IsTrue(TestGetContainer(rpc, validAppAttemptId, validNode, validContainerId, validNMToken, true).Contains(sb.ToString())); // Now lets remove the container from nm-memory nm.GetNodeStatusUpdater().ClearFinishedContainersFromCache(); // This should fail as container is removed from recently tracked finished // containers. sb = new StringBuilder("Container "); sb.Append(validContainerId.ToString()); sb.Append(" is not handled by this NodeManager"); NUnit.Framework.Assert.IsTrue(TestGetContainer(rpc, validAppAttemptId, validNode, validContainerId, validNMToken, false).Contains(sb.ToString())); // using appAttempt-1 NMtoken for launching container for appAttempt-2 should // succeed. ApplicationAttemptId attempt2 = ApplicationAttemptId.NewInstance(appId, 2); Token attempt1NMToken = nmTokenSecretManagerRM.CreateNMToken(validAppAttemptId, validNode , user); Token newContainerToken = containerTokenSecretManager.CreateContainerToken(ContainerId .NewContainerId(attempt2, 1), validNode, user, r, Priority.NewInstance(0), 0); NUnit.Framework.Assert.IsTrue(TestStartContainer(rpc, attempt2, validNode, newContainerToken , attempt1NMToken, false).IsEmpty()); }
/// <exception cref="System.IO.IOException"/> public static FSDataInputStream GetPreviousJobHistoryFileStream(Configuration conf , ApplicationAttemptId applicationAttemptId) { FSDataInputStream @in = null; Path historyFile = null; string jobId = TypeConverter.FromYarn(applicationAttemptId.GetApplicationId()).ToString (); string jobhistoryDir = JobHistoryUtils.GetConfiguredHistoryStagingDirPrefix(conf, jobId); Path histDirPath = FileContext.GetFileContext(conf).MakeQualified(new Path(jobhistoryDir )); FileContext fc = FileContext.GetFileContext(histDirPath.ToUri(), conf); // read the previous history file historyFile = fc.MakeQualified(JobHistoryUtils.GetStagingJobHistoryFile(histDirPath , jobId, (applicationAttemptId.GetAttemptId() - 1))); Log.Info("History file is at " + historyFile); @in = fc.Open(historyFile); return(@in); }