/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> public virtual void TestMRAppMasterMaxAppAttempts() { // No matter what's the maxAppAttempt or attempt id, the isLastRetry always // equals to false bool[] expectedBools = new bool[] { false, false, false }; string applicationAttemptIdStr = "appattempt_1317529182569_0004_000002"; string containerIdStr = "container_1317529182569_0004_000002_1"; string userName = "******"; ApplicationAttemptId applicationAttemptId = ConverterUtils.ToApplicationAttemptId (applicationAttemptIdStr); ContainerId containerId = ConverterUtils.ToContainerId(containerIdStr); JobConf conf = new JobConf(); conf.Set(MRJobConfig.MrAmStagingDir, stagingDir); FilePath stagingDir = new FilePath(MRApps.GetStagingAreaDir(conf, userName).ToString ()); stagingDir.Mkdirs(); for (int i = 0; i < expectedBools.Length; ++i) { MRAppMasterTest appMaster = new MRAppMasterTest(applicationAttemptId, containerId , "host", -1, -1, Runtime.CurrentTimeMillis(), false, true); MRAppMaster.InitAndStartAppMaster(appMaster, conf, userName); NUnit.Framework.Assert.AreEqual("isLastAMRetry is correctly computed.", expectedBools [i], appMaster.IsLastAMRetry()); } }
public virtual void TestMRAppMasterForDifferentUser() { string applicationAttemptIdStr = "appattempt_1317529182569_0004_000001"; string containerIdStr = "container_1317529182569_0004_000001_1"; string userName = "******"; ApplicationAttemptId applicationAttemptId = ConverterUtils.ToApplicationAttemptId (applicationAttemptIdStr); ContainerId containerId = ConverterUtils.ToContainerId(containerIdStr); MRAppMasterTest appMaster = new MRAppMasterTest(applicationAttemptId, containerId , "host", -1, -1, Runtime.CurrentTimeMillis()); JobConf conf = new JobConf(); conf.Set(MRJobConfig.MrAmStagingDir, stagingDir); MRAppMaster.InitAndStartAppMaster(appMaster, conf, userName); Path userPath = new Path(stagingDir, userName); Path userStagingPath = new Path(userPath, ".staging"); NUnit.Framework.Assert.AreEqual(userStagingPath.ToString(), appMaster.stagingDirPath .ToString()); }
public virtual void TestMRAppMasterSuccessLock() { string applicationAttemptIdStr = "appattempt_1317529182569_0004_000002"; string containerIdStr = "container_1317529182569_0004_000002_1"; string userName = "******"; JobConf conf = new JobConf(); conf.Set(MRJobConfig.MrAmStagingDir, stagingDir); ApplicationAttemptId applicationAttemptId = ConverterUtils.ToApplicationAttemptId (applicationAttemptIdStr); JobId jobId = TypeConverter.ToYarn(TypeConverter.FromYarn(applicationAttemptId.GetApplicationId ())); Path start = MRApps.GetStartJobCommitFile(conf, userName, jobId); Path end = MRApps.GetEndJobCommitSuccessFile(conf, userName, jobId); FileSystem fs = FileSystem.Get(conf); fs.Create(start).Close(); fs.Create(end).Close(); ContainerId containerId = ConverterUtils.ToContainerId(containerIdStr); MRAppMaster appMaster = new MRAppMasterTest(applicationAttemptId, containerId, "host" , -1, -1, Runtime.CurrentTimeMillis(), false, false); bool caught = false; try { MRAppMaster.InitAndStartAppMaster(appMaster, conf, userName); } catch (IOException e) { //The IO Exception is expected Log.Info("Caught expected Exception", e); caught = true; } NUnit.Framework.Assert.IsTrue(caught); NUnit.Framework.Assert.IsTrue(appMaster.errorHappenedShutDown); NUnit.Framework.Assert.AreEqual(JobStateInternal.Succeeded, appMaster.forcedState ); appMaster.Stop(); // verify the final status is SUCCEEDED VerifyFailedStatus((MRAppMasterTest)appMaster, "SUCCEEDED"); }
public virtual void TestMRAppMasterMissingStaging() { string applicationAttemptIdStr = "appattempt_1317529182569_0004_000002"; string containerIdStr = "container_1317529182569_0004_000002_1"; string userName = "******"; JobConf conf = new JobConf(); conf.Set(MRJobConfig.MrAmStagingDir, stagingDir); ApplicationAttemptId applicationAttemptId = ConverterUtils.ToApplicationAttemptId (applicationAttemptIdStr); //Delete the staging directory FilePath dir = new FilePath(stagingDir); if (dir.Exists()) { FileUtils.DeleteDirectory(dir); } ContainerId containerId = ConverterUtils.ToContainerId(containerIdStr); MRAppMaster appMaster = new MRAppMasterTest(applicationAttemptId, containerId, "host" , -1, -1, Runtime.CurrentTimeMillis(), false, false); bool caught = false; try { MRAppMaster.InitAndStartAppMaster(appMaster, conf, userName); } catch (IOException e) { //The IO Exception is expected Log.Info("Caught expected Exception", e); caught = true; } NUnit.Framework.Assert.IsTrue(caught); NUnit.Framework.Assert.IsTrue(appMaster.errorHappenedShutDown); //Copying the history file is disabled, but it is not really visible from //here NUnit.Framework.Assert.AreEqual(JobStateInternal.Error, appMaster.forcedState); appMaster.Stop(); }
public virtual void TestMRAppMasterCredentials() { Logger rootLogger = LogManager.GetRootLogger(); rootLogger.SetLevel(Level.Debug); // Simulate credentials passed to AM via client->RM->NM Credentials credentials = new Credentials(); byte[] identifier = Sharpen.Runtime.GetBytesForString("MyIdentifier"); byte[] password = Sharpen.Runtime.GetBytesForString("MyPassword"); Text kind = new Text("MyTokenKind"); Text service = new Text("host:port"); Org.Apache.Hadoop.Security.Token.Token<TokenIdentifier> myToken = new Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier>(identifier, password, kind, service); Text tokenAlias = new Text("myToken"); credentials.AddToken(tokenAlias, myToken); Text appTokenService = new Text("localhost:0"); Org.Apache.Hadoop.Security.Token.Token<AMRMTokenIdentifier> appToken = new Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier>(identifier, password, AMRMTokenIdentifier.KindName, appTokenService ); credentials.AddToken(appTokenService, appToken); Text keyAlias = new Text("mySecretKeyAlias"); credentials.AddSecretKey(keyAlias, Sharpen.Runtime.GetBytesForString("mySecretKey" )); Org.Apache.Hadoop.Security.Token.Token<TokenIdentifier> storedToken = credentials .GetToken(tokenAlias); JobConf conf = new JobConf(); Path tokenFilePath = new Path(testDir.GetAbsolutePath(), "tokens-file"); IDictionary<string, string> newEnv = new Dictionary<string, string>(); newEnv[UserGroupInformation.HadoopTokenFileLocation] = tokenFilePath.ToUri().GetPath (); SetNewEnvironmentHack(newEnv); credentials.WriteTokenStorageFile(tokenFilePath, conf); ApplicationId appId = ApplicationId.NewInstance(12345, 56); ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.NewInstance(appId , 1); ContainerId containerId = ContainerId.NewContainerId(applicationAttemptId, 546); string userName = UserGroupInformation.GetCurrentUser().GetShortUserName(); // Create staging dir, so MRAppMaster doesn't barf. FilePath stagingDir = new FilePath(MRApps.GetStagingAreaDir(conf, userName).ToString ()); stagingDir.Mkdirs(); // Set login-user to null as that is how real world MRApp starts with. // This is null is the reason why token-file is read by UGI. UserGroupInformation.SetLoginUser(null); MRAppMasterTest appMaster = new MRAppMasterTest(applicationAttemptId, containerId , "host", -1, -1, Runtime.CurrentTimeMillis(), false, true); MRAppMaster.InitAndStartAppMaster(appMaster, conf, userName); // Now validate the task credentials Credentials appMasterCreds = appMaster.GetCredentials(); NUnit.Framework.Assert.IsNotNull(appMasterCreds); NUnit.Framework.Assert.AreEqual(1, appMasterCreds.NumberOfSecretKeys()); NUnit.Framework.Assert.AreEqual(1, appMasterCreds.NumberOfTokens()); // Validate the tokens - app token should not be present Org.Apache.Hadoop.Security.Token.Token<TokenIdentifier> usedToken = appMasterCreds .GetToken(tokenAlias); NUnit.Framework.Assert.IsNotNull(usedToken); NUnit.Framework.Assert.AreEqual(storedToken, usedToken); // Validate the keys byte[] usedKey = appMasterCreds.GetSecretKey(keyAlias); NUnit.Framework.Assert.IsNotNull(usedKey); NUnit.Framework.Assert.AreEqual("mySecretKey", Sharpen.Runtime.GetStringForBytes( usedKey)); // The credentials should also be added to conf so that OuputCommitter can // access it - app token should not be present Credentials confCredentials = conf.GetCredentials(); NUnit.Framework.Assert.AreEqual(1, confCredentials.NumberOfSecretKeys()); NUnit.Framework.Assert.AreEqual(1, confCredentials.NumberOfTokens()); NUnit.Framework.Assert.AreEqual(storedToken, confCredentials.GetToken(tokenAlias) ); NUnit.Framework.Assert.AreEqual("mySecretKey", Sharpen.Runtime.GetStringForBytes( confCredentials.GetSecretKey(keyAlias))); // Verify the AM's ugi - app token should be present Credentials ugiCredentials = appMaster.GetUgi().GetCredentials(); NUnit.Framework.Assert.AreEqual(1, ugiCredentials.NumberOfSecretKeys()); NUnit.Framework.Assert.AreEqual(2, ugiCredentials.NumberOfTokens()); NUnit.Framework.Assert.AreEqual(storedToken, ugiCredentials.GetToken(tokenAlias)); NUnit.Framework.Assert.AreEqual(appToken, ugiCredentials.GetToken(appTokenService )); NUnit.Framework.Assert.AreEqual("mySecretKey", Sharpen.Runtime.GetStringForBytes( ugiCredentials.GetSecretKey(keyAlias))); }