public virtual void TestDeletionofStaging() { conf.Set(MRJobConfig.MapreduceJobDir, stagingJobDir); fs = Org.Mockito.Mockito.Mock <FileSystem>(); Org.Mockito.Mockito.When(fs.Delete(Matchers.Any <Path>(), Matchers.AnyBoolean())). ThenReturn(true); //Staging Dir exists string user = UserGroupInformation.GetCurrentUser().GetShortUserName(); Path stagingDir = MRApps.GetStagingAreaDir(conf, user); Org.Mockito.Mockito.When(fs.Exists(stagingDir)).ThenReturn(true); ApplicationId appId = ApplicationId.NewInstance(Runtime.CurrentTimeMillis(), 0); ApplicationAttemptId attemptId = ApplicationAttemptId.NewInstance(appId, 1); JobId jobid = recordFactory.NewRecordInstance <JobId>(); jobid.SetAppId(appId); ContainerAllocator mockAlloc = Org.Mockito.Mockito.Mock <ContainerAllocator>(); NUnit.Framework.Assert.IsTrue(MRJobConfig.DefaultMrAmMaxAttempts > 1); MRAppMaster appMaster = new TestStagingCleanup.TestMRApp(this, attemptId, mockAlloc , JobStateInternal.Running, MRJobConfig.DefaultMrAmMaxAttempts); appMaster.Init(conf); appMaster.Start(); appMaster.ShutDownJob(); //test whether notifyIsLastAMRetry called NUnit.Framework.Assert.AreEqual(true, ((TestStagingCleanup.TestMRApp)appMaster).GetTestIsLastAMRetry ()); Org.Mockito.Mockito.Verify(fs).Delete(stagingJobPath, true); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> public virtual void TestMRAppMasterMaxAppAttempts() { // No matter what's the maxAppAttempt or attempt id, the isLastRetry always // equals to false bool[] expectedBools = new bool[] { false, false, false }; string applicationAttemptIdStr = "appattempt_1317529182569_0004_000002"; string containerIdStr = "container_1317529182569_0004_000002_1"; string userName = "******"; ApplicationAttemptId applicationAttemptId = ConverterUtils.ToApplicationAttemptId (applicationAttemptIdStr); ContainerId containerId = ConverterUtils.ToContainerId(containerIdStr); JobConf conf = new JobConf(); conf.Set(MRJobConfig.MrAmStagingDir, stagingDir); FilePath stagingDir = new FilePath(MRApps.GetStagingAreaDir(conf, userName).ToString ()); stagingDir.Mkdirs(); for (int i = 0; i < expectedBools.Length; ++i) { MRAppMasterTest appMaster = new MRAppMasterTest(applicationAttemptId, containerId , "host", -1, -1, Runtime.CurrentTimeMillis(), false, true); MRAppMaster.InitAndStartAppMaster(appMaster, conf, userName); NUnit.Framework.Assert.AreEqual("isLastAMRetry is correctly computed.", expectedBools [i], appMaster.IsLastAMRetry()); } }
// FIXME: // Disabled this test because currently, when shutdown hook triggered at // lastRetry in RM view, cleanup will not do. This should be supported after // YARN-2261 completed // @Test (timeout = 30000) /// <exception cref="System.IO.IOException"/> public virtual void TestDeletionofStagingOnKillLastTry() { conf.Set(MRJobConfig.MapreduceJobDir, stagingJobDir); fs = Org.Mockito.Mockito.Mock <FileSystem>(); Org.Mockito.Mockito.When(fs.Delete(Matchers.Any <Path>(), Matchers.AnyBoolean())). ThenReturn(true); //Staging Dir exists string user = UserGroupInformation.GetCurrentUser().GetShortUserName(); Path stagingDir = MRApps.GetStagingAreaDir(conf, user); Org.Mockito.Mockito.When(fs.Exists(stagingDir)).ThenReturn(true); ApplicationId appId = ApplicationId.NewInstance(Runtime.CurrentTimeMillis(), 0); ApplicationAttemptId attemptId = ApplicationAttemptId.NewInstance(appId, 1); JobId jobid = recordFactory.NewRecordInstance <JobId>(); jobid.SetAppId(appId); ContainerAllocator mockAlloc = Org.Mockito.Mockito.Mock <ContainerAllocator>(); MRAppMaster appMaster = new TestStagingCleanup.TestMRApp(this, attemptId, mockAlloc ); //no retry appMaster.Init(conf); NUnit.Framework.Assert.IsTrue("appMaster.isLastAMRetry() is false", appMaster.IsLastAMRetry ()); //simulate the process being killed MRAppMaster.MRAppMasterShutdownHook hook = new MRAppMaster.MRAppMasterShutdownHook (appMaster); hook.Run(); NUnit.Framework.Assert.IsTrue("MRAppMaster isn't stopped", appMaster.IsInState(Service.STATE .Stopped)); Org.Mockito.Mockito.Verify(fs).Delete(stagingJobPath, true); }
/// <exception cref="System.Exception"/> protected override void ServiceInit(Configuration conf) { try { //Create the staging directory if it does not exist string user = UserGroupInformation.GetCurrentUser().GetShortUserName(); Path stagingDir = MRApps.GetStagingAreaDir(conf, user); FileSystem fs = GetFileSystem(conf); fs.Mkdirs(stagingDir); } catch (Exception e) { throw new YarnRuntimeException("Error creating staging dir", e); } base.ServiceInit(conf); if (this.clusterInfo != null) { GetContext().GetClusterInfo().SetMaxContainerCapability(this.clusterInfo.GetMaxContainerCapability ()); } else { GetContext().GetClusterInfo().SetMaxContainerCapability(Resource.NewInstance(10240 , 1)); } }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> public virtual string GetStagingAreaDir() { // Path path = new Path(MRJobConstants.JOB_SUBMIT_DIR); string user = UserGroupInformation.GetCurrentUser().GetShortUserName(); Path path = MRApps.GetStagingAreaDir(conf, user); Log.Debug("getStagingAreaDir: dir=" + path); return(path.ToString()); }
/// <summary>Gets the configured directory prefix for In Progress history files.</summary> /// <param name="conf">the configuration for hte job</param> /// <param name="jobId">the id of the job the history file is for.</param> /// <returns>A string representation of the prefix.</returns> /// <exception cref="System.IO.IOException"/> public static string GetConfiguredHistoryStagingDirPrefix(Configuration conf, string jobId) { string user = UserGroupInformation.GetCurrentUser().GetShortUserName(); Path stagingPath = MRApps.GetStagingAreaDir(conf, user); Path path = new Path(stagingPath, jobId); string logDir = path.ToString(); return(EnsurePathInDefaultFileSystem(logDir, conf)); }
/// <exception cref="System.Exception"/> protected override void ServiceStart() { if (overrideStart) { try { UserGroupInformation ugi = UserGroupInformation.GetCurrentUser(); string user = ugi.GetShortUserName(); stagingDirPath = MRApps.GetStagingAreaDir(conf, user); } catch (Exception e) { NUnit.Framework.Assert.Fail(e.Message); } } else { base.ServiceStart(); } }
/// <exception cref="System.IO.IOException"/> private void TestDeletionofStagingOnUnregistrationFailure(int maxAttempts, bool shouldHaveDeleted ) { conf.Set(MRJobConfig.MapreduceJobDir, stagingJobDir); fs = Org.Mockito.Mockito.Mock <FileSystem>(); Org.Mockito.Mockito.When(fs.Delete(Matchers.Any <Path>(), Matchers.AnyBoolean())). ThenReturn(true); //Staging Dir exists string user = UserGroupInformation.GetCurrentUser().GetShortUserName(); Path stagingDir = MRApps.GetStagingAreaDir(conf, user); Org.Mockito.Mockito.When(fs.Exists(stagingDir)).ThenReturn(true); ApplicationId appId = ApplicationId.NewInstance(0, 1); ApplicationAttemptId attemptId = ApplicationAttemptId.NewInstance(appId, 1); JobId jobid = recordFactory.NewRecordInstance <JobId>(); jobid.SetAppId(appId); TestStagingCleanup.TestMRApp appMaster = new TestStagingCleanup.TestMRApp(this, attemptId , null, JobStateInternal.Running, maxAttempts); appMaster.crushUnregistration = true; appMaster.Init(conf); appMaster.Start(); appMaster.ShutDownJob(); ((MRAppMaster.RunningAppContext)appMaster.GetContext()).ResetIsLastAMRetry(); if (shouldHaveDeleted) { NUnit.Framework.Assert.AreEqual(true, appMaster.IsLastAMRetry()); Org.Mockito.Mockito.Verify(fs).Delete(stagingJobPath, true); } else { NUnit.Framework.Assert.AreEqual(false, appMaster.IsLastAMRetry()); Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Never()).Delete(stagingJobPath , true); } }
public virtual void TestMRAppMasterCredentials() { Logger rootLogger = LogManager.GetRootLogger(); rootLogger.SetLevel(Level.Debug); // Simulate credentials passed to AM via client->RM->NM Credentials credentials = new Credentials(); byte[] identifier = Sharpen.Runtime.GetBytesForString("MyIdentifier"); byte[] password = Sharpen.Runtime.GetBytesForString("MyPassword"); Text kind = new Text("MyTokenKind"); Text service = new Text("host:port"); Org.Apache.Hadoop.Security.Token.Token<TokenIdentifier> myToken = new Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier>(identifier, password, kind, service); Text tokenAlias = new Text("myToken"); credentials.AddToken(tokenAlias, myToken); Text appTokenService = new Text("localhost:0"); Org.Apache.Hadoop.Security.Token.Token<AMRMTokenIdentifier> appToken = new Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier>(identifier, password, AMRMTokenIdentifier.KindName, appTokenService ); credentials.AddToken(appTokenService, appToken); Text keyAlias = new Text("mySecretKeyAlias"); credentials.AddSecretKey(keyAlias, Sharpen.Runtime.GetBytesForString("mySecretKey" )); Org.Apache.Hadoop.Security.Token.Token<TokenIdentifier> storedToken = credentials .GetToken(tokenAlias); JobConf conf = new JobConf(); Path tokenFilePath = new Path(testDir.GetAbsolutePath(), "tokens-file"); IDictionary<string, string> newEnv = new Dictionary<string, string>(); newEnv[UserGroupInformation.HadoopTokenFileLocation] = tokenFilePath.ToUri().GetPath (); SetNewEnvironmentHack(newEnv); credentials.WriteTokenStorageFile(tokenFilePath, conf); ApplicationId appId = ApplicationId.NewInstance(12345, 56); ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.NewInstance(appId , 1); ContainerId containerId = ContainerId.NewContainerId(applicationAttemptId, 546); string userName = UserGroupInformation.GetCurrentUser().GetShortUserName(); // Create staging dir, so MRAppMaster doesn't barf. FilePath stagingDir = new FilePath(MRApps.GetStagingAreaDir(conf, userName).ToString ()); stagingDir.Mkdirs(); // Set login-user to null as that is how real world MRApp starts with. // This is null is the reason why token-file is read by UGI. UserGroupInformation.SetLoginUser(null); MRAppMasterTest appMaster = new MRAppMasterTest(applicationAttemptId, containerId , "host", -1, -1, Runtime.CurrentTimeMillis(), false, true); MRAppMaster.InitAndStartAppMaster(appMaster, conf, userName); // Now validate the task credentials Credentials appMasterCreds = appMaster.GetCredentials(); NUnit.Framework.Assert.IsNotNull(appMasterCreds); NUnit.Framework.Assert.AreEqual(1, appMasterCreds.NumberOfSecretKeys()); NUnit.Framework.Assert.AreEqual(1, appMasterCreds.NumberOfTokens()); // Validate the tokens - app token should not be present Org.Apache.Hadoop.Security.Token.Token<TokenIdentifier> usedToken = appMasterCreds .GetToken(tokenAlias); NUnit.Framework.Assert.IsNotNull(usedToken); NUnit.Framework.Assert.AreEqual(storedToken, usedToken); // Validate the keys byte[] usedKey = appMasterCreds.GetSecretKey(keyAlias); NUnit.Framework.Assert.IsNotNull(usedKey); NUnit.Framework.Assert.AreEqual("mySecretKey", Sharpen.Runtime.GetStringForBytes( usedKey)); // The credentials should also be added to conf so that OuputCommitter can // access it - app token should not be present Credentials confCredentials = conf.GetCredentials(); NUnit.Framework.Assert.AreEqual(1, confCredentials.NumberOfSecretKeys()); NUnit.Framework.Assert.AreEqual(1, confCredentials.NumberOfTokens()); NUnit.Framework.Assert.AreEqual(storedToken, confCredentials.GetToken(tokenAlias) ); NUnit.Framework.Assert.AreEqual("mySecretKey", Sharpen.Runtime.GetStringForBytes( confCredentials.GetSecretKey(keyAlias))); // Verify the AM's ugi - app token should be present Credentials ugiCredentials = appMaster.GetUgi().GetCredentials(); NUnit.Framework.Assert.AreEqual(1, ugiCredentials.NumberOfSecretKeys()); NUnit.Framework.Assert.AreEqual(2, ugiCredentials.NumberOfTokens()); NUnit.Framework.Assert.AreEqual(storedToken, ugiCredentials.GetToken(tokenAlias)); NUnit.Framework.Assert.AreEqual(appToken, ugiCredentials.GetToken(appTokenService )); NUnit.Framework.Assert.AreEqual("mySecretKey", Sharpen.Runtime.GetStringForBytes( ugiCredentials.GetSecretKey(keyAlias))); }