public virtual void TestAMStandardEnv() { string AdminLibPath = "foo"; string UserLibPath = "bar"; string UserShell = "shell"; JobConf jobConf = new JobConf(); jobConf.Set(MRJobConfig.MrAmAdminUserEnv, "LD_LIBRARY_PATH=" + AdminLibPath); jobConf.Set(MRJobConfig.MrAmEnv, "LD_LIBRARY_PATH=" + UserLibPath); jobConf.Set(MRJobConfig.MapredAdminUserShell, UserShell); YARNRunner yarnRunner = new YARNRunner(jobConf); ApplicationSubmissionContext appSubCtx = BuildSubmitContext(yarnRunner, jobConf); // make sure PWD is first in the lib path ContainerLaunchContext clc = appSubCtx.GetAMContainerSpec(); IDictionary <string, string> env = clc.GetEnvironment(); string libPath = env[ApplicationConstants.Environment.LdLibraryPath.ToString()]; NUnit.Framework.Assert.IsNotNull("LD_LIBRARY_PATH not set", libPath); string cps = jobConf.GetBoolean(MRConfig.MapreduceAppSubmissionCrossPlatform, MRConfig .DefaultMapreduceAppSubmissionCrossPlatform) ? ApplicationConstants.ClassPathSeparator : FilePath.pathSeparator; NUnit.Framework.Assert.AreEqual("Bad AM LD_LIBRARY_PATH setting", MRApps.CrossPlatformifyMREnv (conf, ApplicationConstants.Environment.Pwd) + cps + AdminLibPath + cps + UserLibPath , libPath); // make sure SHELL is set string shell = env[ApplicationConstants.Environment.Shell.ToString()]; NUnit.Framework.Assert.IsNotNull("SHELL not set", shell); NUnit.Framework.Assert.AreEqual("Bad SHELL setting", UserShell, shell); }
/// <exception cref="System.Exception"/> public virtual void TestWarnCommandOpts() { Logger logger = Logger.GetLogger(typeof(YARNRunner)); ByteArrayOutputStream bout = new ByteArrayOutputStream(); Layout layout = new SimpleLayout(); Appender appender = new WriterAppender(layout, bout); logger.AddAppender(appender); JobConf jobConf = new JobConf(); jobConf.Set(MRJobConfig.MrAmAdminCommandOpts, "-Djava.net.preferIPv4Stack=true -Djava.library.path=foo" ); jobConf.Set(MRJobConfig.MrAmCommandOpts, "-Xmx1024m -Djava.library.path=bar"); YARNRunner yarnRunner = new YARNRunner(jobConf); ApplicationSubmissionContext submissionContext = BuildSubmitContext(yarnRunner, jobConf ); string logMsg = bout.ToString(); NUnit.Framework.Assert.IsTrue(logMsg.Contains("WARN - Usage of -Djava.library.path in " + "yarn.app.mapreduce.am.admin-command-opts can cause programs to no " + "longer function if hadoop native libraries are used. These values " + "should be set as part of the LD_LIBRARY_PATH in the app master JVM " + "env using yarn.app.mapreduce.am.admin.user.env config settings." )); NUnit.Framework.Assert.IsTrue(logMsg.Contains("WARN - Usage of -Djava.library.path in " + "yarn.app.mapreduce.am.command-opts can cause programs to no longer " + "function if hadoop native libraries are used. These values should " + "be set as part of the LD_LIBRARY_PATH in the app master JVM env " + "using yarn.app.mapreduce.am.env config settings." )); }
/// <exception cref="System.Exception"/> public virtual void TestAMAdminCommandOpts() { JobConf jobConf = new JobConf(); jobConf.Set(MRJobConfig.MrAmAdminCommandOpts, "-Djava.net.preferIPv4Stack=true"); jobConf.Set(MRJobConfig.MrAmCommandOpts, "-Xmx1024m"); YARNRunner yarnRunner = new YARNRunner(jobConf); ApplicationSubmissionContext submissionContext = BuildSubmitContext(yarnRunner, jobConf ); ContainerLaunchContext containerSpec = submissionContext.GetAMContainerSpec(); IList <string> commands = containerSpec.GetCommands(); int index = 0; int adminIndex = 0; int adminPos = -1; int userIndex = 0; int userPos = -1; int tmpDirPos = -1; foreach (string command in commands) { if (command != null) { NUnit.Framework.Assert.IsFalse("Profiler should be disabled by default", command. Contains(ProfileParams)); adminPos = command.IndexOf("-Djava.net.preferIPv4Stack=true"); if (adminPos >= 0) { adminIndex = index; } userPos = command.IndexOf("-Xmx1024m"); if (userPos >= 0) { userIndex = index; } tmpDirPos = command.IndexOf("-Djava.io.tmpdir="); } index++; } // Check java.io.tmpdir opts are set in the commands NUnit.Framework.Assert.IsTrue("java.io.tmpdir is not set for AM", tmpDirPos > 0); // Check both admin java opts and user java opts are in the commands NUnit.Framework.Assert.IsTrue("AM admin command opts not in the commands.", adminPos > 0); NUnit.Framework.Assert.IsTrue("AM user command opts not in the commands.", userPos > 0); // Check the admin java opts is before user java opts in the commands if (adminIndex == userIndex) { NUnit.Framework.Assert.IsTrue("AM admin command opts is after user command opts." , adminPos < userPos); } else { NUnit.Framework.Assert.IsTrue("AM admin command opts is after user command opts." , adminIndex < userIndex); } }
/// <exception cref="System.IO.IOException"/> private ApplicationSubmissionContext BuildSubmitContext(YARNRunner yarnRunner, JobConf jobConf) { FilePath jobxml = new FilePath(testWorkDir, MRJobConfig.JobConfFile); OutputStream @out = new FileOutputStream(jobxml); conf.WriteXml(@out); @out.Close(); FilePath jobsplit = new FilePath(testWorkDir, MRJobConfig.JobSplit); @out = new FileOutputStream(jobsplit); @out.Close(); FilePath jobsplitmetainfo = new FilePath(testWorkDir, MRJobConfig.JobSplitMetainfo ); @out = new FileOutputStream(jobsplitmetainfo); @out.Close(); return(yarnRunner.CreateApplicationSubmissionContext(jobConf, testWorkDir.ToString (), new Credentials())); }
protected override void SetUp() { resourceMgrDelegate = Org.Mockito.Mockito.Mock <ResourceMgrDelegate>(); conf = new YarnConfiguration(); conf.Set(YarnConfiguration.RmPrincipal, "mapred/host@REALM"); clientCache = new ClientCache(conf, resourceMgrDelegate); clientCache = Org.Mockito.Mockito.Spy(clientCache); yarnRunner = new YARNRunner(conf, resourceMgrDelegate, clientCache); yarnRunner = Org.Mockito.Mockito.Spy(yarnRunner); submissionContext = Org.Mockito.Mockito.Mock <ApplicationSubmissionContext>(); Org.Mockito.Mockito.DoAnswer(new _Answer_146(this)).When(yarnRunner).CreateApplicationSubmissionContext (Matchers.Any <Configuration>(), Matchers.Any <string>(), Matchers.Any <Credentials >()); appId = ApplicationId.NewInstance(Runtime.CurrentTimeMillis(), 1); jobId = TypeConverter.FromYarn(appId); if (testWorkDir.Exists()) { FileContext.GetLocalFSFileContext().Delete(new Path(testWorkDir.ToString()), true ); } testWorkDir.Mkdirs(); }
/// <exception cref="System.Exception"/> public virtual void TestAMProfiler() { JobConf jobConf = new JobConf(); jobConf.SetBoolean(MRJobConfig.MrAmProfile, true); YARNRunner yarnRunner = new YARNRunner(jobConf); ApplicationSubmissionContext submissionContext = BuildSubmitContext(yarnRunner, jobConf ); ContainerLaunchContext containerSpec = submissionContext.GetAMContainerSpec(); IList <string> commands = containerSpec.GetCommands(); foreach (string command in commands) { if (command != null) { if (command.Contains(ProfileParams)) { return; } } } throw new InvalidOperationException("Profiler opts not found!"); }
/// <exception cref="System.Exception"/> public virtual void TestGetHSDelegationToken() { try { Configuration conf = new Configuration(); // Setup mock service IPEndPoint mockRmAddress = new IPEndPoint("localhost", 4444); Text rmTokenSevice = SecurityUtil.BuildTokenService(mockRmAddress); IPEndPoint mockHsAddress = new IPEndPoint("localhost", 9200); Text hsTokenSevice = SecurityUtil.BuildTokenService(mockHsAddress); // Setup mock rm token RMDelegationTokenIdentifier tokenIdentifier = new RMDelegationTokenIdentifier(new Text("owner"), new Text("renewer"), new Text("real")); Org.Apache.Hadoop.Security.Token.Token <RMDelegationTokenIdentifier> token = new Org.Apache.Hadoop.Security.Token.Token <RMDelegationTokenIdentifier>(new byte[0], new byte[0], tokenIdentifier.GetKind( ), rmTokenSevice); token.SetKind(RMDelegationTokenIdentifier.KindName); // Setup mock history token Org.Apache.Hadoop.Yarn.Api.Records.Token historyToken = Org.Apache.Hadoop.Yarn.Api.Records.Token .NewInstance(new byte[0], MRDelegationTokenIdentifier.KindName.ToString(), new byte [0], hsTokenSevice.ToString()); GetDelegationTokenResponse getDtResponse = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <GetDelegationTokenResponse>(); getDtResponse.SetDelegationToken(historyToken); // mock services MRClientProtocol mockHsProxy = Org.Mockito.Mockito.Mock <MRClientProtocol>(); Org.Mockito.Mockito.DoReturn(mockHsAddress).When(mockHsProxy).GetConnectAddress(); Org.Mockito.Mockito.DoReturn(getDtResponse).When(mockHsProxy).GetDelegationToken( Matchers.Any <GetDelegationTokenRequest>()); ResourceMgrDelegate rmDelegate = Org.Mockito.Mockito.Mock <ResourceMgrDelegate>(); Org.Mockito.Mockito.DoReturn(rmTokenSevice).When(rmDelegate).GetRMDelegationTokenService (); ClientCache clientCache = Org.Mockito.Mockito.Mock <ClientCache>(); Org.Mockito.Mockito.DoReturn(mockHsProxy).When(clientCache).GetInitializedHSProxy (); Credentials creds = new Credentials(); YARNRunner yarnRunner = new YARNRunner(conf, rmDelegate, clientCache); // No HS token if no RM token yarnRunner.AddHistoryToken(creds); Org.Mockito.Mockito.Verify(mockHsProxy, Org.Mockito.Mockito.Times(0)).GetDelegationToken (Matchers.Any <GetDelegationTokenRequest>()); // No HS token if RM token, but secirity disabled. creds.AddToken(new Text("rmdt"), token); yarnRunner.AddHistoryToken(creds); Org.Mockito.Mockito.Verify(mockHsProxy, Org.Mockito.Mockito.Times(0)).GetDelegationToken (Matchers.Any <GetDelegationTokenRequest>()); conf.Set(CommonConfigurationKeys.HadoopSecurityAuthentication, "kerberos"); UserGroupInformation.SetConfiguration(conf); creds = new Credentials(); // No HS token if no RM token, security enabled yarnRunner.AddHistoryToken(creds); Org.Mockito.Mockito.Verify(mockHsProxy, Org.Mockito.Mockito.Times(0)).GetDelegationToken (Matchers.Any <GetDelegationTokenRequest>()); // HS token if RM token present, security enabled creds.AddToken(new Text("rmdt"), token); yarnRunner.AddHistoryToken(creds); Org.Mockito.Mockito.Verify(mockHsProxy, Org.Mockito.Mockito.Times(1)).GetDelegationToken (Matchers.Any <GetDelegationTokenRequest>()); // No additional call to get HS token if RM and HS token present yarnRunner.AddHistoryToken(creds); Org.Mockito.Mockito.Verify(mockHsProxy, Org.Mockito.Mockito.Times(1)).GetDelegationToken (Matchers.Any <GetDelegationTokenRequest>()); } finally { // Back to defaults. UserGroupInformation.SetConfiguration(new Configuration()); } }