public virtual void TestAMStandardEnv()
{
string AdminLibPath = "foo";
string UserLibPath = "bar";
string UserShell = "shell";
JobConf jobConf = new JobConf();
jobConf.Set(MRJobConfig.MrAmAdminUserEnv, "LD_LIBRARY_PATH=" + AdminLibPath);
jobConf.Set(MRJobConfig.MrAmEnv, "LD_LIBRARY_PATH=" + UserLibPath);
jobConf.Set(MRJobConfig.MapredAdminUserShell, UserShell);
YARNRunner yarnRunner = new YARNRunner(jobConf);
ApplicationSubmissionContext appSubCtx = BuildSubmitContext(yarnRunner, jobConf);
// make sure PWD is first in the lib path
ContainerLaunchContext clc = appSubCtx.GetAMContainerSpec();
IDictionary <string, string> env = clc.GetEnvironment();
string libPath = env[ApplicationConstants.Environment.LdLibraryPath.ToString()];
NUnit.Framework.Assert.IsNotNull("LD_LIBRARY_PATH not set", libPath);
string cps = jobConf.GetBoolean(MRConfig.MapreduceAppSubmissionCrossPlatform, MRConfig
.DefaultMapreduceAppSubmissionCrossPlatform) ? ApplicationConstants.ClassPathSeparator
: FilePath.pathSeparator;
NUnit.Framework.Assert.AreEqual("Bad AM LD_LIBRARY_PATH setting", MRApps.CrossPlatformifyMREnv
(conf, ApplicationConstants.Environment.Pwd) + cps + AdminLibPath + cps + UserLibPath
, libPath);
// make sure SHELL is set
string shell = env[ApplicationConstants.Environment.Shell.ToString()];
NUnit.Framework.Assert.IsNotNull("SHELL not set", shell);
NUnit.Framework.Assert.AreEqual("Bad SHELL setting", UserShell, shell);
}
/// <exception cref="System.Exception"/>
public virtual void TestWarnCommandOpts()
{
Logger logger = Logger.GetLogger(typeof(YARNRunner));
ByteArrayOutputStream bout = new ByteArrayOutputStream();
Layout layout = new SimpleLayout();
Appender appender = new WriterAppender(layout, bout);
logger.AddAppender(appender);
JobConf jobConf = new JobConf();
jobConf.Set(MRJobConfig.MrAmAdminCommandOpts, "-Djava.net.preferIPv4Stack=true -Djava.library.path=foo"
);
jobConf.Set(MRJobConfig.MrAmCommandOpts, "-Xmx1024m -Djava.library.path=bar");
YARNRunner yarnRunner = new YARNRunner(jobConf);
ApplicationSubmissionContext submissionContext = BuildSubmitContext(yarnRunner, jobConf
);
string logMsg = bout.ToString();
NUnit.Framework.Assert.IsTrue(logMsg.Contains("WARN - Usage of -Djava.library.path in "
+ "yarn.app.mapreduce.am.admin-command-opts can cause programs to no " + "longer function if hadoop native libraries are used. These values "
+ "should be set as part of the LD_LIBRARY_PATH in the app master JVM " + "env using yarn.app.mapreduce.am.admin.user.env config settings."
));
NUnit.Framework.Assert.IsTrue(logMsg.Contains("WARN - Usage of -Djava.library.path in "
+ "yarn.app.mapreduce.am.command-opts can cause programs to no longer " + "function if hadoop native libraries are used. These values should "
+ "be set as part of the LD_LIBRARY_PATH in the app master JVM env " + "using yarn.app.mapreduce.am.env config settings."
));
}
/// <exception cref="System.Exception"/>
public virtual void TestAMAdminCommandOpts()
{
JobConf jobConf = new JobConf();
jobConf.Set(MRJobConfig.MrAmAdminCommandOpts, "-Djava.net.preferIPv4Stack=true");
jobConf.Set(MRJobConfig.MrAmCommandOpts, "-Xmx1024m");
YARNRunner yarnRunner = new YARNRunner(jobConf);
ApplicationSubmissionContext submissionContext = BuildSubmitContext(yarnRunner, jobConf
);
ContainerLaunchContext containerSpec = submissionContext.GetAMContainerSpec();
IList <string> commands = containerSpec.GetCommands();
int index = 0;
int adminIndex = 0;
int adminPos = -1;
int userIndex = 0;
int userPos = -1;
int tmpDirPos = -1;
foreach (string command in commands)
{
if (command != null)
{
NUnit.Framework.Assert.IsFalse("Profiler should be disabled by default", command.
Contains(ProfileParams));
adminPos = command.IndexOf("-Djava.net.preferIPv4Stack=true");
if (adminPos >= 0)
{
adminIndex = index;
}
userPos = command.IndexOf("-Xmx1024m");
if (userPos >= 0)
{
userIndex = index;
}
tmpDirPos = command.IndexOf("-Djava.io.tmpdir=");
}
index++;
}
// Check java.io.tmpdir opts are set in the commands
NUnit.Framework.Assert.IsTrue("java.io.tmpdir is not set for AM", tmpDirPos > 0);
// Check both admin java opts and user java opts are in the commands
NUnit.Framework.Assert.IsTrue("AM admin command opts not in the commands.", adminPos
> 0);
NUnit.Framework.Assert.IsTrue("AM user command opts not in the commands.", userPos
> 0);
// Check the admin java opts is before user java opts in the commands
if (adminIndex == userIndex)
{
NUnit.Framework.Assert.IsTrue("AM admin command opts is after user command opts."
, adminPos < userPos);
}
else
{
NUnit.Framework.Assert.IsTrue("AM admin command opts is after user command opts."
, adminIndex < userIndex);
}
}
/// <exception cref="System.IO.IOException"/>
private ApplicationSubmissionContext BuildSubmitContext(YARNRunner yarnRunner, JobConf
jobConf)
{
FilePath jobxml = new FilePath(testWorkDir, MRJobConfig.JobConfFile);
OutputStream @out = new FileOutputStream(jobxml);
conf.WriteXml(@out);
@out.Close();
FilePath jobsplit = new FilePath(testWorkDir, MRJobConfig.JobSplit);
@out = new FileOutputStream(jobsplit);
@out.Close();
FilePath jobsplitmetainfo = new FilePath(testWorkDir, MRJobConfig.JobSplitMetainfo
);
@out = new FileOutputStream(jobsplitmetainfo);
@out.Close();
return(yarnRunner.CreateApplicationSubmissionContext(jobConf, testWorkDir.ToString
(), new Credentials()));
}
protected override void SetUp()
{
resourceMgrDelegate = Org.Mockito.Mockito.Mock <ResourceMgrDelegate>();
conf = new YarnConfiguration();
conf.Set(YarnConfiguration.RmPrincipal, "mapred/host@REALM");
clientCache = new ClientCache(conf, resourceMgrDelegate);
clientCache = Org.Mockito.Mockito.Spy(clientCache);
yarnRunner = new YARNRunner(conf, resourceMgrDelegate, clientCache);
yarnRunner = Org.Mockito.Mockito.Spy(yarnRunner);
submissionContext = Org.Mockito.Mockito.Mock <ApplicationSubmissionContext>();
Org.Mockito.Mockito.DoAnswer(new _Answer_146(this)).When(yarnRunner).CreateApplicationSubmissionContext
(Matchers.Any <Configuration>(), Matchers.Any <string>(), Matchers.Any <Credentials
>());
appId = ApplicationId.NewInstance(Runtime.CurrentTimeMillis(), 1);
jobId = TypeConverter.FromYarn(appId);
if (testWorkDir.Exists())
{
FileContext.GetLocalFSFileContext().Delete(new Path(testWorkDir.ToString()), true
);
}
testWorkDir.Mkdirs();
}
/// <exception cref="System.Exception"/>
public virtual void TestAMProfiler()
{
JobConf jobConf = new JobConf();
jobConf.SetBoolean(MRJobConfig.MrAmProfile, true);
YARNRunner yarnRunner = new YARNRunner(jobConf);
ApplicationSubmissionContext submissionContext = BuildSubmitContext(yarnRunner, jobConf
);
ContainerLaunchContext containerSpec = submissionContext.GetAMContainerSpec();
IList <string> commands = containerSpec.GetCommands();
foreach (string command in commands)
{
if (command != null)
{
if (command.Contains(ProfileParams))
{
return;
}
}
}
throw new InvalidOperationException("Profiler opts not found!");
}
/// <exception cref="System.Exception"/>
public virtual void TestGetHSDelegationToken()
{
try
{
Configuration conf = new Configuration();
// Setup mock service
IPEndPoint mockRmAddress = new IPEndPoint("localhost", 4444);
Text rmTokenSevice = SecurityUtil.BuildTokenService(mockRmAddress);
IPEndPoint mockHsAddress = new IPEndPoint("localhost", 9200);
Text hsTokenSevice = SecurityUtil.BuildTokenService(mockHsAddress);
// Setup mock rm token
RMDelegationTokenIdentifier tokenIdentifier = new RMDelegationTokenIdentifier(new
Text("owner"), new Text("renewer"), new Text("real"));
Org.Apache.Hadoop.Security.Token.Token <RMDelegationTokenIdentifier> token = new Org.Apache.Hadoop.Security.Token.Token
<RMDelegationTokenIdentifier>(new byte[0], new byte[0], tokenIdentifier.GetKind(
), rmTokenSevice);
token.SetKind(RMDelegationTokenIdentifier.KindName);
// Setup mock history token
Org.Apache.Hadoop.Yarn.Api.Records.Token historyToken = Org.Apache.Hadoop.Yarn.Api.Records.Token
.NewInstance(new byte[0], MRDelegationTokenIdentifier.KindName.ToString(), new byte
[0], hsTokenSevice.ToString());
GetDelegationTokenResponse getDtResponse = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord
<GetDelegationTokenResponse>();
getDtResponse.SetDelegationToken(historyToken);
// mock services
MRClientProtocol mockHsProxy = Org.Mockito.Mockito.Mock <MRClientProtocol>();
Org.Mockito.Mockito.DoReturn(mockHsAddress).When(mockHsProxy).GetConnectAddress();
Org.Mockito.Mockito.DoReturn(getDtResponse).When(mockHsProxy).GetDelegationToken(
Matchers.Any <GetDelegationTokenRequest>());
ResourceMgrDelegate rmDelegate = Org.Mockito.Mockito.Mock <ResourceMgrDelegate>();
Org.Mockito.Mockito.DoReturn(rmTokenSevice).When(rmDelegate).GetRMDelegationTokenService
();
ClientCache clientCache = Org.Mockito.Mockito.Mock <ClientCache>();
Org.Mockito.Mockito.DoReturn(mockHsProxy).When(clientCache).GetInitializedHSProxy
();
Credentials creds = new Credentials();
YARNRunner yarnRunner = new YARNRunner(conf, rmDelegate, clientCache);
// No HS token if no RM token
yarnRunner.AddHistoryToken(creds);
Org.Mockito.Mockito.Verify(mockHsProxy, Org.Mockito.Mockito.Times(0)).GetDelegationToken
(Matchers.Any <GetDelegationTokenRequest>());
// No HS token if RM token, but secirity disabled.
creds.AddToken(new Text("rmdt"), token);
yarnRunner.AddHistoryToken(creds);
Org.Mockito.Mockito.Verify(mockHsProxy, Org.Mockito.Mockito.Times(0)).GetDelegationToken
(Matchers.Any <GetDelegationTokenRequest>());
conf.Set(CommonConfigurationKeys.HadoopSecurityAuthentication, "kerberos");
UserGroupInformation.SetConfiguration(conf);
creds = new Credentials();
// No HS token if no RM token, security enabled
yarnRunner.AddHistoryToken(creds);
Org.Mockito.Mockito.Verify(mockHsProxy, Org.Mockito.Mockito.Times(0)).GetDelegationToken
(Matchers.Any <GetDelegationTokenRequest>());
// HS token if RM token present, security enabled
creds.AddToken(new Text("rmdt"), token);
yarnRunner.AddHistoryToken(creds);
Org.Mockito.Mockito.Verify(mockHsProxy, Org.Mockito.Mockito.Times(1)).GetDelegationToken
(Matchers.Any <GetDelegationTokenRequest>());
// No additional call to get HS token if RM and HS token present
yarnRunner.AddHistoryToken(creds);
Org.Mockito.Mockito.Verify(mockHsProxy, Org.Mockito.Mockito.Times(1)).GetDelegationToken
(Matchers.Any <GetDelegationTokenRequest>());
}
finally
{
// Back to defaults.
UserGroupInformation.SetConfiguration(new Configuration());
}
}
