public virtual void TestLazyTokenFetchForWebhdfs()
{
// for any(Token.class)
MiniDFSCluster cluster = null;
WebHdfsFileSystem fs = null;
try
{
Configuration clusterConf = new HdfsConfiguration(conf);
SecurityUtil.SetAuthenticationMethod(UserGroupInformation.AuthenticationMethod.Simple
, clusterConf);
clusterConf.SetBoolean(DFSConfigKeys.DfsNamenodeDelegationTokenAlwaysUseKey, true
);
// trick the NN into thinking security is enabled w/o it trying
// to login from a keytab
UserGroupInformation.SetConfiguration(clusterConf);
cluster = new MiniDFSCluster.Builder(clusterConf).NumDataNodes(1).Build();
cluster.WaitActive();
SecurityUtil.SetAuthenticationMethod(UserGroupInformation.AuthenticationMethod.Kerberos
, clusterConf);
UserGroupInformation.SetConfiguration(clusterConf);
uri = DFSUtil.CreateUri("webhdfs", cluster.GetNameNode().GetHttpAddress());
ValidateLazyTokenFetch(clusterConf);
}
finally
{
IOUtils.Cleanup(null, fs);
if (cluster != null)
{
cluster.Shutdown();
}
}
}
public virtual void DisableKerberos()
{
Configuration conf = new Configuration();
conf.Set(CommonConfigurationKeysPublic.HadoopSecurityAuthentication, "simple");
UserGroupInformation.SetConfiguration(conf);
}
public virtual void SetupConf()
{
conf = new Configuration();
conf.SetClass("rpc.engine." + typeof(TestRPC.StoppedProtocol).FullName, typeof(TestRPC.StoppedRpcEngine
), typeof(RpcEngine));
UserGroupInformation.SetConfiguration(conf);
}
/// <exception cref="System.Exception"/>
public virtual void TestResetTimer()
{
YarnConfiguration conf = new YarnConfiguration();
UserGroupInformation.SetConfiguration(conf);
conf.Set(YarnConfiguration.RecoveryEnabled, "true");
conf.Set(YarnConfiguration.RmStore, typeof(MemoryRMStateStore).FullName);
conf.SetBoolean(YarnConfiguration.RmWorkPreservingRecoveryEnabled, true);
conf.SetInt(YarnConfiguration.RmAmExpiryIntervalMs, 6000);
ControlledClock clock = new ControlledClock(new SystemClock());
clock.SetTime(0);
MemoryRMStateStore memStore = new _MemoryRMStateStore_46(clock);
memStore.Init(conf);
ApplicationAttemptId attemptId = Org.Mockito.Mockito.Mock <ApplicationAttemptId>();
Dispatcher dispatcher = Org.Mockito.Mockito.Mock <Dispatcher>();
bool[] expired = new bool[] { false };
AMLivelinessMonitor monitor = new _AMLivelinessMonitor_58(attemptId, expired, dispatcher
, clock);
monitor.Register(attemptId);
MockRM rm = new _MockRM_66(monitor, conf, memStore);
rm.Start();
// make sure that monitor has started
while (monitor.GetServiceState() != Service.STATE.Started)
{
Sharpen.Thread.Sleep(100);
}
// expired[0] would be set to true without resetTimer
NUnit.Framework.Assert.IsFalse(expired[0]);
rm.Stop();
}
public virtual void TestTokenBySuperUser()
{
TestSaslRPC.TestTokenSecretManager sm = new TestSaslRPC.TestTokenSecretManager();
Configuration newConf = new Configuration(masterConf);
SecurityUtil.SetAuthenticationMethod(UserGroupInformation.AuthenticationMethod.Kerberos
, newConf);
UserGroupInformation.SetConfiguration(newConf);
Server server = new RPC.Builder(newConf).SetProtocol(typeof(TestDoAsEffectiveUser.TestProtocol
)).SetInstance(new TestDoAsEffectiveUser.TestImpl(this)).SetBindAddress(Address)
.SetPort(0).SetNumHandlers(5).SetVerbose(true).SetSecretManager(sm).Build();
server.Start();
UserGroupInformation current = UserGroupInformation.CreateUserForTesting(RealUserName
, GroupNames);
RefreshConf(newConf);
IPEndPoint addr = NetUtils.GetConnectAddress(server);
TestSaslRPC.TestTokenIdentifier tokenId = new TestSaslRPC.TestTokenIdentifier(new
Org.Apache.Hadoop.IO.Text(current.GetUserName()), new Org.Apache.Hadoop.IO.Text(
"SomeSuperUser"));
Org.Apache.Hadoop.Security.Token.Token <TestSaslRPC.TestTokenIdentifier> token = new
Org.Apache.Hadoop.Security.Token.Token <TestSaslRPC.TestTokenIdentifier>(tokenId,
sm);
SecurityUtil.SetTokenService(token, addr);
current.AddToken(token);
string retVal = current.DoAs(new _PrivilegedExceptionAction_509(this, addr, newConf
, server));
string expected = RealUserName + " (auth:TOKEN) via SomeSuperUser (auth:SIMPLE)";
Assert.Equal(retVal + "!=" + expected, expected, retVal);
}
/// <exception cref="System.IO.IOException"/>
private void TryLoginAuthenticationMethod(UserGroupInformation.AuthenticationMethod
method, bool expectSuccess)
{
SecurityUtil.SetAuthenticationMethod(method, conf);
UserGroupInformation.SetConfiguration(conf);
// pick up changed auth
UserGroupInformation ugi = null;
Exception ex = null;
try
{
ugi = UserGroupInformation.GetLoginUser();
}
catch (Exception e)
{
ex = e;
}
if (expectSuccess)
{
NUnit.Framework.Assert.IsNotNull(ugi);
Assert.Equal(method, ugi.GetAuthenticationMethod());
}
else
{
NUnit.Framework.Assert.IsNotNull(ex);
Assert.Equal(typeof(NotSupportedException), ex.GetType());
Assert.Equal(method + " login authentication is not supported"
, ex.Message);
}
}
public virtual void TestGetNonProxyUgi()
{
conf.Set(DFSConfigKeys.FsDefaultNameKey, "hdfs://localhost:4321/");
ServletContext context = Org.Mockito.Mockito.Mock <ServletContext>();
string realUser = "******";
string user = "******";
conf.Set(DFSConfigKeys.HadoopSecurityAuthentication, "kerberos");
UserGroupInformation.SetConfiguration(conf);
UserGroupInformation ugi;
HttpServletRequest request;
// have to be auth-ed with remote user
request = GetMockRequest(null, null, null);
try
{
JspHelper.GetUGI(context, request, conf);
NUnit.Framework.Assert.Fail("bad request allowed");
}
catch (IOException ioe)
{
NUnit.Framework.Assert.AreEqual("Security enabled but user not authenticated by filter"
, ioe.Message);
}
request = GetMockRequest(null, realUser, null);
try
{
JspHelper.GetUGI(context, request, conf);
NUnit.Framework.Assert.Fail("bad request allowed");
}
catch (IOException ioe)
{
NUnit.Framework.Assert.AreEqual("Security enabled but user not authenticated by filter"
, ioe.Message);
}
// ugi for remote user
request = GetMockRequest(realUser, null, null);
ugi = JspHelper.GetUGI(context, request, conf);
NUnit.Framework.Assert.IsNull(ugi.GetRealUser());
NUnit.Framework.Assert.AreEqual(ugi.GetShortUserName(), realUser);
CheckUgiFromAuth(ugi);
// ugi for remote user = real user
request = GetMockRequest(realUser, realUser, null);
ugi = JspHelper.GetUGI(context, request, conf);
NUnit.Framework.Assert.IsNull(ugi.GetRealUser());
NUnit.Framework.Assert.AreEqual(ugi.GetShortUserName(), realUser);
CheckUgiFromAuth(ugi);
// ugi for remote user != real user
request = GetMockRequest(realUser, user, null);
try
{
JspHelper.GetUGI(context, request, conf);
NUnit.Framework.Assert.Fail("bad request allowed");
}
catch (IOException ioe)
{
NUnit.Framework.Assert.AreEqual("Usernames not matched: name=" + user + " != expected="
+ realUser, ioe.Message);
}
}
//@Test (timeout = 60000)
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
/// <exception cref="System.TypeLoadException"/>
public virtual void TestSleepJobWithSecurityOn()
{
Log.Info("\n\n\nStarting testSleepJobWithSecurityOn().");
if (!(new FilePath(MiniMRYarnCluster.Appjar)).Exists())
{
return;
}
mrCluster.GetConfig().Set(CommonConfigurationKeysPublic.HadoopSecurityAuthentication
, "kerberos");
mrCluster.GetConfig().Set(YarnConfiguration.RmKeytab, "/etc/krb5.keytab");
mrCluster.GetConfig().Set(YarnConfiguration.NmKeytab, "/etc/krb5.keytab");
mrCluster.GetConfig().Set(YarnConfiguration.RmPrincipal, "rm/sightbusy-lx@LOCALHOST"
);
mrCluster.GetConfig().Set(YarnConfiguration.NmPrincipal, "nm/sightbusy-lx@LOCALHOST"
);
UserGroupInformation.SetConfiguration(mrCluster.GetConfig());
// Keep it in here instead of after RM/NM as multiple user logins happen in
// the same JVM.
UserGroupInformation user = UserGroupInformation.GetCurrentUser();
Log.Info("User name is " + user.GetUserName());
foreach (Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> str in user.GetTokens
())
{
Log.Info("Token is " + str.EncodeToUrlString());
}
user.DoAs(new _PrivilegedExceptionAction_552());
}
public virtual void TestLogin()
{
string userPrincipal = Runtime.GetProperty("user.principal");
string userKeyTab = Runtime.GetProperty("user.keytab");
NUnit.Framework.Assert.IsNotNull("User principal was not specified", userPrincipal
);
NUnit.Framework.Assert.IsNotNull("User keytab was not specified", userKeyTab);
Configuration conf = new Configuration();
conf.Set(CommonConfigurationKeys.HadoopSecurityAuthentication, "kerberos");
UserGroupInformation.SetConfiguration(conf);
UserGroupInformation ugi = UserGroupInformation.LoginUserFromKeytabAndReturnUGI(userPrincipal
, userKeyTab);
Assert.Equal(UserGroupInformation.AuthenticationMethod.Kerberos
, ugi.GetAuthenticationMethod());
try
{
UserGroupInformation.LoginUserFromKeytabAndReturnUGI("*****@*****.**", userKeyTab
);
NUnit.Framework.Assert.Fail("Login should have failed");
}
catch (Exception ex)
{
Runtime.PrintStackTrace(ex);
}
}
public virtual void TestBlockTokenRpcLeak()
{
Configuration conf = new Configuration();
conf.Set(CommonConfigurationKeysPublic.HadoopSecurityAuthentication, "kerberos");
UserGroupInformation.SetConfiguration(conf);
Assume.AssumeTrue(FdDir.Exists());
BlockTokenSecretManager sm = new BlockTokenSecretManager(blockKeyUpdateInterval,
blockTokenLifetime, 0, "fake-pool", null);
Org.Apache.Hadoop.Security.Token.Token <BlockTokenIdentifier> token = sm.GenerateToken
(block3, EnumSet.AllOf <BlockTokenSecretManager.AccessMode>());
Server server = CreateMockDatanode(sm, token, conf);
server.Start();
IPEndPoint addr = NetUtils.GetConnectAddress(server);
DatanodeID fakeDnId = DFSTestUtil.GetLocalDatanodeID(addr.Port);
ExtendedBlock b = new ExtendedBlock("fake-pool", new Org.Apache.Hadoop.Hdfs.Protocol.Block
(12345L));
LocatedBlock fakeBlock = new LocatedBlock(b, new DatanodeInfo[0]);
fakeBlock.SetBlockToken(token);
// Create another RPC proxy with the same configuration - this will never
// attempt to connect anywhere -- but it causes the refcount on the
// RPC "Client" object to stay above 0 such that RPC.stopProxy doesn't
// actually close the TCP connections to the real target DN.
ClientDatanodeProtocol proxyToNoWhere = RPC.GetProxy <ClientDatanodeProtocol>(ClientDatanodeProtocol
.versionID, new IPEndPoint("1.1.1.1", 1), UserGroupInformation.CreateRemoteUser(
"junk"), conf, NetUtils.GetDefaultSocketFactory(conf));
ClientDatanodeProtocol proxy = null;
int fdsAtStart = CountOpenFileDescriptors();
try
{
long endTime = Time.Now() + 3000;
while (Time.Now() < endTime)
{
proxy = DFSUtil.CreateClientDatanodeProtocolProxy(fakeDnId, conf, 1000, false, fakeBlock
);
NUnit.Framework.Assert.AreEqual(block3.GetBlockId(), proxy.GetReplicaVisibleLength
(block3));
if (proxy != null)
{
RPC.StopProxy(proxy);
}
Log.Info("Num open fds:" + CountOpenFileDescriptors());
}
int fdsAtEnd = CountOpenFileDescriptors();
if (fdsAtEnd - fdsAtStart > 50)
{
NUnit.Framework.Assert.Fail("Leaked " + (fdsAtEnd - fdsAtStart) + " fds!");
}
}
finally
{
server.Stop();
}
RPC.StopProxy(proxyToNoWhere);
}
public virtual void CleanUp()
{
jetty.Stop();
// resetting hadoop security to simple
Configuration conf = new Configuration();
UserGroupInformation.SetConfiguration(conf);
}
public virtual void SetUp()
{
// resetting hadoop security to simple
Configuration conf = new Configuration();
UserGroupInformation.SetConfiguration(conf);
jetty = CreateJettyServer();
}
public virtual void TestNonsecureUsernamePattern()
{
try
{
//nonsecure default
Configuration conf = new YarnConfiguration();
conf.Set(CommonConfigurationKeysPublic.HadoopSecurityAuthentication, "simple");
UserGroupInformation.SetConfiguration(conf);
LinuxContainerExecutor lce = new LinuxContainerExecutor();
lce.SetConf(conf);
lce.VerifyUsernamePattern("foo");
try
{
lce.VerifyUsernamePattern("foo/x");
NUnit.Framework.Assert.Fail();
}
catch (ArgumentException)
{
}
catch (Exception ex)
{
//NOP
NUnit.Framework.Assert.Fail(ex.ToString());
}
//nonsecure custom setting
conf.Set(YarnConfiguration.NmNonsecureModeUserPatternKey, "foo");
lce = new LinuxContainerExecutor();
lce.SetConf(conf);
lce.VerifyUsernamePattern("foo");
try
{
lce.VerifyUsernamePattern("bar");
NUnit.Framework.Assert.Fail();
}
catch (ArgumentException)
{
}
catch (Exception ex)
{
//NOP
NUnit.Framework.Assert.Fail(ex.ToString());
}
//secure, pattern matching does not kick in.
conf = new YarnConfiguration();
conf.Set(CommonConfigurationKeysPublic.HadoopSecurityAuthentication, "kerberos");
UserGroupInformation.SetConfiguration(conf);
lce = new LinuxContainerExecutor();
lce.SetConf(conf);
lce.VerifyUsernamePattern("foo");
lce.VerifyUsernamePattern("foo/w");
}
finally
{
Configuration conf = new YarnConfiguration();
conf.Set(CommonConfigurationKeysPublic.HadoopSecurityAuthentication, "simple");
UserGroupInformation.SetConfiguration(conf);
}
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestSecureProxyAuthParamsInUrl()
{
Configuration conf = new Configuration();
// fake turning on security so api thinks it should use tokens
SecurityUtil.SetAuthenticationMethod(UserGroupInformation.AuthenticationMethod.Kerberos
, conf);
UserGroupInformation.SetConfiguration(conf);
UserGroupInformation ugi = UserGroupInformation.CreateRemoteUser("test-user");
ugi.SetAuthenticationMethod(UserGroupInformation.AuthenticationMethod.Kerberos);
ugi = UserGroupInformation.CreateProxyUser("test-proxy-user", ugi);
UserGroupInformation.SetLoginUser(ugi);
WebHdfsFileSystem webhdfs = GetWebHdfsFileSystem(ugi, conf);
Path fsPath = new Path("/");
string tokenString = webhdfs.GetDelegationToken().EncodeToUrlString();
// send real+effective
Uri getTokenUrl = webhdfs.ToUrl(GetOpParam.OP.Getdelegationtoken, fsPath);
CheckQueryParams(new string[] { GetOpParam.OP.Getdelegationtoken.ToQueryString(),
new UserParam(ugi.GetRealUser().GetShortUserName()).ToString(), new DoAsParam(ugi
.GetShortUserName()).ToString() }, getTokenUrl);
// send real+effective
Uri renewTokenUrl = webhdfs.ToUrl(PutOpParam.OP.Renewdelegationtoken, fsPath, new
TokenArgumentParam(tokenString));
CheckQueryParams(new string[] { PutOpParam.OP.Renewdelegationtoken.ToQueryString(
), new UserParam(ugi.GetRealUser().GetShortUserName()).ToString(), new DoAsParam
(ugi.GetShortUserName()).ToString(), new TokenArgumentParam(tokenString).ToString
() }, renewTokenUrl);
// send token
Uri cancelTokenUrl = webhdfs.ToUrl(PutOpParam.OP.Canceldelegationtoken, fsPath, new
TokenArgumentParam(tokenString));
CheckQueryParams(new string[] { PutOpParam.OP.Canceldelegationtoken.ToQueryString
(), new UserParam(ugi.GetRealUser().GetShortUserName()).ToString(), new DoAsParam
(ugi.GetShortUserName()).ToString(), new TokenArgumentParam(tokenString).ToString
() }, cancelTokenUrl);
// send token
Uri fileStatusUrl = webhdfs.ToUrl(GetOpParam.OP.Getfilestatus, fsPath);
CheckQueryParams(new string[] { GetOpParam.OP.Getfilestatus.ToQueryString(), new
DelegationParam(tokenString).ToString() }, fileStatusUrl);
// wipe out internal token to simulate auth always required
webhdfs.SetDelegationToken(null);
// send real+effective
cancelTokenUrl = webhdfs.ToUrl(PutOpParam.OP.Canceldelegationtoken, fsPath, new TokenArgumentParam
(tokenString));
CheckQueryParams(new string[] { PutOpParam.OP.Canceldelegationtoken.ToQueryString
(), new UserParam(ugi.GetRealUser().GetShortUserName()).ToString(), new DoAsParam
(ugi.GetShortUserName()).ToString(), new TokenArgumentParam(tokenString).ToString
() }, cancelTokenUrl);
// send real+effective
fileStatusUrl = webhdfs.ToUrl(GetOpParam.OP.Getfilestatus, fsPath);
CheckQueryParams(new string[] { GetOpParam.OP.Getfilestatus.ToQueryString(), new
UserParam(ugi.GetRealUser().GetShortUserName()).ToString(), new DoAsParam(ugi.GetShortUserName
()).ToString() }, fileStatusUrl);
}
public static void SetUp()
{
conf = new Configuration();
SecurityUtil.SetAuthenticationMethod(UserGroupInformation.AuthenticationMethod.Kerberos
, conf);
UserGroupInformation.SetConfiguration(conf);
UserGroupInformation.SetLoginUser(UserGroupInformation.CreateUserForTesting("LoginUser"
, new string[] { "supergroup" }));
}
public static void Setup()
{
try
{
testMiniKDC = new MiniKdc(MiniKdc.CreateConf(), testRootDir);
testMiniKDC.Start();
testMiniKDC.CreatePrincipal(httpSpnegoKeytabFile, HttpUser + "/localhost");
}
catch (Exception)
{
NUnit.Framework.Assert.IsTrue("Couldn't setup MiniKDC", false);
}
try
{
testTimelineServer = new ApplicationHistoryServer();
conf = new Configuration(false);
conf.SetStrings(TimelineAuthenticationFilterInitializer.Prefix + "type", "kerberos"
);
conf.Set(TimelineAuthenticationFilterInitializer.Prefix + KerberosAuthenticationHandler
.Principal, httpSpnegoPrincipal);
conf.Set(TimelineAuthenticationFilterInitializer.Prefix + KerberosAuthenticationHandler
.Keytab, httpSpnegoKeytabFile.GetAbsolutePath());
conf.Set(CommonConfigurationKeysPublic.HadoopSecurityAuthentication, "kerberos");
conf.Set(YarnConfiguration.TimelineServicePrincipal, httpSpnegoPrincipal);
conf.Set(YarnConfiguration.TimelineServiceKeytab, httpSpnegoKeytabFile.GetAbsolutePath
());
conf.SetBoolean(YarnConfiguration.TimelineServiceEnabled, true);
conf.SetClass(YarnConfiguration.TimelineServiceStore, typeof(MemoryTimelineStore)
, typeof(TimelineStore));
conf.Set(YarnConfiguration.TimelineServiceAddress, "localhost:10200");
conf.Set(YarnConfiguration.TimelineServiceWebappAddress, "localhost:8188");
conf.Set(YarnConfiguration.TimelineServiceWebappHttpsAddress, "localhost:8190");
conf.Set("hadoop.proxyuser.HTTP.hosts", "*");
conf.Set("hadoop.proxyuser.HTTP.users", FooUser);
conf.SetInt(YarnConfiguration.TimelineServiceClientMaxRetries, 1);
if (withSsl)
{
conf.Set(YarnConfiguration.YarnHttpPolicyKey, HttpConfig.Policy.HttpsOnly.ToString
());
FilePath @base = new FilePath(Basedir);
FileUtil.FullyDelete(@base);
@base.Mkdirs();
keystoresDir = new FilePath(Basedir).GetAbsolutePath();
sslConfDir = KeyStoreTestUtil.GetClasspathDir(typeof(Org.Apache.Hadoop.Yarn.Server.Timeline.Security.TestTimelineAuthenticationFilter
));
KeyStoreTestUtil.SetupSSLConfig(keystoresDir, sslConfDir, conf, false);
}
UserGroupInformation.SetConfiguration(conf);
testTimelineServer.Init(conf);
testTimelineServer.Start();
}
catch (Exception)
{
NUnit.Framework.Assert.IsTrue("Couldn't setup TimelineServer", false);
}
}
public virtual void Setup()
{
Logger rootLogger = LogManager.GetRootLogger();
rootLogger.SetLevel(Level.Debug);
conf = new YarnConfiguration();
UserGroupInformation.SetConfiguration(conf);
conf.SetInt(YarnConfiguration.RmAmMaxAttempts, YarnConfiguration.DefaultRmAmMaxAttempts
);
}
public virtual void SetUp()
{
Configuration conf = new YarnConfiguration();
UserGroupInformation.SetConfiguration(conf);
resourceManager = new ResourceManager();
resourceManager.Init(conf);
resourceManager.GetRMContext().GetContainerTokenSecretManager().RollMasterKey();
resourceManager.GetRMContext().GetNMTokenSecretManager().RollMasterKey();
}
public virtual void Setup()
{
Logger rootLogger = LogManager.GetRootLogger();
rootLogger.SetLevel(Level.Debug);
conf = new YarnConfiguration();
UserGroupInformation.SetConfiguration(conf);
conf.Set(YarnConfiguration.RecoveryEnabled, "true");
conf.Set(YarnConfiguration.RmStore, typeof(MemoryRMStateStore).FullName);
NUnit.Framework.Assert.IsTrue(YarnConfiguration.DefaultRmAmMaxAttempts > 1);
}
/// <exception cref="System.Exception"/>
public virtual void TestNMUpdation()
{
YarnConfiguration conf = new YarnConfiguration();
// validating RM NM keys for Unsecured environment
ValidateRMNMKeyExchange(conf);
// validating RM NM keys for secured environment
conf.Set(CommonConfigurationKeysPublic.HadoopSecurityAuthentication, "kerberos");
UserGroupInformation.SetConfiguration(conf);
ValidateRMNMKeyExchange(conf);
}
public virtual void Setup()
{
Logger rootLogger = LogManager.GetRootLogger();
rootLogger.SetLevel(Level.Debug);
ExitUtil.DisableSystemExit();
conf = new YarnConfiguration();
UserGroupInformation.SetConfiguration(conf);
conf.Set(YarnConfiguration.RmStore, typeof(MemoryRMStateStore).FullName);
conf.Set(YarnConfiguration.RmScheduler, typeof(FairScheduler).FullName);
}
public virtual void StartMiniKdc()
{
// This setting below is required. If not enabled, UGI will abort
// any attempt to loginUserFromKeytab.
Configuration conf = new Configuration();
conf.Set(CommonConfigurationKeys.HadoopSecurityAuthentication, "kerberos");
UserGroupInformation.SetConfiguration(conf);
workDir = folder.GetRoot();
kdc = new MiniKdc(MiniKdc.CreateConf(), workDir);
kdc.Start();
}
/// <summary>Start RPC server</summary>
/// <exception cref="System.IO.IOException"/>
internal MiniServer(Configuration conf, string user, string keytabFile)
{
UserGroupInformation.SetConfiguration(conf);
UserGroupInformation.LoginUserFromKeytab(user, keytabFile);
secretManager = new TestDelegationToken.TestDelegationTokenSecretManager(24 * 60
* 60 * 1000, 7 * 24 * 60 * 60 * 1000, 24 * 60 * 60 * 1000, 3600000);
secretManager.StartThreads();
rpcServer = new RPC.Builder(conf).SetProtocol(typeof(MiniRPCBenchmark.MiniProtocol
)).SetInstance(this).SetBindAddress(DefaultServerAddress).SetPort(0).SetNumHandlers
(1).SetVerbose(false).SetSecretManager(secretManager).Build();
rpcServer.Start();
}
/// <exception cref="System.Exception"/>
private void TestKerberosDelegationTokenAuthenticator(bool doAs)
{
string doAsUser = doAs ? OkUser : null;
// setting hadoop security to kerberos
Configuration conf = new Configuration();
conf.Set("hadoop.security.authentication", "kerberos");
UserGroupInformation.SetConfiguration(conf);
FilePath testDir = new FilePath("target/" + UUID.RandomUUID().ToString());
Assert.True(testDir.Mkdirs());
MiniKdc kdc = new MiniKdc(MiniKdc.CreateConf(), testDir);
Org.Mortbay.Jetty.Server jetty = CreateJettyServer();
Context context = new Context();
context.SetContextPath("/foo");
jetty.SetHandler(context);
context.AddFilter(new FilterHolder(typeof(TestWebDelegationToken.KDTAFilter)), "/*"
, 0);
context.AddServlet(new ServletHolder(typeof(TestWebDelegationToken.UserServlet)),
"/bar");
try
{
kdc.Start();
FilePath keytabFile = new FilePath(testDir, "test.keytab");
kdc.CreatePrincipal(keytabFile, "client", "HTTP/localhost");
TestWebDelegationToken.KDTAFilter.keytabFile = keytabFile.GetAbsolutePath();
jetty.Start();
DelegationTokenAuthenticatedURL.Token token = new DelegationTokenAuthenticatedURL.Token
();
DelegationTokenAuthenticatedURL aUrl = new DelegationTokenAuthenticatedURL();
Uri url = new Uri(GetJettyURL() + "/foo/bar");
try
{
aUrl.GetDelegationToken(url, token, FooUser, doAsUser);
NUnit.Framework.Assert.Fail();
}
catch (AuthenticationException ex)
{
Assert.True(ex.Message.Contains("GSSException"));
}
DoAsKerberosUser("client", keytabFile.GetAbsolutePath(), new _Callable_778(aUrl,
url, token, doAs, doAsUser));
}
finally
{
// Make sure the token belongs to the right owner
jetty.Stop();
kdc.Stop();
}
}
public virtual void TestLazyTokenFetchForSWebhdfs()
{
// for any(Token.class)
MiniDFSCluster cluster = null;
SWebHdfsFileSystem fs = null;
try
{
Configuration clusterConf = new HdfsConfiguration(conf);
SecurityUtil.SetAuthenticationMethod(UserGroupInformation.AuthenticationMethod.Simple
, clusterConf);
clusterConf.SetBoolean(DFSConfigKeys.DfsNamenodeDelegationTokenAlwaysUseKey, true
);
string Basedir = Runtime.GetProperty("test.build.dir", "target/test-dir") + "/" +
typeof(TestWebHdfsTokens).Name;
string keystoresDir;
string sslConfDir;
clusterConf.SetBoolean(DFSConfigKeys.DfsWebhdfsEnabledKey, true);
clusterConf.Set(DFSConfigKeys.DfsHttpPolicyKey, HttpConfig.Policy.HttpsOnly.ToString
());
clusterConf.Set(DFSConfigKeys.DfsNamenodeHttpsAddressKey, "localhost:0");
clusterConf.Set(DFSConfigKeys.DfsDatanodeHttpsAddressKey, "localhost:0");
FilePath @base = new FilePath(Basedir);
FileUtil.FullyDelete(@base);
@base.Mkdirs();
keystoresDir = new FilePath(Basedir).GetAbsolutePath();
sslConfDir = KeyStoreTestUtil.GetClasspathDir(typeof(TestWebHdfsTokens));
KeyStoreTestUtil.SetupSSLConfig(keystoresDir, sslConfDir, clusterConf, false);
// trick the NN into thinking security is enabled w/o it trying
// to login from a keytab
UserGroupInformation.SetConfiguration(clusterConf);
cluster = new MiniDFSCluster.Builder(clusterConf).NumDataNodes(1).Build();
cluster.WaitActive();
IPEndPoint addr = cluster.GetNameNode().GetHttpsAddress();
string nnAddr = NetUtils.GetHostPortString(addr);
clusterConf.Set(DFSConfigKeys.DfsNamenodeHttpsAddressKey, nnAddr);
SecurityUtil.SetAuthenticationMethod(UserGroupInformation.AuthenticationMethod.Kerberos
, clusterConf);
UserGroupInformation.SetConfiguration(clusterConf);
uri = DFSUtil.CreateUri("swebhdfs", cluster.GetNameNode().GetHttpsAddress());
ValidateLazyTokenFetch(clusterConf);
}
finally
{
IOUtils.Cleanup(null, fs);
if (cluster != null)
{
cluster.Shutdown();
}
}
}
public static void Init()
{
baseDir = new FilePath(Runtime.GetProperty("test.build.dir", "target/test-dir"),
typeof(TestSecureNNWithQJM).Name);
FileUtil.FullyDelete(baseDir);
NUnit.Framework.Assert.IsTrue(baseDir.Mkdirs());
Properties kdcConf = MiniKdc.CreateConf();
kdc = new MiniKdc(kdcConf, baseDir);
kdc.Start();
baseConf = new HdfsConfiguration();
SecurityUtil.SetAuthenticationMethod(UserGroupInformation.AuthenticationMethod.Kerberos
, baseConf);
UserGroupInformation.SetConfiguration(baseConf);
NUnit.Framework.Assert.IsTrue("Expected configuration to enable security", UserGroupInformation
.IsSecurityEnabled());
string userName = UserGroupInformation.GetLoginUser().GetShortUserName();
FilePath keytabFile = new FilePath(baseDir, userName + ".keytab");
string keytab = keytabFile.GetAbsolutePath();
// Windows will not reverse name lookup "127.0.0.1" to "localhost".
string krbInstance = Path.Windows ? "127.0.0.1" : "localhost";
kdc.CreatePrincipal(keytabFile, userName + "/" + krbInstance, "HTTP/" + krbInstance
);
string hdfsPrincipal = userName + "/" + krbInstance + "@" + kdc.GetRealm();
string spnegoPrincipal = "HTTP/" + krbInstance + "@" + kdc.GetRealm();
baseConf.Set(DFSConfigKeys.DfsNamenodeKerberosPrincipalKey, hdfsPrincipal);
baseConf.Set(DFSConfigKeys.DfsNamenodeKeytabFileKey, keytab);
baseConf.Set(DFSConfigKeys.DfsDatanodeKerberosPrincipalKey, hdfsPrincipal);
baseConf.Set(DFSConfigKeys.DfsDatanodeKeytabFileKey, keytab);
baseConf.Set(DFSConfigKeys.DfsWebAuthenticationKerberosPrincipalKey, spnegoPrincipal
);
baseConf.Set(DFSConfigKeys.DfsJournalnodeKeytabFileKey, keytab);
baseConf.Set(DFSConfigKeys.DfsJournalnodeKerberosPrincipalKey, hdfsPrincipal);
baseConf.Set(DFSConfigKeys.DfsJournalnodeKerberosInternalSpnegoPrincipalKey, spnegoPrincipal
);
baseConf.SetBoolean(DFSConfigKeys.DfsBlockAccessTokenEnableKey, true);
baseConf.Set(DFSConfigKeys.DfsDataTransferProtectionKey, "authentication");
baseConf.Set(DFSConfigKeys.DfsHttpPolicyKey, HttpConfig.Policy.HttpsOnly.ToString
());
baseConf.Set(DFSConfigKeys.DfsNamenodeHttpsAddressKey, "localhost:0");
baseConf.Set(DFSConfigKeys.DfsDatanodeHttpsAddressKey, "localhost:0");
baseConf.Set(DFSConfigKeys.DfsJournalnodeHttpsAddressKey, "localhost:0");
baseConf.SetInt(CommonConfigurationKeys.IpcClientConnectMaxRetriesOnSaslKey, 10);
string keystoresDir = baseDir.GetAbsolutePath();
string sslConfDir = KeyStoreTestUtil.GetClasspathDir(typeof(TestSecureNNWithQJM));
KeyStoreTestUtil.SetupSSLConfig(keystoresDir, sslConfDir, baseConf, false);
}
/// <summary>test constructor</summary>
/// <exception cref="System.Exception"/>
public virtual void TestConstructorWithKerberos()
{
// security on, default is remove default realm
SecurityUtil.SetAuthenticationMethod(UserGroupInformation.AuthenticationMethod.Kerberos
, conf);
UserGroupInformation.SetConfiguration(conf);
TestConstructorSuccess("user1", "user1");
TestConstructorSuccess("*****@*****.**", "user2");
TestConstructorSuccess("user3/[email protected]", "user3");
// failure test
TestConstructorFailures("*****@*****.**");
TestConstructorFailures("user5/[email protected]");
TestConstructorFailures(null);
TestConstructorFailures(string.Empty);
}
/// <summary>test constructor</summary>
/// <exception cref="System.Exception"/>
public virtual void TestConstructorWithRules()
{
// security off, but use rules if explicitly set
conf.Set(CommonConfigurationKeysPublic.HadoopSecurityAuthToLocal, "RULE:[1:$1@$0](.*@OTHER.REALM)s/(.*)@.*/other-$1/"
);
UserGroupInformation.SetConfiguration(conf);
TestConstructorSuccess("user1", "user1");
TestConstructorSuccess("*****@*****.**", "other-user4");
// failure test
TestConstructorFailures("*****@*****.**");
TestConstructorFailures("user3/[email protected]");
TestConstructorFailures("user5/[email protected]");
TestConstructorFailures(null);
TestConstructorFailures(string.Empty);
}
/// <summary>Run MiniBenchmark using delegation token authentication.</summary>
/// <param name="conf">- configuration</param>
/// <param name="count">- connect this many times</param>
/// <param name="keytabKey">- key for keytab file in the configuration</param>
/// <param name="userNameKey">- key for user name in the configuration</param>
/// <returns>average time to connect</returns>
/// <exception cref="System.IO.IOException"/>
internal virtual long RunMiniBenchmarkWithDelegationToken(Configuration conf, int
count, string keytabKey, string userNameKey)
{
// get login information
string user = Runtime.GetProperty("user.name");
if (userNameKey != null)
{
user = conf.Get(userNameKey, user);
}
string keytabFile = null;
if (keytabKey != null)
{
keytabFile = conf.Get(keytabKey, keytabFile);
}
MiniRPCBenchmark.MiniServer miniServer = null;
UserGroupInformation.SetConfiguration(conf);
string shortUserName = UserGroupInformation.CreateRemoteUser(user).GetShortUserName
();
try
{
conf.SetStrings(DefaultImpersonationProvider.GetTestProvider().GetProxySuperuserGroupConfKey
(shortUserName), GroupName1);
ConfigureSuperUserIPAddresses(conf, shortUserName);
// start the server
miniServer = new MiniRPCBenchmark.MiniServer(conf, user, keytabFile);
IPEndPoint addr = miniServer.GetAddress();
ConnectToServerAndGetDelegationToken(conf, addr);
// connect to the server count times
SetLoggingLevel(logLevel);
long elapsed = 0L;
for (int idx = 0; idx < count; idx++)
{
elapsed += ConnectToServerUsingDelegationToken(conf, addr);
}
return(elapsed);
}
finally
{
if (miniServer != null)
{
miniServer.Stop();
}
}
}
/// <exception cref="System.IO.IOException"/>
public RpcProgramMountd(NfsConfiguration config, DatagramSocket registrationSocket
, bool allowInsecurePorts)
: base("mountd", "localhost", config.GetInt(NfsConfigKeys.DfsNfsMountdPortKey, NfsConfigKeys
.DfsNfsMountdPortDefault), Program, Version1, Version3, registrationSocket, allowInsecurePorts
)
{
// Note that RPC cache is not enabled
exports = new AList <string>();
exports.AddItem(config.Get(NfsConfigKeys.DfsNfsExportPointKey, NfsConfigKeys.DfsNfsExportPointDefault
));
this.hostsMatcher = NfsExports.GetInstance(config);
this.mounts = Sharpen.Collections.SynchronizedList(new AList <MountEntry>());
UserGroupInformation.SetConfiguration(config);
SecurityUtil.Login(config, NfsConfigKeys.DfsNfsKeytabFileKey, NfsConfigKeys.DfsNfsKerberosPrincipalKey
);
this.dfsClient = new DFSClient(NameNode.GetAddress(config), config);
}
