public virtual void TestBlockTokenRpcLeak()
{
Configuration conf = new Configuration();
conf.Set(CommonConfigurationKeysPublic.HadoopSecurityAuthentication, "kerberos");
UserGroupInformation.SetConfiguration(conf);
Assume.AssumeTrue(FdDir.Exists());
BlockTokenSecretManager sm = new BlockTokenSecretManager(blockKeyUpdateInterval,
blockTokenLifetime, 0, "fake-pool", null);
Org.Apache.Hadoop.Security.Token.Token <BlockTokenIdentifier> token = sm.GenerateToken
(block3, EnumSet.AllOf <BlockTokenSecretManager.AccessMode>());
Server server = CreateMockDatanode(sm, token, conf);
server.Start();
IPEndPoint addr = NetUtils.GetConnectAddress(server);
DatanodeID fakeDnId = DFSTestUtil.GetLocalDatanodeID(addr.Port);
ExtendedBlock b = new ExtendedBlock("fake-pool", new Org.Apache.Hadoop.Hdfs.Protocol.Block
(12345L));
LocatedBlock fakeBlock = new LocatedBlock(b, new DatanodeInfo[0]);
fakeBlock.SetBlockToken(token);
// Create another RPC proxy with the same configuration - this will never
// attempt to connect anywhere -- but it causes the refcount on the
// RPC "Client" object to stay above 0 such that RPC.stopProxy doesn't
// actually close the TCP connections to the real target DN.
ClientDatanodeProtocol proxyToNoWhere = RPC.GetProxy <ClientDatanodeProtocol>(ClientDatanodeProtocol
.versionID, new IPEndPoint("1.1.1.1", 1), UserGroupInformation.CreateRemoteUser(
"junk"), conf, NetUtils.GetDefaultSocketFactory(conf));
ClientDatanodeProtocol proxy = null;
int fdsAtStart = CountOpenFileDescriptors();
try
{
long endTime = Time.Now() + 3000;
while (Time.Now() < endTime)
{
proxy = DFSUtil.CreateClientDatanodeProtocolProxy(fakeDnId, conf, 1000, false, fakeBlock
);
NUnit.Framework.Assert.AreEqual(block3.GetBlockId(), proxy.GetReplicaVisibleLength
(block3));
if (proxy != null)
{
RPC.StopProxy(proxy);
}
Log.Info("Num open fds:" + CountOpenFileDescriptors());
}
int fdsAtEnd = CountOpenFileDescriptors();
if (fdsAtEnd - fdsAtStart > 50)
{
NUnit.Framework.Assert.Fail("Leaked " + (fdsAtEnd - fdsAtStart) + " fds!");
}
}
finally
{
server.Stop();
}
RPC.StopProxy(proxyToNoWhere);
}
private LocatedBlock CreateLocatedBlockNoStorageMedia()
{
DatanodeInfo[] dnInfos = new DatanodeInfo[] { DFSTestUtil.GetLocalDatanodeInfo("127.0.0.1"
, "h1", DatanodeInfo.AdminStates.DecommissionInprogress), DFSTestUtil.GetLocalDatanodeInfo
("127.0.0.1", "h2", DatanodeInfo.AdminStates.Decommissioned), DFSTestUtil.GetLocalDatanodeInfo
("127.0.0.1", "h3", DatanodeInfo.AdminStates.Normal) };
LocatedBlock lb = new LocatedBlock(new ExtendedBlock("bp12", 12345, 10, 53), dnInfos
, 5, false);
lb.SetBlockToken(new Org.Apache.Hadoop.Security.Token.Token <BlockTokenIdentifier>
(Sharpen.Runtime.GetBytesForString("identifier"), Sharpen.Runtime.GetBytesForString
("password"), new Text("kind"), new Text("service")));
return(lb);
}
private LocatedBlock CreateLocatedBlock()
{
DatanodeInfo[] dnInfos = new DatanodeInfo[] { DFSTestUtil.GetLocalDatanodeInfo("127.0.0.1"
, "h1", DatanodeInfo.AdminStates.DecommissionInprogress), DFSTestUtil.GetLocalDatanodeInfo
("127.0.0.1", "h2", DatanodeInfo.AdminStates.Decommissioned), DFSTestUtil.GetLocalDatanodeInfo
("127.0.0.1", "h3", DatanodeInfo.AdminStates.Normal), DFSTestUtil.GetLocalDatanodeInfo
("127.0.0.1", "h4", DatanodeInfo.AdminStates.Normal) };
string[] storageIDs = new string[] { "s1", "s2", "s3", "s4" };
StorageType[] media = new StorageType[] { StorageType.Disk, StorageType.Ssd, StorageType
.Disk, StorageType.RamDisk };
LocatedBlock lb = new LocatedBlock(new ExtendedBlock("bp12", 12345, 10, 53), dnInfos
, storageIDs, media, 5, false, new DatanodeInfo[] { });
lb.SetBlockToken(new Org.Apache.Hadoop.Security.Token.Token <BlockTokenIdentifier>
(Sharpen.Runtime.GetBytesForString("identifier"), Sharpen.Runtime.GetBytesForString
("password"), new Text("kind"), new Text("service")));
return(lb);
}
/// <summary>Convert a Json map to LocatedBlock.</summary>
/// <exception cref="System.IO.IOException"/>
private static LocatedBlock ToLocatedBlock <_T0>(IDictionary <_T0> m)
{
if (m == null)
{
return(null);
}
ExtendedBlock b = ToExtendedBlock((IDictionary <object, object>)m["block"]);
DatanodeInfo[] locations = ToDatanodeInfoArray(GetList(m, "locations"));
long startOffset = ((Number)m["startOffset"]);
bool isCorrupt = (bool)m["isCorrupt"];
DatanodeInfo[] cachedLocations = ToDatanodeInfoArray(GetList(m, "cachedLocations"
));
LocatedBlock locatedblock = new LocatedBlock(b, locations, null, null, startOffset
, isCorrupt, cachedLocations);
locatedblock.SetBlockToken(ToBlockToken((IDictionary <object, object>)m["blockToken"
]));
return(locatedblock);
}
public virtual void TestRead()
{
MiniDFSCluster cluster = null;
int numDataNodes = 2;
Configuration conf = GetConf(numDataNodes);
try
{
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(numDataNodes).Build();
cluster.WaitActive();
NUnit.Framework.Assert.AreEqual(numDataNodes, cluster.GetDataNodes().Count);
NameNode nn = cluster.GetNameNode();
NamenodeProtocols nnProto = nn.GetRpcServer();
BlockManager bm = nn.GetNamesystem().GetBlockManager();
BlockTokenSecretManager sm = bm.GetBlockTokenSecretManager();
// set a short token lifetime (1 second) initially
SecurityTestUtil.SetBlockTokenLifetime(sm, 1000L);
Path fileToRead = new Path(FileToRead);
FileSystem fs = cluster.GetFileSystem();
CreateFile(fs, fileToRead);
/*
* setup for testing expiration handling of cached tokens
*/
// read using blockSeekTo(). Acquired tokens are cached in in1
FSDataInputStream in1 = fs.Open(fileToRead);
NUnit.Framework.Assert.IsTrue(CheckFile1(in1));
// read using blockSeekTo(). Acquired tokens are cached in in2
FSDataInputStream in2 = fs.Open(fileToRead);
NUnit.Framework.Assert.IsTrue(CheckFile1(in2));
// read using fetchBlockByteRange(). Acquired tokens are cached in in3
FSDataInputStream in3 = fs.Open(fileToRead);
NUnit.Framework.Assert.IsTrue(CheckFile2(in3));
/*
* testing READ interface on DN using a BlockReader
*/
DFSClient client = null;
try
{
client = new DFSClient(new IPEndPoint("localhost", cluster.GetNameNodePort()), conf
);
}
finally
{
if (client != null)
{
client.Close();
}
}
IList <LocatedBlock> locatedBlocks = nnProto.GetBlockLocations(FileToRead, 0, FileSize
).GetLocatedBlocks();
LocatedBlock lblock = locatedBlocks[0];
// first block
Org.Apache.Hadoop.Security.Token.Token <BlockTokenIdentifier> myToken = lblock.GetBlockToken
();
// verify token is not expired
NUnit.Framework.Assert.IsFalse(SecurityTestUtil.IsBlockTokenExpired(myToken));
// read with valid token, should succeed
TryRead(conf, lblock, true);
/*
* wait till myToken and all cached tokens in in1, in2 and in3 expire
*/
while (!SecurityTestUtil.IsBlockTokenExpired(myToken))
{
try
{
Sharpen.Thread.Sleep(10);
}
catch (Exception)
{
}
}
/*
* continue testing READ interface on DN using a BlockReader
*/
// verify token is expired
NUnit.Framework.Assert.IsTrue(SecurityTestUtil.IsBlockTokenExpired(myToken));
// read should fail
TryRead(conf, lblock, false);
// use a valid new token
lblock.SetBlockToken(sm.GenerateToken(lblock.GetBlock(), EnumSet.Of(BlockTokenSecretManager.AccessMode
.Read)));
// read should succeed
TryRead(conf, lblock, true);
// use a token with wrong blockID
ExtendedBlock wrongBlock = new ExtendedBlock(lblock.GetBlock().GetBlockPoolId(),
lblock.GetBlock().GetBlockId() + 1);
lblock.SetBlockToken(sm.GenerateToken(wrongBlock, EnumSet.Of(BlockTokenSecretManager.AccessMode
.Read)));
// read should fail
TryRead(conf, lblock, false);
// use a token with wrong access modes
lblock.SetBlockToken(sm.GenerateToken(lblock.GetBlock(), EnumSet.Of(BlockTokenSecretManager.AccessMode
.Write, BlockTokenSecretManager.AccessMode.Copy, BlockTokenSecretManager.AccessMode
.Replace)));
// read should fail
TryRead(conf, lblock, false);
// set a long token lifetime for future tokens
SecurityTestUtil.SetBlockTokenLifetime(sm, 600 * 1000L);
/*
* testing that when cached tokens are expired, DFSClient will re-fetch
* tokens transparently for READ.
*/
// confirm all tokens cached in in1 are expired by now
IList <LocatedBlock> lblocks = DFSTestUtil.GetAllBlocks(in1);
foreach (LocatedBlock blk in lblocks)
{
NUnit.Framework.Assert.IsTrue(SecurityTestUtil.IsBlockTokenExpired(blk.GetBlockToken
()));
}
// verify blockSeekTo() is able to re-fetch token transparently
in1.Seek(0);
NUnit.Framework.Assert.IsTrue(CheckFile1(in1));
// confirm all tokens cached in in2 are expired by now
IList <LocatedBlock> lblocks2 = DFSTestUtil.GetAllBlocks(in2);
foreach (LocatedBlock blk_1 in lblocks2)
{
NUnit.Framework.Assert.IsTrue(SecurityTestUtil.IsBlockTokenExpired(blk_1.GetBlockToken
()));
}
// verify blockSeekTo() is able to re-fetch token transparently (testing
// via another interface method)
NUnit.Framework.Assert.IsTrue(in2.SeekToNewSource(0));
NUnit.Framework.Assert.IsTrue(CheckFile1(in2));
// confirm all tokens cached in in3 are expired by now
IList <LocatedBlock> lblocks3 = DFSTestUtil.GetAllBlocks(in3);
foreach (LocatedBlock blk_2 in lblocks3)
{
NUnit.Framework.Assert.IsTrue(SecurityTestUtil.IsBlockTokenExpired(blk_2.GetBlockToken
()));
}
// verify fetchBlockByteRange() is able to re-fetch token transparently
NUnit.Framework.Assert.IsTrue(CheckFile2(in3));
/*
* testing that after datanodes are restarted on the same ports, cached
* tokens should still work and there is no need to fetch new tokens from
* namenode. This test should run while namenode is down (to make sure no
* new tokens can be fetched from namenode).
*/
// restart datanodes on the same ports that they currently use
NUnit.Framework.Assert.IsTrue(cluster.RestartDataNodes(true));
cluster.WaitActive();
NUnit.Framework.Assert.AreEqual(numDataNodes, cluster.GetDataNodes().Count);
cluster.ShutdownNameNode(0);
// confirm tokens cached in in1 are still valid
lblocks = DFSTestUtil.GetAllBlocks(in1);
foreach (LocatedBlock blk_3 in lblocks)
{
NUnit.Framework.Assert.IsFalse(SecurityTestUtil.IsBlockTokenExpired(blk_3.GetBlockToken
()));
}
// verify blockSeekTo() still works (forced to use cached tokens)
in1.Seek(0);
NUnit.Framework.Assert.IsTrue(CheckFile1(in1));
// confirm tokens cached in in2 are still valid
lblocks2 = DFSTestUtil.GetAllBlocks(in2);
foreach (LocatedBlock blk_4 in lblocks2)
{
NUnit.Framework.Assert.IsFalse(SecurityTestUtil.IsBlockTokenExpired(blk_4.GetBlockToken
()));
}
// verify blockSeekTo() still works (forced to use cached tokens)
in2.SeekToNewSource(0);
NUnit.Framework.Assert.IsTrue(CheckFile1(in2));
// confirm tokens cached in in3 are still valid
lblocks3 = DFSTestUtil.GetAllBlocks(in3);
foreach (LocatedBlock blk_5 in lblocks3)
{
NUnit.Framework.Assert.IsFalse(SecurityTestUtil.IsBlockTokenExpired(blk_5.GetBlockToken
()));
}
// verify fetchBlockByteRange() still works (forced to use cached tokens)
NUnit.Framework.Assert.IsTrue(CheckFile2(in3));
/*
* testing that when namenode is restarted, cached tokens should still
* work and there is no need to fetch new tokens from namenode. Like the
* previous test, this test should also run while namenode is down. The
* setup for this test depends on the previous test.
*/
// restart the namenode and then shut it down for test
cluster.RestartNameNode(0);
cluster.ShutdownNameNode(0);
// verify blockSeekTo() still works (forced to use cached tokens)
in1.Seek(0);
NUnit.Framework.Assert.IsTrue(CheckFile1(in1));
// verify again blockSeekTo() still works (forced to use cached tokens)
in2.SeekToNewSource(0);
NUnit.Framework.Assert.IsTrue(CheckFile1(in2));
// verify fetchBlockByteRange() still works (forced to use cached tokens)
NUnit.Framework.Assert.IsTrue(CheckFile2(in3));
/*
* testing that after both namenode and datanodes got restarted (namenode
* first, followed by datanodes), DFSClient can't access DN without
* re-fetching tokens and is able to re-fetch tokens transparently. The
* setup of this test depends on the previous test.
*/
// restore the cluster and restart the datanodes for test
cluster.RestartNameNode(0);
NUnit.Framework.Assert.IsTrue(cluster.RestartDataNodes(true));
cluster.WaitActive();
NUnit.Framework.Assert.AreEqual(numDataNodes, cluster.GetDataNodes().Count);
// shutdown namenode so that DFSClient can't get new tokens from namenode
cluster.ShutdownNameNode(0);
// verify blockSeekTo() fails (cached tokens become invalid)
in1.Seek(0);
NUnit.Framework.Assert.IsFalse(CheckFile1(in1));
// verify fetchBlockByteRange() fails (cached tokens become invalid)
NUnit.Framework.Assert.IsFalse(CheckFile2(in3));
// restart the namenode to allow DFSClient to re-fetch tokens
cluster.RestartNameNode(0);
// verify blockSeekTo() works again (by transparently re-fetching
// tokens from namenode)
in1.Seek(0);
NUnit.Framework.Assert.IsTrue(CheckFile1(in1));
in2.SeekToNewSource(0);
NUnit.Framework.Assert.IsTrue(CheckFile1(in2));
// verify fetchBlockByteRange() works again (by transparently
// re-fetching tokens from namenode)
NUnit.Framework.Assert.IsTrue(CheckFile2(in3));
/*
* testing that when datanodes are restarted on different ports, DFSClient
* is able to re-fetch tokens transparently to connect to them
*/
// restart datanodes on newly assigned ports
NUnit.Framework.Assert.IsTrue(cluster.RestartDataNodes(false));
cluster.WaitActive();
NUnit.Framework.Assert.AreEqual(numDataNodes, cluster.GetDataNodes().Count);
// verify blockSeekTo() is able to re-fetch token transparently
in1.Seek(0);
NUnit.Framework.Assert.IsTrue(CheckFile1(in1));
// verify blockSeekTo() is able to re-fetch token transparently
in2.SeekToNewSource(0);
NUnit.Framework.Assert.IsTrue(CheckFile1(in2));
// verify fetchBlockByteRange() is able to re-fetch token transparently
NUnit.Framework.Assert.IsTrue(CheckFile2(in3));
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}