/// <exception cref="System.IO.IOException"/>
internal static ClientDatanodeProtocolPB CreateClientDatanodeProtocolProxy(DatanodeID
datanodeid, Configuration conf, int socketTimeout, bool connectToDnViaHostname,
LocatedBlock locatedBlock)
{
string dnAddr = datanodeid.GetIpcAddr(connectToDnViaHostname);
IPEndPoint addr = NetUtils.CreateSocketAddr(dnAddr);
if (Log.IsDebugEnabled())
{
Log.Debug("Connecting to datanode " + dnAddr + " addr=" + addr);
}
// Since we're creating a new UserGroupInformation here, we know that no
// future RPC proxies will be able to re-use the same connection. And
// usages of this proxy tend to be one-off calls.
//
// This is a temporary fix: callers should really achieve this by using
// RPC.stopProxy() on the resulting object, but this is currently not
// working in trunk. See the discussion on HDFS-1965.
Configuration confWithNoIpcIdle = new Configuration(conf);
confWithNoIpcIdle.SetInt(CommonConfigurationKeysPublic.IpcClientConnectionMaxidletimeKey
, 0);
UserGroupInformation ticket = UserGroupInformation.CreateRemoteUser(locatedBlock.
GetBlock().GetLocalBlock().ToString());
ticket.AddToken(locatedBlock.GetBlockToken());
return(CreateClientDatanodeProtocolProxy(addr, ticket, confWithNoIpcIdle, NetUtils
.GetDefaultSocketFactory(conf), socketTimeout));
}
/// <exception cref="System.Exception"/>
public virtual void TestBlockReaderLocalLegacyWithAppend()
{
short ReplFactor = 1;
HdfsConfiguration conf = GetConfiguration(null);
conf.SetBoolean(DFSConfigKeys.DfsClientUseLegacyBlockreaderlocal, true);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();
cluster.WaitActive();
DistributedFileSystem dfs = cluster.GetFileSystem();
Path path = new Path("/testBlockReaderLocalLegacy");
DFSTestUtil.CreateFile(dfs, path, 10, ReplFactor, 0);
DFSTestUtil.WaitReplication(dfs, path, ReplFactor);
ClientDatanodeProtocol proxy;
Org.Apache.Hadoop.Security.Token.Token <BlockTokenIdentifier> token;
ExtendedBlock originalBlock;
long originalGS;
{
LocatedBlock lb = cluster.GetNameNode().GetRpcServer().GetBlockLocations(path.ToString
(), 0, 1).Get(0);
proxy = DFSUtil.CreateClientDatanodeProtocolProxy(lb.GetLocations()[0], conf, 60000
, false);
token = lb.GetBlockToken();
// get block and generation stamp
ExtendedBlock blk = new ExtendedBlock(lb.GetBlock());
originalBlock = new ExtendedBlock(blk);
originalGS = originalBlock.GetGenerationStamp();
// test getBlockLocalPathInfo
BlockLocalPathInfo info = proxy.GetBlockLocalPathInfo(blk, token);
NUnit.Framework.Assert.AreEqual(originalGS, info.GetBlock().GetGenerationStamp());
}
{
// append one byte
FSDataOutputStream @out = dfs.Append(path);
@out.Write(1);
@out.Close();
}
{
// get new generation stamp
LocatedBlock lb = cluster.GetNameNode().GetRpcServer().GetBlockLocations(path.ToString
(), 0, 1).Get(0);
long newGS = lb.GetBlock().GetGenerationStamp();
NUnit.Framework.Assert.IsTrue(newGS > originalGS);
// getBlockLocalPathInfo using the original block.
NUnit.Framework.Assert.AreEqual(originalGS, originalBlock.GetGenerationStamp());
BlockLocalPathInfo info = proxy.GetBlockLocalPathInfo(originalBlock, token);
NUnit.Framework.Assert.AreEqual(newGS, info.GetBlock().GetGenerationStamp());
}
cluster.Shutdown();
}
private void Compare(LocatedBlock expected, LocatedBlock actual)
{
NUnit.Framework.Assert.AreEqual(expected.GetBlock(), actual.GetBlock());
Compare(expected.GetBlockToken(), actual.GetBlockToken());
NUnit.Framework.Assert.AreEqual(expected.GetStartOffset(), actual.GetStartOffset(
));
NUnit.Framework.Assert.AreEqual(expected.IsCorrupt(), actual.IsCorrupt());
DatanodeInfo[] ei = expected.GetLocations();
DatanodeInfo[] ai = actual.GetLocations();
NUnit.Framework.Assert.AreEqual(ei.Length, ai.Length);
for (int i = 0; i < ei.Length; i++)
{
Compare(ei[i], ai[i]);
}
}
/// <summary>Convert a LocatedBlock to a Json map.</summary>
/// <exception cref="System.IO.IOException"/>
private static IDictionary <string, object> ToJsonMap(LocatedBlock locatedblock)
{
if (locatedblock == null)
{
return(null);
}
IDictionary <string, object> m = new SortedDictionary <string, object>();
m["blockToken"] = ToJsonMap(locatedblock.GetBlockToken());
m["isCorrupt"] = locatedblock.IsCorrupt();
m["startOffset"] = locatedblock.GetStartOffset();
m["block"] = ToJsonMap(locatedblock.GetBlock());
m["locations"] = ToJsonArray(locatedblock.GetLocations());
m["cachedLocations"] = ToJsonArray(locatedblock.GetCachedLocations());
return(m);
}
/// <summary>Get a BlockReader for the given block.</summary>
/// <exception cref="System.IO.IOException"/>
public static BlockReader GetBlockReader(MiniDFSCluster cluster, LocatedBlock testBlock
, int offset, int lenToRead)
{
IPEndPoint targetAddr = null;
ExtendedBlock block = testBlock.GetBlock();
DatanodeInfo[] nodes = testBlock.GetLocations();
targetAddr = NetUtils.CreateSocketAddr(nodes[0].GetXferAddr());
DistributedFileSystem fs = cluster.GetFileSystem();
return(new BlockReaderFactory(fs.GetClient().GetConf()).SetInetSocketAddress(targetAddr
).SetBlock(block).SetFileName(targetAddr.ToString() + ":" + block.GetBlockId()).
SetBlockToken(testBlock.GetBlockToken()).SetStartOffset(offset).SetLength(lenToRead
).SetVerifyChecksum(true).SetClientName("BlockReaderTestUtil").SetDatanodeInfo(nodes
[0]).SetClientCacheContext(ClientContext.GetFromConf(fs.GetConf())).SetCachingStrategy
(CachingStrategy.NewDefaultStrategy()).SetConfiguration(fs.GetConf()).SetAllowShortCircuitLocalReads
(true).SetRemotePeerFactory(new _RemotePeerFactory_196(fs)).Build());
}
/// <summary>try to access a block on a data node.</summary>
/// <remarks>try to access a block on a data node. If fails - throws exception</remarks>
/// <param name="datanode"/>
/// <param name="lblock"/>
/// <exception cref="System.IO.IOException"/>
private void AccessBlock(DatanodeInfo datanode, LocatedBlock lblock)
{
IPEndPoint targetAddr = null;
ExtendedBlock block = lblock.GetBlock();
targetAddr = NetUtils.CreateSocketAddr(datanode.GetXferAddr());
BlockReader blockReader = new BlockReaderFactory(new DFSClient.Conf(conf)).SetInetSocketAddress
(targetAddr).SetBlock(block).SetFileName(BlockReaderFactory.GetFileName(targetAddr
, "test-blockpoolid", block.GetBlockId())).SetBlockToken(lblock.GetBlockToken())
.SetStartOffset(0).SetLength(-1).SetVerifyChecksum(true).SetClientName("TestDataNodeVolumeFailure"
).SetDatanodeInfo(datanode).SetCachingStrategy(CachingStrategy.NewDefaultStrategy
()).SetClientCacheContext(ClientContext.GetFromConf(conf)).SetConfiguration(conf
).SetRemotePeerFactory(new _RemotePeerFactory_422(this)).Build();
blockReader.Close();
}
public virtual void TestRead()
{
MiniDFSCluster cluster = null;
int numDataNodes = 2;
Configuration conf = GetConf(numDataNodes);
try
{
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(numDataNodes).Build();
cluster.WaitActive();
NUnit.Framework.Assert.AreEqual(numDataNodes, cluster.GetDataNodes().Count);
NameNode nn = cluster.GetNameNode();
NamenodeProtocols nnProto = nn.GetRpcServer();
BlockManager bm = nn.GetNamesystem().GetBlockManager();
BlockTokenSecretManager sm = bm.GetBlockTokenSecretManager();
// set a short token lifetime (1 second) initially
SecurityTestUtil.SetBlockTokenLifetime(sm, 1000L);
Path fileToRead = new Path(FileToRead);
FileSystem fs = cluster.GetFileSystem();
CreateFile(fs, fileToRead);
/*
* setup for testing expiration handling of cached tokens
*/
// read using blockSeekTo(). Acquired tokens are cached in in1
FSDataInputStream in1 = fs.Open(fileToRead);
NUnit.Framework.Assert.IsTrue(CheckFile1(in1));
// read using blockSeekTo(). Acquired tokens are cached in in2
FSDataInputStream in2 = fs.Open(fileToRead);
NUnit.Framework.Assert.IsTrue(CheckFile1(in2));
// read using fetchBlockByteRange(). Acquired tokens are cached in in3
FSDataInputStream in3 = fs.Open(fileToRead);
NUnit.Framework.Assert.IsTrue(CheckFile2(in3));
/*
* testing READ interface on DN using a BlockReader
*/
DFSClient client = null;
try
{
client = new DFSClient(new IPEndPoint("localhost", cluster.GetNameNodePort()), conf
);
}
finally
{
if (client != null)
{
client.Close();
}
}
IList <LocatedBlock> locatedBlocks = nnProto.GetBlockLocations(FileToRead, 0, FileSize
).GetLocatedBlocks();
LocatedBlock lblock = locatedBlocks[0];
// first block
Org.Apache.Hadoop.Security.Token.Token <BlockTokenIdentifier> myToken = lblock.GetBlockToken
();
// verify token is not expired
NUnit.Framework.Assert.IsFalse(SecurityTestUtil.IsBlockTokenExpired(myToken));
// read with valid token, should succeed
TryRead(conf, lblock, true);
/*
* wait till myToken and all cached tokens in in1, in2 and in3 expire
*/
while (!SecurityTestUtil.IsBlockTokenExpired(myToken))
{
try
{
Sharpen.Thread.Sleep(10);
}
catch (Exception)
{
}
}
/*
* continue testing READ interface on DN using a BlockReader
*/
// verify token is expired
NUnit.Framework.Assert.IsTrue(SecurityTestUtil.IsBlockTokenExpired(myToken));
// read should fail
TryRead(conf, lblock, false);
// use a valid new token
lblock.SetBlockToken(sm.GenerateToken(lblock.GetBlock(), EnumSet.Of(BlockTokenSecretManager.AccessMode
.Read)));
// read should succeed
TryRead(conf, lblock, true);
// use a token with wrong blockID
ExtendedBlock wrongBlock = new ExtendedBlock(lblock.GetBlock().GetBlockPoolId(),
lblock.GetBlock().GetBlockId() + 1);
lblock.SetBlockToken(sm.GenerateToken(wrongBlock, EnumSet.Of(BlockTokenSecretManager.AccessMode
.Read)));
// read should fail
TryRead(conf, lblock, false);
// use a token with wrong access modes
lblock.SetBlockToken(sm.GenerateToken(lblock.GetBlock(), EnumSet.Of(BlockTokenSecretManager.AccessMode
.Write, BlockTokenSecretManager.AccessMode.Copy, BlockTokenSecretManager.AccessMode
.Replace)));
// read should fail
TryRead(conf, lblock, false);
// set a long token lifetime for future tokens
SecurityTestUtil.SetBlockTokenLifetime(sm, 600 * 1000L);
/*
* testing that when cached tokens are expired, DFSClient will re-fetch
* tokens transparently for READ.
*/
// confirm all tokens cached in in1 are expired by now
IList <LocatedBlock> lblocks = DFSTestUtil.GetAllBlocks(in1);
foreach (LocatedBlock blk in lblocks)
{
NUnit.Framework.Assert.IsTrue(SecurityTestUtil.IsBlockTokenExpired(blk.GetBlockToken
()));
}
// verify blockSeekTo() is able to re-fetch token transparently
in1.Seek(0);
NUnit.Framework.Assert.IsTrue(CheckFile1(in1));
// confirm all tokens cached in in2 are expired by now
IList <LocatedBlock> lblocks2 = DFSTestUtil.GetAllBlocks(in2);
foreach (LocatedBlock blk_1 in lblocks2)
{
NUnit.Framework.Assert.IsTrue(SecurityTestUtil.IsBlockTokenExpired(blk_1.GetBlockToken
()));
}
// verify blockSeekTo() is able to re-fetch token transparently (testing
// via another interface method)
NUnit.Framework.Assert.IsTrue(in2.SeekToNewSource(0));
NUnit.Framework.Assert.IsTrue(CheckFile1(in2));
// confirm all tokens cached in in3 are expired by now
IList <LocatedBlock> lblocks3 = DFSTestUtil.GetAllBlocks(in3);
foreach (LocatedBlock blk_2 in lblocks3)
{
NUnit.Framework.Assert.IsTrue(SecurityTestUtil.IsBlockTokenExpired(blk_2.GetBlockToken
()));
}
// verify fetchBlockByteRange() is able to re-fetch token transparently
NUnit.Framework.Assert.IsTrue(CheckFile2(in3));
/*
* testing that after datanodes are restarted on the same ports, cached
* tokens should still work and there is no need to fetch new tokens from
* namenode. This test should run while namenode is down (to make sure no
* new tokens can be fetched from namenode).
*/
// restart datanodes on the same ports that they currently use
NUnit.Framework.Assert.IsTrue(cluster.RestartDataNodes(true));
cluster.WaitActive();
NUnit.Framework.Assert.AreEqual(numDataNodes, cluster.GetDataNodes().Count);
cluster.ShutdownNameNode(0);
// confirm tokens cached in in1 are still valid
lblocks = DFSTestUtil.GetAllBlocks(in1);
foreach (LocatedBlock blk_3 in lblocks)
{
NUnit.Framework.Assert.IsFalse(SecurityTestUtil.IsBlockTokenExpired(blk_3.GetBlockToken
()));
}
// verify blockSeekTo() still works (forced to use cached tokens)
in1.Seek(0);
NUnit.Framework.Assert.IsTrue(CheckFile1(in1));
// confirm tokens cached in in2 are still valid
lblocks2 = DFSTestUtil.GetAllBlocks(in2);
foreach (LocatedBlock blk_4 in lblocks2)
{
NUnit.Framework.Assert.IsFalse(SecurityTestUtil.IsBlockTokenExpired(blk_4.GetBlockToken
()));
}
// verify blockSeekTo() still works (forced to use cached tokens)
in2.SeekToNewSource(0);
NUnit.Framework.Assert.IsTrue(CheckFile1(in2));
// confirm tokens cached in in3 are still valid
lblocks3 = DFSTestUtil.GetAllBlocks(in3);
foreach (LocatedBlock blk_5 in lblocks3)
{
NUnit.Framework.Assert.IsFalse(SecurityTestUtil.IsBlockTokenExpired(blk_5.GetBlockToken
()));
}
// verify fetchBlockByteRange() still works (forced to use cached tokens)
NUnit.Framework.Assert.IsTrue(CheckFile2(in3));
/*
* testing that when namenode is restarted, cached tokens should still
* work and there is no need to fetch new tokens from namenode. Like the
* previous test, this test should also run while namenode is down. The
* setup for this test depends on the previous test.
*/
// restart the namenode and then shut it down for test
cluster.RestartNameNode(0);
cluster.ShutdownNameNode(0);
// verify blockSeekTo() still works (forced to use cached tokens)
in1.Seek(0);
NUnit.Framework.Assert.IsTrue(CheckFile1(in1));
// verify again blockSeekTo() still works (forced to use cached tokens)
in2.SeekToNewSource(0);
NUnit.Framework.Assert.IsTrue(CheckFile1(in2));
// verify fetchBlockByteRange() still works (forced to use cached tokens)
NUnit.Framework.Assert.IsTrue(CheckFile2(in3));
/*
* testing that after both namenode and datanodes got restarted (namenode
* first, followed by datanodes), DFSClient can't access DN without
* re-fetching tokens and is able to re-fetch tokens transparently. The
* setup of this test depends on the previous test.
*/
// restore the cluster and restart the datanodes for test
cluster.RestartNameNode(0);
NUnit.Framework.Assert.IsTrue(cluster.RestartDataNodes(true));
cluster.WaitActive();
NUnit.Framework.Assert.AreEqual(numDataNodes, cluster.GetDataNodes().Count);
// shutdown namenode so that DFSClient can't get new tokens from namenode
cluster.ShutdownNameNode(0);
// verify blockSeekTo() fails (cached tokens become invalid)
in1.Seek(0);
NUnit.Framework.Assert.IsFalse(CheckFile1(in1));
// verify fetchBlockByteRange() fails (cached tokens become invalid)
NUnit.Framework.Assert.IsFalse(CheckFile2(in3));
// restart the namenode to allow DFSClient to re-fetch tokens
cluster.RestartNameNode(0);
// verify blockSeekTo() works again (by transparently re-fetching
// tokens from namenode)
in1.Seek(0);
NUnit.Framework.Assert.IsTrue(CheckFile1(in1));
in2.SeekToNewSource(0);
NUnit.Framework.Assert.IsTrue(CheckFile1(in2));
// verify fetchBlockByteRange() works again (by transparently
// re-fetching tokens from namenode)
NUnit.Framework.Assert.IsTrue(CheckFile2(in3));
/*
* testing that when datanodes are restarted on different ports, DFSClient
* is able to re-fetch tokens transparently to connect to them
*/
// restart datanodes on newly assigned ports
NUnit.Framework.Assert.IsTrue(cluster.RestartDataNodes(false));
cluster.WaitActive();
NUnit.Framework.Assert.AreEqual(numDataNodes, cluster.GetDataNodes().Count);
// verify blockSeekTo() is able to re-fetch token transparently
in1.Seek(0);
NUnit.Framework.Assert.IsTrue(CheckFile1(in1));
// verify blockSeekTo() is able to re-fetch token transparently
in2.SeekToNewSource(0);
NUnit.Framework.Assert.IsTrue(CheckFile1(in2));
// verify fetchBlockByteRange() is able to re-fetch token transparently
NUnit.Framework.Assert.IsTrue(CheckFile2(in3));
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
// try reading a block using a BlockReader directly
private static void TryRead(Configuration conf, LocatedBlock lblock, bool shouldSucceed
)
{
IPEndPoint targetAddr = null;
IOException ioe = null;
BlockReader blockReader = null;
ExtendedBlock block = lblock.GetBlock();
try
{
DatanodeInfo[] nodes = lblock.GetLocations();
targetAddr = NetUtils.CreateSocketAddr(nodes[0].GetXferAddr());
blockReader = new BlockReaderFactory(new DFSClient.Conf(conf)).SetFileName(BlockReaderFactory
.GetFileName(targetAddr, "test-blockpoolid", block.GetBlockId())).SetBlock(block
).SetBlockToken(lblock.GetBlockToken()).SetInetSocketAddress(targetAddr).SetStartOffset
(0).SetLength(-1).SetVerifyChecksum(true).SetClientName("TestBlockTokenWithDFS")
.SetDatanodeInfo(nodes[0]).SetCachingStrategy(CachingStrategy.NewDefaultStrategy
()).SetClientCacheContext(ClientContext.GetFromConf(conf)).SetConfiguration(conf
).SetRemotePeerFactory(new _RemotePeerFactory_162(conf)).Build();
}
catch (IOException ex)
{
ioe = ex;
}
finally
{
if (blockReader != null)
{
try
{
blockReader.Close();
}
catch (IOException e)
{
throw new RuntimeException(e);
}
}
}
if (shouldSucceed)
{
NUnit.Framework.Assert.IsNotNull("OP_READ_BLOCK: access token is invalid, " + "when it is expected to be valid"
, blockReader);
}
else
{
NUnit.Framework.Assert.IsNotNull("OP_READ_BLOCK: access token is valid, " + "when it is expected to be invalid"
, ioe);
NUnit.Framework.Assert.IsTrue("OP_READ_BLOCK failed due to reasons other than access token: "
, ioe is InvalidBlockTokenException);
}
}