/// <exception cref="System.Exception"/>
public virtual HdfsBlocksMetadata Call()
{
HdfsBlocksMetadata metadata = null;
// Create the RPC proxy and make the RPC
ClientDatanodeProtocol cdp = null;
TraceScope scope = Trace.StartSpan("getHdfsBlocksMetadata", parentSpan);
try
{
cdp = DFSUtil.CreateClientDatanodeProtocolProxy(datanode, configuration, timeout,
connectToDnViaHostname);
metadata = cdp.GetHdfsBlocksMetadata(poolId, blockIds, dnTokens);
}
catch (IOException e)
{
// Bubble this up to the caller, handle with the Future
throw;
}
finally
{
scope.Close();
if (cdp != null)
{
RPC.StopProxy(cdp);
}
}
return(metadata);
}
public virtual void TestBlockTokenRpcLeak()
{
Configuration conf = new Configuration();
conf.Set(CommonConfigurationKeysPublic.HadoopSecurityAuthentication, "kerberos");
UserGroupInformation.SetConfiguration(conf);
Assume.AssumeTrue(FdDir.Exists());
BlockTokenSecretManager sm = new BlockTokenSecretManager(blockKeyUpdateInterval,
blockTokenLifetime, 0, "fake-pool", null);
Org.Apache.Hadoop.Security.Token.Token <BlockTokenIdentifier> token = sm.GenerateToken
(block3, EnumSet.AllOf <BlockTokenSecretManager.AccessMode>());
Server server = CreateMockDatanode(sm, token, conf);
server.Start();
IPEndPoint addr = NetUtils.GetConnectAddress(server);
DatanodeID fakeDnId = DFSTestUtil.GetLocalDatanodeID(addr.Port);
ExtendedBlock b = new ExtendedBlock("fake-pool", new Org.Apache.Hadoop.Hdfs.Protocol.Block
(12345L));
LocatedBlock fakeBlock = new LocatedBlock(b, new DatanodeInfo[0]);
fakeBlock.SetBlockToken(token);
// Create another RPC proxy with the same configuration - this will never
// attempt to connect anywhere -- but it causes the refcount on the
// RPC "Client" object to stay above 0 such that RPC.stopProxy doesn't
// actually close the TCP connections to the real target DN.
ClientDatanodeProtocol proxyToNoWhere = RPC.GetProxy <ClientDatanodeProtocol>(ClientDatanodeProtocol
.versionID, new IPEndPoint("1.1.1.1", 1), UserGroupInformation.CreateRemoteUser(
"junk"), conf, NetUtils.GetDefaultSocketFactory(conf));
ClientDatanodeProtocol proxy = null;
int fdsAtStart = CountOpenFileDescriptors();
try
{
long endTime = Time.Now() + 3000;
while (Time.Now() < endTime)
{
proxy = DFSUtil.CreateClientDatanodeProtocolProxy(fakeDnId, conf, 1000, false, fakeBlock
);
NUnit.Framework.Assert.AreEqual(block3.GetBlockId(), proxy.GetReplicaVisibleLength
(block3));
if (proxy != null)
{
RPC.StopProxy(proxy);
}
Log.Info("Num open fds:" + CountOpenFileDescriptors());
}
int fdsAtEnd = CountOpenFileDescriptors();
if (fdsAtEnd - fdsAtStart > 50)
{
NUnit.Framework.Assert.Fail("Leaked " + (fdsAtEnd - fdsAtStart) + " fds!");
}
}
finally
{
server.Stop();
}
RPC.StopProxy(proxyToNoWhere);
}
/// <exception cref="System.Exception"/>
public virtual void TestBlockReaderLocalLegacyWithAppend()
{
short ReplFactor = 1;
HdfsConfiguration conf = GetConfiguration(null);
conf.SetBoolean(DFSConfigKeys.DfsClientUseLegacyBlockreaderlocal, true);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();
cluster.WaitActive();
DistributedFileSystem dfs = cluster.GetFileSystem();
Path path = new Path("/testBlockReaderLocalLegacy");
DFSTestUtil.CreateFile(dfs, path, 10, ReplFactor, 0);
DFSTestUtil.WaitReplication(dfs, path, ReplFactor);
ClientDatanodeProtocol proxy;
Org.Apache.Hadoop.Security.Token.Token <BlockTokenIdentifier> token;
ExtendedBlock originalBlock;
long originalGS;
{
LocatedBlock lb = cluster.GetNameNode().GetRpcServer().GetBlockLocations(path.ToString
(), 0, 1).Get(0);
proxy = DFSUtil.CreateClientDatanodeProtocolProxy(lb.GetLocations()[0], conf, 60000
, false);
token = lb.GetBlockToken();
// get block and generation stamp
ExtendedBlock blk = new ExtendedBlock(lb.GetBlock());
originalBlock = new ExtendedBlock(blk);
originalGS = originalBlock.GetGenerationStamp();
// test getBlockLocalPathInfo
BlockLocalPathInfo info = proxy.GetBlockLocalPathInfo(blk, token);
NUnit.Framework.Assert.AreEqual(originalGS, info.GetBlock().GetGenerationStamp());
}
{
// append one byte
FSDataOutputStream @out = dfs.Append(path);
@out.Write(1);
@out.Close();
}
{
// get new generation stamp
LocatedBlock lb = cluster.GetNameNode().GetRpcServer().GetBlockLocations(path.ToString
(), 0, 1).Get(0);
long newGS = lb.GetBlock().GetGenerationStamp();
NUnit.Framework.Assert.IsTrue(newGS > originalGS);
// getBlockLocalPathInfo using the original block.
NUnit.Framework.Assert.AreEqual(originalGS, originalBlock.GetGenerationStamp());
BlockLocalPathInfo info = proxy.GetBlockLocalPathInfo(originalBlock, token);
NUnit.Framework.Assert.AreEqual(newGS, info.GetBlock().GetGenerationStamp());
}
cluster.Shutdown();
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestDeprecatedGetBlockLocalPathInfoRpc()
{
Configuration conf = new Configuration();
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Format(
true).Build();
cluster.WaitActive();
FileSystem fs = cluster.GetFileSystem();
try
{
DFSTestUtil.CreateFile(fs, new Path("/tmp/x"), 16, (short)1, 23);
LocatedBlocks lb = cluster.GetNameNode().GetRpcServer().GetBlockLocations("/tmp/x"
, 0, 16);
// Create a new block object, because the block inside LocatedBlock at
// namenode is of type BlockInfo.
ExtendedBlock blk = new ExtendedBlock(lb.Get(0).GetBlock());
Org.Apache.Hadoop.Security.Token.Token <BlockTokenIdentifier> token = lb.Get(0).GetBlockToken
();
DatanodeInfo dnInfo = lb.Get(0).GetLocations()[0];
ClientDatanodeProtocol proxy = DFSUtil.CreateClientDatanodeProtocolProxy(dnInfo,
conf, 60000, false);
try
{
proxy.GetBlockLocalPathInfo(blk, token);
NUnit.Framework.Assert.Fail("The call should have failed as this user " + " is not allowed to call getBlockLocalPathInfo"
);
}
catch (IOException ex)
{
NUnit.Framework.Assert.IsTrue(ex.Message.Contains("not allowed to call getBlockLocalPathInfo"
));
}
}
finally
{
fs.Close();
cluster.Shutdown();
}
}
public virtual void TestBlockTokenRpc()
{
Configuration conf = new Configuration();
conf.Set(CommonConfigurationKeysPublic.HadoopSecurityAuthentication, "kerberos");
UserGroupInformation.SetConfiguration(conf);
BlockTokenSecretManager sm = new BlockTokenSecretManager(blockKeyUpdateInterval,
blockTokenLifetime, 0, "fake-pool", null);
Org.Apache.Hadoop.Security.Token.Token <BlockTokenIdentifier> token = sm.GenerateToken
(block3, EnumSet.AllOf <BlockTokenSecretManager.AccessMode>());
Server server = CreateMockDatanode(sm, token, conf);
server.Start();
IPEndPoint addr = NetUtils.GetConnectAddress(server);
UserGroupInformation ticket = UserGroupInformation.CreateRemoteUser(block3.ToString
());
ticket.AddToken(token);
ClientDatanodeProtocol proxy = null;
try
{
proxy = DFSUtil.CreateClientDatanodeProtocolProxy(addr, ticket, conf, NetUtils.GetDefaultSocketFactory
(conf));
NUnit.Framework.Assert.AreEqual(block3.GetBlockId(), proxy.GetReplicaVisibleLength
(block3));
}
finally
{
server.Stop();
if (proxy != null)
{
RPC.StopProxy(proxy);
}
}
}
/// <exception cref="System.Exception"/>
public ClientDatanodeProtocol Run()
{
return(DFSUtil.CreateClientDatanodeProtocolProxy(node, conf, socketTimeout, connectToDnViaHostname
));
}
