/// <exception cref="System.Exception"/>
public virtual HdfsBlocksMetadata Call()
{
HdfsBlocksMetadata metadata = null;
// Create the RPC proxy and make the RPC
ClientDatanodeProtocol cdp = null;
TraceScope scope = Trace.StartSpan("getHdfsBlocksMetadata", parentSpan);
try
{
cdp = DFSUtil.CreateClientDatanodeProtocolProxy(datanode, configuration, timeout,
connectToDnViaHostname);
metadata = cdp.GetHdfsBlocksMetadata(poolId, blockIds, dnTokens);
}
catch (IOException e)
{
// Bubble this up to the caller, handle with the Future
throw;
}
finally
{
scope.Close();
if (cdp != null)
{
RPC.StopProxy(cdp);
}
}
return(metadata);
}
public virtual void TestBlockTokenRpcLeak()
{
Configuration conf = new Configuration();
conf.Set(CommonConfigurationKeysPublic.HadoopSecurityAuthentication, "kerberos");
UserGroupInformation.SetConfiguration(conf);
Assume.AssumeTrue(FdDir.Exists());
BlockTokenSecretManager sm = new BlockTokenSecretManager(blockKeyUpdateInterval,
blockTokenLifetime, 0, "fake-pool", null);
Org.Apache.Hadoop.Security.Token.Token <BlockTokenIdentifier> token = sm.GenerateToken
(block3, EnumSet.AllOf <BlockTokenSecretManager.AccessMode>());
Server server = CreateMockDatanode(sm, token, conf);
server.Start();
IPEndPoint addr = NetUtils.GetConnectAddress(server);
DatanodeID fakeDnId = DFSTestUtil.GetLocalDatanodeID(addr.Port);
ExtendedBlock b = new ExtendedBlock("fake-pool", new Org.Apache.Hadoop.Hdfs.Protocol.Block
(12345L));
LocatedBlock fakeBlock = new LocatedBlock(b, new DatanodeInfo[0]);
fakeBlock.SetBlockToken(token);
// Create another RPC proxy with the same configuration - this will never
// attempt to connect anywhere -- but it causes the refcount on the
// RPC "Client" object to stay above 0 such that RPC.stopProxy doesn't
// actually close the TCP connections to the real target DN.
ClientDatanodeProtocol proxyToNoWhere = RPC.GetProxy <ClientDatanodeProtocol>(ClientDatanodeProtocol
.versionID, new IPEndPoint("1.1.1.1", 1), UserGroupInformation.CreateRemoteUser(
"junk"), conf, NetUtils.GetDefaultSocketFactory(conf));
ClientDatanodeProtocol proxy = null;
int fdsAtStart = CountOpenFileDescriptors();
try
{
long endTime = Time.Now() + 3000;
while (Time.Now() < endTime)
{
proxy = DFSUtil.CreateClientDatanodeProtocolProxy(fakeDnId, conf, 1000, false, fakeBlock
);
NUnit.Framework.Assert.AreEqual(block3.GetBlockId(), proxy.GetReplicaVisibleLength
(block3));
if (proxy != null)
{
RPC.StopProxy(proxy);
}
Log.Info("Num open fds:" + CountOpenFileDescriptors());
}
int fdsAtEnd = CountOpenFileDescriptors();
if (fdsAtEnd - fdsAtStart > 50)
{
NUnit.Framework.Assert.Fail("Leaked " + (fdsAtEnd - fdsAtStart) + " fds!");
}
}
finally
{
server.Stop();
}
RPC.StopProxy(proxyToNoWhere);
}
private void ResetDatanodeProxy()
{
lock (this)
{
if (null != proxy)
{
RPC.StopProxy(proxy);
proxy = null;
}
}
}
/// <exception cref="System.IO.IOException"/>
private ClientDatanodeProtocol GetDatanodeProxy(UserGroupInformation ugi, DatanodeInfo
node, Configuration conf, int socketTimeout, bool connectToDnViaHostname)
{
lock (this)
{
if (proxy == null)
{
try
{
proxy = ugi.DoAs(new _PrivilegedExceptionAction_107(node, conf, socketTimeout, connectToDnViaHostname
));
}
catch (Exception e)
{
Log.Warn("encountered exception ", e);
}
}
return(proxy);
}
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestDeprecatedGetBlockLocalPathInfoRpc()
{
Configuration conf = new Configuration();
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Format(
true).Build();
cluster.WaitActive();
FileSystem fs = cluster.GetFileSystem();
try
{
DFSTestUtil.CreateFile(fs, new Path("/tmp/x"), 16, (short)1, 23);
LocatedBlocks lb = cluster.GetNameNode().GetRpcServer().GetBlockLocations("/tmp/x"
, 0, 16);
// Create a new block object, because the block inside LocatedBlock at
// namenode is of type BlockInfo.
ExtendedBlock blk = new ExtendedBlock(lb.Get(0).GetBlock());
Org.Apache.Hadoop.Security.Token.Token <BlockTokenIdentifier> token = lb.Get(0).GetBlockToken
();
DatanodeInfo dnInfo = lb.Get(0).GetLocations()[0];
ClientDatanodeProtocol proxy = DFSUtil.CreateClientDatanodeProtocolProxy(dnInfo,
conf, 60000, false);
try
{
proxy.GetBlockLocalPathInfo(blk, token);
NUnit.Framework.Assert.Fail("The call should have failed as this user " + " is not allowed to call getBlockLocalPathInfo"
);
}
catch (IOException ex)
{
NUnit.Framework.Assert.IsTrue(ex.Message.Contains("not allowed to call getBlockLocalPathInfo"
));
}
}
finally
{
fs.Close();
cluster.Shutdown();
}
}
/// <exception cref="System.IO.IOException"/>
private static BlockLocalPathInfo GetBlockPathInfo(UserGroupInformation ugi, ExtendedBlock
blk, DatanodeInfo node, Configuration conf, int timeout, Org.Apache.Hadoop.Security.Token.Token
<BlockTokenIdentifier> token, bool connectToDnViaHostname, StorageType storageType
)
{
BlockReaderLocalLegacy.LocalDatanodeInfo localDatanodeInfo = GetLocalDatanodeInfo
(node.GetIpcPort());
BlockLocalPathInfo pathinfo = null;
ClientDatanodeProtocol proxy = localDatanodeInfo.GetDatanodeProxy(ugi, node, conf
, timeout, connectToDnViaHostname);
try
{
// make RPC to local datanode to find local pathnames of blocks
pathinfo = proxy.GetBlockLocalPathInfo(blk, token);
// We cannot cache the path information for a replica on transient storage.
// If the replica gets evicted, then it moves to a different path. Then,
// our next attempt to read from the cached path would fail to find the
// file. Additionally, the failure would cause us to disable legacy
// short-circuit read for all subsequent use in the ClientContext. Unlike
// the newer short-circuit read implementation, we have no communication
// channel for the DataNode to notify the client that the path has been
// invalidated. Therefore, our only option is to skip caching.
if (pathinfo != null && !storageType.IsTransient())
{
if (Log.IsDebugEnabled())
{
Log.Debug("Cached location of block " + blk + " as " + pathinfo);
}
localDatanodeInfo.SetBlockLocalPathInfo(blk, pathinfo);
}
}
catch (IOException e)
{
localDatanodeInfo.ResetDatanodeProxy();
// Reset proxy on error
throw;
}
return(pathinfo);
}
public virtual void TestBlockTokenRpc()
{
Configuration conf = new Configuration();
conf.Set(CommonConfigurationKeysPublic.HadoopSecurityAuthentication, "kerberos");
UserGroupInformation.SetConfiguration(conf);
BlockTokenSecretManager sm = new BlockTokenSecretManager(blockKeyUpdateInterval,
blockTokenLifetime, 0, "fake-pool", null);
Org.Apache.Hadoop.Security.Token.Token <BlockTokenIdentifier> token = sm.GenerateToken
(block3, EnumSet.AllOf <BlockTokenSecretManager.AccessMode>());
Server server = CreateMockDatanode(sm, token, conf);
server.Start();
IPEndPoint addr = NetUtils.GetConnectAddress(server);
UserGroupInformation ticket = UserGroupInformation.CreateRemoteUser(block3.ToString
());
ticket.AddToken(token);
ClientDatanodeProtocol proxy = null;
try
{
proxy = DFSUtil.CreateClientDatanodeProtocolProxy(addr, ticket, conf, NetUtils.GetDefaultSocketFactory
(conf));
NUnit.Framework.Assert.AreEqual(block3.GetBlockId(), proxy.GetReplicaVisibleLength
(block3));
}
finally
{
server.Stop();
if (proxy != null)
{
RPC.StopProxy(proxy);
}
}
}
public ClientDatanodeProtocolServerSideTranslatorPB(ClientDatanodeProtocol impl)
{
this.impl = impl;
}
