/// <exception cref="System.Exception"/> public virtual HdfsBlocksMetadata Call() { HdfsBlocksMetadata metadata = null; // Create the RPC proxy and make the RPC ClientDatanodeProtocol cdp = null; TraceScope scope = Trace.StartSpan("getHdfsBlocksMetadata", parentSpan); try { cdp = DFSUtil.CreateClientDatanodeProtocolProxy(datanode, configuration, timeout, connectToDnViaHostname); metadata = cdp.GetHdfsBlocksMetadata(poolId, blockIds, dnTokens); } catch (IOException e) { // Bubble this up to the caller, handle with the Future throw; } finally { scope.Close(); if (cdp != null) { RPC.StopProxy(cdp); } } return(metadata); }
public virtual void TestBlockTokenRpcLeak() { Configuration conf = new Configuration(); conf.Set(CommonConfigurationKeysPublic.HadoopSecurityAuthentication, "kerberos"); UserGroupInformation.SetConfiguration(conf); Assume.AssumeTrue(FdDir.Exists()); BlockTokenSecretManager sm = new BlockTokenSecretManager(blockKeyUpdateInterval, blockTokenLifetime, 0, "fake-pool", null); Org.Apache.Hadoop.Security.Token.Token <BlockTokenIdentifier> token = sm.GenerateToken (block3, EnumSet.AllOf <BlockTokenSecretManager.AccessMode>()); Server server = CreateMockDatanode(sm, token, conf); server.Start(); IPEndPoint addr = NetUtils.GetConnectAddress(server); DatanodeID fakeDnId = DFSTestUtil.GetLocalDatanodeID(addr.Port); ExtendedBlock b = new ExtendedBlock("fake-pool", new Org.Apache.Hadoop.Hdfs.Protocol.Block (12345L)); LocatedBlock fakeBlock = new LocatedBlock(b, new DatanodeInfo[0]); fakeBlock.SetBlockToken(token); // Create another RPC proxy with the same configuration - this will never // attempt to connect anywhere -- but it causes the refcount on the // RPC "Client" object to stay above 0 such that RPC.stopProxy doesn't // actually close the TCP connections to the real target DN. ClientDatanodeProtocol proxyToNoWhere = RPC.GetProxy <ClientDatanodeProtocol>(ClientDatanodeProtocol .versionID, new IPEndPoint("1.1.1.1", 1), UserGroupInformation.CreateRemoteUser( "junk"), conf, NetUtils.GetDefaultSocketFactory(conf)); ClientDatanodeProtocol proxy = null; int fdsAtStart = CountOpenFileDescriptors(); try { long endTime = Time.Now() + 3000; while (Time.Now() < endTime) { proxy = DFSUtil.CreateClientDatanodeProtocolProxy(fakeDnId, conf, 1000, false, fakeBlock ); NUnit.Framework.Assert.AreEqual(block3.GetBlockId(), proxy.GetReplicaVisibleLength (block3)); if (proxy != null) { RPC.StopProxy(proxy); } Log.Info("Num open fds:" + CountOpenFileDescriptors()); } int fdsAtEnd = CountOpenFileDescriptors(); if (fdsAtEnd - fdsAtStart > 50) { NUnit.Framework.Assert.Fail("Leaked " + (fdsAtEnd - fdsAtStart) + " fds!"); } } finally { server.Stop(); } RPC.StopProxy(proxyToNoWhere); }
private void ResetDatanodeProxy() { lock (this) { if (null != proxy) { RPC.StopProxy(proxy); proxy = null; } } }
/// <exception cref="System.IO.IOException"/> private ClientDatanodeProtocol GetDatanodeProxy(UserGroupInformation ugi, DatanodeInfo node, Configuration conf, int socketTimeout, bool connectToDnViaHostname) { lock (this) { if (proxy == null) { try { proxy = ugi.DoAs(new _PrivilegedExceptionAction_107(node, conf, socketTimeout, connectToDnViaHostname )); } catch (Exception e) { Log.Warn("encountered exception ", e); } } return(proxy); } }
/// <exception cref="System.IO.IOException"/> public virtual void TestDeprecatedGetBlockLocalPathInfoRpc() { Configuration conf = new Configuration(); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Format( true).Build(); cluster.WaitActive(); FileSystem fs = cluster.GetFileSystem(); try { DFSTestUtil.CreateFile(fs, new Path("/tmp/x"), 16, (short)1, 23); LocatedBlocks lb = cluster.GetNameNode().GetRpcServer().GetBlockLocations("/tmp/x" , 0, 16); // Create a new block object, because the block inside LocatedBlock at // namenode is of type BlockInfo. ExtendedBlock blk = new ExtendedBlock(lb.Get(0).GetBlock()); Org.Apache.Hadoop.Security.Token.Token <BlockTokenIdentifier> token = lb.Get(0).GetBlockToken (); DatanodeInfo dnInfo = lb.Get(0).GetLocations()[0]; ClientDatanodeProtocol proxy = DFSUtil.CreateClientDatanodeProtocolProxy(dnInfo, conf, 60000, false); try { proxy.GetBlockLocalPathInfo(blk, token); NUnit.Framework.Assert.Fail("The call should have failed as this user " + " is not allowed to call getBlockLocalPathInfo" ); } catch (IOException ex) { NUnit.Framework.Assert.IsTrue(ex.Message.Contains("not allowed to call getBlockLocalPathInfo" )); } } finally { fs.Close(); cluster.Shutdown(); } }
/// <exception cref="System.IO.IOException"/> private static BlockLocalPathInfo GetBlockPathInfo(UserGroupInformation ugi, ExtendedBlock blk, DatanodeInfo node, Configuration conf, int timeout, Org.Apache.Hadoop.Security.Token.Token <BlockTokenIdentifier> token, bool connectToDnViaHostname, StorageType storageType ) { BlockReaderLocalLegacy.LocalDatanodeInfo localDatanodeInfo = GetLocalDatanodeInfo (node.GetIpcPort()); BlockLocalPathInfo pathinfo = null; ClientDatanodeProtocol proxy = localDatanodeInfo.GetDatanodeProxy(ugi, node, conf , timeout, connectToDnViaHostname); try { // make RPC to local datanode to find local pathnames of blocks pathinfo = proxy.GetBlockLocalPathInfo(blk, token); // We cannot cache the path information for a replica on transient storage. // If the replica gets evicted, then it moves to a different path. Then, // our next attempt to read from the cached path would fail to find the // file. Additionally, the failure would cause us to disable legacy // short-circuit read for all subsequent use in the ClientContext. Unlike // the newer short-circuit read implementation, we have no communication // channel for the DataNode to notify the client that the path has been // invalidated. Therefore, our only option is to skip caching. if (pathinfo != null && !storageType.IsTransient()) { if (Log.IsDebugEnabled()) { Log.Debug("Cached location of block " + blk + " as " + pathinfo); } localDatanodeInfo.SetBlockLocalPathInfo(blk, pathinfo); } } catch (IOException e) { localDatanodeInfo.ResetDatanodeProxy(); // Reset proxy on error throw; } return(pathinfo); }
public virtual void TestBlockTokenRpc() { Configuration conf = new Configuration(); conf.Set(CommonConfigurationKeysPublic.HadoopSecurityAuthentication, "kerberos"); UserGroupInformation.SetConfiguration(conf); BlockTokenSecretManager sm = new BlockTokenSecretManager(blockKeyUpdateInterval, blockTokenLifetime, 0, "fake-pool", null); Org.Apache.Hadoop.Security.Token.Token <BlockTokenIdentifier> token = sm.GenerateToken (block3, EnumSet.AllOf <BlockTokenSecretManager.AccessMode>()); Server server = CreateMockDatanode(sm, token, conf); server.Start(); IPEndPoint addr = NetUtils.GetConnectAddress(server); UserGroupInformation ticket = UserGroupInformation.CreateRemoteUser(block3.ToString ()); ticket.AddToken(token); ClientDatanodeProtocol proxy = null; try { proxy = DFSUtil.CreateClientDatanodeProtocolProxy(addr, ticket, conf, NetUtils.GetDefaultSocketFactory (conf)); NUnit.Framework.Assert.AreEqual(block3.GetBlockId(), proxy.GetReplicaVisibleLength (block3)); } finally { server.Stop(); if (proxy != null) { RPC.StopProxy(proxy); } } }
public ClientDatanodeProtocolServerSideTranslatorPB(ClientDatanodeProtocol impl) { this.impl = impl; }