/// <exception cref="System.IO.IOException"/> internal static ClientDatanodeProtocolPB CreateClientDatanodeProtocolProxy(DatanodeID datanodeid, Configuration conf, int socketTimeout, bool connectToDnViaHostname, LocatedBlock locatedBlock) { string dnAddr = datanodeid.GetIpcAddr(connectToDnViaHostname); IPEndPoint addr = NetUtils.CreateSocketAddr(dnAddr); if (Log.IsDebugEnabled()) { Log.Debug("Connecting to datanode " + dnAddr + " addr=" + addr); } // Since we're creating a new UserGroupInformation here, we know that no // future RPC proxies will be able to re-use the same connection. And // usages of this proxy tend to be one-off calls. // // This is a temporary fix: callers should really achieve this by using // RPC.stopProxy() on the resulting object, but this is currently not // working in trunk. See the discussion on HDFS-1965. Configuration confWithNoIpcIdle = new Configuration(conf); confWithNoIpcIdle.SetInt(CommonConfigurationKeysPublic.IpcClientConnectionMaxidletimeKey , 0); UserGroupInformation ticket = UserGroupInformation.CreateRemoteUser(locatedBlock. GetBlock().GetLocalBlock().ToString()); ticket.AddToken(locatedBlock.GetBlockToken()); return(CreateClientDatanodeProtocolProxy(addr, ticket, confWithNoIpcIdle, NetUtils .GetDefaultSocketFactory(conf), socketTimeout)); }
/// <summary>DatanodeDescriptor constructor</summary> /// <param name="nodeID">id of the data node</param> public DatanodeDescriptor(DatanodeID nodeID) : base(nodeID) { decommissioningStatus = new DatanodeDescriptor.DecommissioningStatus(this); pendingCached = new DatanodeDescriptor.CachedBlocksList(this, DatanodeDescriptor.CachedBlocksList.Type .PendingCached); cached = new DatanodeDescriptor.CachedBlocksList(this, DatanodeDescriptor.CachedBlocksList.Type .Cached); pendingUncached = new DatanodeDescriptor.CachedBlocksList(this, DatanodeDescriptor.CachedBlocksList.Type .PendingUncached); // isAlive == heartbeats.contains(this) // This is an optimization, because contains takes O(n) time on Arraylist // A system administrator can tune the balancer bandwidth parameter // (dfs.balance.bandwidthPerSec) dynamically by calling // "dfsadmin -setBalanacerBandwidth <newbandwidth>", at which point the // following 'bandwidth' variable gets updated with the new value for each // node. Once the heartbeat command is issued to update the value on the // specified datanode, this value will be set back to 0. /* Variables for maintaining number of blocks scheduled to be written to * this storage. This count is approximate and might be slightly bigger * in case of errors (e.g. datanode does not report if an error occurs * while writing the block). */ //10min // The number of replication work pending before targets are determined // HB processing can use it to tell if it is the first HB since DN restarted UpdateHeartbeatState(StorageReport.EmptyArray, 0L, 0L, 0, 0, null); }
/// <summary> /// Test to verify that InterDatanode RPC timesout as expected when /// the server DN does not respond. /// </summary> /// <exception cref="System.Exception"/> public virtual void TestInterDNProtocolTimeout() { Org.Apache.Hadoop.Ipc.Server server = new TestInterDatanodeProtocol.TestServer(1, true); server.Start(); IPEndPoint addr = NetUtils.GetConnectAddress(server); DatanodeID fakeDnId = DFSTestUtil.GetLocalDatanodeID(addr.Port); DatanodeInfo dInfo = new DatanodeInfo(fakeDnId); InterDatanodeProtocol proxy = null; try { proxy = DataNode.CreateInterDataNodeProtocolProxy(dInfo, conf, 500, false); proxy.InitReplicaRecovery(new BlockRecoveryCommand.RecoveringBlock(new ExtendedBlock ("bpid", 1), null, 100)); NUnit.Framework.Assert.Fail("Expected SocketTimeoutException exception, but did not get." ); } finally { if (proxy != null) { RPC.StopProxy(proxy); } server.Stop(); } }
/// <summary>Create a redirection URL</summary> /// <exception cref="System.IO.IOException"/> private Uri CreateRedirectURL(string path, string encodedPath, HdfsFileStatus status , UserGroupInformation ugi, ClientProtocol nnproxy, HttpServletRequest request, string dt) { string scheme = request.GetScheme(); LocatedBlocks blks = nnproxy.GetBlockLocations(status.GetFullPath(new Path(path)) .ToUri().GetPath(), 0, 1); Configuration conf = NameNodeHttpServer.GetConfFromContext(GetServletContext()); DatanodeID host = PickSrcDatanode(blks, status, conf); string hostname; if (host is DatanodeInfo) { hostname = host.GetHostName(); } else { hostname = host.GetIpAddr(); } int port = "https".Equals(scheme) ? host.GetInfoSecurePort() : host.GetInfoPort(); string dtParam = string.Empty; if (dt != null) { dtParam = JspHelper.GetDelegationTokenUrlParam(dt); } // Add namenode address to the url params NameNode nn = NameNodeHttpServer.GetNameNodeFromContext(GetServletContext()); string addr = nn.GetNameNodeAddressHostPortString(); string addrParam = JspHelper.GetUrlParam(JspHelper.NamenodeAddress, addr); return(new Uri(scheme, hostname, port, "/streamFile" + encodedPath + '?' + "ugi=" + ServletUtil.EncodeQueryValue(ugi.GetShortUserName()) + dtParam + addrParam)); }
private RemoteBlockReader(string file, string bpid, long blockId, DataInputStream @in, DataChecksum checksum, bool verifyChecksum, long startOffset, long firstChunkOffset , long bytesToRead, Peer peer, DatanodeID datanodeID, PeerCache peerCache) : base(new Path("/" + Block.BlockFilePrefix + blockId + ":" + bpid + ":of:" + file ), 1, verifyChecksum, checksum.GetChecksumSize() > 0 ? checksum : null, checksum .GetBytesPerChecksum(), checksum.GetChecksumSize()) { // Path is used only for printing block and file information in debug /*too non path-like?*/ this.isLocal = DFSClient.IsLocalAddress(NetUtils.CreateSocketAddr(datanodeID.GetXferAddr ())); this.peer = peer; this.datanodeID = datanodeID; this.@in = @in; this.checksum = checksum; this.startOffset = Math.Max(startOffset, 0); this.blockId = blockId; // The total number of bytes that we need to transfer from the DN is // the amount that the user wants (bytesToRead), plus the padding at // the beginning in order to chunk-align. Note that the DN may elect // to send more than this amount if the read starts/ends mid-chunk. this.bytesNeededToFinish = bytesToRead + (startOffset - firstChunkOffset); this.firstChunkOffset = firstChunkOffset; lastChunkOffset = firstChunkOffset; lastChunkLen = -1; bytesPerChecksum = this.checksum.GetBytesPerChecksum(); checksumSize = this.checksum.GetChecksumSize(); this.peerCache = peerCache; }
internal virtual bool IsExcluded(DatanodeID dn) { lock (this) { return(excludes.Match(ResolvedAddressFromDatanodeID(dn))); } }
public virtual void TestCommitBlockSynchronization() { INodeFile file = MockFileUnderConstruction(); Block block = new Block(blockId, length, genStamp); FSNamesystem namesystemSpy = MakeNameSystemSpy(block, file); DatanodeID[] newTargets = new DatanodeID[0]; ExtendedBlock lastBlock = new ExtendedBlock(); namesystemSpy.CommitBlockSynchronization(lastBlock, genStamp, length, false, false , newTargets, null); // Repeat the call to make sure it does not throw namesystemSpy.CommitBlockSynchronization(lastBlock, genStamp, length, false, false , newTargets, null); // Simulate 'completing' the block. BlockInfoContiguous completedBlockInfo = new BlockInfoContiguous(block, (short)1); completedBlockInfo.SetBlockCollection(file); completedBlockInfo.SetGenerationStamp(genStamp); Org.Mockito.Mockito.DoReturn(completedBlockInfo).When(namesystemSpy).GetStoredBlock (Matchers.Any <Block>()); Org.Mockito.Mockito.DoReturn(completedBlockInfo).When(file).GetLastBlock(); // Repeat the call to make sure it does not throw namesystemSpy.CommitBlockSynchronization(lastBlock, genStamp, length, false, false , newTargets, null); }
/// <summary>Create a redirection URL</summary> /// <exception cref="System.IO.IOException"/> private Uri CreateRedirectURL(UserGroupInformation ugi, DatanodeID host, HttpServletRequest request, NameNode nn) { string hostname = host is DatanodeInfo?host.GetHostName() : host.GetIpAddr(); string scheme = request.GetScheme(); int port = host.GetInfoPort(); if ("https".Equals(scheme)) { int portObject = (int)GetServletContext().GetAttribute(DFSConfigKeys.DfsDatanodeHttpsPortKey ); if (portObject != null) { port = portObject; } } string encodedPath = ServletUtil.GetRawPath(request, "/fileChecksum"); string dtParam = string.Empty; if (UserGroupInformation.IsSecurityEnabled()) { string tokenString = ugi.GetTokens().GetEnumerator().Next().EncodeToUrlString(); dtParam = JspHelper.GetDelegationTokenUrlParam(tokenString); } string addr = nn.GetNameNodeAddressHostPortString(); string addrParam = JspHelper.GetUrlParam(JspHelper.NamenodeAddress, addr); return(new Uri(scheme, hostname, port, "/getFileChecksum" + encodedPath + '?' + "ugi=" + ServletUtil.EncodeQueryValue(ugi.GetShortUserName()) + dtParam + addrParam)); }
// If the includes list is empty, act as if everything is in the // includes list. internal virtual bool IsIncluded(DatanodeID dn) { lock (this) { return(includes.IsEmpty() || includes.Match(ResolvedAddressFromDatanodeID(dn))); } }
public virtual void TestMultiplePeersWithSameKey() { int Capacity = 3; PeerCache cache = new PeerCache(Capacity, 100000); DatanodeID dnId = new DatanodeID("192.168.0.1", "fakehostname", "fake_datanode_id" , 100, 101, 102, 103); HashMultiset <TestPeerCache.FakePeer> peers = HashMultiset.Create(Capacity); for (int i = 0; i < Capacity; ++i) { TestPeerCache.FakePeer peer = new TestPeerCache.FakePeer(dnId, false); peers.AddItem(peer); cache.Put(dnId, peer); } // Check that all of the peers ended up in the cache NUnit.Framework.Assert.AreEqual(Capacity, cache.Size()); while (!peers.IsEmpty()) { Peer peer = cache.Get(dnId, false); NUnit.Framework.Assert.IsTrue(peer != null); NUnit.Framework.Assert.IsTrue(!peer.IsClosed()); peers.Remove(peer); } NUnit.Framework.Assert.AreEqual(0, cache.Size()); cache.Close(); }
public virtual void TestEviction() { int Capacity = 3; PeerCache cache = new PeerCache(Capacity, 100000); DatanodeID[] dnIds = new DatanodeID[Capacity + 1]; TestPeerCache.FakePeer[] peers = new TestPeerCache.FakePeer[Capacity + 1]; for (int i = 0; i < dnIds.Length; ++i) { dnIds[i] = new DatanodeID("192.168.0.1", "fakehostname_" + i, "fake_datanode_id_" + i, 100, 101, 102, 103); peers[i] = new TestPeerCache.FakePeer(dnIds[i], false); } for (int i_1 = 0; i_1 < Capacity; ++i_1) { cache.Put(dnIds[i_1], peers[i_1]); } // Check that the peers are cached NUnit.Framework.Assert.AreEqual(Capacity, cache.Size()); // Add another entry and check that the first entry was evicted cache.Put(dnIds[Capacity], peers[Capacity]); NUnit.Framework.Assert.AreEqual(Capacity, cache.Size()); NUnit.Framework.Assert.AreSame(null, cache.Get(dnIds[0], false)); // Make sure that the other entries are still there for (int i_2 = 1; i_2 < Capacity; ++i_2) { Peer peer = cache.Get(dnIds[i_2], false); NUnit.Framework.Assert.AreSame(peers[i_2], peer); NUnit.Framework.Assert.IsTrue(!peer.IsClosed()); peer.Close(); } NUnit.Framework.Assert.AreEqual(1, cache.Size()); cache.Close(); }
public virtual void TestExpiry() { int Capacity = 3; int ExpiryPeriod = 10; PeerCache cache = new PeerCache(Capacity, ExpiryPeriod); DatanodeID[] dnIds = new DatanodeID[Capacity]; TestPeerCache.FakePeer[] peers = new TestPeerCache.FakePeer[Capacity]; for (int i = 0; i < Capacity; ++i) { dnIds[i] = new DatanodeID("192.168.0.1", "fakehostname_" + i, "fake_datanode_id", 100, 101, 102, 103); peers[i] = new TestPeerCache.FakePeer(dnIds[i], false); } for (int i_1 = 0; i_1 < Capacity; ++i_1) { cache.Put(dnIds[i_1], peers[i_1]); } // Wait for the peers to expire Sharpen.Thread.Sleep(ExpiryPeriod * 50); NUnit.Framework.Assert.AreEqual(0, cache.Size()); // make sure that the peers were closed when they were expired for (int i_2 = 0; i_2 < Capacity; ++i_2) { NUnit.Framework.Assert.IsTrue(peers[i_2].IsClosed()); } // sleep for another second and see if // the daemon thread runs fine on empty cache Sharpen.Thread.Sleep(ExpiryPeriod * 50); cache.Close(); }
/// <summary>Verify the support for decommissioning a datanode that is already dead.</summary> /// <remarks> /// Verify the support for decommissioning a datanode that is already dead. /// Under this scenario the datanode should immediately be marked as /// DECOMMISSIONED /// </remarks> /// <exception cref="System.Exception"/> public virtual void TestDecommissionDeadDN() { Logger log = Logger.GetLogger(typeof(DecommissionManager)); log.SetLevel(Level.Debug); DatanodeID dnID = cluster.GetDataNodes()[0].GetDatanodeId(); string dnName = dnID.GetXferAddr(); MiniDFSCluster.DataNodeProperties stoppedDN = cluster.StopDataNode(0); DFSTestUtil.WaitForDatanodeState(cluster, dnID.GetDatanodeUuid(), false, 30000); FSNamesystem fsn = cluster.GetNamesystem(); DatanodeManager dm = fsn.GetBlockManager().GetDatanodeManager(); DatanodeDescriptor dnDescriptor = dm.GetDatanode(dnID); DecommissionNode(fsn, localFileSys, dnName); dm.RefreshNodes(conf); BlockManagerTestUtil.RecheckDecommissionState(dm); NUnit.Framework.Assert.IsTrue(dnDescriptor.IsDecommissioned()); // Add the node back cluster.RestartDataNode(stoppedDN, true); cluster.WaitActive(); // Call refreshNodes on FSNamesystem with empty exclude file to remove the // datanode from decommissioning list and make it available again. WriteConfigFile(localFileSys, excludeFile, null); dm.RefreshNodes(conf); }
/// <exception cref="Com.Google.Protobuf.ServiceException"/> public virtual DatanodeProtocolProtos.CommitBlockSynchronizationResponseProto CommitBlockSynchronization (RpcController controller, DatanodeProtocolProtos.CommitBlockSynchronizationRequestProto request) { IList <HdfsProtos.DatanodeIDProto> dnprotos = request.GetNewTaragetsList(); DatanodeID[] dns = new DatanodeID[dnprotos.Count]; for (int i = 0; i < dnprotos.Count; i++) { dns[i] = PBHelper.Convert(dnprotos[i]); } IList <string> sidprotos = request.GetNewTargetStoragesList(); string[] storageIDs = Sharpen.Collections.ToArray(sidprotos, new string[sidprotos .Count]); try { impl.CommitBlockSynchronization(PBHelper.Convert(request.GetBlock()), request.GetNewGenStamp (), request.GetNewLength(), request.GetCloseFile(), request.GetDeleteBlock(), dns , storageIDs); } catch (IOException e) { throw new ServiceException(e); } return(VoidCommitBlockSynchronizationResponseProto); }
public virtual void TestBlockTokenRpcLeak() { Configuration conf = new Configuration(); conf.Set(CommonConfigurationKeysPublic.HadoopSecurityAuthentication, "kerberos"); UserGroupInformation.SetConfiguration(conf); Assume.AssumeTrue(FdDir.Exists()); BlockTokenSecretManager sm = new BlockTokenSecretManager(blockKeyUpdateInterval, blockTokenLifetime, 0, "fake-pool", null); Org.Apache.Hadoop.Security.Token.Token <BlockTokenIdentifier> token = sm.GenerateToken (block3, EnumSet.AllOf <BlockTokenSecretManager.AccessMode>()); Server server = CreateMockDatanode(sm, token, conf); server.Start(); IPEndPoint addr = NetUtils.GetConnectAddress(server); DatanodeID fakeDnId = DFSTestUtil.GetLocalDatanodeID(addr.Port); ExtendedBlock b = new ExtendedBlock("fake-pool", new Org.Apache.Hadoop.Hdfs.Protocol.Block (12345L)); LocatedBlock fakeBlock = new LocatedBlock(b, new DatanodeInfo[0]); fakeBlock.SetBlockToken(token); // Create another RPC proxy with the same configuration - this will never // attempt to connect anywhere -- but it causes the refcount on the // RPC "Client" object to stay above 0 such that RPC.stopProxy doesn't // actually close the TCP connections to the real target DN. ClientDatanodeProtocol proxyToNoWhere = RPC.GetProxy <ClientDatanodeProtocol>(ClientDatanodeProtocol .versionID, new IPEndPoint("1.1.1.1", 1), UserGroupInformation.CreateRemoteUser( "junk"), conf, NetUtils.GetDefaultSocketFactory(conf)); ClientDatanodeProtocol proxy = null; int fdsAtStart = CountOpenFileDescriptors(); try { long endTime = Time.Now() + 3000; while (Time.Now() < endTime) { proxy = DFSUtil.CreateClientDatanodeProtocolProxy(fakeDnId, conf, 1000, false, fakeBlock ); NUnit.Framework.Assert.AreEqual(block3.GetBlockId(), proxy.GetReplicaVisibleLength (block3)); if (proxy != null) { RPC.StopProxy(proxy); } Log.Info("Num open fds:" + CountOpenFileDescriptors()); } int fdsAtEnd = CountOpenFileDescriptors(); if (fdsAtEnd - fdsAtStart > 50) { NUnit.Framework.Assert.Fail("Leaked " + (fdsAtEnd - fdsAtStart) + " fds!"); } } finally { server.Stop(); } RPC.StopProxy(proxyToNoWhere); }
/// <summary>Receives SASL negotiation from a peer on behalf of a server.</summary> /// <param name="peer">connection peer</param> /// <param name="underlyingOut">connection output stream</param> /// <param name="underlyingIn">connection input stream</param> /// <?/> /// <param name="datanodeId">ID of DataNode accepting connection</param> /// <returns>new pair of streams, wrapped after SASL negotiation</returns> /// <exception cref="System.IO.IOException">for any error</exception> public virtual IOStreamPair Receive(Peer peer, OutputStream underlyingOut, InputStream underlyingIn, int xferPort, DatanodeID datanodeId) { if (dnConf.GetEncryptDataTransfer()) { Log.Debug("SASL server doing encrypted handshake for peer = {}, datanodeId = {}", peer, datanodeId); return(GetEncryptedStreams(peer, underlyingOut, underlyingIn)); } else { if (!UserGroupInformation.IsSecurityEnabled()) { Log.Debug("SASL server skipping handshake in unsecured configuration for " + "peer = {}, datanodeId = {}" , peer, datanodeId); return(new IOStreamPair(underlyingIn, underlyingOut)); } else { if (SecurityUtil.IsPrivilegedPort(xferPort)) { Log.Debug("SASL server skipping handshake in secured configuration for " + "peer = {}, datanodeId = {}" , peer, datanodeId); return(new IOStreamPair(underlyingIn, underlyingOut)); } else { if (dnConf.GetSaslPropsResolver() != null) { Log.Debug("SASL server doing general handshake for peer = {}, datanodeId = {}", peer , datanodeId); return(GetSaslStreams(peer, underlyingOut, underlyingIn)); } else { if (dnConf.GetIgnoreSecurePortsForTesting()) { // It's a secured cluster using non-privileged ports, but no SASL. The // only way this can happen is if the DataNode has // ignore.secure.ports.for.testing configured, so this is a rare edge case. Log.Debug("SASL server skipping handshake in secured configuration with no SASL " + "protection configured for peer = {}, datanodeId = {}", peer, datanodeId); return(new IOStreamPair(underlyingIn, underlyingOut)); } else { // The error message here intentionally does not mention // ignore.secure.ports.for.testing. That's intended for dev use only. // This code path is not expected to execute ever, because DataNode startup // checks for invalid configuration and aborts. throw new IOException(string.Format("Cannot create a secured " + "connection if DataNode listens on unprivileged port (%d) and no " + "protection is defined in configuration property %s.", datanodeId.GetXferPort (), DFSConfigKeys.DfsDataTransferProtectionKey)); } } } } } }
public DatanodeRegistration(DatanodeID dn, StorageInfo info, ExportedBlockKeys keys , string softwareVersion) : base(dn) { this.storageInfo = info; this.exportedKeys = keys; this.softwareVersion = softwareVersion; }
/// <summary>Set the marker if the DatanodeID is matched.</summary> /// <exception cref="System.IO.IOException"/> public override void Run(DatanodeID datanodeid) { DataTransferTestUtil.DataTransferTest test = GetDataTransferTest(); if (test.IsNotSuccessAndLastPipelineContains(index, datanodeid)) { marker.Mark(); } }
/// <summary>Get a cached peer connected to the given DataNode.</summary> /// <param name="dnId">The DataNode to get a Peer for.</param> /// <param name="isDomain">Whether to retrieve a DomainPeer or not.</param> /// <returns> /// An open Peer connected to the DN, or null if none /// was found. /// </returns> public virtual Peer Get(DatanodeID dnId, bool isDomain) { if (capacity <= 0) { // disabled return(null); } return(GetInternal(dnId, isDomain)); }
/// <summary>Sends client SASL negotiation for a socket if required.</summary> /// <param name="socket">connection socket</param> /// <param name="underlyingOut">connection output stream</param> /// <param name="underlyingIn">connection input stream</param> /// <param name="encryptionKeyFactory">for creation of an encryption key</param> /// <param name="accessToken">connection block access token</param> /// <param name="datanodeId">ID of destination DataNode</param> /// <returns>new pair of streams, wrapped after SASL negotiation</returns> /// <exception cref="System.IO.IOException">for any error</exception> public virtual IOStreamPair SocketSend(Socket socket, OutputStream underlyingOut, InputStream underlyingIn, DataEncryptionKeyFactory encryptionKeyFactory, Org.Apache.Hadoop.Security.Token.Token <BlockTokenIdentifier> accessToken, DatanodeID datanodeId) { IOStreamPair ios = CheckTrustAndSend(socket.GetInetAddress(), underlyingOut, underlyingIn , encryptionKeyFactory, accessToken, datanodeId); return(ios != null ? ios : new IOStreamPair(underlyingIn, underlyingOut)); }
internal virtual void Compare(DatanodeID dn, DatanodeID dn2) { NUnit.Framework.Assert.AreEqual(dn.GetIpAddr(), dn2.GetIpAddr()); NUnit.Framework.Assert.AreEqual(dn.GetHostName(), dn2.GetHostName()); NUnit.Framework.Assert.AreEqual(dn.GetDatanodeUuid(), dn2.GetDatanodeUuid()); NUnit.Framework.Assert.AreEqual(dn.GetXferPort(), dn2.GetXferPort()); NUnit.Framework.Assert.AreEqual(dn.GetInfoPort(), dn2.GetInfoPort()); NUnit.Framework.Assert.AreEqual(dn.GetIpcPort(), dn2.GetIpcPort()); }
public virtual void TestConvertDatanodeID() { DatanodeID dn = DFSTestUtil.GetLocalDatanodeID(); HdfsProtos.DatanodeIDProto dnProto = PBHelper.Convert(dn); DatanodeID dn2 = PBHelper.Convert(dnProto); Compare(dn, dn2); }
/// <summary>Sends client SASL negotiation if required.</summary> /// <remarks> /// Sends client SASL negotiation if required. Determines the correct type of /// SASL handshake based on configuration. /// </remarks> /// <param name="addr">connection address</param> /// <param name="underlyingOut">connection output stream</param> /// <param name="underlyingIn">connection input stream</param> /// <param name="encryptionKey">for an encrypted SASL handshake</param> /// <param name="accessToken">connection block access token</param> /// <param name="datanodeId">ID of destination DataNode</param> /// <returns>new pair of streams, wrapped after SASL negotiation</returns> /// <exception cref="System.IO.IOException">for any error</exception> private IOStreamPair Send(IPAddress addr, OutputStream underlyingOut, InputStream underlyingIn, DataEncryptionKey encryptionKey, Org.Apache.Hadoop.Security.Token.Token <BlockTokenIdentifier> accessToken, DatanodeID datanodeId) { if (encryptionKey != null) { Log.Debug("SASL client doing encrypted handshake for addr = {}, datanodeId = {}", addr, datanodeId); return(GetEncryptedStreams(underlyingOut, underlyingIn, encryptionKey)); } else { if (!UserGroupInformation.IsSecurityEnabled()) { Log.Debug("SASL client skipping handshake in unsecured configuration for " + "addr = {}, datanodeId = {}" , addr, datanodeId); return(null); } else { if (SecurityUtil.IsPrivilegedPort(datanodeId.GetXferPort())) { Log.Debug("SASL client skipping handshake in secured configuration with " + "privileged port for addr = {}, datanodeId = {}" , addr, datanodeId); return(null); } else { if (fallbackToSimpleAuth != null && fallbackToSimpleAuth.Get()) { Log.Debug("SASL client skipping handshake in secured configuration with " + "unsecured cluster for addr = {}, datanodeId = {}" , addr, datanodeId); return(null); } else { if (saslPropsResolver != null) { Log.Debug("SASL client doing general handshake for addr = {}, datanodeId = {}", addr , datanodeId); return(GetSaslStreams(addr, underlyingOut, underlyingIn, accessToken, datanodeId)); } else { // It's a secured cluster using non-privileged ports, but no SASL. The // only way this can happen is if the DataNode has // ignore.secure.ports.for.testing configured, so this is a rare edge case. Log.Debug("SASL client skipping handshake in secured configuration with no SASL " + "protection configured for addr = {}, datanodeId = {}", addr, datanodeId); return(null); } } } } } }
/// <exception cref="System.IO.IOException"/> public override void Run(DatanodeID id) { DataTransferTestUtil.DataTransferTest test = GetDataTransferTest(); if (test.IsNotSuccessAndLastPipelineContains(index, id)) { string s = ToString(id); FiTestUtil.Log.Info(s); throw new IOException(s); } }
/// <param name="nodeReg">DatanodeID to update registration for.</param> public override void UpdateRegInfo(DatanodeID nodeReg) { base.UpdateRegInfo(nodeReg); // must re-process IBR after re-registration foreach (DatanodeStorageInfo storage in GetStorageInfos()) { storage.SetBlockReportCount(0); } heartbeatedSinceRegistration = false; }
/// <exception cref="System.IO.IOException"/> public static InterDatanodeProtocol CreateInterDatanodeProtocolProxy(DataNode dn, DatanodeID datanodeid, Configuration conf, bool connectToDnViaHostname) { if (connectToDnViaHostname != dn.GetDnConf().connectToDnViaHostname) { throw new Exception("Unexpected DN hostname configuration"); } return(DataNode.CreateInterDataNodeProtocolProxy(datanodeid, conf, dn.GetDnConf() .socketTimeout, dn.GetDnConf().connectToDnViaHostname)); }
/// <exception cref="Org.Apache.Hadoop.Util.DiskChecker.DiskOutOfSpaceException"/> public override void Run(DatanodeID id) { DataTransferTestUtil.DataTransferTest test = GetDataTransferTest(); if (test.IsNotSuccessAndLastPipelineContains(index, id) && countdown.IsSatisfied( )) { string s = ToString(id); FiTestUtil.Log.Info(s); throw new DiskChecker.DiskOutOfSpaceException(s); } }
/// <summary> /// Is the test not yet success /// and the last pipeline contains the given datanode? /// </summary> private bool IsNotSuccessAndLastPipelineContains(int index, DatanodeID id) { lock (this) { if (IsSuccess()) { return(false); } int n = pipelines.Count; return(n == 0 ? false : pipelines[n - 1].Contains(index, id)); } }
/// <summary>Return the datanode descriptor for the given datanode.</summary> /// <exception cref="System.IO.IOException"/> public static DatanodeDescriptor GetDatanode(FSNamesystem ns, DatanodeID id) { ns.ReadLock(); try { return(ns.GetBlockManager().GetDatanodeManager().GetDatanode(id)); } finally { ns.ReadUnlock(); } }
/// <summary>DatanodeDescriptor constructor</summary> /// <param name="nodeID">id of the data node</param> /// <param name="networkLocation">location of the data node in network</param> public DatanodeDescriptor(DatanodeID nodeID, string networkLocation) : base(nodeID, networkLocation) { decommissioningStatus = new DatanodeDescriptor.DecommissioningStatus(this); pendingCached = new DatanodeDescriptor.CachedBlocksList(this, DatanodeDescriptor.CachedBlocksList.Type .PendingCached); cached = new DatanodeDescriptor.CachedBlocksList(this, DatanodeDescriptor.CachedBlocksList.Type .Cached); pendingUncached = new DatanodeDescriptor.CachedBlocksList(this, DatanodeDescriptor.CachedBlocksList.Type .PendingUncached); UpdateHeartbeatState(StorageReport.EmptyArray, 0L, 0L, 0, 0, null); }