/// <exception cref="System.IO.IOException"/>
internal static ClientDatanodeProtocolPB CreateClientDatanodeProtocolProxy(DatanodeID
datanodeid, Configuration conf, int socketTimeout, bool connectToDnViaHostname,
LocatedBlock locatedBlock)
{
string dnAddr = datanodeid.GetIpcAddr(connectToDnViaHostname);
IPEndPoint addr = NetUtils.CreateSocketAddr(dnAddr);
if (Log.IsDebugEnabled())
{
Log.Debug("Connecting to datanode " + dnAddr + " addr=" + addr);
}
// Since we're creating a new UserGroupInformation here, we know that no
// future RPC proxies will be able to re-use the same connection. And
// usages of this proxy tend to be one-off calls.
//
// This is a temporary fix: callers should really achieve this by using
// RPC.stopProxy() on the resulting object, but this is currently not
// working in trunk. See the discussion on HDFS-1965.
Configuration confWithNoIpcIdle = new Configuration(conf);
confWithNoIpcIdle.SetInt(CommonConfigurationKeysPublic.IpcClientConnectionMaxidletimeKey
, 0);
UserGroupInformation ticket = UserGroupInformation.CreateRemoteUser(locatedBlock.
GetBlock().GetLocalBlock().ToString());
ticket.AddToken(locatedBlock.GetBlockToken());
return(CreateClientDatanodeProtocolProxy(addr, ticket, confWithNoIpcIdle, NetUtils
.GetDefaultSocketFactory(conf), socketTimeout));
}
/// <summary>DatanodeDescriptor constructor</summary>
/// <param name="nodeID">id of the data node</param>
public DatanodeDescriptor(DatanodeID nodeID)
: base(nodeID)
{
decommissioningStatus = new DatanodeDescriptor.DecommissioningStatus(this);
pendingCached = new DatanodeDescriptor.CachedBlocksList(this, DatanodeDescriptor.CachedBlocksList.Type
.PendingCached);
cached = new DatanodeDescriptor.CachedBlocksList(this, DatanodeDescriptor.CachedBlocksList.Type
.Cached);
pendingUncached = new DatanodeDescriptor.CachedBlocksList(this, DatanodeDescriptor.CachedBlocksList.Type
.PendingUncached);
// isAlive == heartbeats.contains(this)
// This is an optimization, because contains takes O(n) time on Arraylist
// A system administrator can tune the balancer bandwidth parameter
// (dfs.balance.bandwidthPerSec) dynamically by calling
// "dfsadmin -setBalanacerBandwidth <newbandwidth>", at which point the
// following 'bandwidth' variable gets updated with the new value for each
// node. Once the heartbeat command is issued to update the value on the
// specified datanode, this value will be set back to 0.
/* Variables for maintaining number of blocks scheduled to be written to
* this storage. This count is approximate and might be slightly bigger
* in case of errors (e.g. datanode does not report if an error occurs
* while writing the block).
*/
//10min
// The number of replication work pending before targets are determined
// HB processing can use it to tell if it is the first HB since DN restarted
UpdateHeartbeatState(StorageReport.EmptyArray, 0L, 0L, 0, 0, null);
}
/// <summary>
/// Test to verify that InterDatanode RPC timesout as expected when
/// the server DN does not respond.
/// </summary>
/// <exception cref="System.Exception"/>
public virtual void TestInterDNProtocolTimeout()
{
Org.Apache.Hadoop.Ipc.Server server = new TestInterDatanodeProtocol.TestServer(1,
true);
server.Start();
IPEndPoint addr = NetUtils.GetConnectAddress(server);
DatanodeID fakeDnId = DFSTestUtil.GetLocalDatanodeID(addr.Port);
DatanodeInfo dInfo = new DatanodeInfo(fakeDnId);
InterDatanodeProtocol proxy = null;
try
{
proxy = DataNode.CreateInterDataNodeProtocolProxy(dInfo, conf, 500, false);
proxy.InitReplicaRecovery(new BlockRecoveryCommand.RecoveringBlock(new ExtendedBlock
("bpid", 1), null, 100));
NUnit.Framework.Assert.Fail("Expected SocketTimeoutException exception, but did not get."
);
}
finally
{
if (proxy != null)
{
RPC.StopProxy(proxy);
}
server.Stop();
}
}
/// <summary>Create a redirection URL</summary>
/// <exception cref="System.IO.IOException"/>
private Uri CreateRedirectURL(string path, string encodedPath, HdfsFileStatus status
, UserGroupInformation ugi, ClientProtocol nnproxy, HttpServletRequest request,
string dt)
{
string scheme = request.GetScheme();
LocatedBlocks blks = nnproxy.GetBlockLocations(status.GetFullPath(new Path(path))
.ToUri().GetPath(), 0, 1);
Configuration conf = NameNodeHttpServer.GetConfFromContext(GetServletContext());
DatanodeID host = PickSrcDatanode(blks, status, conf);
string hostname;
if (host is DatanodeInfo)
{
hostname = host.GetHostName();
}
else
{
hostname = host.GetIpAddr();
}
int port = "https".Equals(scheme) ? host.GetInfoSecurePort() : host.GetInfoPort();
string dtParam = string.Empty;
if (dt != null)
{
dtParam = JspHelper.GetDelegationTokenUrlParam(dt);
}
// Add namenode address to the url params
NameNode nn = NameNodeHttpServer.GetNameNodeFromContext(GetServletContext());
string addr = nn.GetNameNodeAddressHostPortString();
string addrParam = JspHelper.GetUrlParam(JspHelper.NamenodeAddress, addr);
return(new Uri(scheme, hostname, port, "/streamFile" + encodedPath + '?' + "ugi="
+ ServletUtil.EncodeQueryValue(ugi.GetShortUserName()) + dtParam + addrParam));
}
private RemoteBlockReader(string file, string bpid, long blockId, DataInputStream
@in, DataChecksum checksum, bool verifyChecksum, long startOffset, long firstChunkOffset
, long bytesToRead, Peer peer, DatanodeID datanodeID, PeerCache peerCache)
: base(new Path("/" + Block.BlockFilePrefix + blockId + ":" + bpid + ":of:" + file
), 1, verifyChecksum, checksum.GetChecksumSize() > 0 ? checksum : null, checksum
.GetBytesPerChecksum(), checksum.GetChecksumSize())
{
// Path is used only for printing block and file information in debug
/*too non path-like?*/
this.isLocal = DFSClient.IsLocalAddress(NetUtils.CreateSocketAddr(datanodeID.GetXferAddr
()));
this.peer = peer;
this.datanodeID = datanodeID;
this.@in = @in;
this.checksum = checksum;
this.startOffset = Math.Max(startOffset, 0);
this.blockId = blockId;
// The total number of bytes that we need to transfer from the DN is
// the amount that the user wants (bytesToRead), plus the padding at
// the beginning in order to chunk-align. Note that the DN may elect
// to send more than this amount if the read starts/ends mid-chunk.
this.bytesNeededToFinish = bytesToRead + (startOffset - firstChunkOffset);
this.firstChunkOffset = firstChunkOffset;
lastChunkOffset = firstChunkOffset;
lastChunkLen = -1;
bytesPerChecksum = this.checksum.GetBytesPerChecksum();
checksumSize = this.checksum.GetChecksumSize();
this.peerCache = peerCache;
}
internal virtual bool IsExcluded(DatanodeID dn)
{
lock (this)
{
return(excludes.Match(ResolvedAddressFromDatanodeID(dn)));
}
}
public virtual void TestCommitBlockSynchronization()
{
INodeFile file = MockFileUnderConstruction();
Block block = new Block(blockId, length, genStamp);
FSNamesystem namesystemSpy = MakeNameSystemSpy(block, file);
DatanodeID[] newTargets = new DatanodeID[0];
ExtendedBlock lastBlock = new ExtendedBlock();
namesystemSpy.CommitBlockSynchronization(lastBlock, genStamp, length, false, false
, newTargets, null);
// Repeat the call to make sure it does not throw
namesystemSpy.CommitBlockSynchronization(lastBlock, genStamp, length, false, false
, newTargets, null);
// Simulate 'completing' the block.
BlockInfoContiguous completedBlockInfo = new BlockInfoContiguous(block, (short)1);
completedBlockInfo.SetBlockCollection(file);
completedBlockInfo.SetGenerationStamp(genStamp);
Org.Mockito.Mockito.DoReturn(completedBlockInfo).When(namesystemSpy).GetStoredBlock
(Matchers.Any <Block>());
Org.Mockito.Mockito.DoReturn(completedBlockInfo).When(file).GetLastBlock();
// Repeat the call to make sure it does not throw
namesystemSpy.CommitBlockSynchronization(lastBlock, genStamp, length, false, false
, newTargets, null);
}
/// <summary>Create a redirection URL</summary>
/// <exception cref="System.IO.IOException"/>
private Uri CreateRedirectURL(UserGroupInformation ugi, DatanodeID host, HttpServletRequest
request, NameNode nn)
{
string hostname = host is DatanodeInfo?host.GetHostName() : host.GetIpAddr();
string scheme = request.GetScheme();
int port = host.GetInfoPort();
if ("https".Equals(scheme))
{
int portObject = (int)GetServletContext().GetAttribute(DFSConfigKeys.DfsDatanodeHttpsPortKey
);
if (portObject != null)
{
port = portObject;
}
}
string encodedPath = ServletUtil.GetRawPath(request, "/fileChecksum");
string dtParam = string.Empty;
if (UserGroupInformation.IsSecurityEnabled())
{
string tokenString = ugi.GetTokens().GetEnumerator().Next().EncodeToUrlString();
dtParam = JspHelper.GetDelegationTokenUrlParam(tokenString);
}
string addr = nn.GetNameNodeAddressHostPortString();
string addrParam = JspHelper.GetUrlParam(JspHelper.NamenodeAddress, addr);
return(new Uri(scheme, hostname, port, "/getFileChecksum" + encodedPath + '?' + "ugi="
+ ServletUtil.EncodeQueryValue(ugi.GetShortUserName()) + dtParam + addrParam));
}
// If the includes list is empty, act as if everything is in the
// includes list.
internal virtual bool IsIncluded(DatanodeID dn)
{
lock (this)
{
return(includes.IsEmpty() || includes.Match(ResolvedAddressFromDatanodeID(dn)));
}
}
public virtual void TestMultiplePeersWithSameKey()
{
int Capacity = 3;
PeerCache cache = new PeerCache(Capacity, 100000);
DatanodeID dnId = new DatanodeID("192.168.0.1", "fakehostname", "fake_datanode_id"
, 100, 101, 102, 103);
HashMultiset <TestPeerCache.FakePeer> peers = HashMultiset.Create(Capacity);
for (int i = 0; i < Capacity; ++i)
{
TestPeerCache.FakePeer peer = new TestPeerCache.FakePeer(dnId, false);
peers.AddItem(peer);
cache.Put(dnId, peer);
}
// Check that all of the peers ended up in the cache
NUnit.Framework.Assert.AreEqual(Capacity, cache.Size());
while (!peers.IsEmpty())
{
Peer peer = cache.Get(dnId, false);
NUnit.Framework.Assert.IsTrue(peer != null);
NUnit.Framework.Assert.IsTrue(!peer.IsClosed());
peers.Remove(peer);
}
NUnit.Framework.Assert.AreEqual(0, cache.Size());
cache.Close();
}
public virtual void TestEviction()
{
int Capacity = 3;
PeerCache cache = new PeerCache(Capacity, 100000);
DatanodeID[] dnIds = new DatanodeID[Capacity + 1];
TestPeerCache.FakePeer[] peers = new TestPeerCache.FakePeer[Capacity + 1];
for (int i = 0; i < dnIds.Length; ++i)
{
dnIds[i] = new DatanodeID("192.168.0.1", "fakehostname_" + i, "fake_datanode_id_"
+ i, 100, 101, 102, 103);
peers[i] = new TestPeerCache.FakePeer(dnIds[i], false);
}
for (int i_1 = 0; i_1 < Capacity; ++i_1)
{
cache.Put(dnIds[i_1], peers[i_1]);
}
// Check that the peers are cached
NUnit.Framework.Assert.AreEqual(Capacity, cache.Size());
// Add another entry and check that the first entry was evicted
cache.Put(dnIds[Capacity], peers[Capacity]);
NUnit.Framework.Assert.AreEqual(Capacity, cache.Size());
NUnit.Framework.Assert.AreSame(null, cache.Get(dnIds[0], false));
// Make sure that the other entries are still there
for (int i_2 = 1; i_2 < Capacity; ++i_2)
{
Peer peer = cache.Get(dnIds[i_2], false);
NUnit.Framework.Assert.AreSame(peers[i_2], peer);
NUnit.Framework.Assert.IsTrue(!peer.IsClosed());
peer.Close();
}
NUnit.Framework.Assert.AreEqual(1, cache.Size());
cache.Close();
}
public virtual void TestExpiry()
{
int Capacity = 3;
int ExpiryPeriod = 10;
PeerCache cache = new PeerCache(Capacity, ExpiryPeriod);
DatanodeID[] dnIds = new DatanodeID[Capacity];
TestPeerCache.FakePeer[] peers = new TestPeerCache.FakePeer[Capacity];
for (int i = 0; i < Capacity; ++i)
{
dnIds[i] = new DatanodeID("192.168.0.1", "fakehostname_" + i, "fake_datanode_id",
100, 101, 102, 103);
peers[i] = new TestPeerCache.FakePeer(dnIds[i], false);
}
for (int i_1 = 0; i_1 < Capacity; ++i_1)
{
cache.Put(dnIds[i_1], peers[i_1]);
}
// Wait for the peers to expire
Sharpen.Thread.Sleep(ExpiryPeriod * 50);
NUnit.Framework.Assert.AreEqual(0, cache.Size());
// make sure that the peers were closed when they were expired
for (int i_2 = 0; i_2 < Capacity; ++i_2)
{
NUnit.Framework.Assert.IsTrue(peers[i_2].IsClosed());
}
// sleep for another second and see if
// the daemon thread runs fine on empty cache
Sharpen.Thread.Sleep(ExpiryPeriod * 50);
cache.Close();
}
/// <summary>Verify the support for decommissioning a datanode that is already dead.</summary>
/// <remarks>
/// Verify the support for decommissioning a datanode that is already dead.
/// Under this scenario the datanode should immediately be marked as
/// DECOMMISSIONED
/// </remarks>
/// <exception cref="System.Exception"/>
public virtual void TestDecommissionDeadDN()
{
Logger log = Logger.GetLogger(typeof(DecommissionManager));
log.SetLevel(Level.Debug);
DatanodeID dnID = cluster.GetDataNodes()[0].GetDatanodeId();
string dnName = dnID.GetXferAddr();
MiniDFSCluster.DataNodeProperties stoppedDN = cluster.StopDataNode(0);
DFSTestUtil.WaitForDatanodeState(cluster, dnID.GetDatanodeUuid(), false, 30000);
FSNamesystem fsn = cluster.GetNamesystem();
DatanodeManager dm = fsn.GetBlockManager().GetDatanodeManager();
DatanodeDescriptor dnDescriptor = dm.GetDatanode(dnID);
DecommissionNode(fsn, localFileSys, dnName);
dm.RefreshNodes(conf);
BlockManagerTestUtil.RecheckDecommissionState(dm);
NUnit.Framework.Assert.IsTrue(dnDescriptor.IsDecommissioned());
// Add the node back
cluster.RestartDataNode(stoppedDN, true);
cluster.WaitActive();
// Call refreshNodes on FSNamesystem with empty exclude file to remove the
// datanode from decommissioning list and make it available again.
WriteConfigFile(localFileSys, excludeFile, null);
dm.RefreshNodes(conf);
}
/// <exception cref="Com.Google.Protobuf.ServiceException"/>
public virtual DatanodeProtocolProtos.CommitBlockSynchronizationResponseProto CommitBlockSynchronization
(RpcController controller, DatanodeProtocolProtos.CommitBlockSynchronizationRequestProto
request)
{
IList <HdfsProtos.DatanodeIDProto> dnprotos = request.GetNewTaragetsList();
DatanodeID[] dns = new DatanodeID[dnprotos.Count];
for (int i = 0; i < dnprotos.Count; i++)
{
dns[i] = PBHelper.Convert(dnprotos[i]);
}
IList <string> sidprotos = request.GetNewTargetStoragesList();
string[] storageIDs = Sharpen.Collections.ToArray(sidprotos, new string[sidprotos
.Count]);
try
{
impl.CommitBlockSynchronization(PBHelper.Convert(request.GetBlock()), request.GetNewGenStamp
(), request.GetNewLength(), request.GetCloseFile(), request.GetDeleteBlock(), dns
, storageIDs);
}
catch (IOException e)
{
throw new ServiceException(e);
}
return(VoidCommitBlockSynchronizationResponseProto);
}
public virtual void TestBlockTokenRpcLeak()
{
Configuration conf = new Configuration();
conf.Set(CommonConfigurationKeysPublic.HadoopSecurityAuthentication, "kerberos");
UserGroupInformation.SetConfiguration(conf);
Assume.AssumeTrue(FdDir.Exists());
BlockTokenSecretManager sm = new BlockTokenSecretManager(blockKeyUpdateInterval,
blockTokenLifetime, 0, "fake-pool", null);
Org.Apache.Hadoop.Security.Token.Token <BlockTokenIdentifier> token = sm.GenerateToken
(block3, EnumSet.AllOf <BlockTokenSecretManager.AccessMode>());
Server server = CreateMockDatanode(sm, token, conf);
server.Start();
IPEndPoint addr = NetUtils.GetConnectAddress(server);
DatanodeID fakeDnId = DFSTestUtil.GetLocalDatanodeID(addr.Port);
ExtendedBlock b = new ExtendedBlock("fake-pool", new Org.Apache.Hadoop.Hdfs.Protocol.Block
(12345L));
LocatedBlock fakeBlock = new LocatedBlock(b, new DatanodeInfo[0]);
fakeBlock.SetBlockToken(token);
// Create another RPC proxy with the same configuration - this will never
// attempt to connect anywhere -- but it causes the refcount on the
// RPC "Client" object to stay above 0 such that RPC.stopProxy doesn't
// actually close the TCP connections to the real target DN.
ClientDatanodeProtocol proxyToNoWhere = RPC.GetProxy <ClientDatanodeProtocol>(ClientDatanodeProtocol
.versionID, new IPEndPoint("1.1.1.1", 1), UserGroupInformation.CreateRemoteUser(
"junk"), conf, NetUtils.GetDefaultSocketFactory(conf));
ClientDatanodeProtocol proxy = null;
int fdsAtStart = CountOpenFileDescriptors();
try
{
long endTime = Time.Now() + 3000;
while (Time.Now() < endTime)
{
proxy = DFSUtil.CreateClientDatanodeProtocolProxy(fakeDnId, conf, 1000, false, fakeBlock
);
NUnit.Framework.Assert.AreEqual(block3.GetBlockId(), proxy.GetReplicaVisibleLength
(block3));
if (proxy != null)
{
RPC.StopProxy(proxy);
}
Log.Info("Num open fds:" + CountOpenFileDescriptors());
}
int fdsAtEnd = CountOpenFileDescriptors();
if (fdsAtEnd - fdsAtStart > 50)
{
NUnit.Framework.Assert.Fail("Leaked " + (fdsAtEnd - fdsAtStart) + " fds!");
}
}
finally
{
server.Stop();
}
RPC.StopProxy(proxyToNoWhere);
}
/// <summary>Receives SASL negotiation from a peer on behalf of a server.</summary>
/// <param name="peer">connection peer</param>
/// <param name="underlyingOut">connection output stream</param>
/// <param name="underlyingIn">connection input stream</param>
/// <?/>
/// <param name="datanodeId">ID of DataNode accepting connection</param>
/// <returns>new pair of streams, wrapped after SASL negotiation</returns>
/// <exception cref="System.IO.IOException">for any error</exception>
public virtual IOStreamPair Receive(Peer peer, OutputStream underlyingOut, InputStream
underlyingIn, int xferPort, DatanodeID datanodeId)
{
if (dnConf.GetEncryptDataTransfer())
{
Log.Debug("SASL server doing encrypted handshake for peer = {}, datanodeId = {}",
peer, datanodeId);
return(GetEncryptedStreams(peer, underlyingOut, underlyingIn));
}
else
{
if (!UserGroupInformation.IsSecurityEnabled())
{
Log.Debug("SASL server skipping handshake in unsecured configuration for " + "peer = {}, datanodeId = {}"
, peer, datanodeId);
return(new IOStreamPair(underlyingIn, underlyingOut));
}
else
{
if (SecurityUtil.IsPrivilegedPort(xferPort))
{
Log.Debug("SASL server skipping handshake in secured configuration for " + "peer = {}, datanodeId = {}"
, peer, datanodeId);
return(new IOStreamPair(underlyingIn, underlyingOut));
}
else
{
if (dnConf.GetSaslPropsResolver() != null)
{
Log.Debug("SASL server doing general handshake for peer = {}, datanodeId = {}", peer
, datanodeId);
return(GetSaslStreams(peer, underlyingOut, underlyingIn));
}
else
{
if (dnConf.GetIgnoreSecurePortsForTesting())
{
// It's a secured cluster using non-privileged ports, but no SASL. The
// only way this can happen is if the DataNode has
// ignore.secure.ports.for.testing configured, so this is a rare edge case.
Log.Debug("SASL server skipping handshake in secured configuration with no SASL "
+ "protection configured for peer = {}, datanodeId = {}", peer, datanodeId);
return(new IOStreamPair(underlyingIn, underlyingOut));
}
else
{
// The error message here intentionally does not mention
// ignore.secure.ports.for.testing. That's intended for dev use only.
// This code path is not expected to execute ever, because DataNode startup
// checks for invalid configuration and aborts.
throw new IOException(string.Format("Cannot create a secured " + "connection if DataNode listens on unprivileged port (%d) and no "
+ "protection is defined in configuration property %s.", datanodeId.GetXferPort
(), DFSConfigKeys.DfsDataTransferProtectionKey));
}
}
}
}
}
}
public DatanodeRegistration(DatanodeID dn, StorageInfo info, ExportedBlockKeys keys
, string softwareVersion)
: base(dn)
{
this.storageInfo = info;
this.exportedKeys = keys;
this.softwareVersion = softwareVersion;
}
/// <summary>Set the marker if the DatanodeID is matched.</summary>
/// <exception cref="System.IO.IOException"/>
public override void Run(DatanodeID datanodeid)
{
DataTransferTestUtil.DataTransferTest test = GetDataTransferTest();
if (test.IsNotSuccessAndLastPipelineContains(index, datanodeid))
{
marker.Mark();
}
}
/// <summary>Get a cached peer connected to the given DataNode.</summary>
/// <param name="dnId">The DataNode to get a Peer for.</param>
/// <param name="isDomain">Whether to retrieve a DomainPeer or not.</param>
/// <returns>
/// An open Peer connected to the DN, or null if none
/// was found.
/// </returns>
public virtual Peer Get(DatanodeID dnId, bool isDomain)
{
if (capacity <= 0)
{
// disabled
return(null);
}
return(GetInternal(dnId, isDomain));
}
/// <summary>Sends client SASL negotiation for a socket if required.</summary>
/// <param name="socket">connection socket</param>
/// <param name="underlyingOut">connection output stream</param>
/// <param name="underlyingIn">connection input stream</param>
/// <param name="encryptionKeyFactory">for creation of an encryption key</param>
/// <param name="accessToken">connection block access token</param>
/// <param name="datanodeId">ID of destination DataNode</param>
/// <returns>new pair of streams, wrapped after SASL negotiation</returns>
/// <exception cref="System.IO.IOException">for any error</exception>
public virtual IOStreamPair SocketSend(Socket socket, OutputStream underlyingOut,
InputStream underlyingIn, DataEncryptionKeyFactory encryptionKeyFactory, Org.Apache.Hadoop.Security.Token.Token
<BlockTokenIdentifier> accessToken, DatanodeID datanodeId)
{
IOStreamPair ios = CheckTrustAndSend(socket.GetInetAddress(), underlyingOut, underlyingIn
, encryptionKeyFactory, accessToken, datanodeId);
return(ios != null ? ios : new IOStreamPair(underlyingIn, underlyingOut));
}
internal virtual void Compare(DatanodeID dn, DatanodeID dn2)
{
NUnit.Framework.Assert.AreEqual(dn.GetIpAddr(), dn2.GetIpAddr());
NUnit.Framework.Assert.AreEqual(dn.GetHostName(), dn2.GetHostName());
NUnit.Framework.Assert.AreEqual(dn.GetDatanodeUuid(), dn2.GetDatanodeUuid());
NUnit.Framework.Assert.AreEqual(dn.GetXferPort(), dn2.GetXferPort());
NUnit.Framework.Assert.AreEqual(dn.GetInfoPort(), dn2.GetInfoPort());
NUnit.Framework.Assert.AreEqual(dn.GetIpcPort(), dn2.GetIpcPort());
}
public virtual void TestConvertDatanodeID()
{
DatanodeID dn = DFSTestUtil.GetLocalDatanodeID();
HdfsProtos.DatanodeIDProto dnProto = PBHelper.Convert(dn);
DatanodeID dn2 = PBHelper.Convert(dnProto);
Compare(dn, dn2);
}
/// <summary>Sends client SASL negotiation if required.</summary>
/// <remarks>
/// Sends client SASL negotiation if required. Determines the correct type of
/// SASL handshake based on configuration.
/// </remarks>
/// <param name="addr">connection address</param>
/// <param name="underlyingOut">connection output stream</param>
/// <param name="underlyingIn">connection input stream</param>
/// <param name="encryptionKey">for an encrypted SASL handshake</param>
/// <param name="accessToken">connection block access token</param>
/// <param name="datanodeId">ID of destination DataNode</param>
/// <returns>new pair of streams, wrapped after SASL negotiation</returns>
/// <exception cref="System.IO.IOException">for any error</exception>
private IOStreamPair Send(IPAddress addr, OutputStream underlyingOut, InputStream
underlyingIn, DataEncryptionKey encryptionKey, Org.Apache.Hadoop.Security.Token.Token
<BlockTokenIdentifier> accessToken, DatanodeID datanodeId)
{
if (encryptionKey != null)
{
Log.Debug("SASL client doing encrypted handshake for addr = {}, datanodeId = {}",
addr, datanodeId);
return(GetEncryptedStreams(underlyingOut, underlyingIn, encryptionKey));
}
else
{
if (!UserGroupInformation.IsSecurityEnabled())
{
Log.Debug("SASL client skipping handshake in unsecured configuration for " + "addr = {}, datanodeId = {}"
, addr, datanodeId);
return(null);
}
else
{
if (SecurityUtil.IsPrivilegedPort(datanodeId.GetXferPort()))
{
Log.Debug("SASL client skipping handshake in secured configuration with " + "privileged port for addr = {}, datanodeId = {}"
, addr, datanodeId);
return(null);
}
else
{
if (fallbackToSimpleAuth != null && fallbackToSimpleAuth.Get())
{
Log.Debug("SASL client skipping handshake in secured configuration with " + "unsecured cluster for addr = {}, datanodeId = {}"
, addr, datanodeId);
return(null);
}
else
{
if (saslPropsResolver != null)
{
Log.Debug("SASL client doing general handshake for addr = {}, datanodeId = {}", addr
, datanodeId);
return(GetSaslStreams(addr, underlyingOut, underlyingIn, accessToken, datanodeId));
}
else
{
// It's a secured cluster using non-privileged ports, but no SASL. The
// only way this can happen is if the DataNode has
// ignore.secure.ports.for.testing configured, so this is a rare edge case.
Log.Debug("SASL client skipping handshake in secured configuration with no SASL "
+ "protection configured for addr = {}, datanodeId = {}", addr, datanodeId);
return(null);
}
}
}
}
}
}
/// <exception cref="System.IO.IOException"/>
public override void Run(DatanodeID id)
{
DataTransferTestUtil.DataTransferTest test = GetDataTransferTest();
if (test.IsNotSuccessAndLastPipelineContains(index, id))
{
string s = ToString(id);
FiTestUtil.Log.Info(s);
throw new IOException(s);
}
}
/// <param name="nodeReg">DatanodeID to update registration for.</param>
public override void UpdateRegInfo(DatanodeID nodeReg)
{
base.UpdateRegInfo(nodeReg);
// must re-process IBR after re-registration
foreach (DatanodeStorageInfo storage in GetStorageInfos())
{
storage.SetBlockReportCount(0);
}
heartbeatedSinceRegistration = false;
}
/// <exception cref="System.IO.IOException"/>
public static InterDatanodeProtocol CreateInterDatanodeProtocolProxy(DataNode dn,
DatanodeID datanodeid, Configuration conf, bool connectToDnViaHostname)
{
if (connectToDnViaHostname != dn.GetDnConf().connectToDnViaHostname)
{
throw new Exception("Unexpected DN hostname configuration");
}
return(DataNode.CreateInterDataNodeProtocolProxy(datanodeid, conf, dn.GetDnConf()
.socketTimeout, dn.GetDnConf().connectToDnViaHostname));
}
/// <exception cref="Org.Apache.Hadoop.Util.DiskChecker.DiskOutOfSpaceException"/>
public override void Run(DatanodeID id)
{
DataTransferTestUtil.DataTransferTest test = GetDataTransferTest();
if (test.IsNotSuccessAndLastPipelineContains(index, id) && countdown.IsSatisfied(
))
{
string s = ToString(id);
FiTestUtil.Log.Info(s);
throw new DiskChecker.DiskOutOfSpaceException(s);
}
}
/// <summary>
/// Is the test not yet success
/// and the last pipeline contains the given datanode?
/// </summary>
private bool IsNotSuccessAndLastPipelineContains(int index, DatanodeID id)
{
lock (this)
{
if (IsSuccess())
{
return(false);
}
int n = pipelines.Count;
return(n == 0 ? false : pipelines[n - 1].Contains(index, id));
}
}
/// <summary>Return the datanode descriptor for the given datanode.</summary>
/// <exception cref="System.IO.IOException"/>
public static DatanodeDescriptor GetDatanode(FSNamesystem ns, DatanodeID id)
{
ns.ReadLock();
try
{
return(ns.GetBlockManager().GetDatanodeManager().GetDatanode(id));
}
finally
{
ns.ReadUnlock();
}
}
/// <summary>DatanodeDescriptor constructor</summary>
/// <param name="nodeID">id of the data node</param>
/// <param name="networkLocation">location of the data node in network</param>
public DatanodeDescriptor(DatanodeID nodeID, string networkLocation)
: base(nodeID, networkLocation)
{
decommissioningStatus = new DatanodeDescriptor.DecommissioningStatus(this);
pendingCached = new DatanodeDescriptor.CachedBlocksList(this, DatanodeDescriptor.CachedBlocksList.Type
.PendingCached);
cached = new DatanodeDescriptor.CachedBlocksList(this, DatanodeDescriptor.CachedBlocksList.Type
.Cached);
pendingUncached = new DatanodeDescriptor.CachedBlocksList(this, DatanodeDescriptor.CachedBlocksList.Type
.PendingUncached);
UpdateHeartbeatState(StorageReport.EmptyArray, 0L, 0L, 0, 0, null);
}
