private void Subtract(DatanodeDescriptor node)
{
capacityUsed -= node.GetDfsUsed();
blockPoolUsed -= node.GetBlockPoolUsed();
xceiverCount -= node.GetXceiverCount();
if (!(node.IsDecommissionInProgress() || node.IsDecommissioned()))
{
nodesInService--;
nodesInServiceXceiverCount -= node.GetXceiverCount();
capacityTotal -= node.GetCapacity();
capacityRemaining -= node.GetRemaining();
}
else
{
capacityTotal -= node.GetDfsUsed();
}
cacheCapacity -= node.GetCacheCapacity();
cacheUsed -= node.GetCacheUsed();
}
public virtual void StopDecommission(DatanodeDescriptor node)
{
if (node.IsDecommissionInProgress() || node.IsDecommissioned())
{
// Update DN stats maintained by HeartbeatManager
hbManager.StopDecommission(node);
// Over-replicated blocks will be detected and processed when
// the dead node comes back and send in its full block report.
if (node.isAlive)
{
blockManager.ProcessOverReplicatedBlocksOnReCommission(node);
}
// Remove from tracking in DecommissionManager
pendingNodes.Remove(node);
Sharpen.Collections.Remove(decomNodeBlocks, node);
}
else
{
Log.Trace("stopDecommission: Node {} in {}, nothing to do." + node, node.GetAdminState
());
}
}
/// <returns>
/// the number of racks over which a given block is replicated
/// decommissioning/decommissioned nodes are not counted. corrupt replicas
/// are also ignored
/// </returns>
private static int GetNumberOfRacks(BlockManager blockManager, Block b)
{
ICollection <string> rackSet = new HashSet <string>(0);
ICollection <DatanodeDescriptor> corruptNodes = GetCorruptReplicas(blockManager).GetNodes
(b);
foreach (DatanodeStorageInfo storage in blockManager.blocksMap.GetStorages(b))
{
DatanodeDescriptor cur = storage.GetDatanodeDescriptor();
if (!cur.IsDecommissionInProgress() && !cur.IsDecommissioned())
{
if ((corruptNodes == null) || !corruptNodes.Contains(cur))
{
string rackName = cur.GetNetworkLocation();
if (!rackSet.Contains(rackName))
{
rackSet.AddItem(rackName);
}
}
}
}
return(rackSet.Count);
}
public virtual void StartDecommission(DatanodeDescriptor node)
{
if (!node.IsDecommissionInProgress() && !node.IsDecommissioned())
{
// Update DN stats maintained by HeartbeatManager
hbManager.StartDecommission(node);
// hbManager.startDecommission will set dead node to decommissioned.
if (node.IsDecommissionInProgress())
{
foreach (DatanodeStorageInfo storage in node.GetStorageInfos())
{
Log.Info("Starting decommission of {} {} with {} blocks", node, storage, storage.
NumBlocks());
}
node.decommissioningStatus.SetStartTime(Time.MonotonicNow());
pendingNodes.AddItem(node);
}
}
else
{
Log.Trace("startDecommission: Node {} in {}, nothing to do." + node, node.GetAdminState
());
}
}
/// <summary>Add new entries to the PendingCached list.</summary>
/// <param name="neededCached">The number of replicas that need to be cached.</param>
/// <param name="cachedBlock">The block which needs to be cached.</param>
/// <param name="cached">A list of DataNodes currently caching the block.</param>
/// <param name="pendingCached">
/// A list of DataNodes that will soon cache the
/// block.
/// </param>
private void AddNewPendingCached(int neededCached, CachedBlock cachedBlock, IList
<DatanodeDescriptor> cached, IList <DatanodeDescriptor> pendingCached)
{
// To figure out which replicas can be cached, we consult the
// blocksMap. We don't want to try to cache a corrupt replica, though.
BlockInfoContiguous blockInfo = blockManager.GetStoredBlock(new Block(cachedBlock
.GetBlockId()));
if (blockInfo == null)
{
Log.Debug("Block {}: can't add new cached replicas," + " because there is no record of this block "
+ "on the NameNode.", cachedBlock.GetBlockId());
return;
}
if (!blockInfo.IsComplete())
{
Log.Debug("Block {}: can't cache this block, because it is not yet" + " complete."
, cachedBlock.GetBlockId());
return;
}
// Filter the list of replicas to only the valid targets
IList <DatanodeDescriptor> possibilities = new List <DatanodeDescriptor>();
int numReplicas = blockInfo.GetCapacity();
ICollection <DatanodeDescriptor> corrupt = blockManager.GetCorruptReplicas(blockInfo
);
int outOfCapacity = 0;
for (int i = 0; i < numReplicas; i++)
{
DatanodeDescriptor datanode = blockInfo.GetDatanode(i);
if (datanode == null)
{
continue;
}
if (datanode.IsDecommissioned() || datanode.IsDecommissionInProgress())
{
continue;
}
if (corrupt != null && corrupt.Contains(datanode))
{
continue;
}
if (pendingCached.Contains(datanode) || cached.Contains(datanode))
{
continue;
}
long pendingBytes = 0;
// Subtract pending cached blocks from effective capacity
IEnumerator <CachedBlock> it = datanode.GetPendingCached().GetEnumerator();
while (it.HasNext())
{
CachedBlock cBlock = it.Next();
BlockInfoContiguous info = blockManager.GetStoredBlock(new Block(cBlock.GetBlockId
()));
if (info != null)
{
pendingBytes -= info.GetNumBytes();
}
}
it = datanode.GetPendingUncached().GetEnumerator();
// Add pending uncached blocks from effective capacity
while (it.HasNext())
{
CachedBlock cBlock = it.Next();
BlockInfoContiguous info = blockManager.GetStoredBlock(new Block(cBlock.GetBlockId
()));
if (info != null)
{
pendingBytes += info.GetNumBytes();
}
}
long pendingCapacity = pendingBytes + datanode.GetCacheRemaining();
if (pendingCapacity < blockInfo.GetNumBytes())
{
Log.Trace("Block {}: DataNode {} is not a valid possibility " + "because the block has size {}, but the DataNode only has {}"
+ "bytes of cache remaining ({} pending bytes, {} already cached.", blockInfo.GetBlockId
(), datanode.GetDatanodeUuid(), blockInfo.GetNumBytes(), pendingCapacity, pendingBytes
, datanode.GetCacheRemaining());
outOfCapacity++;
continue;
}
possibilities.AddItem(datanode);
}
IList <DatanodeDescriptor> chosen = ChooseDatanodesForCaching(possibilities, neededCached
, blockManager.GetDatanodeManager().GetStaleInterval());
foreach (DatanodeDescriptor datanode_1 in chosen)
{
Log.Trace("Block {}: added to PENDING_CACHED on DataNode {}", blockInfo.GetBlockId
(), datanode_1.GetDatanodeUuid());
pendingCached.AddItem(datanode_1);
bool added = datanode_1.GetPendingCached().AddItem(cachedBlock);
System.Diagnostics.Debug.Assert(added);
}
// We were unable to satisfy the requested replication factor
if (neededCached > chosen.Count)
{
Log.Debug("Block {}: we only have {} of {} cached replicas." + " {} DataNodes have insufficient cache capacity."
, blockInfo.GetBlockId(), (cachedBlock.GetReplication() - neededCached + chosen.
Count), cachedBlock.GetReplication(), outOfCapacity);
}
}