/// <summary>Scan through the cached block map.</summary> /// <remarks> /// Scan through the cached block map. /// Any blocks which are under-replicated should be assigned new Datanodes. /// Blocks that are over-replicated should be removed from Datanodes. /// </remarks> private void RescanCachedBlockMap() { for (IEnumerator <CachedBlock> cbIter = cachedBlocks.GetEnumerator(); cbIter.HasNext ();) { scannedBlocks++; CachedBlock cblock = cbIter.Next(); IList <DatanodeDescriptor> pendingCached = cblock.GetDatanodes(DatanodeDescriptor.CachedBlocksList.Type .PendingCached); IList <DatanodeDescriptor> cached = cblock.GetDatanodes(DatanodeDescriptor.CachedBlocksList.Type .Cached); IList <DatanodeDescriptor> pendingUncached = cblock.GetDatanodes(DatanodeDescriptor.CachedBlocksList.Type .PendingUncached); // Remove nodes from PENDING_UNCACHED if they were actually uncached. for (IEnumerator <DatanodeDescriptor> iter = pendingUncached.GetEnumerator(); iter .HasNext();) { DatanodeDescriptor datanode = iter.Next(); if (!cblock.IsInList(datanode.GetCached())) { Log.Trace("Block {}: removing from PENDING_UNCACHED for node {} " + "because the DataNode uncached it." , cblock.GetBlockId(), datanode.GetDatanodeUuid()); datanode.GetPendingUncached().Remove(cblock); iter.Remove(); } } BlockInfoContiguous blockInfo = blockManager.GetStoredBlock(new Block(cblock.GetBlockId ())); string reason = FindReasonForNotCaching(cblock, blockInfo); int neededCached = 0; if (reason != null) { Log.Trace("Block {}: can't cache block because it is {}", cblock.GetBlockId(), reason ); } else { neededCached = cblock.GetReplication(); } int numCached = cached.Count; if (numCached >= neededCached) { // If we have enough replicas, drop all pending cached. for (IEnumerator <DatanodeDescriptor> iter_1 = pendingCached.GetEnumerator(); iter_1 .HasNext();) { DatanodeDescriptor datanode = iter_1.Next(); datanode.GetPendingCached().Remove(cblock); iter_1.Remove(); Log.Trace("Block {}: removing from PENDING_CACHED for node {}" + "because we already have {} cached replicas and we only" + " need {}", cblock.GetBlockId(), datanode.GetDatanodeUuid(), numCached, neededCached ); } } if (numCached < neededCached) { // If we don't have enough replicas, drop all pending uncached. for (IEnumerator <DatanodeDescriptor> iter_1 = pendingUncached.GetEnumerator(); iter_1 .HasNext();) { DatanodeDescriptor datanode = iter_1.Next(); datanode.GetPendingUncached().Remove(cblock); iter_1.Remove(); Log.Trace("Block {}: removing from PENDING_UNCACHED for node {} " + "because we only have {} cached replicas and we need " + "{}", cblock.GetBlockId(), datanode.GetDatanodeUuid(), numCached, neededCached ); } } int neededUncached = numCached - (pendingUncached.Count + neededCached); if (neededUncached > 0) { AddNewPendingUncached(neededUncached, cblock, cached, pendingUncached); } else { int additionalCachedNeeded = neededCached - (numCached + pendingCached.Count); if (additionalCachedNeeded > 0) { AddNewPendingCached(additionalCachedNeeded, cblock, cached, pendingCached); } } if ((neededCached == 0) && pendingUncached.IsEmpty() && pendingCached.IsEmpty()) { // we have nothing more to do with this block. Log.Trace("Block {}: removing from cachedBlocks, since neededCached " + "== 0, and pendingUncached and pendingCached are empty." , cblock.GetBlockId()); cbIter.Remove(); } } }
public virtual IEnumerator <INodeWithAdditionalFields> GetMapIterator() { return(map.GetEnumerator()); }