private string FindReasonForNotCaching(CachedBlock cblock, BlockInfoContiguous blockInfo ) { if (blockInfo == null) { // Somehow, a cache report with the block arrived, but the block // reports from the DataNode haven't (yet?) described such a block. // Alternately, the NameNode might have invalidated the block, but the // DataNode hasn't caught up. In any case, we want to tell the DN // to uncache this. return("not tracked by the BlockManager"); } else { if (!blockInfo.IsComplete()) { // When a cached block changes state from complete to some other state // on the DataNode (perhaps because of append), it will begin the // uncaching process. However, the uncaching process is not // instantaneous, especially if clients have pinned the block. So // there may be a period of time when incomplete blocks remain cached // on the DataNodes. return("not complete"); } else { if (cblock.GetReplication() == 0) { // Since 0 is not a valid value for a cache directive's replication // field, seeing a replication of 0 on a CacheBlock means that it // has never been reached by any sweep. return("not needed by any directives"); } else { if (cblock.GetMark() != mark) { // Although the block was needed in the past, we didn't reach it during // the current sweep. Therefore, it doesn't need to be cached any more. // Need to set the replication to 0 so it doesn't flip back to cached // when the mark flips on the next scan cblock.SetReplicationAndMark((short)0, mark); return("no longer needed by any directives"); } } } } return(null); }
/// <summary>Add new entries to the PendingCached list.</summary> /// <param name="neededCached">The number of replicas that need to be cached.</param> /// <param name="cachedBlock">The block which needs to be cached.</param> /// <param name="cached">A list of DataNodes currently caching the block.</param> /// <param name="pendingCached"> /// A list of DataNodes that will soon cache the /// block. /// </param> private void AddNewPendingCached(int neededCached, CachedBlock cachedBlock, IList <DatanodeDescriptor> cached, IList <DatanodeDescriptor> pendingCached) { // To figure out which replicas can be cached, we consult the // blocksMap. We don't want to try to cache a corrupt replica, though. BlockInfoContiguous blockInfo = blockManager.GetStoredBlock(new Block(cachedBlock .GetBlockId())); if (blockInfo == null) { Log.Debug("Block {}: can't add new cached replicas," + " because there is no record of this block " + "on the NameNode.", cachedBlock.GetBlockId()); return; } if (!blockInfo.IsComplete()) { Log.Debug("Block {}: can't cache this block, because it is not yet" + " complete." , cachedBlock.GetBlockId()); return; } // Filter the list of replicas to only the valid targets IList <DatanodeDescriptor> possibilities = new List <DatanodeDescriptor>(); int numReplicas = blockInfo.GetCapacity(); ICollection <DatanodeDescriptor> corrupt = blockManager.GetCorruptReplicas(blockInfo ); int outOfCapacity = 0; for (int i = 0; i < numReplicas; i++) { DatanodeDescriptor datanode = blockInfo.GetDatanode(i); if (datanode == null) { continue; } if (datanode.IsDecommissioned() || datanode.IsDecommissionInProgress()) { continue; } if (corrupt != null && corrupt.Contains(datanode)) { continue; } if (pendingCached.Contains(datanode) || cached.Contains(datanode)) { continue; } long pendingBytes = 0; // Subtract pending cached blocks from effective capacity IEnumerator <CachedBlock> it = datanode.GetPendingCached().GetEnumerator(); while (it.HasNext()) { CachedBlock cBlock = it.Next(); BlockInfoContiguous info = blockManager.GetStoredBlock(new Block(cBlock.GetBlockId ())); if (info != null) { pendingBytes -= info.GetNumBytes(); } } it = datanode.GetPendingUncached().GetEnumerator(); // Add pending uncached blocks from effective capacity while (it.HasNext()) { CachedBlock cBlock = it.Next(); BlockInfoContiguous info = blockManager.GetStoredBlock(new Block(cBlock.GetBlockId ())); if (info != null) { pendingBytes += info.GetNumBytes(); } } long pendingCapacity = pendingBytes + datanode.GetCacheRemaining(); if (pendingCapacity < blockInfo.GetNumBytes()) { Log.Trace("Block {}: DataNode {} is not a valid possibility " + "because the block has size {}, but the DataNode only has {}" + "bytes of cache remaining ({} pending bytes, {} already cached.", blockInfo.GetBlockId (), datanode.GetDatanodeUuid(), blockInfo.GetNumBytes(), pendingCapacity, pendingBytes , datanode.GetCacheRemaining()); outOfCapacity++; continue; } possibilities.AddItem(datanode); } IList <DatanodeDescriptor> chosen = ChooseDatanodesForCaching(possibilities, neededCached , blockManager.GetDatanodeManager().GetStaleInterval()); foreach (DatanodeDescriptor datanode_1 in chosen) { Log.Trace("Block {}: added to PENDING_CACHED on DataNode {}", blockInfo.GetBlockId (), datanode_1.GetDatanodeUuid()); pendingCached.AddItem(datanode_1); bool added = datanode_1.GetPendingCached().AddItem(cachedBlock); System.Diagnostics.Debug.Assert(added); } // We were unable to satisfy the requested replication factor if (neededCached > chosen.Count) { Log.Debug("Block {}: we only have {} of {} cached replicas." + " {} DataNodes have insufficient cache capacity." , blockInfo.GetBlockId(), (cachedBlock.GetReplication() - neededCached + chosen. Count), cachedBlock.GetReplication(), outOfCapacity); } }