private string FindReasonForNotCaching(CachedBlock cblock, BlockInfoContiguous blockInfo
)
{
if (blockInfo == null)
{
// Somehow, a cache report with the block arrived, but the block
// reports from the DataNode haven't (yet?) described such a block.
// Alternately, the NameNode might have invalidated the block, but the
// DataNode hasn't caught up. In any case, we want to tell the DN
// to uncache this.
return("not tracked by the BlockManager");
}
else
{
if (!blockInfo.IsComplete())
{
// When a cached block changes state from complete to some other state
// on the DataNode (perhaps because of append), it will begin the
// uncaching process. However, the uncaching process is not
// instantaneous, especially if clients have pinned the block. So
// there may be a period of time when incomplete blocks remain cached
// on the DataNodes.
return("not complete");
}
else
{
if (cblock.GetReplication() == 0)
{
// Since 0 is not a valid value for a cache directive's replication
// field, seeing a replication of 0 on a CacheBlock means that it
// has never been reached by any sweep.
return("not needed by any directives");
}
else
{
if (cblock.GetMark() != mark)
{
// Although the block was needed in the past, we didn't reach it during
// the current sweep. Therefore, it doesn't need to be cached any more.
// Need to set the replication to 0 so it doesn't flip back to cached
// when the mark flips on the next scan
cblock.SetReplicationAndMark((short)0, mark);
return("no longer needed by any directives");
}
}
}
}
return(null);
}
/// <summary>Apply a CacheDirective to a file.</summary>
/// <param name="directive">The CacheDirective to apply.</param>
/// <param name="file">The file.</param>
private void RescanFile(CacheDirective directive, INodeFile file)
{
BlockInfoContiguous[] blockInfos = file.GetBlocks();
// Increment the "needed" statistics
directive.AddFilesNeeded(1);
// We don't cache UC blocks, don't add them to the total here
long neededTotal = file.ComputeFileSizeNotIncludingLastUcBlock() * directive.GetReplication
();
directive.AddBytesNeeded(neededTotal);
// The pool's bytesNeeded is incremented as we scan. If the demand
// thus far plus the demand of this file would exceed the pool's limit,
// do not cache this file.
CachePool pool = directive.GetPool();
if (pool.GetBytesNeeded() > pool.GetLimit())
{
Log.Debug("Directive {}: not scanning file {} because " + "bytesNeeded for pool {} is {}, but the pool's limit is {}"
, directive.GetId(), file.GetFullPathName(), pool.GetPoolName(), pool.GetBytesNeeded
(), pool.GetLimit());
return;
}
long cachedTotal = 0;
foreach (BlockInfoContiguous blockInfo in blockInfos)
{
if (!blockInfo.GetBlockUCState().Equals(HdfsServerConstants.BlockUCState.Complete
))
{
// We don't try to cache blocks that are under construction.
Log.Trace("Directive {}: can't cache block {} because it is in state " + "{}, not COMPLETE."
, directive.GetId(), blockInfo, blockInfo.GetBlockUCState());
continue;
}
Block block = new Block(blockInfo.GetBlockId());
CachedBlock ncblock = new CachedBlock(block.GetBlockId(), directive.GetReplication
(), mark);
CachedBlock ocblock = cachedBlocks.Get(ncblock);
if (ocblock == null)
{
cachedBlocks.Put(ncblock);
ocblock = ncblock;
}
else
{
// Update bytesUsed using the current replication levels.
// Assumptions: we assume that all the blocks are the same length
// on each datanode. We can assume this because we're only caching
// blocks in state COMPLETE.
// Note that if two directives are caching the same block(s), they will
// both get them added to their bytesCached.
IList <DatanodeDescriptor> cachedOn = ocblock.GetDatanodes(DatanodeDescriptor.CachedBlocksList.Type
.Cached);
long cachedByBlock = Math.Min(cachedOn.Count, directive.GetReplication()) * blockInfo
.GetNumBytes();
cachedTotal += cachedByBlock;
if ((mark != ocblock.GetMark()) || (ocblock.GetReplication() < directive.GetReplication
()))
{
//
// Overwrite the block's replication and mark in two cases:
//
// 1. If the mark on the CachedBlock is different from the mark for
// this scan, that means the block hasn't been updated during this
// scan, and we should overwrite whatever is there, since it is no
// longer valid.
//
// 2. If the replication in the CachedBlock is less than what the
// directive asks for, we want to increase the block's replication
// field to what the directive asks for.
//
ocblock.SetReplicationAndMark(directive.GetReplication(), mark);
}
}
Log.Trace("Directive {}: setting replication for block {} to {}", directive.GetId
(), blockInfo, ocblock.GetReplication());
}
// Increment the "cached" statistics
directive.AddBytesCached(cachedTotal);
if (cachedTotal == neededTotal)
{
directive.AddFilesCached(1);
}
Log.Debug("Directive {}: caching {}: {}/{} bytes", directive.GetId(), file.GetFullPathName
(), cachedTotal, neededTotal);
}
