/// <summary>
/// Returns a
/// <see cref="CacheDirectiveInfo"/>
/// based on this CacheDirective.
/// <p>
/// This always sets an absolute expiry time, never a relative TTL.
/// </summary>
public CacheDirectiveInfo ToInfo()
{
return(new CacheDirectiveInfo.Builder().SetId(id).SetPath(new Path(path)).SetReplication
(replication).SetPool(pool.GetPoolName()).SetExpiration(CacheDirectiveInfo.Expiration
.NewAbsolute(expiryTime)).Build());
}
/// <summary>Apply a CacheDirective to a file.</summary>
/// <param name="directive">The CacheDirective to apply.</param>
/// <param name="file">The file.</param>
private void RescanFile(CacheDirective directive, INodeFile file)
{
BlockInfoContiguous[] blockInfos = file.GetBlocks();
// Increment the "needed" statistics
directive.AddFilesNeeded(1);
// We don't cache UC blocks, don't add them to the total here
long neededTotal = file.ComputeFileSizeNotIncludingLastUcBlock() * directive.GetReplication
();
directive.AddBytesNeeded(neededTotal);
// The pool's bytesNeeded is incremented as we scan. If the demand
// thus far plus the demand of this file would exceed the pool's limit,
// do not cache this file.
CachePool pool = directive.GetPool();
if (pool.GetBytesNeeded() > pool.GetLimit())
{
Log.Debug("Directive {}: not scanning file {} because " + "bytesNeeded for pool {} is {}, but the pool's limit is {}"
, directive.GetId(), file.GetFullPathName(), pool.GetPoolName(), pool.GetBytesNeeded
(), pool.GetLimit());
return;
}
long cachedTotal = 0;
foreach (BlockInfoContiguous blockInfo in blockInfos)
{
if (!blockInfo.GetBlockUCState().Equals(HdfsServerConstants.BlockUCState.Complete
))
{
// We don't try to cache blocks that are under construction.
Log.Trace("Directive {}: can't cache block {} because it is in state " + "{}, not COMPLETE."
, directive.GetId(), blockInfo, blockInfo.GetBlockUCState());
continue;
}
Block block = new Block(blockInfo.GetBlockId());
CachedBlock ncblock = new CachedBlock(block.GetBlockId(), directive.GetReplication
(), mark);
CachedBlock ocblock = cachedBlocks.Get(ncblock);
if (ocblock == null)
{
cachedBlocks.Put(ncblock);
ocblock = ncblock;
}
else
{
// Update bytesUsed using the current replication levels.
// Assumptions: we assume that all the blocks are the same length
// on each datanode. We can assume this because we're only caching
// blocks in state COMPLETE.
// Note that if two directives are caching the same block(s), they will
// both get them added to their bytesCached.
IList <DatanodeDescriptor> cachedOn = ocblock.GetDatanodes(DatanodeDescriptor.CachedBlocksList.Type
.Cached);
long cachedByBlock = Math.Min(cachedOn.Count, directive.GetReplication()) * blockInfo
.GetNumBytes();
cachedTotal += cachedByBlock;
if ((mark != ocblock.GetMark()) || (ocblock.GetReplication() < directive.GetReplication
()))
{
//
// Overwrite the block's replication and mark in two cases:
//
// 1. If the mark on the CachedBlock is different from the mark for
// this scan, that means the block hasn't been updated during this
// scan, and we should overwrite whatever is there, since it is no
// longer valid.
//
// 2. If the replication in the CachedBlock is less than what the
// directive asks for, we want to increase the block's replication
// field to what the directive asks for.
//
ocblock.SetReplicationAndMark(directive.GetReplication(), mark);
}
}
Log.Trace("Directive {}: setting replication for block {} to {}", directive.GetId
(), blockInfo, ocblock.GetReplication());
}
// Increment the "cached" statistics
directive.AddBytesCached(cachedTotal);
if (cachedTotal == neededTotal)
{
directive.AddFilesCached(1);
}
Log.Debug("Directive {}: caching {}: {}/{} bytes", directive.GetId(), file.GetFullPathName
(), cachedTotal, neededTotal);
}