Ejemplo n.º 1
0
 /// <summary>
 /// Returns a
 /// <see cref="CacheDirectiveInfo"/>
 /// based on this CacheDirective.
 /// <p>
 /// This always sets an absolute expiry time, never a relative TTL.
 /// </summary>
 public CacheDirectiveInfo ToInfo()
 {
     return(new CacheDirectiveInfo.Builder().SetId(id).SetPath(new Path(path)).SetReplication
                (replication).SetPool(pool.GetPoolName()).SetExpiration(CacheDirectiveInfo.Expiration
                                                                        .NewAbsolute(expiryTime)).Build());
 }
Ejemplo n.º 2
0
        /// <summary>Apply a CacheDirective to a file.</summary>
        /// <param name="directive">The CacheDirective to apply.</param>
        /// <param name="file">The file.</param>
        private void RescanFile(CacheDirective directive, INodeFile file)
        {
            BlockInfoContiguous[] blockInfos = file.GetBlocks();
            // Increment the "needed" statistics
            directive.AddFilesNeeded(1);
            // We don't cache UC blocks, don't add them to the total here
            long neededTotal = file.ComputeFileSizeNotIncludingLastUcBlock() * directive.GetReplication
                                   ();

            directive.AddBytesNeeded(neededTotal);
            // The pool's bytesNeeded is incremented as we scan. If the demand
            // thus far plus the demand of this file would exceed the pool's limit,
            // do not cache this file.
            CachePool pool = directive.GetPool();

            if (pool.GetBytesNeeded() > pool.GetLimit())
            {
                Log.Debug("Directive {}: not scanning file {} because " + "bytesNeeded for pool {} is {}, but the pool's limit is {}"
                          , directive.GetId(), file.GetFullPathName(), pool.GetPoolName(), pool.GetBytesNeeded
                              (), pool.GetLimit());
                return;
            }
            long cachedTotal = 0;

            foreach (BlockInfoContiguous blockInfo in blockInfos)
            {
                if (!blockInfo.GetBlockUCState().Equals(HdfsServerConstants.BlockUCState.Complete
                                                        ))
                {
                    // We don't try to cache blocks that are under construction.
                    Log.Trace("Directive {}: can't cache block {} because it is in state " + "{}, not COMPLETE."
                              , directive.GetId(), blockInfo, blockInfo.GetBlockUCState());
                    continue;
                }
                Block       block   = new Block(blockInfo.GetBlockId());
                CachedBlock ncblock = new CachedBlock(block.GetBlockId(), directive.GetReplication
                                                          (), mark);
                CachedBlock ocblock = cachedBlocks.Get(ncblock);
                if (ocblock == null)
                {
                    cachedBlocks.Put(ncblock);
                    ocblock = ncblock;
                }
                else
                {
                    // Update bytesUsed using the current replication levels.
                    // Assumptions: we assume that all the blocks are the same length
                    // on each datanode.  We can assume this because we're only caching
                    // blocks in state COMPLETE.
                    // Note that if two directives are caching the same block(s), they will
                    // both get them added to their bytesCached.
                    IList <DatanodeDescriptor> cachedOn = ocblock.GetDatanodes(DatanodeDescriptor.CachedBlocksList.Type
                                                                               .Cached);
                    long cachedByBlock = Math.Min(cachedOn.Count, directive.GetReplication()) * blockInfo
                                         .GetNumBytes();
                    cachedTotal += cachedByBlock;
                    if ((mark != ocblock.GetMark()) || (ocblock.GetReplication() < directive.GetReplication
                                                            ()))
                    {
                        //
                        // Overwrite the block's replication and mark in two cases:
                        //
                        // 1. If the mark on the CachedBlock is different from the mark for
                        // this scan, that means the block hasn't been updated during this
                        // scan, and we should overwrite whatever is there, since it is no
                        // longer valid.
                        //
                        // 2. If the replication in the CachedBlock is less than what the
                        // directive asks for, we want to increase the block's replication
                        // field to what the directive asks for.
                        //
                        ocblock.SetReplicationAndMark(directive.GetReplication(), mark);
                    }
                }
                Log.Trace("Directive {}: setting replication for block {} to {}", directive.GetId
                              (), blockInfo, ocblock.GetReplication());
            }
            // Increment the "cached" statistics
            directive.AddBytesCached(cachedTotal);
            if (cachedTotal == neededTotal)
            {
                directive.AddFilesCached(1);
            }
            Log.Debug("Directive {}: caching {}: {}/{} bytes", directive.GetId(), file.GetFullPathName
                          (), cachedTotal, neededTotal);
        }