Esempio n. 1
0
        /// <returns>the FileInputStream for the meta data of the given block.</returns>
        /// <exception cref="System.IO.FileNotFoundException">if the file not found.</exception>
        /// <exception cref="System.InvalidCastException">if the underlying input stream is not a FileInputStream.
        ///     </exception>
        /// <exception cref="System.IO.IOException"/>
        public static FileInputStream GetMetaDataInputStream <_T0>(ExtendedBlock b, FsDatasetSpi
                                                                   <_T0> data)
            where _T0 : FsVolumeSpi
        {
            LengthInputStream lin = data.GetMetaDataInputStream(b);

            if (lin == null)
            {
                throw new FileNotFoundException("Meta file for " + b + " not found.");
            }
            return((FileInputStream)lin.GetWrappedStream());
        }
Esempio n. 2
0
        /// <summary>Constructor</summary>
        /// <param name="block">Block that is being read</param>
        /// <param name="startOffset">starting offset to read from</param>
        /// <param name="length">length of data to read</param>
        /// <param name="corruptChecksumOk">if true, corrupt checksum is okay</param>
        /// <param name="verifyChecksum">verify checksum while reading the data</param>
        /// <param name="sendChecksum">send checksum to client.</param>
        /// <param name="datanode">datanode from which the block is being read</param>
        /// <param name="clientTraceFmt">format string used to print client trace logs</param>
        /// <exception cref="System.IO.IOException"/>
        internal BlockSender(ExtendedBlock block, long startOffset, long length, bool corruptChecksumOk
                             , bool verifyChecksum, bool sendChecksum, DataNode datanode, string clientTraceFmt
                             , CachingStrategy cachingStrategy)
        {
            // Cache-management related fields
            // 1MB
            try
            {
                this.block             = block;
                this.corruptChecksumOk = corruptChecksumOk;
                this.verifyChecksum    = verifyChecksum;
                this.clientTraceFmt    = clientTraceFmt;

                /*
                 * If the client asked for the cache to be dropped behind all reads,
                 * we honor that.  Otherwise, we use the DataNode defaults.
                 * When using DataNode defaults, we use a heuristic where we only
                 * drop the cache for large reads.
                 */
                if (cachingStrategy.GetDropBehind() == null)
                {
                    this.dropCacheBehindAllReads   = false;
                    this.dropCacheBehindLargeReads = datanode.GetDnConf().dropCacheBehindReads;
                }
                else
                {
                    this.dropCacheBehindAllReads = this.dropCacheBehindLargeReads = cachingStrategy.GetDropBehind
                                                                                        ();
                }

                /*
                 * Similarly, if readahead was explicitly requested, we always do it.
                 * Otherwise, we read ahead based on the DataNode settings, and only
                 * when the reads are large.
                 */
                if (cachingStrategy.GetReadahead() == null)
                {
                    this.alwaysReadahead = false;
                    this.readaheadLength = datanode.GetDnConf().readaheadLength;
                }
                else
                {
                    this.alwaysReadahead = true;
                    this.readaheadLength = cachingStrategy.GetReadahead();
                }
                this.datanode = datanode;
                if (verifyChecksum)
                {
                    // To simplify implementation, callers may not specify verification
                    // without sending.
                    Preconditions.CheckArgument(sendChecksum, "If verifying checksum, currently must also send it."
                                                );
                }
                Replica replica;
                long    replicaVisibleLength;
                lock (datanode.data)
                {
                    replica = GetReplica(block, datanode);
                    replicaVisibleLength = replica.GetVisibleLength();
                }
                // if there is a write in progress
                ChunkChecksum chunkChecksum = null;
                if (replica is ReplicaBeingWritten)
                {
                    ReplicaBeingWritten rbw = (ReplicaBeingWritten)replica;
                    WaitForMinLength(rbw, startOffset + length);
                    chunkChecksum = rbw.GetLastChecksumAndDataLen();
                }
                if (replica.GetGenerationStamp() < block.GetGenerationStamp())
                {
                    throw new IOException("Replica gen stamp < block genstamp, block=" + block + ", replica="
                                          + replica);
                }
                else
                {
                    if (replica.GetGenerationStamp() > block.GetGenerationStamp())
                    {
                        if (DataNode.Log.IsDebugEnabled())
                        {
                            DataNode.Log.Debug("Bumping up the client provided" + " block's genstamp to latest "
                                               + replica.GetGenerationStamp() + " for block " + block);
                        }
                        block.SetGenerationStamp(replica.GetGenerationStamp());
                    }
                }
                if (replicaVisibleLength < 0)
                {
                    throw new IOException("Replica is not readable, block=" + block + ", replica=" +
                                          replica);
                }
                if (DataNode.Log.IsDebugEnabled())
                {
                    DataNode.Log.Debug("block=" + block + ", replica=" + replica);
                }
                // transferToFully() fails on 32 bit platforms for block sizes >= 2GB,
                // use normal transfer in those cases
                this.transferToAllowed = datanode.GetDnConf().transferToAllowed&& (!is32Bit || length
                                                                                   <= int.MaxValue);
                // Obtain a reference before reading data
                this.volumeRef = datanode.data.GetVolume(block).ObtainReference();

                /*
                 * (corruptChecksumOK, meta_file_exist): operation
                 * True,   True: will verify checksum
                 * True,  False: No verify, e.g., need to read data from a corrupted file
                 * False,  True: will verify checksum
                 * False, False: throws IOException file not found
                 */
                DataChecksum csum = null;
                if (verifyChecksum || sendChecksum)
                {
                    LengthInputStream metaIn = null;
                    bool keepMetaInOpen      = false;
                    try
                    {
                        metaIn = datanode.data.GetMetaDataInputStream(block);
                        if (!corruptChecksumOk || metaIn != null)
                        {
                            if (metaIn == null)
                            {
                                //need checksum but meta-data not found
                                throw new FileNotFoundException("Meta-data not found for " + block);
                            }
                            // The meta file will contain only the header if the NULL checksum
                            // type was used, or if the replica was written to transient storage.
                            // Checksum verification is not performed for replicas on transient
                            // storage.  The header is important for determining the checksum
                            // type later when lazy persistence copies the block to non-transient
                            // storage and computes the checksum.
                            if (metaIn.GetLength() > BlockMetadataHeader.GetHeaderSize())
                            {
                                checksumIn = new DataInputStream(new BufferedInputStream(metaIn, HdfsConstants.IoFileBufferSize
                                                                                         ));
                                csum           = BlockMetadataHeader.ReadDataChecksum(checksumIn, block);
                                keepMetaInOpen = true;
                            }
                        }
                        else
                        {
                            Log.Warn("Could not find metadata file for " + block);
                        }
                    }
                    finally
                    {
                        if (!keepMetaInOpen)
                        {
                            IOUtils.CloseStream(metaIn);
                        }
                    }
                }
                if (csum == null)
                {
                    // The number of bytes per checksum here determines the alignment
                    // of reads: we always start reading at a checksum chunk boundary,
                    // even if the checksum type is NULL. So, choosing too big of a value
                    // would risk sending too much unnecessary data. 512 (1 disk sector)
                    // is likely to result in minimal extra IO.
                    csum = DataChecksum.NewDataChecksum(DataChecksum.Type.Null, 512);
                }

                /*
                 * If chunkSize is very large, then the metadata file is mostly
                 * corrupted. For now just truncate bytesPerchecksum to blockLength.
                 */
                int size = csum.GetBytesPerChecksum();
                if (size > 10 * 1024 * 1024 && size > replicaVisibleLength)
                {
                    csum = DataChecksum.NewDataChecksum(csum.GetChecksumType(), Math.Max((int)replicaVisibleLength
                                                                                         , 10 * 1024 * 1024));
                    size = csum.GetBytesPerChecksum();
                }
                chunkSize    = size;
                checksum     = csum;
                checksumSize = checksum.GetChecksumSize();
                length       = length < 0 ? replicaVisibleLength : length;
                // end is either last byte on disk or the length for which we have a
                // checksum
                long end = chunkChecksum != null?chunkChecksum.GetDataLength() : replica.GetBytesOnDisk
                                   ();

                if (startOffset < 0 || startOffset > end || (length + startOffset) > end)
                {
                    string msg = " Offset " + startOffset + " and length " + length + " don't match block "
                                 + block + " ( blockLen " + end + " )";
                    Log.Warn(datanode.GetDNRegistrationForBP(block.GetBlockPoolId()) + ":sendBlock() : "
                             + msg);
                    throw new IOException(msg);
                }
                // Ensure read offset is position at the beginning of chunk
                offset = startOffset - (startOffset % chunkSize);
                if (length >= 0)
                {
                    // Ensure endOffset points to end of chunk.
                    long tmpLen = startOffset + length;
                    if (tmpLen % chunkSize != 0)
                    {
                        tmpLen += (chunkSize - tmpLen % chunkSize);
                    }
                    if (tmpLen < end)
                    {
                        // will use on-disk checksum here since the end is a stable chunk
                        end = tmpLen;
                    }
                    else
                    {
                        if (chunkChecksum != null)
                        {
                            // last chunk is changing. flag that we need to use in-memory checksum
                            this.lastChunkChecksum = chunkChecksum;
                        }
                    }
                }
                endOffset = end;
                // seek to the right offsets
                if (offset > 0 && checksumIn != null)
                {
                    long checksumSkip = (offset / chunkSize) * checksumSize;
                    // note blockInStream is seeked when created below
                    if (checksumSkip > 0)
                    {
                        // Should we use seek() for checksum file as well?
                        IOUtils.SkipFully(checksumIn, checksumSkip);
                    }
                }
                seqno = 0;
                if (DataNode.Log.IsDebugEnabled())
                {
                    DataNode.Log.Debug("replica=" + replica);
                }
                blockIn = datanode.data.GetBlockInputStream(block, offset);
                // seek to offset
                if (blockIn is FileInputStream)
                {
                    blockInFd = ((FileInputStream)blockIn).GetFD();
                }
                else
                {
                    blockInFd = null;
                }
            }
            catch (IOException ioe)
            {
                IOUtils.CloseStream(this);
                IOUtils.CloseStream(blockIn);
                throw;
            }
        }