/// <summary>Find out the number of bytes in the block that match its crc.</summary>
/// <remarks>
/// Find out the number of bytes in the block that match its crc.
/// This algorithm assumes that data corruption caused by unexpected
/// datanode shutdown occurs only in the last crc chunk. So it checks
/// only the last chunk.
/// </remarks>
/// <param name="blockFile">the block file</param>
/// <param name="genStamp">generation stamp of the block</param>
/// <returns>the number of valid bytes</returns>
private long ValidateIntegrityAndSetLength(FilePath blockFile, long genStamp)
{
DataInputStream checksumIn = null;
InputStream blockIn = null;
try
{
FilePath metaFile = FsDatasetUtil.GetMetaFile(blockFile, genStamp);
long blockFileLen = blockFile.Length();
long metaFileLen = metaFile.Length();
int crcHeaderLen = DataChecksum.GetChecksumHeaderSize();
if (!blockFile.Exists() || blockFileLen == 0 || !metaFile.Exists() || metaFileLen
< crcHeaderLen)
{
return(0);
}
checksumIn = new DataInputStream(new BufferedInputStream(new FileInputStream(metaFile
), HdfsConstants.IoFileBufferSize));
// read and handle the common header here. For now just a version
DataChecksum checksum = BlockMetadataHeader.ReadDataChecksum(checksumIn, metaFile
);
int bytesPerChecksum = checksum.GetBytesPerChecksum();
int checksumSize = checksum.GetChecksumSize();
long numChunks = Math.Min((blockFileLen + bytesPerChecksum - 1) / bytesPerChecksum
, (metaFileLen - crcHeaderLen) / checksumSize);
if (numChunks == 0)
{
return(0);
}
IOUtils.SkipFully(checksumIn, (numChunks - 1) * checksumSize);
blockIn = new FileInputStream(blockFile);
long lastChunkStartPos = (numChunks - 1) * bytesPerChecksum;
IOUtils.SkipFully(blockIn, lastChunkStartPos);
int lastChunkSize = (int)Math.Min(bytesPerChecksum, blockFileLen - lastChunkStartPos
);
byte[] buf = new byte[lastChunkSize + checksumSize];
checksumIn.ReadFully(buf, lastChunkSize, checksumSize);
IOUtils.ReadFully(blockIn, buf, 0, lastChunkSize);
checksum.Update(buf, 0, lastChunkSize);
long validFileLength;
if (checksum.Compare(buf, lastChunkSize))
{
// last chunk matches crc
validFileLength = lastChunkStartPos + lastChunkSize;
}
else
{
// last chunck is corrupt
validFileLength = lastChunkStartPos;
}
// truncate if extra bytes are present without CRC
if (blockFile.Length() > validFileLength)
{
RandomAccessFile blockRAF = new RandomAccessFile(blockFile, "rw");
try
{
// truncate blockFile
blockRAF.SetLength(validFileLength);
}
finally
{
blockRAF.Close();
}
}
return(validFileLength);
}
catch (IOException e)
{
FsDatasetImpl.Log.Warn(e);
return(0);
}
finally
{
IOUtils.CloseStream(checksumIn);
IOUtils.CloseStream(blockIn);
}
}
// Multiple datanodes could be running on the local machine. Store proxies in
// a map keyed by the ipc port of the datanode.
// reader for the data file
// reader for the checksum file
/// <summary>The only way this object can be instantiated.</summary>
/// <exception cref="System.IO.IOException"/>
internal static BlockReaderLocalLegacy NewBlockReader(DFSClient.Conf conf, UserGroupInformation
userGroupInformation, Configuration configuration, string file, ExtendedBlock blk
, Org.Apache.Hadoop.Security.Token.Token <BlockTokenIdentifier> token, DatanodeInfo
node, long startOffset, long length, StorageType storageType)
{
BlockReaderLocalLegacy.LocalDatanodeInfo localDatanodeInfo = GetLocalDatanodeInfo
(node.GetIpcPort());
// check the cache first
BlockLocalPathInfo pathinfo = localDatanodeInfo.GetBlockLocalPathInfo(blk);
if (pathinfo == null)
{
if (userGroupInformation == null)
{
userGroupInformation = UserGroupInformation.GetCurrentUser();
}
pathinfo = GetBlockPathInfo(userGroupInformation, blk, node, configuration, conf.
socketTimeout, token, conf.connectToDnViaHostname, storageType);
}
// check to see if the file exists. It may so happen that the
// HDFS file has been deleted and this block-lookup is occurring
// on behalf of a new HDFS file. This time, the block file could
// be residing in a different portion of the fs.data.dir directory.
// In this case, we remove this entry from the cache. The next
// call to this method will re-populate the cache.
FileInputStream dataIn = null;
FileInputStream checksumIn = null;
BlockReaderLocalLegacy localBlockReader = null;
bool skipChecksumCheck = conf.skipShortCircuitChecksums || storageType.IsTransient
();
try
{
// get a local file system
FilePath blkfile = new FilePath(pathinfo.GetBlockPath());
dataIn = new FileInputStream(blkfile);
if (Log.IsDebugEnabled())
{
Log.Debug("New BlockReaderLocalLegacy for file " + blkfile + " of size " + blkfile
.Length() + " startOffset " + startOffset + " length " + length + " short circuit checksum "
+ !skipChecksumCheck);
}
if (!skipChecksumCheck)
{
// get the metadata file
FilePath metafile = new FilePath(pathinfo.GetMetaPath());
checksumIn = new FileInputStream(metafile);
DataChecksum checksum = BlockMetadataHeader.ReadDataChecksum(new DataInputStream(
checksumIn), blk);
long firstChunkOffset = startOffset - (startOffset % checksum.GetBytesPerChecksum
());
localBlockReader = new BlockReaderLocalLegacy(conf, file, blk, token, startOffset
, length, pathinfo, checksum, true, dataIn, firstChunkOffset, checksumIn);
}
else
{
localBlockReader = new BlockReaderLocalLegacy(conf, file, blk, token, startOffset
, length, pathinfo, dataIn);
}
}
catch (IOException e)
{
// remove from cache
localDatanodeInfo.RemoveBlockLocalPathInfo(blk);
DFSClient.Log.Warn("BlockReaderLocalLegacy: Removing " + blk + " from cache because local file "
+ pathinfo.GetBlockPath() + " could not be opened.");
throw;
}
finally
{
if (localBlockReader == null)
{
if (dataIn != null)
{
dataIn.Close();
}
if (checksumIn != null)
{
checksumIn.Close();
}
}
}
return(localBlockReader);
}