/// <exception cref="System.IO.IOException"/>
public static FilePath GetMetaFile <_T0>(FsDatasetSpi <_T0> fsd, string bpid, Block
b)
where _T0 : FsVolumeSpi
{
return(FsDatasetUtil.GetMetaFile(GetBlockFile(fsd, bpid, b), b.GetGenerationStamp
()));
}
/// <exception cref="System.IO.IOException"/>
internal virtual FilePath AddBlock(Block b, FilePath f)
{
FilePath blockDir = DatanodeUtil.IdToBlockDir(finalizedDir, b.GetBlockId());
if (!blockDir.Exists())
{
if (!blockDir.Mkdirs())
{
throw new IOException("Failed to mkdirs " + blockDir);
}
}
FilePath blockFile = FsDatasetImpl.MoveBlockFiles(b, f, blockDir);
FilePath metaFile = FsDatasetUtil.GetMetaFile(blockFile, b.GetGenerationStamp());
dfsUsage.IncDfsUsed(b.GetNumBytes() + metaFile.Length());
return(blockFile);
}
/// <summary>Recover an unlinked tmp file on datanode restart.</summary>
/// <remarks>
/// Recover an unlinked tmp file on datanode restart. If the original block
/// does not exist, then the tmp file is renamed to be the
/// original file name and the original name is returned; otherwise the tmp
/// file is deleted and null is returned.
/// </remarks>
/// <exception cref="System.IO.IOException"/>
internal virtual FilePath RecoverTempUnlinkedBlock(FilePath unlinkedTmp)
{
FilePath blockFile = FsDatasetUtil.GetOrigFile(unlinkedTmp);
if (blockFile.Exists())
{
// If the original block file still exists, then no recovery is needed.
if (!unlinkedTmp.Delete())
{
throw new IOException("Unable to cleanup unlinked tmp file " + unlinkedTmp);
}
return(null);
}
else
{
if (!unlinkedTmp.RenameTo(blockFile))
{
throw new IOException("Unable to rename unlinked tmp file " + unlinkedTmp);
}
return(blockFile);
}
}
/// <summary>Find out the number of bytes in the block that match its crc.</summary>
/// <remarks>
/// Find out the number of bytes in the block that match its crc.
/// This algorithm assumes that data corruption caused by unexpected
/// datanode shutdown occurs only in the last crc chunk. So it checks
/// only the last chunk.
/// </remarks>
/// <param name="blockFile">the block file</param>
/// <param name="genStamp">generation stamp of the block</param>
/// <returns>the number of valid bytes</returns>
private long ValidateIntegrityAndSetLength(FilePath blockFile, long genStamp)
{
DataInputStream checksumIn = null;
InputStream blockIn = null;
try
{
FilePath metaFile = FsDatasetUtil.GetMetaFile(blockFile, genStamp);
long blockFileLen = blockFile.Length();
long metaFileLen = metaFile.Length();
int crcHeaderLen = DataChecksum.GetChecksumHeaderSize();
if (!blockFile.Exists() || blockFileLen == 0 || !metaFile.Exists() || metaFileLen
< crcHeaderLen)
{
return(0);
}
checksumIn = new DataInputStream(new BufferedInputStream(new FileInputStream(metaFile
), HdfsConstants.IoFileBufferSize));
// read and handle the common header here. For now just a version
DataChecksum checksum = BlockMetadataHeader.ReadDataChecksum(checksumIn, metaFile
);
int bytesPerChecksum = checksum.GetBytesPerChecksum();
int checksumSize = checksum.GetChecksumSize();
long numChunks = Math.Min((blockFileLen + bytesPerChecksum - 1) / bytesPerChecksum
, (metaFileLen - crcHeaderLen) / checksumSize);
if (numChunks == 0)
{
return(0);
}
IOUtils.SkipFully(checksumIn, (numChunks - 1) * checksumSize);
blockIn = new FileInputStream(blockFile);
long lastChunkStartPos = (numChunks - 1) * bytesPerChecksum;
IOUtils.SkipFully(blockIn, lastChunkStartPos);
int lastChunkSize = (int)Math.Min(bytesPerChecksum, blockFileLen - lastChunkStartPos
);
byte[] buf = new byte[lastChunkSize + checksumSize];
checksumIn.ReadFully(buf, lastChunkSize, checksumSize);
IOUtils.ReadFully(blockIn, buf, 0, lastChunkSize);
checksum.Update(buf, 0, lastChunkSize);
long validFileLength;
if (checksum.Compare(buf, lastChunkSize))
{
// last chunk matches crc
validFileLength = lastChunkStartPos + lastChunkSize;
}
else
{
// last chunck is corrupt
validFileLength = lastChunkStartPos;
}
// truncate if extra bytes are present without CRC
if (blockFile.Length() > validFileLength)
{
RandomAccessFile blockRAF = new RandomAccessFile(blockFile, "rw");
try
{
// truncate blockFile
blockRAF.SetLength(validFileLength);
}
finally
{
blockRAF.Close();
}
}
return(validFileLength);
}
catch (IOException e)
{
FsDatasetImpl.Log.Warn(e);
return(0);
}
finally
{
IOUtils.CloseStream(checksumIn);
IOUtils.CloseStream(blockIn);
}
}
/// <summary>Add replicas under the given directory to the volume map</summary>
/// <param name="volumeMap">the replicas map</param>
/// <param name="dir">an input directory</param>
/// <param name="lazyWriteReplicaMap">
/// Map of replicas on transient
/// storage.
/// </param>
/// <param name="isFinalized">
/// true if the directory has finalized replicas;
/// false if the directory has rbw replicas
/// </param>
/// <exception cref="System.IO.IOException"/>
internal virtual void AddToReplicasMap(ReplicaMap volumeMap, FilePath dir, RamDiskReplicaTracker
lazyWriteReplicaMap, bool isFinalized)
{
FilePath[] files = FileUtil.ListFiles(dir);
foreach (FilePath file in files)
{
if (file.IsDirectory())
{
AddToReplicasMap(volumeMap, file, lazyWriteReplicaMap, isFinalized);
}
if (isFinalized && FsDatasetUtil.IsUnlinkTmpFile(file))
{
file = RecoverTempUnlinkedBlock(file);
if (file == null)
{
// the original block still exists, so we cover it
// in another iteration and can continue here
continue;
}
}
if (!Block.IsBlockFilename(file))
{
continue;
}
long genStamp = FsDatasetUtil.GetGenerationStampFromFile(files, file);
long blockId = Block.Filename2id(file.GetName());
ReplicaInfo newReplica = null;
if (isFinalized)
{
newReplica = new FinalizedReplica(blockId, file.Length(), genStamp, volume, file.
GetParentFile());
}
else
{
bool loadRwr = true;
FilePath restartMeta = new FilePath(file.GetParent() + FilePath.pathSeparator + "."
+ file.GetName() + ".restart");
Scanner sc = null;
try
{
sc = new Scanner(restartMeta, "UTF-8");
// The restart meta file exists
if (sc.HasNextLong() && (sc.NextLong() > Time.Now()))
{
// It didn't expire. Load the replica as a RBW.
// We don't know the expected block length, so just use 0
// and don't reserve any more space for writes.
newReplica = new ReplicaBeingWritten(blockId, ValidateIntegrityAndSetLength(file,
genStamp), genStamp, volume, file.GetParentFile(), null, 0);
loadRwr = false;
}
sc.Close();
if (!restartMeta.Delete())
{
FsDatasetImpl.Log.Warn("Failed to delete restart meta file: " + restartMeta.GetPath
());
}
}
catch (FileNotFoundException)
{
}
finally
{
// nothing to do hereFile dir =
if (sc != null)
{
sc.Close();
}
}
// Restart meta doesn't exist or expired.
if (loadRwr)
{
newReplica = new ReplicaWaitingToBeRecovered(blockId, ValidateIntegrityAndSetLength
(file, genStamp), genStamp, volume, file.GetParentFile());
}
}
ReplicaInfo oldReplica = volumeMap.Get(bpid, newReplica.GetBlockId());
if (oldReplica == null)
{
volumeMap.Add(bpid, newReplica);
}
else
{
// We have multiple replicas of the same block so decide which one
// to keep.
newReplica = ResolveDuplicateReplicas(newReplica, oldReplica, volumeMap);
}
// If we are retaining a replica on transient storage make sure
// it is in the lazyWriteReplicaMap so it can be persisted
// eventually.
if (newReplica.GetVolume().IsTransientStorage())
{
lazyWriteReplicaMap.AddReplica(bpid, blockId, (FsVolumeImpl)newReplica.GetVolume(
));
}
else
{
lazyWriteReplicaMap.DiscardReplica(bpid, blockId, false);
}
}
}
