/// <exception cref="System.IO.IOException"/> public static FilePath GetMetaFile <_T0>(FsDatasetSpi <_T0> fsd, string bpid, Block b) where _T0 : FsVolumeSpi { return(FsDatasetUtil.GetMetaFile(GetBlockFile(fsd, bpid, b), b.GetGenerationStamp ())); }
/// <exception cref="System.IO.IOException"/> internal virtual FilePath AddBlock(Block b, FilePath f) { FilePath blockDir = DatanodeUtil.IdToBlockDir(finalizedDir, b.GetBlockId()); if (!blockDir.Exists()) { if (!blockDir.Mkdirs()) { throw new IOException("Failed to mkdirs " + blockDir); } } FilePath blockFile = FsDatasetImpl.MoveBlockFiles(b, f, blockDir); FilePath metaFile = FsDatasetUtil.GetMetaFile(blockFile, b.GetGenerationStamp()); dfsUsage.IncDfsUsed(b.GetNumBytes() + metaFile.Length()); return(blockFile); }
/// <summary>Recover an unlinked tmp file on datanode restart.</summary> /// <remarks> /// Recover an unlinked tmp file on datanode restart. If the original block /// does not exist, then the tmp file is renamed to be the /// original file name and the original name is returned; otherwise the tmp /// file is deleted and null is returned. /// </remarks> /// <exception cref="System.IO.IOException"/> internal virtual FilePath RecoverTempUnlinkedBlock(FilePath unlinkedTmp) { FilePath blockFile = FsDatasetUtil.GetOrigFile(unlinkedTmp); if (blockFile.Exists()) { // If the original block file still exists, then no recovery is needed. if (!unlinkedTmp.Delete()) { throw new IOException("Unable to cleanup unlinked tmp file " + unlinkedTmp); } return(null); } else { if (!unlinkedTmp.RenameTo(blockFile)) { throw new IOException("Unable to rename unlinked tmp file " + unlinkedTmp); } return(blockFile); } }
/// <summary>Find out the number of bytes in the block that match its crc.</summary> /// <remarks> /// Find out the number of bytes in the block that match its crc. /// This algorithm assumes that data corruption caused by unexpected /// datanode shutdown occurs only in the last crc chunk. So it checks /// only the last chunk. /// </remarks> /// <param name="blockFile">the block file</param> /// <param name="genStamp">generation stamp of the block</param> /// <returns>the number of valid bytes</returns> private long ValidateIntegrityAndSetLength(FilePath blockFile, long genStamp) { DataInputStream checksumIn = null; InputStream blockIn = null; try { FilePath metaFile = FsDatasetUtil.GetMetaFile(blockFile, genStamp); long blockFileLen = blockFile.Length(); long metaFileLen = metaFile.Length(); int crcHeaderLen = DataChecksum.GetChecksumHeaderSize(); if (!blockFile.Exists() || blockFileLen == 0 || !metaFile.Exists() || metaFileLen < crcHeaderLen) { return(0); } checksumIn = new DataInputStream(new BufferedInputStream(new FileInputStream(metaFile ), HdfsConstants.IoFileBufferSize)); // read and handle the common header here. For now just a version DataChecksum checksum = BlockMetadataHeader.ReadDataChecksum(checksumIn, metaFile ); int bytesPerChecksum = checksum.GetBytesPerChecksum(); int checksumSize = checksum.GetChecksumSize(); long numChunks = Math.Min((blockFileLen + bytesPerChecksum - 1) / bytesPerChecksum , (metaFileLen - crcHeaderLen) / checksumSize); if (numChunks == 0) { return(0); } IOUtils.SkipFully(checksumIn, (numChunks - 1) * checksumSize); blockIn = new FileInputStream(blockFile); long lastChunkStartPos = (numChunks - 1) * bytesPerChecksum; IOUtils.SkipFully(blockIn, lastChunkStartPos); int lastChunkSize = (int)Math.Min(bytesPerChecksum, blockFileLen - lastChunkStartPos ); byte[] buf = new byte[lastChunkSize + checksumSize]; checksumIn.ReadFully(buf, lastChunkSize, checksumSize); IOUtils.ReadFully(blockIn, buf, 0, lastChunkSize); checksum.Update(buf, 0, lastChunkSize); long validFileLength; if (checksum.Compare(buf, lastChunkSize)) { // last chunk matches crc validFileLength = lastChunkStartPos + lastChunkSize; } else { // last chunck is corrupt validFileLength = lastChunkStartPos; } // truncate if extra bytes are present without CRC if (blockFile.Length() > validFileLength) { RandomAccessFile blockRAF = new RandomAccessFile(blockFile, "rw"); try { // truncate blockFile blockRAF.SetLength(validFileLength); } finally { blockRAF.Close(); } } return(validFileLength); } catch (IOException e) { FsDatasetImpl.Log.Warn(e); return(0); } finally { IOUtils.CloseStream(checksumIn); IOUtils.CloseStream(blockIn); } }
/// <summary>Add replicas under the given directory to the volume map</summary> /// <param name="volumeMap">the replicas map</param> /// <param name="dir">an input directory</param> /// <param name="lazyWriteReplicaMap"> /// Map of replicas on transient /// storage. /// </param> /// <param name="isFinalized"> /// true if the directory has finalized replicas; /// false if the directory has rbw replicas /// </param> /// <exception cref="System.IO.IOException"/> internal virtual void AddToReplicasMap(ReplicaMap volumeMap, FilePath dir, RamDiskReplicaTracker lazyWriteReplicaMap, bool isFinalized) { FilePath[] files = FileUtil.ListFiles(dir); foreach (FilePath file in files) { if (file.IsDirectory()) { AddToReplicasMap(volumeMap, file, lazyWriteReplicaMap, isFinalized); } if (isFinalized && FsDatasetUtil.IsUnlinkTmpFile(file)) { file = RecoverTempUnlinkedBlock(file); if (file == null) { // the original block still exists, so we cover it // in another iteration and can continue here continue; } } if (!Block.IsBlockFilename(file)) { continue; } long genStamp = FsDatasetUtil.GetGenerationStampFromFile(files, file); long blockId = Block.Filename2id(file.GetName()); ReplicaInfo newReplica = null; if (isFinalized) { newReplica = new FinalizedReplica(blockId, file.Length(), genStamp, volume, file. GetParentFile()); } else { bool loadRwr = true; FilePath restartMeta = new FilePath(file.GetParent() + FilePath.pathSeparator + "." + file.GetName() + ".restart"); Scanner sc = null; try { sc = new Scanner(restartMeta, "UTF-8"); // The restart meta file exists if (sc.HasNextLong() && (sc.NextLong() > Time.Now())) { // It didn't expire. Load the replica as a RBW. // We don't know the expected block length, so just use 0 // and don't reserve any more space for writes. newReplica = new ReplicaBeingWritten(blockId, ValidateIntegrityAndSetLength(file, genStamp), genStamp, volume, file.GetParentFile(), null, 0); loadRwr = false; } sc.Close(); if (!restartMeta.Delete()) { FsDatasetImpl.Log.Warn("Failed to delete restart meta file: " + restartMeta.GetPath ()); } } catch (FileNotFoundException) { } finally { // nothing to do hereFile dir = if (sc != null) { sc.Close(); } } // Restart meta doesn't exist or expired. if (loadRwr) { newReplica = new ReplicaWaitingToBeRecovered(blockId, ValidateIntegrityAndSetLength (file, genStamp), genStamp, volume, file.GetParentFile()); } } ReplicaInfo oldReplica = volumeMap.Get(bpid, newReplica.GetBlockId()); if (oldReplica == null) { volumeMap.Add(bpid, newReplica); } else { // We have multiple replicas of the same block so decide which one // to keep. newReplica = ResolveDuplicateReplicas(newReplica, oldReplica, volumeMap); } // If we are retaining a replica on transient storage make sure // it is in the lazyWriteReplicaMap so it can be persisted // eventually. if (newReplica.GetVolume().IsTransientStorage()) { lazyWriteReplicaMap.AddReplica(bpid, blockId, (FsVolumeImpl)newReplica.GetVolume( )); } else { lazyWriteReplicaMap.DiscardReplica(bpid, blockId, false); } } }