// do not store locations of last block /// <summary> /// Serialize a /// <see cref="INodeFile"/> /// node /// </summary> /// <param name="node">The node to write</param> /// <param name="out"> /// The /// <see cref="System.IO.DataOutputStream"/> /// where the fields are written /// </param> /// <param name="writeBlock">Whether to write block information</param> /// <exception cref="System.IO.IOException"/> public static void WriteINodeFile(INodeFile file, DataOutput @out, bool writeUnderConstruction ) { WriteLocalName(file, @out); @out.WriteLong(file.GetId()); @out.WriteShort(file.GetFileReplication()); @out.WriteLong(file.GetModificationTime()); @out.WriteLong(file.GetAccessTime()); @out.WriteLong(file.GetPreferredBlockSize()); WriteBlocks(file.GetBlocks(), @out); SnapshotFSImageFormat.SaveFileDiffList(file, @out); if (writeUnderConstruction) { if (file.IsUnderConstruction()) { @out.WriteBoolean(true); FileUnderConstructionFeature uc = file.GetFileUnderConstructionFeature(); WriteString(uc.GetClientName(), @out); WriteString(uc.GetClientMachine(), @out); } else { @out.WriteBoolean(false); } } WritePermissionStatus(file, @out); }
/// <summary>Create FileStatus with location info by file INode</summary> /// <exception cref="System.IO.IOException"/> private static HdfsLocatedFileStatus CreateLocatedFileStatus(FSDirectory fsd, string fullPath, byte[] path, INode node, byte storagePolicy, int snapshot, bool isRawPath , INodesInPath iip) { System.Diagnostics.Debug.Assert(fsd.HasReadLock()); long size = 0; // length is zero for directories short replication = 0; long blocksize = 0; LocatedBlocks loc = null; bool isEncrypted; FileEncryptionInfo feInfo = isRawPath ? null : fsd.GetFileEncryptionInfo(node, snapshot , iip); if (node.IsFile()) { INodeFile fileNode = node.AsFile(); size = fileNode.ComputeFileSize(snapshot); replication = fileNode.GetFileReplication(snapshot); blocksize = fileNode.GetPreferredBlockSize(); bool inSnapshot = snapshot != Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot .CurrentStateId; bool isUc = !inSnapshot && fileNode.IsUnderConstruction(); long fileSize = !inSnapshot && isUc?fileNode.ComputeFileSizeNotIncludingLastUcBlock () : size; loc = fsd.GetFSNamesystem().GetBlockManager().CreateLocatedBlocks(fileNode.GetBlocks (snapshot), fileSize, isUc, 0L, size, false, inSnapshot, feInfo); if (loc == null) { loc = new LocatedBlocks(); } isEncrypted = (feInfo != null) || (isRawPath && fsd.IsInAnEZ(INodesInPath.FromINode (node))); } else { isEncrypted = fsd.IsInAnEZ(INodesInPath.FromINode(node)); } int childrenNum = node.IsDirectory() ? node.AsDirectory().GetChildrenNum(snapshot ) : 0; INodeAttributes nodeAttrs = fsd.GetAttributes(fullPath, path, node, snapshot); HdfsLocatedFileStatus status = new HdfsLocatedFileStatus(size, node.IsDirectory() , replication, blocksize, node.GetModificationTime(snapshot), node.GetAccessTime (snapshot), GetPermissionForFileStatus(nodeAttrs, isEncrypted), nodeAttrs.GetUserName (), nodeAttrs.GetGroupName(), node.IsSymlink() ? node.AsSymlink().GetSymlink() : null, path, node.GetId(), loc, childrenNum, feInfo, storagePolicy); // Set caching information for the located blocks. if (loc != null) { CacheManager cacheManager = fsd.GetFSNamesystem().GetCacheManager(); foreach (LocatedBlock lb in loc.GetLocatedBlocks()) { cacheManager.SetCachedLocations(lb); } } return(status); }
public virtual void TestAddBlock() { DistributedFileSystem fs = cluster.GetFileSystem(); Path file1 = new Path("/file1"); Path file2 = new Path("/file2"); Path file3 = new Path("/file3"); Path file4 = new Path("/file4"); DFSTestUtil.CreateFile(fs, file1, Blocksize - 1, Replication, 0L); DFSTestUtil.CreateFile(fs, file2, Blocksize, Replication, 0L); DFSTestUtil.CreateFile(fs, file3, Blocksize * 2 - 1, Replication, 0L); DFSTestUtil.CreateFile(fs, file4, Blocksize * 2, Replication, 0L); // restart NameNode cluster.RestartNameNode(true); FSDirectory fsdir = cluster.GetNamesystem().GetFSDirectory(); // check file1 INodeFile file1Node = fsdir.GetINode4Write(file1.ToString()).AsFile(); BlockInfoContiguous[] file1Blocks = file1Node.GetBlocks(); NUnit.Framework.Assert.AreEqual(1, file1Blocks.Length); NUnit.Framework.Assert.AreEqual(Blocksize - 1, file1Blocks[0].GetNumBytes()); NUnit.Framework.Assert.AreEqual(HdfsServerConstants.BlockUCState.Complete, file1Blocks [0].GetBlockUCState()); // check file2 INodeFile file2Node = fsdir.GetINode4Write(file2.ToString()).AsFile(); BlockInfoContiguous[] file2Blocks = file2Node.GetBlocks(); NUnit.Framework.Assert.AreEqual(1, file2Blocks.Length); NUnit.Framework.Assert.AreEqual(Blocksize, file2Blocks[0].GetNumBytes()); NUnit.Framework.Assert.AreEqual(HdfsServerConstants.BlockUCState.Complete, file2Blocks [0].GetBlockUCState()); // check file3 INodeFile file3Node = fsdir.GetINode4Write(file3.ToString()).AsFile(); BlockInfoContiguous[] file3Blocks = file3Node.GetBlocks(); NUnit.Framework.Assert.AreEqual(2, file3Blocks.Length); NUnit.Framework.Assert.AreEqual(Blocksize, file3Blocks[0].GetNumBytes()); NUnit.Framework.Assert.AreEqual(HdfsServerConstants.BlockUCState.Complete, file3Blocks [0].GetBlockUCState()); NUnit.Framework.Assert.AreEqual(Blocksize - 1, file3Blocks[1].GetNumBytes()); NUnit.Framework.Assert.AreEqual(HdfsServerConstants.BlockUCState.Complete, file3Blocks [1].GetBlockUCState()); // check file4 INodeFile file4Node = fsdir.GetINode4Write(file4.ToString()).AsFile(); BlockInfoContiguous[] file4Blocks = file4Node.GetBlocks(); NUnit.Framework.Assert.AreEqual(2, file4Blocks.Length); NUnit.Framework.Assert.AreEqual(Blocksize, file4Blocks[0].GetNumBytes()); NUnit.Framework.Assert.AreEqual(HdfsServerConstants.BlockUCState.Complete, file4Blocks [0].GetBlockUCState()); NUnit.Framework.Assert.AreEqual(Blocksize, file4Blocks[1].GetNumBytes()); NUnit.Framework.Assert.AreEqual(HdfsServerConstants.BlockUCState.Complete, file4Blocks [1].GetBlockUCState()); }
public static void UpdateBlocksMap(INodeFile file, BlockManager bm) { // Add file->block mapping BlockInfoContiguous[] blocks = file.GetBlocks(); if (blocks != null) { for (int i = 0; i < blocks.Length; i++) { file.SetBlock(i, bm.AddBlockCollection(blocks[i], file)); } } }
/// <exception cref="System.IO.IOException"/> private void TestPersistHelper(Configuration conf) { MiniDFSCluster cluster = null; try { cluster = new MiniDFSCluster.Builder(conf).Build(); cluster.WaitActive(); FSNamesystem fsn = cluster.GetNamesystem(); DistributedFileSystem fs = cluster.GetFileSystem(); Path dir = new Path("/abc/def"); Path file1 = new Path(dir, "f1"); Path file2 = new Path(dir, "f2"); // create an empty file f1 fs.Create(file1).Close(); // create an under-construction file f2 FSDataOutputStream @out = fs.Create(file2); @out.WriteBytes("hello"); ((DFSOutputStream)@out.GetWrappedStream()).Hsync(EnumSet.Of(HdfsDataOutputStream.SyncFlag .UpdateLength)); // checkpoint fs.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter); fs.SaveNamespace(); fs.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeLeave); cluster.RestartNameNode(); cluster.WaitActive(); fs = cluster.GetFileSystem(); NUnit.Framework.Assert.IsTrue(fs.IsDirectory(dir)); NUnit.Framework.Assert.IsTrue(fs.Exists(file1)); NUnit.Framework.Assert.IsTrue(fs.Exists(file2)); // check internals of file2 INodeFile file2Node = fsn.dir.GetINode4Write(file2.ToString()).AsFile(); NUnit.Framework.Assert.AreEqual("hello".Length, file2Node.ComputeFileSize()); NUnit.Framework.Assert.IsTrue(file2Node.IsUnderConstruction()); BlockInfoContiguous[] blks = file2Node.GetBlocks(); NUnit.Framework.Assert.AreEqual(1, blks.Length); NUnit.Framework.Assert.AreEqual(HdfsServerConstants.BlockUCState.UnderConstruction , blks[0].GetBlockUCState()); // check lease manager LeaseManager.Lease lease = fsn.leaseManager.GetLeaseByPath(file2.ToString()); NUnit.Framework.Assert.IsNotNull(lease); } finally { if (cluster != null) { cluster.Shutdown(); } } }
/// <exception cref="System.IO.IOException"/> private void Save(OutputStream @out, INodeFile n) { FsImageProto.INodeSection.INodeFile.Builder b = BuildINodeFile(n, parent.GetSaverContext ()); if (n.GetBlocks() != null) { foreach (Block block in n.GetBlocks()) { b.AddBlocks(PBHelper.Convert(block)); } } FileUnderConstructionFeature uc = n.GetFileUnderConstructionFeature(); if (uc != null) { FsImageProto.INodeSection.FileUnderConstructionFeature f = ((FsImageProto.INodeSection.FileUnderConstructionFeature )FsImageProto.INodeSection.FileUnderConstructionFeature.NewBuilder().SetClientName (uc.GetClientName()).SetClientMachine(uc.GetClientMachine()).Build()); b.SetFileUC(f); } FsImageProto.INodeSection.INode r = ((FsImageProto.INodeSection.INode)BuildINodeCommon (n).SetType(FsImageProto.INodeSection.INode.Type.File).SetFile(b).Build()); r.WriteDelimitedTo(@out); }
/// <exception cref="System.IO.IOException"/> private void VerifyFileBlocks(string file, bool isFileOpen) { FSNamesystem ns = cluster.GetNamesystem(); INodeFile inode = INodeFile.ValueOf(ns.dir.GetINode(file), file); NUnit.Framework.Assert.IsTrue("File " + inode.ToString() + " isUnderConstruction = " + inode.IsUnderConstruction() + " expected to be " + isFileOpen, inode.IsUnderConstruction () == isFileOpen); BlockInfoContiguous[] blocks = inode.GetBlocks(); NUnit.Framework.Assert.IsTrue("File does not have blocks: " + inode.ToString(), blocks != null && blocks.Length > 0); int idx = 0; BlockInfoContiguous curBlock; // all blocks but the last two should be regular blocks for (; idx < blocks.Length - 2; idx++) { curBlock = blocks[idx]; NUnit.Framework.Assert.IsTrue("Block is not complete: " + curBlock, curBlock.IsComplete ()); NUnit.Framework.Assert.IsTrue("Block is not in BlocksMap: " + curBlock, ns.GetBlockManager ().GetStoredBlock(curBlock) == curBlock); } // the penultimate block is either complete or // committed if the file is not closed if (idx > 0) { curBlock = blocks[idx - 1]; // penultimate block NUnit.Framework.Assert.IsTrue("Block " + curBlock + " isUnderConstruction = " + inode .IsUnderConstruction() + " expected to be " + isFileOpen, (isFileOpen && curBlock .IsComplete()) || (!isFileOpen && !curBlock.IsComplete() == (curBlock.GetBlockUCState () == HdfsServerConstants.BlockUCState.Committed))); NUnit.Framework.Assert.IsTrue("Block is not in BlocksMap: " + curBlock, ns.GetBlockManager ().GetStoredBlock(curBlock) == curBlock); } // The last block is complete if the file is closed. // If the file is open, the last block may be complete or not. curBlock = blocks[idx]; // last block if (!isFileOpen) { NUnit.Framework.Assert.IsTrue("Block " + curBlock + ", isFileOpen = " + isFileOpen , curBlock.IsComplete()); } NUnit.Framework.Assert.IsTrue("Block is not in BlocksMap: " + curBlock, ns.GetBlockManager ().GetStoredBlock(curBlock) == curBlock); }
/// <summary> /// When deleting a file in the current fs directory, and the file is contained /// in a snapshot, we should delete the last block if it's under construction /// and its size is 0. /// </summary> internal virtual void CleanZeroSizeBlock(INodeFile f, INode.BlocksMapUpdateInfo collectedBlocks ) { BlockInfoContiguous[] blocks = f.GetBlocks(); if (blocks != null && blocks.Length > 0 && blocks[blocks.Length - 1] is BlockInfoContiguousUnderConstruction) { BlockInfoContiguousUnderConstruction lastUC = (BlockInfoContiguousUnderConstruction )blocks[blocks.Length - 1]; if (lastUC.GetNumBytes() == 0) { // this is a 0-sized block. do not need check its UC state here collectedBlocks.AddDeleteBlock(lastUC); f.RemoveLastBlock(lastUC); } } }
// Helper function that writes an INodeUnderConstruction // into the output stream // /// <exception cref="System.IO.IOException"/> internal static void WriteINodeUnderConstruction(DataOutputStream @out, INodeFile cons, string path) { WriteString(path, @out); @out.WriteLong(cons.GetId()); @out.WriteShort(cons.GetFileReplication()); @out.WriteLong(cons.GetModificationTime()); @out.WriteLong(cons.GetPreferredBlockSize()); WriteBlocks(cons.GetBlocks(), @out); cons.GetPermissionStatus().Write(@out); FileUnderConstructionFeature uc = cons.GetFileUnderConstructionFeature(); WriteString(uc.GetClientName(), @out); WriteString(uc.GetClientMachine(), @out); @out.WriteInt(0); }
public virtual void TestAddBlockUC() { DistributedFileSystem fs = cluster.GetFileSystem(); Path file1 = new Path("/file1"); DFSTestUtil.CreateFile(fs, file1, Blocksize - 1, Replication, 0L); FSDataOutputStream @out = null; try { // append files without closing the streams @out = fs.Append(file1); string appendContent = "appending-content"; @out.WriteBytes(appendContent); ((DFSOutputStream)@out.GetWrappedStream()).Hsync(EnumSet.Of(HdfsDataOutputStream.SyncFlag .UpdateLength)); // restart NN cluster.RestartNameNode(true); FSDirectory fsdir = cluster.GetNamesystem().GetFSDirectory(); INodeFile fileNode = fsdir.GetINode4Write(file1.ToString()).AsFile(); BlockInfoContiguous[] fileBlocks = fileNode.GetBlocks(); NUnit.Framework.Assert.AreEqual(2, fileBlocks.Length); NUnit.Framework.Assert.AreEqual(Blocksize, fileBlocks[0].GetNumBytes()); NUnit.Framework.Assert.AreEqual(HdfsServerConstants.BlockUCState.Complete, fileBlocks [0].GetBlockUCState()); NUnit.Framework.Assert.AreEqual(appendContent.Length - 1, fileBlocks[1].GetNumBytes ()); NUnit.Framework.Assert.AreEqual(HdfsServerConstants.BlockUCState.UnderConstruction , fileBlocks[1].GetBlockUCState()); } finally { if (@out != null) { @out.Close(); } } }
/// <exception cref="Org.Apache.Hadoop.Hdfs.Protocol.QuotaExceededException"/> /// <exception cref="Org.Apache.Hadoop.FS.UnresolvedLinkException"/> /// <exception cref="Org.Apache.Hadoop.Hdfs.Protocol.SnapshotAccessControlException"/ /// > internal static Block[] UnprotectedSetReplication(FSDirectory fsd, string src, short replication, short[] blockRepls) { System.Diagnostics.Debug.Assert(fsd.HasWriteLock()); INodesInPath iip = fsd.GetINodesInPath4Write(src, true); INode inode = iip.GetLastINode(); if (inode == null || !inode.IsFile()) { return(null); } INodeFile file = inode.AsFile(); short oldBR = file.GetBlockReplication(); // before setFileReplication, check for increasing block replication. // if replication > oldBR, then newBR == replication. // if replication < oldBR, we don't know newBR yet. if (replication > oldBR) { long dsDelta = file.StoragespaceConsumed() / oldBR; fsd.UpdateCount(iip, 0L, dsDelta, oldBR, replication, true); } file.SetFileReplication(replication, iip.GetLatestSnapshotId()); short newBR = file.GetBlockReplication(); // check newBR < oldBR case. if (newBR < oldBR) { long dsDelta = file.StoragespaceConsumed() / newBR; fsd.UpdateCount(iip, 0L, dsDelta, oldBR, newBR, true); } if (blockRepls != null) { blockRepls[0] = oldBR; blockRepls[1] = newBR; } return(file.GetBlocks()); }