// do not store locations of last block
/// <summary>
/// Serialize a
/// <see cref="INodeFile"/>
/// node
/// </summary>
/// <param name="node">The node to write</param>
/// <param name="out">
/// The
/// <see cref="System.IO.DataOutputStream"/>
/// where the fields are written
/// </param>
/// <param name="writeBlock">Whether to write block information</param>
/// <exception cref="System.IO.IOException"/>
public static void WriteINodeFile(INodeFile file, DataOutput @out, bool writeUnderConstruction
)
{
WriteLocalName(file, @out);
@out.WriteLong(file.GetId());
@out.WriteShort(file.GetFileReplication());
@out.WriteLong(file.GetModificationTime());
@out.WriteLong(file.GetAccessTime());
@out.WriteLong(file.GetPreferredBlockSize());
WriteBlocks(file.GetBlocks(), @out);
SnapshotFSImageFormat.SaveFileDiffList(file, @out);
if (writeUnderConstruction)
{
if (file.IsUnderConstruction())
{
@out.WriteBoolean(true);
FileUnderConstructionFeature uc = file.GetFileUnderConstructionFeature();
WriteString(uc.GetClientName(), @out);
WriteString(uc.GetClientMachine(), @out);
}
else
{
@out.WriteBoolean(false);
}
}
WritePermissionStatus(file, @out);
}
/// <summary>Create FileStatus with location info by file INode</summary>
/// <exception cref="System.IO.IOException"/>
private static HdfsLocatedFileStatus CreateLocatedFileStatus(FSDirectory fsd, string
fullPath, byte[] path, INode node, byte storagePolicy, int snapshot, bool isRawPath
, INodesInPath iip)
{
System.Diagnostics.Debug.Assert(fsd.HasReadLock());
long size = 0;
// length is zero for directories
short replication = 0;
long blocksize = 0;
LocatedBlocks loc = null;
bool isEncrypted;
FileEncryptionInfo feInfo = isRawPath ? null : fsd.GetFileEncryptionInfo(node, snapshot
, iip);
if (node.IsFile())
{
INodeFile fileNode = node.AsFile();
size = fileNode.ComputeFileSize(snapshot);
replication = fileNode.GetFileReplication(snapshot);
blocksize = fileNode.GetPreferredBlockSize();
bool inSnapshot = snapshot != Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot
.CurrentStateId;
bool isUc = !inSnapshot && fileNode.IsUnderConstruction();
long fileSize = !inSnapshot && isUc?fileNode.ComputeFileSizeNotIncludingLastUcBlock
() : size;
loc = fsd.GetFSNamesystem().GetBlockManager().CreateLocatedBlocks(fileNode.GetBlocks
(snapshot), fileSize, isUc, 0L, size, false, inSnapshot, feInfo);
if (loc == null)
{
loc = new LocatedBlocks();
}
isEncrypted = (feInfo != null) || (isRawPath && fsd.IsInAnEZ(INodesInPath.FromINode
(node)));
}
else
{
isEncrypted = fsd.IsInAnEZ(INodesInPath.FromINode(node));
}
int childrenNum = node.IsDirectory() ? node.AsDirectory().GetChildrenNum(snapshot
) : 0;
INodeAttributes nodeAttrs = fsd.GetAttributes(fullPath, path, node, snapshot);
HdfsLocatedFileStatus status = new HdfsLocatedFileStatus(size, node.IsDirectory()
, replication, blocksize, node.GetModificationTime(snapshot), node.GetAccessTime
(snapshot), GetPermissionForFileStatus(nodeAttrs, isEncrypted), nodeAttrs.GetUserName
(), nodeAttrs.GetGroupName(), node.IsSymlink() ? node.AsSymlink().GetSymlink() :
null, path, node.GetId(), loc, childrenNum, feInfo, storagePolicy);
// Set caching information for the located blocks.
if (loc != null)
{
CacheManager cacheManager = fsd.GetFSNamesystem().GetCacheManager();
foreach (LocatedBlock lb in loc.GetLocatedBlocks())
{
cacheManager.SetCachedLocations(lb);
}
}
return(status);
}
public virtual void TestAddBlock()
{
DistributedFileSystem fs = cluster.GetFileSystem();
Path file1 = new Path("/file1");
Path file2 = new Path("/file2");
Path file3 = new Path("/file3");
Path file4 = new Path("/file4");
DFSTestUtil.CreateFile(fs, file1, Blocksize - 1, Replication, 0L);
DFSTestUtil.CreateFile(fs, file2, Blocksize, Replication, 0L);
DFSTestUtil.CreateFile(fs, file3, Blocksize * 2 - 1, Replication, 0L);
DFSTestUtil.CreateFile(fs, file4, Blocksize * 2, Replication, 0L);
// restart NameNode
cluster.RestartNameNode(true);
FSDirectory fsdir = cluster.GetNamesystem().GetFSDirectory();
// check file1
INodeFile file1Node = fsdir.GetINode4Write(file1.ToString()).AsFile();
BlockInfoContiguous[] file1Blocks = file1Node.GetBlocks();
NUnit.Framework.Assert.AreEqual(1, file1Blocks.Length);
NUnit.Framework.Assert.AreEqual(Blocksize - 1, file1Blocks[0].GetNumBytes());
NUnit.Framework.Assert.AreEqual(HdfsServerConstants.BlockUCState.Complete, file1Blocks
[0].GetBlockUCState());
// check file2
INodeFile file2Node = fsdir.GetINode4Write(file2.ToString()).AsFile();
BlockInfoContiguous[] file2Blocks = file2Node.GetBlocks();
NUnit.Framework.Assert.AreEqual(1, file2Blocks.Length);
NUnit.Framework.Assert.AreEqual(Blocksize, file2Blocks[0].GetNumBytes());
NUnit.Framework.Assert.AreEqual(HdfsServerConstants.BlockUCState.Complete, file2Blocks
[0].GetBlockUCState());
// check file3
INodeFile file3Node = fsdir.GetINode4Write(file3.ToString()).AsFile();
BlockInfoContiguous[] file3Blocks = file3Node.GetBlocks();
NUnit.Framework.Assert.AreEqual(2, file3Blocks.Length);
NUnit.Framework.Assert.AreEqual(Blocksize, file3Blocks[0].GetNumBytes());
NUnit.Framework.Assert.AreEqual(HdfsServerConstants.BlockUCState.Complete, file3Blocks
[0].GetBlockUCState());
NUnit.Framework.Assert.AreEqual(Blocksize - 1, file3Blocks[1].GetNumBytes());
NUnit.Framework.Assert.AreEqual(HdfsServerConstants.BlockUCState.Complete, file3Blocks
[1].GetBlockUCState());
// check file4
INodeFile file4Node = fsdir.GetINode4Write(file4.ToString()).AsFile();
BlockInfoContiguous[] file4Blocks = file4Node.GetBlocks();
NUnit.Framework.Assert.AreEqual(2, file4Blocks.Length);
NUnit.Framework.Assert.AreEqual(Blocksize, file4Blocks[0].GetNumBytes());
NUnit.Framework.Assert.AreEqual(HdfsServerConstants.BlockUCState.Complete, file4Blocks
[0].GetBlockUCState());
NUnit.Framework.Assert.AreEqual(Blocksize, file4Blocks[1].GetNumBytes());
NUnit.Framework.Assert.AreEqual(HdfsServerConstants.BlockUCState.Complete, file4Blocks
[1].GetBlockUCState());
}
public static void UpdateBlocksMap(INodeFile file, BlockManager bm)
{
// Add file->block mapping
BlockInfoContiguous[] blocks = file.GetBlocks();
if (blocks != null)
{
for (int i = 0; i < blocks.Length; i++)
{
file.SetBlock(i, bm.AddBlockCollection(blocks[i], file));
}
}
}
/// <exception cref="System.IO.IOException"/>
private void TestPersistHelper(Configuration conf)
{
MiniDFSCluster cluster = null;
try
{
cluster = new MiniDFSCluster.Builder(conf).Build();
cluster.WaitActive();
FSNamesystem fsn = cluster.GetNamesystem();
DistributedFileSystem fs = cluster.GetFileSystem();
Path dir = new Path("/abc/def");
Path file1 = new Path(dir, "f1");
Path file2 = new Path(dir, "f2");
// create an empty file f1
fs.Create(file1).Close();
// create an under-construction file f2
FSDataOutputStream @out = fs.Create(file2);
@out.WriteBytes("hello");
((DFSOutputStream)@out.GetWrappedStream()).Hsync(EnumSet.Of(HdfsDataOutputStream.SyncFlag
.UpdateLength));
// checkpoint
fs.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter);
fs.SaveNamespace();
fs.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeLeave);
cluster.RestartNameNode();
cluster.WaitActive();
fs = cluster.GetFileSystem();
NUnit.Framework.Assert.IsTrue(fs.IsDirectory(dir));
NUnit.Framework.Assert.IsTrue(fs.Exists(file1));
NUnit.Framework.Assert.IsTrue(fs.Exists(file2));
// check internals of file2
INodeFile file2Node = fsn.dir.GetINode4Write(file2.ToString()).AsFile();
NUnit.Framework.Assert.AreEqual("hello".Length, file2Node.ComputeFileSize());
NUnit.Framework.Assert.IsTrue(file2Node.IsUnderConstruction());
BlockInfoContiguous[] blks = file2Node.GetBlocks();
NUnit.Framework.Assert.AreEqual(1, blks.Length);
NUnit.Framework.Assert.AreEqual(HdfsServerConstants.BlockUCState.UnderConstruction
, blks[0].GetBlockUCState());
// check lease manager
LeaseManager.Lease lease = fsn.leaseManager.GetLeaseByPath(file2.ToString());
NUnit.Framework.Assert.IsNotNull(lease);
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
/// <exception cref="System.IO.IOException"/>
private void Save(OutputStream @out, INodeFile n)
{
FsImageProto.INodeSection.INodeFile.Builder b = BuildINodeFile(n, parent.GetSaverContext
());
if (n.GetBlocks() != null)
{
foreach (Block block in n.GetBlocks())
{
b.AddBlocks(PBHelper.Convert(block));
}
}
FileUnderConstructionFeature uc = n.GetFileUnderConstructionFeature();
if (uc != null)
{
FsImageProto.INodeSection.FileUnderConstructionFeature f = ((FsImageProto.INodeSection.FileUnderConstructionFeature
)FsImageProto.INodeSection.FileUnderConstructionFeature.NewBuilder().SetClientName
(uc.GetClientName()).SetClientMachine(uc.GetClientMachine()).Build());
b.SetFileUC(f);
}
FsImageProto.INodeSection.INode r = ((FsImageProto.INodeSection.INode)BuildINodeCommon
(n).SetType(FsImageProto.INodeSection.INode.Type.File).SetFile(b).Build());
r.WriteDelimitedTo(@out);
}
/// <exception cref="System.IO.IOException"/>
private void VerifyFileBlocks(string file, bool isFileOpen)
{
FSNamesystem ns = cluster.GetNamesystem();
INodeFile inode = INodeFile.ValueOf(ns.dir.GetINode(file), file);
NUnit.Framework.Assert.IsTrue("File " + inode.ToString() + " isUnderConstruction = "
+ inode.IsUnderConstruction() + " expected to be " + isFileOpen, inode.IsUnderConstruction
() == isFileOpen);
BlockInfoContiguous[] blocks = inode.GetBlocks();
NUnit.Framework.Assert.IsTrue("File does not have blocks: " + inode.ToString(), blocks
!= null && blocks.Length > 0);
int idx = 0;
BlockInfoContiguous curBlock;
// all blocks but the last two should be regular blocks
for (; idx < blocks.Length - 2; idx++)
{
curBlock = blocks[idx];
NUnit.Framework.Assert.IsTrue("Block is not complete: " + curBlock, curBlock.IsComplete
());
NUnit.Framework.Assert.IsTrue("Block is not in BlocksMap: " + curBlock, ns.GetBlockManager
().GetStoredBlock(curBlock) == curBlock);
}
// the penultimate block is either complete or
// committed if the file is not closed
if (idx > 0)
{
curBlock = blocks[idx - 1];
// penultimate block
NUnit.Framework.Assert.IsTrue("Block " + curBlock + " isUnderConstruction = " + inode
.IsUnderConstruction() + " expected to be " + isFileOpen, (isFileOpen && curBlock
.IsComplete()) || (!isFileOpen && !curBlock.IsComplete() == (curBlock.GetBlockUCState
() == HdfsServerConstants.BlockUCState.Committed)));
NUnit.Framework.Assert.IsTrue("Block is not in BlocksMap: " + curBlock, ns.GetBlockManager
().GetStoredBlock(curBlock) == curBlock);
}
// The last block is complete if the file is closed.
// If the file is open, the last block may be complete or not.
curBlock = blocks[idx];
// last block
if (!isFileOpen)
{
NUnit.Framework.Assert.IsTrue("Block " + curBlock + ", isFileOpen = " + isFileOpen
, curBlock.IsComplete());
}
NUnit.Framework.Assert.IsTrue("Block is not in BlocksMap: " + curBlock, ns.GetBlockManager
().GetStoredBlock(curBlock) == curBlock);
}
/// <summary>
/// When deleting a file in the current fs directory, and the file is contained
/// in a snapshot, we should delete the last block if it's under construction
/// and its size is 0.
/// </summary>
internal virtual void CleanZeroSizeBlock(INodeFile f, INode.BlocksMapUpdateInfo collectedBlocks
)
{
BlockInfoContiguous[] blocks = f.GetBlocks();
if (blocks != null && blocks.Length > 0 && blocks[blocks.Length - 1] is BlockInfoContiguousUnderConstruction)
{
BlockInfoContiguousUnderConstruction lastUC = (BlockInfoContiguousUnderConstruction
)blocks[blocks.Length - 1];
if (lastUC.GetNumBytes() == 0)
{
// this is a 0-sized block. do not need check its UC state here
collectedBlocks.AddDeleteBlock(lastUC);
f.RemoveLastBlock(lastUC);
}
}
}
// Helper function that writes an INodeUnderConstruction
// into the output stream
//
/// <exception cref="System.IO.IOException"/>
internal static void WriteINodeUnderConstruction(DataOutputStream @out, INodeFile
cons, string path)
{
WriteString(path, @out);
@out.WriteLong(cons.GetId());
@out.WriteShort(cons.GetFileReplication());
@out.WriteLong(cons.GetModificationTime());
@out.WriteLong(cons.GetPreferredBlockSize());
WriteBlocks(cons.GetBlocks(), @out);
cons.GetPermissionStatus().Write(@out);
FileUnderConstructionFeature uc = cons.GetFileUnderConstructionFeature();
WriteString(uc.GetClientName(), @out);
WriteString(uc.GetClientMachine(), @out);
@out.WriteInt(0);
}
public virtual void TestAddBlockUC()
{
DistributedFileSystem fs = cluster.GetFileSystem();
Path file1 = new Path("/file1");
DFSTestUtil.CreateFile(fs, file1, Blocksize - 1, Replication, 0L);
FSDataOutputStream @out = null;
try
{
// append files without closing the streams
@out = fs.Append(file1);
string appendContent = "appending-content";
@out.WriteBytes(appendContent);
((DFSOutputStream)@out.GetWrappedStream()).Hsync(EnumSet.Of(HdfsDataOutputStream.SyncFlag
.UpdateLength));
// restart NN
cluster.RestartNameNode(true);
FSDirectory fsdir = cluster.GetNamesystem().GetFSDirectory();
INodeFile fileNode = fsdir.GetINode4Write(file1.ToString()).AsFile();
BlockInfoContiguous[] fileBlocks = fileNode.GetBlocks();
NUnit.Framework.Assert.AreEqual(2, fileBlocks.Length);
NUnit.Framework.Assert.AreEqual(Blocksize, fileBlocks[0].GetNumBytes());
NUnit.Framework.Assert.AreEqual(HdfsServerConstants.BlockUCState.Complete, fileBlocks
[0].GetBlockUCState());
NUnit.Framework.Assert.AreEqual(appendContent.Length - 1, fileBlocks[1].GetNumBytes
());
NUnit.Framework.Assert.AreEqual(HdfsServerConstants.BlockUCState.UnderConstruction
, fileBlocks[1].GetBlockUCState());
}
finally
{
if (@out != null)
{
@out.Close();
}
}
}
/// <exception cref="Org.Apache.Hadoop.Hdfs.Protocol.QuotaExceededException"/>
/// <exception cref="Org.Apache.Hadoop.FS.UnresolvedLinkException"/>
/// <exception cref="Org.Apache.Hadoop.Hdfs.Protocol.SnapshotAccessControlException"/
/// >
internal static Block[] UnprotectedSetReplication(FSDirectory fsd, string src, short
replication, short[] blockRepls)
{
System.Diagnostics.Debug.Assert(fsd.HasWriteLock());
INodesInPath iip = fsd.GetINodesInPath4Write(src, true);
INode inode = iip.GetLastINode();
if (inode == null || !inode.IsFile())
{
return(null);
}
INodeFile file = inode.AsFile();
short oldBR = file.GetBlockReplication();
// before setFileReplication, check for increasing block replication.
// if replication > oldBR, then newBR == replication.
// if replication < oldBR, we don't know newBR yet.
if (replication > oldBR)
{
long dsDelta = file.StoragespaceConsumed() / oldBR;
fsd.UpdateCount(iip, 0L, dsDelta, oldBR, replication, true);
}
file.SetFileReplication(replication, iip.GetLatestSnapshotId());
short newBR = file.GetBlockReplication();
// check newBR < oldBR case.
if (newBR < oldBR)
{
long dsDelta = file.StoragespaceConsumed() / newBR;
fsd.UpdateCount(iip, 0L, dsDelta, oldBR, newBR, true);
}
if (blockRepls != null)
{
blockRepls[0] = oldBR;
blockRepls[1] = newBR;
}
return(file.GetBlocks());
}
