public virtual void TestCommitBlockSynchronization()
{
INodeFile file = MockFileUnderConstruction();
Block block = new Block(blockId, length, genStamp);
FSNamesystem namesystemSpy = MakeNameSystemSpy(block, file);
DatanodeID[] newTargets = new DatanodeID[0];
ExtendedBlock lastBlock = new ExtendedBlock();
namesystemSpy.CommitBlockSynchronization(lastBlock, genStamp, length, false, false
, newTargets, null);
// Repeat the call to make sure it does not throw
namesystemSpy.CommitBlockSynchronization(lastBlock, genStamp, length, false, false
, newTargets, null);
// Simulate 'completing' the block.
BlockInfoContiguous completedBlockInfo = new BlockInfoContiguous(block, (short)1);
completedBlockInfo.SetBlockCollection(file);
completedBlockInfo.SetGenerationStamp(genStamp);
Org.Mockito.Mockito.DoReturn(completedBlockInfo).When(namesystemSpy).GetStoredBlock
(Matchers.Any <Block>());
Org.Mockito.Mockito.DoReturn(completedBlockInfo).When(file).GetLastBlock();
// Repeat the call to make sure it does not throw
namesystemSpy.CommitBlockSynchronization(lastBlock, genStamp, length, false, false
, newTargets, null);
}
// do not store locations of last block
/// <summary>
/// Serialize a
/// <see cref="INodeFile"/>
/// node
/// </summary>
/// <param name="node">The node to write</param>
/// <param name="out">
/// The
/// <see cref="System.IO.DataOutputStream"/>
/// where the fields are written
/// </param>
/// <param name="writeBlock">Whether to write block information</param>
/// <exception cref="System.IO.IOException"/>
public static void WriteINodeFile(INodeFile file, DataOutput @out, bool writeUnderConstruction
)
{
WriteLocalName(file, @out);
@out.WriteLong(file.GetId());
@out.WriteShort(file.GetFileReplication());
@out.WriteLong(file.GetModificationTime());
@out.WriteLong(file.GetAccessTime());
@out.WriteLong(file.GetPreferredBlockSize());
WriteBlocks(file.GetBlocks(), @out);
SnapshotFSImageFormat.SaveFileDiffList(file, @out);
if (writeUnderConstruction)
{
if (file.IsUnderConstruction())
{
@out.WriteBoolean(true);
FileUnderConstructionFeature uc = file.GetFileUnderConstructionFeature();
WriteString(uc.GetClientName(), @out);
WriteString(uc.GetClientMachine(), @out);
}
else
{
@out.WriteBoolean(false);
}
}
WritePermissionStatus(file, @out);
}
public virtual void TestAclGroupTraverseDenyOnlyDefaultEntries()
{
INodeDirectory inodeDir = CreateINodeDirectory(inodeRoot, "dir1", "bruce", "execs"
, (short)0x1ed);
INodeFile inodeFile = CreateINodeFile(inodeDir, "file1", "bruce", "execs", (short
)0x1a4);
AddAcl(inodeDir, AclTestHelpers.AclEntry(AclEntryScope.Access, AclEntryType.User,
FsAction.All), AclTestHelpers.AclEntry(AclEntryScope.Access, AclEntryType.Group,
FsAction.None), AclTestHelpers.AclEntry(AclEntryScope.Access, AclEntryType.Other
, FsAction.ReadExecute), AclTestHelpers.AclEntry(AclEntryScope.Default, AclEntryType
.User, FsAction.All), AclTestHelpers.AclEntry(AclEntryScope.Default, AclEntryType
.Group, "sales", FsAction.None), AclTestHelpers.AclEntry(AclEntryScope.Default,
AclEntryType.Group, FsAction.None), AclTestHelpers.AclEntry(AclEntryScope.Default
, AclEntryType.Other, FsAction.ReadExecute));
AssertPermissionGranted(Bruce, "/dir1/file1", FsAction.ReadWrite);
AssertPermissionGranted(Diana, "/dir1/file1", FsAction.Read);
AssertPermissionDenied(Clark, "/dir1/file1", FsAction.Read);
AssertPermissionDenied(Clark, "/dir1/file1", FsAction.Write);
AssertPermissionDenied(Clark, "/dir1/file1", FsAction.Execute);
AssertPermissionDenied(Clark, "/dir1/file1", FsAction.ReadWrite);
AssertPermissionDenied(Clark, "/dir1/file1", FsAction.ReadExecute);
AssertPermissionDenied(Clark, "/dir1/file1", FsAction.WriteExecute);
AssertPermissionDenied(Clark, "/dir1/file1", FsAction.All);
}
/// <summary>Get the list of inodes corresponding to valid leases.</summary>
/// <returns>list of inodes</returns>
internal virtual IDictionary <string, INodeFile> GetINodesUnderConstruction()
{
IDictionary <string, INodeFile> inodes = new SortedDictionary <string, INodeFile>();
foreach (string p in sortedLeasesByPath.Keys)
{
// verify that path exists in namespace
try
{
INodeFile node = INodeFile.ValueOf(fsnamesystem.dir.GetINode(p), p);
if (node.IsUnderConstruction())
{
inodes[p] = node;
}
else
{
Log.Warn("Ignore the lease of file " + p + " for checkpoint since the file is not under construction"
);
}
}
catch (IOException ioe)
{
Log.Error(ioe);
}
}
return(inodes);
}
public INodeFile(INodeFile that)
: base(that)
{
this.header = that.header;
this.blocks = that.blocks;
this.features = that.features;
}
/// <summary>Create FileStatus by file INode</summary>
/// <exception cref="System.IO.IOException"/>
internal static HdfsFileStatus CreateFileStatus(FSDirectory fsd, string fullPath,
byte[] path, INode node, byte storagePolicy, int snapshot, bool isRawPath, INodesInPath
iip)
{
long size = 0;
// length is zero for directories
short replication = 0;
long blocksize = 0;
bool isEncrypted;
FileEncryptionInfo feInfo = isRawPath ? null : fsd.GetFileEncryptionInfo(node, snapshot
, iip);
if (node.IsFile())
{
INodeFile fileNode = node.AsFile();
size = fileNode.ComputeFileSize(snapshot);
replication = fileNode.GetFileReplication(snapshot);
blocksize = fileNode.GetPreferredBlockSize();
isEncrypted = (feInfo != null) || (isRawPath && fsd.IsInAnEZ(INodesInPath.FromINode
(node)));
}
else
{
isEncrypted = fsd.IsInAnEZ(INodesInPath.FromINode(node));
}
int childrenNum = node.IsDirectory() ? node.AsDirectory().GetChildrenNum(snapshot
) : 0;
INodeAttributes nodeAttrs = fsd.GetAttributes(fullPath, path, node, snapshot);
return(new HdfsFileStatus(size, node.IsDirectory(), replication, blocksize, node.
GetModificationTime(snapshot), node.GetAccessTime(snapshot), GetPermissionForFileStatus
(nodeAttrs, isEncrypted), nodeAttrs.GetUserName(), nodeAttrs.GetGroupName(), node
.IsSymlink() ? node.AsSymlink().GetSymlink() : null, path, node.GetId(), childrenNum
, feInfo, storagePolicy));
}
/// <summary>Create FileStatus with location info by file INode</summary>
/// <exception cref="System.IO.IOException"/>
private static HdfsLocatedFileStatus CreateLocatedFileStatus(FSDirectory fsd, string
fullPath, byte[] path, INode node, byte storagePolicy, int snapshot, bool isRawPath
, INodesInPath iip)
{
System.Diagnostics.Debug.Assert(fsd.HasReadLock());
long size = 0;
// length is zero for directories
short replication = 0;
long blocksize = 0;
LocatedBlocks loc = null;
bool isEncrypted;
FileEncryptionInfo feInfo = isRawPath ? null : fsd.GetFileEncryptionInfo(node, snapshot
, iip);
if (node.IsFile())
{
INodeFile fileNode = node.AsFile();
size = fileNode.ComputeFileSize(snapshot);
replication = fileNode.GetFileReplication(snapshot);
blocksize = fileNode.GetPreferredBlockSize();
bool inSnapshot = snapshot != Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot
.CurrentStateId;
bool isUc = !inSnapshot && fileNode.IsUnderConstruction();
long fileSize = !inSnapshot && isUc?fileNode.ComputeFileSizeNotIncludingLastUcBlock
() : size;
loc = fsd.GetFSNamesystem().GetBlockManager().CreateLocatedBlocks(fileNode.GetBlocks
(snapshot), fileSize, isUc, 0L, size, false, inSnapshot, feInfo);
if (loc == null)
{
loc = new LocatedBlocks();
}
isEncrypted = (feInfo != null) || (isRawPath && fsd.IsInAnEZ(INodesInPath.FromINode
(node)));
}
else
{
isEncrypted = fsd.IsInAnEZ(INodesInPath.FromINode(node));
}
int childrenNum = node.IsDirectory() ? node.AsDirectory().GetChildrenNum(snapshot
) : 0;
INodeAttributes nodeAttrs = fsd.GetAttributes(fullPath, path, node, snapshot);
HdfsLocatedFileStatus status = new HdfsLocatedFileStatus(size, node.IsDirectory()
, replication, blocksize, node.GetModificationTime(snapshot), node.GetAccessTime
(snapshot), GetPermissionForFileStatus(nodeAttrs, isEncrypted), nodeAttrs.GetUserName
(), nodeAttrs.GetGroupName(), node.IsSymlink() ? node.AsSymlink().GetSymlink() :
null, path, node.GetId(), loc, childrenNum, feInfo, storagePolicy);
// Set caching information for the located blocks.
if (loc != null)
{
CacheManager cacheManager = fsd.GetFSNamesystem().GetCacheManager();
foreach (LocatedBlock lb in loc.GetLocatedBlocks())
{
cacheManager.SetCachedLocations(lb);
}
}
return(status);
}
internal static void AddFiles(FSEditLog editLog, int numFiles, short replication,
int blocksPerFile, long startingBlockId, long blockSize, FileNameGenerator nameGenerator
)
{
PermissionStatus p = new PermissionStatus("joeDoe", "people", new FsPermission((short
)0x1ff));
INodeId inodeId = new INodeId();
INodeDirectory dirInode = new INodeDirectory(inodeId.NextValue(), null, p, 0L);
editLog.LogMkDir(BasePath, dirInode);
BlockInfoContiguous[] blocks = new BlockInfoContiguous[blocksPerFile];
for (int iB = 0; iB < blocksPerFile; ++iB)
{
blocks[iB] = new BlockInfoContiguous(new Block(0, blockSize, BlockGenerationStamp
), replication);
}
long currentBlockId = startingBlockId;
long bidAtSync = startingBlockId;
for (int iF = 0; iF < numFiles; iF++)
{
for (int iB_1 = 0; iB_1 < blocksPerFile; ++iB_1)
{
blocks[iB_1].SetBlockId(currentBlockId++);
}
INodeFile inode = new INodeFile(inodeId.NextValue(), null, p, 0L, 0L, blocks, replication
, blockSize, unchecked ((byte)0));
inode.ToUnderConstruction(string.Empty, string.Empty);
// Append path to filename with information about blockIDs
string path = "_" + iF + "_B" + blocks[0].GetBlockId() + "_to_B" + blocks[blocksPerFile
- 1].GetBlockId() + "_";
string filePath = nameGenerator.GetNextFileName(string.Empty);
filePath = filePath + path;
// Log the new sub directory in edits
if ((iF % nameGenerator.GetFilesPerDirectory()) == 0)
{
string currentDir = nameGenerator.GetCurrentDir();
dirInode = new INodeDirectory(inodeId.NextValue(), null, p, 0L);
editLog.LogMkDir(currentDir, dirInode);
}
INodeFile fileUc = new INodeFile(inodeId.NextValue(), null, p, 0L, 0L, BlockInfoContiguous
.EmptyArray, replication, blockSize);
fileUc.ToUnderConstruction(string.Empty, string.Empty);
editLog.LogOpenFile(filePath, fileUc, false, false);
editLog.LogCloseFile(filePath, inode);
if (currentBlockId - bidAtSync >= 2000)
{
// sync every 2K blocks
editLog.LogSync();
bidAtSync = currentBlockId;
}
}
System.Console.Out.WriteLine("Created edits log in directory " + edits_dir);
System.Console.Out.WriteLine(" containing " + numFiles + " File-Creates, each file with "
+ blocksPerFile + " blocks");
System.Console.Out.WriteLine(" blocks range: " + startingBlockId + " to " + (currentBlockId
- 1));
}
/// <summary>Update the length for the last block</summary>
/// <param name="lastBlockLength">The length of the last block reported from client</param>
/// <exception cref="System.IO.IOException"/>
internal virtual void UpdateLengthOfLastBlock(INodeFile f, long lastBlockLength)
{
BlockInfoContiguous lastBlock = f.GetLastBlock();
System.Diagnostics.Debug.Assert((lastBlock != null), "The last block for path " +
f.GetFullPathName() + " is null when updating its length");
System.Diagnostics.Debug.Assert((lastBlock is BlockInfoContiguousUnderConstruction
), "The last block for path " + f.GetFullPathName() + " is not a BlockInfoUnderConstruction when updating its length"
);
lastBlock.SetNumBytes(lastBlockLength);
}
public virtual void TestAddBlock()
{
DistributedFileSystem fs = cluster.GetFileSystem();
Path file1 = new Path("/file1");
Path file2 = new Path("/file2");
Path file3 = new Path("/file3");
Path file4 = new Path("/file4");
DFSTestUtil.CreateFile(fs, file1, Blocksize - 1, Replication, 0L);
DFSTestUtil.CreateFile(fs, file2, Blocksize, Replication, 0L);
DFSTestUtil.CreateFile(fs, file3, Blocksize * 2 - 1, Replication, 0L);
DFSTestUtil.CreateFile(fs, file4, Blocksize * 2, Replication, 0L);
// restart NameNode
cluster.RestartNameNode(true);
FSDirectory fsdir = cluster.GetNamesystem().GetFSDirectory();
// check file1
INodeFile file1Node = fsdir.GetINode4Write(file1.ToString()).AsFile();
BlockInfoContiguous[] file1Blocks = file1Node.GetBlocks();
NUnit.Framework.Assert.AreEqual(1, file1Blocks.Length);
NUnit.Framework.Assert.AreEqual(Blocksize - 1, file1Blocks[0].GetNumBytes());
NUnit.Framework.Assert.AreEqual(HdfsServerConstants.BlockUCState.Complete, file1Blocks
[0].GetBlockUCState());
// check file2
INodeFile file2Node = fsdir.GetINode4Write(file2.ToString()).AsFile();
BlockInfoContiguous[] file2Blocks = file2Node.GetBlocks();
NUnit.Framework.Assert.AreEqual(1, file2Blocks.Length);
NUnit.Framework.Assert.AreEqual(Blocksize, file2Blocks[0].GetNumBytes());
NUnit.Framework.Assert.AreEqual(HdfsServerConstants.BlockUCState.Complete, file2Blocks
[0].GetBlockUCState());
// check file3
INodeFile file3Node = fsdir.GetINode4Write(file3.ToString()).AsFile();
BlockInfoContiguous[] file3Blocks = file3Node.GetBlocks();
NUnit.Framework.Assert.AreEqual(2, file3Blocks.Length);
NUnit.Framework.Assert.AreEqual(Blocksize, file3Blocks[0].GetNumBytes());
NUnit.Framework.Assert.AreEqual(HdfsServerConstants.BlockUCState.Complete, file3Blocks
[0].GetBlockUCState());
NUnit.Framework.Assert.AreEqual(Blocksize - 1, file3Blocks[1].GetNumBytes());
NUnit.Framework.Assert.AreEqual(HdfsServerConstants.BlockUCState.Complete, file3Blocks
[1].GetBlockUCState());
// check file4
INodeFile file4Node = fsdir.GetINode4Write(file4.ToString()).AsFile();
BlockInfoContiguous[] file4Blocks = file4Node.GetBlocks();
NUnit.Framework.Assert.AreEqual(2, file4Blocks.Length);
NUnit.Framework.Assert.AreEqual(Blocksize, file4Blocks[0].GetNumBytes());
NUnit.Framework.Assert.AreEqual(HdfsServerConstants.BlockUCState.Complete, file4Blocks
[0].GetBlockUCState());
NUnit.Framework.Assert.AreEqual(Blocksize, file4Blocks[1].GetNumBytes());
NUnit.Framework.Assert.AreEqual(HdfsServerConstants.BlockUCState.Complete, file4Blocks
[1].GetBlockUCState());
}
/// <summary>
/// <inheritDoc/>
/// <br/>
/// To destroy a DstReference node, we first remove its link with the
/// referred node. If the reference number of the referred node is <= 0, we
/// destroy the subtree of the referred node. Otherwise, we clean the
/// referred node's subtree and delete everything created after the last
/// rename operation, i.e., everything outside of the scope of the prior
/// WithName nodes.
/// </summary>
public override void DestroyAndCollectBlocks(BlockStoragePolicySuite bsps, INode.BlocksMapUpdateInfo
collectedBlocks, IList <INode> removedINodes)
{
if (RemoveReference(this) <= 0)
{
GetReferredINode().DestroyAndCollectBlocks(bsps, collectedBlocks, removedINodes);
}
else
{
// we will clean everything, including files, directories, and
// snapshots, that were created after this prior snapshot
int prior = GetPriorSnapshot(this);
// prior must be non-null, otherwise we do not have any previous
// WithName nodes, and the reference number will be 0.
Preconditions.CheckState(prior != Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot
.NoSnapshotId);
// identify the snapshot created after prior
int snapshot = GetSelfSnapshot(prior);
INode referred = GetReferredINode().AsReference().GetReferredINode();
if (referred.IsFile())
{
// if referred is a file, it must be a file with snapshot since we did
// recordModification before the rename
INodeFile file = referred.AsFile();
Preconditions.CheckState(file.IsWithSnapshot());
// make sure we mark the file as deleted
file.GetFileWithSnapshotFeature().DeleteCurrentFile();
// when calling cleanSubtree of the referred node, since we
// compute quota usage updates before calling this destroy
// function, we use true for countDiffChange
referred.CleanSubtree(bsps, snapshot, prior, collectedBlocks, removedINodes);
}
else
{
if (referred.IsDirectory())
{
// similarly, if referred is a directory, it must be an
// INodeDirectory with snapshot
INodeDirectory dir = referred.AsDirectory();
Preconditions.CheckState(dir.IsWithSnapshot());
try
{
DirectoryWithSnapshotFeature.DestroyDstSubtree(bsps, dir, snapshot, prior, collectedBlocks
, removedINodes);
}
catch (QuotaExceededException e)
{
Log.Error("should not exceed quota while snapshot deletion", e);
}
}
}
}
}
/// <exception cref="System.IO.IOException"/>
private static INodeFile CreateINodeFile(INodeDirectory parent, string name, string
owner, string group, short perm)
{
PermissionStatus permStatus = PermissionStatus.CreateImmutable(owner, group, FsPermission
.CreateImmutable(perm));
INodeFile inodeFile = new INodeFile(INodeId.GrandfatherInodeId, Sharpen.Runtime.GetBytesForString
(name, "UTF-8"), permStatus, 0L, 0L, null, Replication, PreferredBlockSize, unchecked (
(byte)0));
parent.AddChild(inodeFile);
return(inodeFile);
}
public static void UpdateBlocksMap(INodeFile file, BlockManager bm)
{
// Add file->block mapping
BlockInfoContiguous[] blocks = file.GetBlocks();
if (blocks != null)
{
for (int i = 0; i < blocks.Length; i++)
{
file.SetBlock(i, bm.AddBlockCollection(blocks[i], file));
}
}
}
/// <exception cref="System.IO.IOException"/>
private void TestPersistHelper(Configuration conf)
{
MiniDFSCluster cluster = null;
try
{
cluster = new MiniDFSCluster.Builder(conf).Build();
cluster.WaitActive();
FSNamesystem fsn = cluster.GetNamesystem();
DistributedFileSystem fs = cluster.GetFileSystem();
Path dir = new Path("/abc/def");
Path file1 = new Path(dir, "f1");
Path file2 = new Path(dir, "f2");
// create an empty file f1
fs.Create(file1).Close();
// create an under-construction file f2
FSDataOutputStream @out = fs.Create(file2);
@out.WriteBytes("hello");
((DFSOutputStream)@out.GetWrappedStream()).Hsync(EnumSet.Of(HdfsDataOutputStream.SyncFlag
.UpdateLength));
// checkpoint
fs.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter);
fs.SaveNamespace();
fs.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeLeave);
cluster.RestartNameNode();
cluster.WaitActive();
fs = cluster.GetFileSystem();
NUnit.Framework.Assert.IsTrue(fs.IsDirectory(dir));
NUnit.Framework.Assert.IsTrue(fs.Exists(file1));
NUnit.Framework.Assert.IsTrue(fs.Exists(file2));
// check internals of file2
INodeFile file2Node = fsn.dir.GetINode4Write(file2.ToString()).AsFile();
NUnit.Framework.Assert.AreEqual("hello".Length, file2Node.ComputeFileSize());
NUnit.Framework.Assert.IsTrue(file2Node.IsUnderConstruction());
BlockInfoContiguous[] blks = file2Node.GetBlocks();
NUnit.Framework.Assert.AreEqual(1, blks.Length);
NUnit.Framework.Assert.AreEqual(HdfsServerConstants.BlockUCState.UnderConstruction
, blks[0].GetBlockUCState());
// check lease manager
LeaseManager.Lease lease = fsn.leaseManager.GetLeaseByPath(file2.ToString());
NUnit.Framework.Assert.IsNotNull(lease);
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
/// <summary>Returns true if the file is closed</summary>
/// <exception cref="System.IO.IOException"/>
internal static bool IsFileClosed(FSDirectory fsd, string src)
{
FSPermissionChecker pc = fsd.GetPermissionChecker();
byte[][] pathComponents = FSDirectory.GetPathComponentsForReservedPath(src);
src = fsd.ResolvePath(pc, src, pathComponents);
INodesInPath iip = fsd.GetINodesInPath(src, true);
if (fsd.IsPermissionEnabled())
{
fsd.CheckTraverse(pc, iip);
}
return(!INodeFile.ValueOf(iip.GetLastINode(), src).IsUnderConstruction());
}
public virtual void TestAclNamedUserDeny()
{
INodeFile inodeFile = CreateINodeFile(inodeRoot, "file1", "bruce", "execs", (short
)0x1a4);
AddAcl(inodeFile, AclTestHelpers.AclEntry(AclEntryScope.Access, AclEntryType.User
, FsAction.ReadWrite), AclTestHelpers.AclEntry(AclEntryScope.Access, AclEntryType
.User, "diana", FsAction.None), AclTestHelpers.AclEntry(AclEntryScope.Access, AclEntryType
.Group, FsAction.Read), AclTestHelpers.AclEntry(AclEntryScope.Access, AclEntryType
.Mask, FsAction.Read), AclTestHelpers.AclEntry(AclEntryScope.Access, AclEntryType
.Other, FsAction.Read));
AssertPermissionGranted(Bruce, "/file1", FsAction.ReadWrite);
AssertPermissionGranted(Clark, "/file1", FsAction.Read);
AssertPermissionDenied(Diana, "/file1", FsAction.Read);
}
/// <exception cref="System.IO.IOException"/>
private void VerifyFileBlocks(string file, bool isFileOpen)
{
FSNamesystem ns = cluster.GetNamesystem();
INodeFile inode = INodeFile.ValueOf(ns.dir.GetINode(file), file);
NUnit.Framework.Assert.IsTrue("File " + inode.ToString() + " isUnderConstruction = "
+ inode.IsUnderConstruction() + " expected to be " + isFileOpen, inode.IsUnderConstruction
() == isFileOpen);
BlockInfoContiguous[] blocks = inode.GetBlocks();
NUnit.Framework.Assert.IsTrue("File does not have blocks: " + inode.ToString(), blocks
!= null && blocks.Length > 0);
int idx = 0;
BlockInfoContiguous curBlock;
// all blocks but the last two should be regular blocks
for (; idx < blocks.Length - 2; idx++)
{
curBlock = blocks[idx];
NUnit.Framework.Assert.IsTrue("Block is not complete: " + curBlock, curBlock.IsComplete
());
NUnit.Framework.Assert.IsTrue("Block is not in BlocksMap: " + curBlock, ns.GetBlockManager
().GetStoredBlock(curBlock) == curBlock);
}
// the penultimate block is either complete or
// committed if the file is not closed
if (idx > 0)
{
curBlock = blocks[idx - 1];
// penultimate block
NUnit.Framework.Assert.IsTrue("Block " + curBlock + " isUnderConstruction = " + inode
.IsUnderConstruction() + " expected to be " + isFileOpen, (isFileOpen && curBlock
.IsComplete()) || (!isFileOpen && !curBlock.IsComplete() == (curBlock.GetBlockUCState
() == HdfsServerConstants.BlockUCState.Committed)));
NUnit.Framework.Assert.IsTrue("Block is not in BlocksMap: " + curBlock, ns.GetBlockManager
().GetStoredBlock(curBlock) == curBlock);
}
// The last block is complete if the file is closed.
// If the file is open, the last block may be complete or not.
curBlock = blocks[idx];
// last block
if (!isFileOpen)
{
NUnit.Framework.Assert.IsTrue("Block " + curBlock + ", isFileOpen = " + isFileOpen
, curBlock.IsComplete());
}
NUnit.Framework.Assert.IsTrue("Block is not in BlocksMap: " + curBlock, ns.GetBlockManager
().GetStoredBlock(curBlock) == curBlock);
}
/// <summary>
/// When deleting a file in the current fs directory, and the file is contained
/// in a snapshot, we should delete the last block if it's under construction
/// and its size is 0.
/// </summary>
internal virtual void CleanZeroSizeBlock(INodeFile f, INode.BlocksMapUpdateInfo collectedBlocks
)
{
BlockInfoContiguous[] blocks = f.GetBlocks();
if (blocks != null && blocks.Length > 0 && blocks[blocks.Length - 1] is BlockInfoContiguousUnderConstruction)
{
BlockInfoContiguousUnderConstruction lastUC = (BlockInfoContiguousUnderConstruction
)blocks[blocks.Length - 1];
if (lastUC.GetNumBytes() == 0)
{
// this is a 0-sized block. do not need check its UC state here
collectedBlocks.AddDeleteBlock(lastUC);
f.RemoveLastBlock(lastUC);
}
}
}
public virtual void TestCommitBlockSynchronizationWithCloseAndNonExistantTarget()
{
INodeFile file = MockFileUnderConstruction();
Block block = new Block(blockId, length, genStamp);
FSNamesystem namesystemSpy = MakeNameSystemSpy(block, file);
DatanodeID[] newTargets = new DatanodeID[] { new DatanodeID("0.0.0.0", "nonexistantHost"
, "1", 0, 0, 0, 0) };
ExtendedBlock lastBlock = new ExtendedBlock();
namesystemSpy.CommitBlockSynchronization(lastBlock, genStamp, length, true, false
, newTargets, null);
// Repeat the call to make sure it returns true
namesystemSpy.CommitBlockSynchronization(lastBlock, genStamp, length, true, false
, newTargets, null);
}
// Helper function that writes an INodeUnderConstruction
// into the output stream
//
/// <exception cref="System.IO.IOException"/>
internal static void WriteINodeUnderConstruction(DataOutputStream @out, INodeFile
cons, string path)
{
WriteString(path, @out);
@out.WriteLong(cons.GetId());
@out.WriteShort(cons.GetFileReplication());
@out.WriteLong(cons.GetModificationTime());
@out.WriteLong(cons.GetPreferredBlockSize());
WriteBlocks(cons.GetBlocks(), @out);
cons.GetPermissionStatus().Write(@out);
FileUnderConstructionFeature uc = cons.GetFileUnderConstructionFeature();
WriteString(uc.GetClientName(), @out);
WriteString(uc.GetClientMachine(), @out);
@out.WriteInt(0);
}
/// <exception cref="System.IO.IOException"/>
private static void VerifyTargetFile(FSDirectory fsd, string target, INodesInPath
targetIIP)
{
// check the target
if (fsd.GetEZForPath(targetIIP) != null)
{
throw new HadoopIllegalArgumentException("concat can not be called for files in an encryption zone."
);
}
INodeFile targetINode = INodeFile.ValueOf(targetIIP.GetLastINode(), target);
if (targetINode.IsUnderConstruction())
{
throw new HadoopIllegalArgumentException("concat: target file " + target + " is under construction"
);
}
}
private INodeFile LoadINodeFile(FsImageProto.INodeSection.INode n)
{
System.Diagnostics.Debug.Assert(n.GetType() == FsImageProto.INodeSection.INode.Type
.File);
FsImageProto.INodeSection.INodeFile f = n.GetFile();
IList <HdfsProtos.BlockProto> bp = f.GetBlocksList();
short replication = (short)f.GetReplication();
FSImageFormatProtobuf.LoaderContext state = parent.GetLoaderContext();
BlockInfoContiguous[] blocks = new BlockInfoContiguous[bp.Count];
for (int i = 0; i < e; ++i)
{
blocks[i] = new BlockInfoContiguous(PBHelper.Convert(bp[i]), replication);
}
PermissionStatus permissions = LoadPermission(f.GetPermission(), parent.GetLoaderContext
().GetStringTable());
INodeFile file = new INodeFile(n.GetId(), n.GetName().ToByteArray(), permissions,
f.GetModificationTime(), f.GetAccessTime(), blocks, replication, f.GetPreferredBlockSize
(), unchecked ((byte)f.GetStoragePolicyID()));
if (f.HasAcl())
{
int[] entries = AclEntryStatusFormat.ToInt(LoadAclEntries(f.GetAcl(), state.GetStringTable
()));
file.AddAclFeature(new AclFeature(entries));
}
if (f.HasXAttrs())
{
file.AddXAttrFeature(new XAttrFeature(LoadXAttrs(f.GetXAttrs(), state.GetStringTable
())));
}
// under-construction information
if (f.HasFileUC())
{
FsImageProto.INodeSection.FileUnderConstructionFeature uc = f.GetFileUC();
file.ToUnderConstruction(uc.GetClientName(), uc.GetClientMachine());
if (blocks.Length > 0)
{
BlockInfoContiguous lastBlk = file.GetLastBlock();
// replace the last block of file
file.SetBlock(file.NumBlocks() - 1, new BlockInfoContiguousUnderConstruction(lastBlk
, replication));
}
}
return(file);
}
public virtual void TestAclGroup()
{
INodeFile inodeFile = CreateINodeFile(inodeRoot, "file1", "bruce", "execs", (short
)0x1a0);
AddAcl(inodeFile, AclTestHelpers.AclEntry(AclEntryScope.Access, AclEntryType.User
, FsAction.ReadWrite), AclTestHelpers.AclEntry(AclEntryScope.Access, AclEntryType
.Group, FsAction.Read), AclTestHelpers.AclEntry(AclEntryScope.Access, AclEntryType
.Mask, FsAction.Read), AclTestHelpers.AclEntry(AclEntryScope.Access, AclEntryType
.Other, FsAction.None));
AssertPermissionGranted(Clark, "/file1", FsAction.Read);
AssertPermissionDenied(Clark, "/file1", FsAction.Write);
AssertPermissionDenied(Clark, "/file1", FsAction.Execute);
AssertPermissionDenied(Clark, "/file1", FsAction.ReadWrite);
AssertPermissionDenied(Clark, "/file1", FsAction.ReadExecute);
AssertPermissionDenied(Clark, "/file1", FsAction.WriteExecute);
AssertPermissionDenied(Clark, "/file1", FsAction.All);
}
public virtual void TestCommitBlockSynchronizationWithDelete()
{
INodeFile file = MockFileUnderConstruction();
Block block = new Block(blockId, length, genStamp);
FSNamesystem namesystemSpy = MakeNameSystemSpy(block, file);
DatanodeID[] newTargets = new DatanodeID[0];
ExtendedBlock lastBlock = new ExtendedBlock();
namesystemSpy.CommitBlockSynchronization(lastBlock, genStamp, length, false, true
, newTargets, null);
// Simulate removing the last block from the file.
Org.Mockito.Mockito.DoReturn(false).When(file).RemoveLastBlock(Matchers.Any <Block
>());
// Repeat the call to make sure it does not throw
namesystemSpy.CommitBlockSynchronization(lastBlock, genStamp, length, false, true
, newTargets, null);
}
/// <exception cref="System.IO.IOException"/>
internal void SerializeFilesUCSection(OutputStream @out)
{
IDictionary <string, INodeFile> ucMap = fsn.GetFilesUnderConstruction();
foreach (KeyValuePair <string, INodeFile> entry in ucMap)
{
string path = entry.Key;
INodeFile file = entry.Value;
FsImageProto.FilesUnderConstructionSection.FileUnderConstructionEntry.Builder b =
FsImageProto.FilesUnderConstructionSection.FileUnderConstructionEntry.NewBuilder
().SetInodeId(file.GetId()).SetFullPath(path);
FsImageProto.FilesUnderConstructionSection.FileUnderConstructionEntry e = ((FsImageProto.FilesUnderConstructionSection.FileUnderConstructionEntry
)b.Build());
e.WriteDelimitedTo(@out);
}
parent.CommitSection(summary, FSImageFormatProtobuf.SectionName.FilesUnderconstruction
);
}
/// <summary>Load the under-construction files section, and update the lease map</summary>
/// <exception cref="System.IO.IOException"/>
internal void LoadFilesUnderConstructionSection(InputStream @in)
{
while (true)
{
FsImageProto.FilesUnderConstructionSection.FileUnderConstructionEntry entry = FsImageProto.FilesUnderConstructionSection.FileUnderConstructionEntry
.ParseDelimitedFrom(@in);
if (entry == null)
{
break;
}
// update the lease manager
INodeFile file = dir.GetInode(entry.GetInodeId()).AsFile();
FileUnderConstructionFeature uc = file.GetFileUnderConstructionFeature();
Preconditions.CheckState(uc != null);
// file must be under-construction
fsn.leaseManager.AddLease(uc.GetClientName(), entry.GetFullPath());
}
}
// Helper function that reads in an INodeUnderConstruction
// from the input stream
//
/// <exception cref="System.IO.IOException"/>
internal static INodeFile ReadINodeUnderConstruction(DataInput @in, FSNamesystem
fsNamesys, int imgVersion)
{
byte[] name = ReadBytes(@in);
long inodeId = NameNodeLayoutVersion.Supports(LayoutVersion.Feature.AddInodeId, imgVersion
) ? @in.ReadLong() : fsNamesys.dir.AllocateNewInodeId();
short blockReplication = @in.ReadShort();
long modificationTime = @in.ReadLong();
long preferredBlockSize = @in.ReadLong();
int numBlocks = @in.ReadInt();
BlockInfoContiguous[] blocks = new BlockInfoContiguous[numBlocks];
Block blk = new Block();
int i = 0;
for (; i < numBlocks - 1; i++)
{
blk.ReadFields(@in);
blocks[i] = new BlockInfoContiguous(blk, blockReplication);
}
// last block is UNDER_CONSTRUCTION
if (numBlocks > 0)
{
blk.ReadFields(@in);
blocks[i] = new BlockInfoContiguousUnderConstruction(blk, blockReplication, HdfsServerConstants.BlockUCState
.UnderConstruction, null);
}
PermissionStatus perm = PermissionStatus.Read(@in);
string clientName = ReadString(@in);
string clientMachine = ReadString(@in);
// We previously stored locations for the last block, now we
// just record that there are none
int numLocs = @in.ReadInt();
System.Diagnostics.Debug.Assert(numLocs == 0, "Unexpected block locations");
// Images in the pre-protobuf format will not have the lazyPersist flag,
// so it is safe to pass false always.
INodeFile file = new INodeFile(inodeId, name, perm, modificationTime, modificationTime
, blocks, blockReplication, preferredBlockSize, unchecked ((byte)0));
file.ToUnderConstruction(clientName, clientMachine);
return(file);
}
/// <summary>
/// Test append over a specific type of storage quota does not mark file as
/// UC or create a lease
/// </summary>
/// <exception cref="System.Exception"/>
public virtual void TestAppendOverTypeQuota()
{
Path dir = new Path("/TestAppendOverTypeQuota");
Path file = new Path(dir, "file");
// create partial block file
dfs.Mkdirs(dir);
// set the storage policy on dir
dfs.SetStoragePolicy(dir, HdfsConstants.OnessdStoragePolicyName);
DFSTestUtil.CreateFile(dfs, file, Blocksize / 2, Replication, seed);
// set quota of SSD to 1L
dfs.SetQuotaByStorageType(dir, StorageType.Ssd, 1L);
INodeDirectory dirNode = fsdir.GetINode4Write(dir.ToString()).AsDirectory();
long spaceUsed = dirNode.GetDirectoryWithQuotaFeature().GetSpaceConsumed().GetStorageSpace
();
try
{
DFSTestUtil.AppendFile(dfs, file, Blocksize);
NUnit.Framework.Assert.Fail("append didn't fail");
}
catch (RemoteException e)
{
NUnit.Framework.Assert.IsTrue(e.GetClassName().Contains("QuotaByStorageTypeExceededException"
));
}
// check that the file exists, isn't UC, and has no dangling lease
INodeFile inode = fsdir.GetINode(file.ToString()).AsFile();
NUnit.Framework.Assert.IsNotNull(inode);
NUnit.Framework.Assert.IsFalse("should not be UC", inode.IsUnderConstruction());
NUnit.Framework.Assert.IsNull("should not have a lease", cluster.GetNamesystem().
GetLeaseManager().GetLeaseByPath(file.ToString()));
// make sure the quota usage is unchanged
long newSpaceUsed = dirNode.GetDirectoryWithQuotaFeature().GetSpaceConsumed().GetStorageSpace
();
NUnit.Framework.Assert.AreEqual(spaceUsed, newSpaceUsed);
// make sure edits aren't corrupted
dfs.RecoverLease(file);
cluster.RestartNameNodes();
}
/// <exception cref="System.IO.IOException"/>
private FSNamesystem MakeNameSystemSpy(Block block, INodeFile file)
{
Configuration conf = new Configuration();
FSImage image = new FSImage(conf);
DatanodeStorageInfo[] targets = new DatanodeStorageInfo[] { };
FSNamesystem namesystem = new FSNamesystem(conf, image);
namesystem.SetImageLoaded(true);
// set file's parent as root and put the file to inodeMap, so
// FSNamesystem's isFileDeleted() method will return false on this file
if (file.GetParent() == null)
{
INodeDirectory mparent = Org.Mockito.Mockito.Mock <INodeDirectory>();
INodeDirectory parent = new INodeDirectory(mparent.GetId(), new byte[0], mparent.
GetPermissionStatus(), mparent.GetAccessTime());
parent.SetLocalName(new byte[0]);
parent.AddChild(file);
file.SetParent(parent);
}
namesystem.dir.GetINodeMap().Put(file);
FSNamesystem namesystemSpy = Org.Mockito.Mockito.Spy(namesystem);
BlockInfoContiguousUnderConstruction blockInfo = new BlockInfoContiguousUnderConstruction
(block, (short)1, HdfsServerConstants.BlockUCState.UnderConstruction, targets);
blockInfo.SetBlockCollection(file);
blockInfo.SetGenerationStamp(genStamp);
blockInfo.InitializeBlockRecovery(genStamp);
Org.Mockito.Mockito.DoReturn(true).When(file).RemoveLastBlock(Matchers.Any <Block>
());
Org.Mockito.Mockito.DoReturn(true).When(file).IsUnderConstruction();
Org.Mockito.Mockito.DoReturn(new BlockInfoContiguous[1]).When(file).GetBlocks();
Org.Mockito.Mockito.DoReturn(blockInfo).When(namesystemSpy).GetStoredBlock(Matchers.Any
<Block>());
Org.Mockito.Mockito.DoReturn(blockInfo).When(file).GetLastBlock();
Org.Mockito.Mockito.DoReturn(string.Empty).When(namesystemSpy).CloseFileCommitBlocks
(Matchers.Any <INodeFile>(), Matchers.Any <BlockInfoContiguous>());
Org.Mockito.Mockito.DoReturn(Org.Mockito.Mockito.Mock <FSEditLog>()).When(namesystemSpy
).GetEditLog();
return(namesystemSpy);
}
/// <exception cref="System.IO.IOException"/>
internal static long GetPreferredBlockSize(FSDirectory fsd, string src)
{
FSPermissionChecker pc = fsd.GetPermissionChecker();
byte[][] pathComponents = FSDirectory.GetPathComponentsForReservedPath(src);
fsd.ReadLock();
try
{
src = fsd.ResolvePath(pc, src, pathComponents);
INodesInPath iip = fsd.GetINodesInPath(src, false);
if (fsd.IsPermissionEnabled())
{
fsd.CheckTraverse(pc, iip);
}
return(INodeFile.ValueOf(iip.GetLastINode(), src).GetPreferredBlockSize());
}
finally
{
fsd.ReadUnlock();
}
}
