private int GetSelfSnapshot(int prior)
{
INodeReference.WithCount wc = (INodeReference.WithCount)GetReferredINode().AsReference
();
INode referred = wc.GetReferredINode();
int lastSnapshot = Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.CurrentStateId;
if (referred.IsFile() && referred.AsFile().IsWithSnapshot())
{
lastSnapshot = referred.AsFile().GetDiffs().GetLastSnapshotId();
}
else
{
if (referred.IsDirectory())
{
DirectoryWithSnapshotFeature sf = referred.AsDirectory().GetDirectoryWithSnapshotFeature
();
if (sf != null)
{
lastSnapshot = sf.GetLastSnapshotId();
}
}
}
if (lastSnapshot != Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.CurrentStateId &&
lastSnapshot != prior)
{
return(lastSnapshot);
}
else
{
return(Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.CurrentStateId);
}
}
/// <summary>Create FileStatus by file INode</summary>
/// <exception cref="System.IO.IOException"/>
internal static HdfsFileStatus CreateFileStatus(FSDirectory fsd, string fullPath,
byte[] path, INode node, byte storagePolicy, int snapshot, bool isRawPath, INodesInPath
iip)
{
long size = 0;
// length is zero for directories
short replication = 0;
long blocksize = 0;
bool isEncrypted;
FileEncryptionInfo feInfo = isRawPath ? null : fsd.GetFileEncryptionInfo(node, snapshot
, iip);
if (node.IsFile())
{
INodeFile fileNode = node.AsFile();
size = fileNode.ComputeFileSize(snapshot);
replication = fileNode.GetFileReplication(snapshot);
blocksize = fileNode.GetPreferredBlockSize();
isEncrypted = (feInfo != null) || (isRawPath && fsd.IsInAnEZ(INodesInPath.FromINode
(node)));
}
else
{
isEncrypted = fsd.IsInAnEZ(INodesInPath.FromINode(node));
}
int childrenNum = node.IsDirectory() ? node.AsDirectory().GetChildrenNum(snapshot
) : 0;
INodeAttributes nodeAttrs = fsd.GetAttributes(fullPath, path, node, snapshot);
return(new HdfsFileStatus(size, node.IsDirectory(), replication, blocksize, node.
GetModificationTime(snapshot), node.GetAccessTime(snapshot), GetPermissionForFileStatus
(nodeAttrs, isEncrypted), nodeAttrs.GetUserName(), nodeAttrs.GetGroupName(), node
.IsSymlink() ? node.AsSymlink().GetSymlink() : null, path, node.GetId(), childrenNum
, feInfo, storagePolicy));
}
/// <summary>Save one inode's attributes to the image.</summary>
/// <exception cref="System.IO.IOException"/>
public static void SaveINode2Image(INode node, DataOutput @out, bool writeUnderConstruction
, SnapshotFSImageFormat.ReferenceMap referenceMap)
{
if (node.IsReference())
{
WriteINodeReference(node.AsReference(), @out, writeUnderConstruction, referenceMap
);
}
else
{
if (node.IsDirectory())
{
WriteINodeDirectory(node.AsDirectory(), @out);
}
else
{
if (node.IsSymlink())
{
WriteINodeSymlink(node.AsSymlink(), @out);
}
else
{
if (node.IsFile())
{
WriteINodeFile(node.AsFile(), @out, writeUnderConstruction);
}
}
}
}
}
/// <summary>Create FileStatus with location info by file INode</summary>
/// <exception cref="System.IO.IOException"/>
private static HdfsLocatedFileStatus CreateLocatedFileStatus(FSDirectory fsd, string
fullPath, byte[] path, INode node, byte storagePolicy, int snapshot, bool isRawPath
, INodesInPath iip)
{
System.Diagnostics.Debug.Assert(fsd.HasReadLock());
long size = 0;
// length is zero for directories
short replication = 0;
long blocksize = 0;
LocatedBlocks loc = null;
bool isEncrypted;
FileEncryptionInfo feInfo = isRawPath ? null : fsd.GetFileEncryptionInfo(node, snapshot
, iip);
if (node.IsFile())
{
INodeFile fileNode = node.AsFile();
size = fileNode.ComputeFileSize(snapshot);
replication = fileNode.GetFileReplication(snapshot);
blocksize = fileNode.GetPreferredBlockSize();
bool inSnapshot = snapshot != Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot
.CurrentStateId;
bool isUc = !inSnapshot && fileNode.IsUnderConstruction();
long fileSize = !inSnapshot && isUc?fileNode.ComputeFileSizeNotIncludingLastUcBlock
() : size;
loc = fsd.GetFSNamesystem().GetBlockManager().CreateLocatedBlocks(fileNode.GetBlocks
(snapshot), fileSize, isUc, 0L, size, false, inSnapshot, feInfo);
if (loc == null)
{
loc = new LocatedBlocks();
}
isEncrypted = (feInfo != null) || (isRawPath && fsd.IsInAnEZ(INodesInPath.FromINode
(node)));
}
else
{
isEncrypted = fsd.IsInAnEZ(INodesInPath.FromINode(node));
}
int childrenNum = node.IsDirectory() ? node.AsDirectory().GetChildrenNum(snapshot
) : 0;
INodeAttributes nodeAttrs = fsd.GetAttributes(fullPath, path, node, snapshot);
HdfsLocatedFileStatus status = new HdfsLocatedFileStatus(size, node.IsDirectory()
, replication, blocksize, node.GetModificationTime(snapshot), node.GetAccessTime
(snapshot), GetPermissionForFileStatus(nodeAttrs, isEncrypted), nodeAttrs.GetUserName
(), nodeAttrs.GetGroupName(), node.IsSymlink() ? node.AsSymlink().GetSymlink() :
null, path, node.GetId(), loc, childrenNum, feInfo, storagePolicy);
// Set caching information for the located blocks.
if (loc != null)
{
CacheManager cacheManager = fsd.GetFSNamesystem().GetCacheManager();
foreach (LocatedBlock lb in loc.GetLocatedBlocks())
{
cacheManager.SetCachedLocations(lb);
}
}
return(status);
}
/// <summary>
/// <inheritDoc/>
/// <br/>
/// To destroy a DstReference node, we first remove its link with the
/// referred node. If the reference number of the referred node is <= 0, we
/// destroy the subtree of the referred node. Otherwise, we clean the
/// referred node's subtree and delete everything created after the last
/// rename operation, i.e., everything outside of the scope of the prior
/// WithName nodes.
/// </summary>
public override void DestroyAndCollectBlocks(BlockStoragePolicySuite bsps, INode.BlocksMapUpdateInfo
collectedBlocks, IList <INode> removedINodes)
{
if (RemoveReference(this) <= 0)
{
GetReferredINode().DestroyAndCollectBlocks(bsps, collectedBlocks, removedINodes);
}
else
{
// we will clean everything, including files, directories, and
// snapshots, that were created after this prior snapshot
int prior = GetPriorSnapshot(this);
// prior must be non-null, otherwise we do not have any previous
// WithName nodes, and the reference number will be 0.
Preconditions.CheckState(prior != Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot
.NoSnapshotId);
// identify the snapshot created after prior
int snapshot = GetSelfSnapshot(prior);
INode referred = GetReferredINode().AsReference().GetReferredINode();
if (referred.IsFile())
{
// if referred is a file, it must be a file with snapshot since we did
// recordModification before the rename
INodeFile file = referred.AsFile();
Preconditions.CheckState(file.IsWithSnapshot());
// make sure we mark the file as deleted
file.GetFileWithSnapshotFeature().DeleteCurrentFile();
// when calling cleanSubtree of the referred node, since we
// compute quota usage updates before calling this destroy
// function, we use true for countDiffChange
referred.CleanSubtree(bsps, snapshot, prior, collectedBlocks, removedINodes);
}
else
{
if (referred.IsDirectory())
{
// similarly, if referred is a directory, it must be an
// INodeDirectory with snapshot
INodeDirectory dir = referred.AsDirectory();
Preconditions.CheckState(dir.IsWithSnapshot());
try
{
DirectoryWithSnapshotFeature.DestroyDstSubtree(bsps, dir, snapshot, prior, collectedBlocks
, removedINodes);
}
catch (QuotaExceededException e)
{
Log.Error("should not exceed quota while snapshot deletion", e);
}
}
}
}
}
private void AddToParent(INodeDirectory parent, INode child)
{
if (parent == dir.rootDir && FSDirectory.IsReservedName(child))
{
throw new HadoopIllegalArgumentException("File name \"" + child.GetLocalName() +
"\" is reserved. Please " + " change the name of the existing file or directory to another "
+ "name before upgrading to this release.");
}
// NOTE: This does not update space counts for parents
if (!parent.AddChild(child))
{
return;
}
dir.CacheName(child);
if (child.IsFile())
{
UpdateBlocksMap(child.AsFile(), fsn.GetBlockManager());
}
}
/// <exception cref="System.IO.IOException"/>
internal static void UnprotectedSetStoragePolicy(FSDirectory fsd, BlockManager bm
, INodesInPath iip, byte policyId)
{
System.Diagnostics.Debug.Assert(fsd.HasWriteLock());
INode inode = iip.GetLastINode();
if (inode == null)
{
throw new FileNotFoundException("File/Directory does not exist: " + iip.GetPath()
);
}
int snapshotId = iip.GetLatestSnapshotId();
if (inode.IsFile())
{
BlockStoragePolicy newPolicy = bm.GetStoragePolicy(policyId);
if (newPolicy.IsCopyOnCreateFile())
{
throw new HadoopIllegalArgumentException("Policy " + newPolicy + " cannot be set after file creation."
);
}
BlockStoragePolicy currentPolicy = bm.GetStoragePolicy(inode.GetLocalStoragePolicyID
());
if (currentPolicy != null && currentPolicy.IsCopyOnCreateFile())
{
throw new HadoopIllegalArgumentException("Existing policy " + currentPolicy.GetName
() + " cannot be changed after file creation.");
}
inode.AsFile().SetStoragePolicyID(policyId, snapshotId);
}
else
{
if (inode.IsDirectory())
{
SetDirStoragePolicy(fsd, inode.AsDirectory(), policyId, snapshotId);
}
else
{
throw new FileNotFoundException(iip.GetPath() + " is not a file or directory");
}
}
}
/// <summary>Cast INode to INodeFile.</summary>
/// <exception cref="System.IO.FileNotFoundException"/>
public static Org.Apache.Hadoop.Hdfs.Server.Namenode.INodeFile ValueOf(INode inode
, string path, bool acceptNull)
{
if (inode == null)
{
if (acceptNull)
{
return(null);
}
else
{
throw new FileNotFoundException("File does not exist: " + path);
}
}
if (!inode.IsFile())
{
throw new FileNotFoundException("Path is not a file: " + path);
}
return(inode.AsFile());
}
/// <exception cref="System.IO.IOException"/>
private void Save(OutputStream @out, INode n)
{
if (n.IsDirectory())
{
Save(@out, n.AsDirectory());
}
else
{
if (n.IsFile())
{
Save(@out, n.AsFile());
}
else
{
if (n.IsSymlink())
{
Save(@out, n.AsSymlink());
}
}
}
}
/// <exception cref="Org.Apache.Hadoop.Hdfs.Protocol.QuotaExceededException"/>
/// <exception cref="Org.Apache.Hadoop.FS.UnresolvedLinkException"/>
/// <exception cref="Org.Apache.Hadoop.Hdfs.Protocol.SnapshotAccessControlException"/
/// >
internal static Block[] UnprotectedSetReplication(FSDirectory fsd, string src, short
replication, short[] blockRepls)
{
System.Diagnostics.Debug.Assert(fsd.HasWriteLock());
INodesInPath iip = fsd.GetINodesInPath4Write(src, true);
INode inode = iip.GetLastINode();
if (inode == null || !inode.IsFile())
{
return(null);
}
INodeFile file = inode.AsFile();
short oldBR = file.GetBlockReplication();
// before setFileReplication, check for increasing block replication.
// if replication > oldBR, then newBR == replication.
// if replication < oldBR, we don't know newBR yet.
if (replication > oldBR)
{
long dsDelta = file.StoragespaceConsumed() / oldBR;
fsd.UpdateCount(iip, 0L, dsDelta, oldBR, replication, true);
}
file.SetFileReplication(replication, iip.GetLatestSnapshotId());
short newBR = file.GetBlockReplication();
// check newBR < oldBR case.
if (newBR < oldBR)
{
long dsDelta = file.StoragespaceConsumed() / newBR;
fsd.UpdateCount(iip, 0L, dsDelta, oldBR, newBR, true);
}
if (blockRepls != null)
{
blockRepls[0] = oldBR;
blockRepls[1] = newBR;
}
return(file.GetBlocks());
}
/// <summary>
/// When destroying a reference node (WithName or DstReference), we call this
/// method to identify the snapshot which is the latest snapshot before the
/// reference node's creation.
/// </summary>
internal static int GetPriorSnapshot(Org.Apache.Hadoop.Hdfs.Server.Namenode.INodeReference
@ref)
{
INodeReference.WithCount wc = (INodeReference.WithCount)@ref.GetReferredINode();
INodeReference.WithName wn = null;
if (@ref is INodeReference.DstReference)
{
wn = wc.GetLastWithName();
}
else
{
if (@ref is INodeReference.WithName)
{
wn = wc.GetPriorWithName((INodeReference.WithName)@ref);
}
}
if (wn != null)
{
INode referred = wc.GetReferredINode();
if (referred.IsFile() && referred.AsFile().IsWithSnapshot())
{
return(referred.AsFile().GetDiffs().GetPrior(wn.lastSnapshotId));
}
else
{
if (referred.IsDirectory())
{
DirectoryWithSnapshotFeature sf = referred.AsDirectory().GetDirectoryWithSnapshotFeature
();
if (sf != null)
{
return(sf.GetDiffs().GetPrior(wn.lastSnapshotId));
}
}
}
}
return(Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.NoSnapshotId);
}
private int GetSelfSnapshot()
{
INode referred = GetReferredINode().AsReference().GetReferredINode();
int snapshot = Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.NoSnapshotId;
if (referred.IsFile() && referred.AsFile().IsWithSnapshot())
{
snapshot = referred.AsFile().GetDiffs().GetPrior(lastSnapshotId);
}
else
{
if (referred.IsDirectory())
{
DirectoryWithSnapshotFeature sf = referred.AsDirectory().GetDirectoryWithSnapshotFeature
();
if (sf != null)
{
snapshot = sf.GetDiffs().GetPrior(lastSnapshotId);
}
}
}
return(snapshot);
}
/// <exception cref="System.IO.IOException"/>
internal static INode UnprotectedSetXAttrs(FSDirectory fsd, string src, IList <XAttr
> xAttrs, EnumSet <XAttrSetFlag> flag)
{
System.Diagnostics.Debug.Assert(fsd.HasWriteLock());
INodesInPath iip = fsd.GetINodesInPath4Write(FSDirectory.NormalizePath(src), true
);
INode inode = FSDirectory.ResolveLastINode(iip);
int snapshotId = iip.GetLatestSnapshotId();
IList <XAttr> existingXAttrs = XAttrStorage.ReadINodeXAttrs(inode);
IList <XAttr> newXAttrs = SetINodeXAttrs(fsd, existingXAttrs, xAttrs, flag);
bool isFile = inode.IsFile();
foreach (XAttr xattr in newXAttrs)
{
string xaName = XAttrHelper.GetPrefixName(xattr);
/*
* If we're adding the encryption zone xattr, then add src to the list
* of encryption zones.
*/
if (HdfsServerConstants.CryptoXattrEncryptionZone.Equals(xaName))
{
HdfsProtos.ZoneEncryptionInfoProto ezProto = HdfsProtos.ZoneEncryptionInfoProto.ParseFrom
(xattr.GetValue());
fsd.ezManager.AddEncryptionZone(inode.GetId(), PBHelper.Convert(ezProto.GetSuite(
)), PBHelper.Convert(ezProto.GetCryptoProtocolVersion()), ezProto.GetKeyName());
}
if (!isFile && HdfsServerConstants.SecurityXattrUnreadableBySuperuser.Equals(xaName
))
{
throw new IOException("Can only set '" + HdfsServerConstants.SecurityXattrUnreadableBySuperuser
+ "' on a file.");
}
}
XAttrStorage.UpdateINodeXAttrs(inode, newXAttrs, snapshotId);
return(inode);
}
/// <exception cref="System.IO.IOException"/>
internal static HdfsFileStatus Mkdirs(FSNamesystem fsn, string src, PermissionStatus
permissions, bool createParent)
{
FSDirectory fsd = fsn.GetFSDirectory();
if (NameNode.stateChangeLog.IsDebugEnabled())
{
NameNode.stateChangeLog.Debug("DIR* NameSystem.mkdirs: " + src);
}
if (!DFSUtil.IsValidName(src))
{
throw new InvalidPathException(src);
}
FSPermissionChecker pc = fsd.GetPermissionChecker();
byte[][] pathComponents = FSDirectory.GetPathComponentsForReservedPath(src);
fsd.WriteLock();
try
{
src = fsd.ResolvePath(pc, src, pathComponents);
INodesInPath iip = fsd.GetINodesInPath4Write(src);
if (fsd.IsPermissionEnabled())
{
fsd.CheckTraverse(pc, iip);
}
INode lastINode = iip.GetLastINode();
if (lastINode != null && lastINode.IsFile())
{
throw new FileAlreadyExistsException("Path is not a directory: " + src);
}
INodesInPath existing = lastINode != null ? iip : iip.GetExistingINodes();
if (lastINode == null)
{
if (fsd.IsPermissionEnabled())
{
fsd.CheckAncestorAccess(pc, iip, FsAction.Write);
}
if (!createParent)
{
fsd.VerifyParentDir(iip, src);
}
// validate that we have enough inodes. This is, at best, a
// heuristic because the mkdirs() operation might need to
// create multiple inodes.
fsn.CheckFsObjectLimit();
IList <string> nonExisting = iip.GetPath(existing.Length(), iip.Length() - existing
.Length());
int length = nonExisting.Count;
if (length > 1)
{
IList <string> ancestors = nonExisting.SubList(0, length - 1);
// Ensure that the user can traversal the path by adding implicit
// u+wx permission to all ancestor directories
existing = CreateChildrenDirectories(fsd, existing, ancestors, AddImplicitUwx(permissions
, permissions));
if (existing == null)
{
throw new IOException("Failed to create directory: " + src);
}
}
if ((existing = CreateChildrenDirectories(fsd, existing, nonExisting.SubList(length
- 1, length), permissions)) == null)
{
throw new IOException("Failed to create directory: " + src);
}
}
return(fsd.GetAuditFileInfo(existing));
}
finally
{
fsd.WriteUnlock();
}
}
/// <summary>
/// If a default ACL is defined on a parent directory, then copies that default
/// ACL to a newly created child file or directory.
/// </summary>
/// <param name="child">INode newly created child</param>
public static void CopyINodeDefaultAcl(INode child)
{
INodeDirectory parent = child.GetParent();
AclFeature parentAclFeature = parent.GetAclFeature();
if (parentAclFeature == null || !(child.IsFile() || child.IsDirectory()))
{
return;
}
// Split parent's entries into access vs. default.
IList <AclEntry> featureEntries = GetEntriesFromAclFeature(parent.GetAclFeature());
ScopedAclEntries scopedEntries = new ScopedAclEntries(featureEntries);
IList <AclEntry> parentDefaultEntries = scopedEntries.GetDefaultEntries();
// The parent may have an access ACL but no default ACL. If so, exit.
if (parentDefaultEntries.IsEmpty())
{
return;
}
// Pre-allocate list size for access entries to copy from parent.
IList <AclEntry> accessEntries = Lists.NewArrayListWithCapacity(parentDefaultEntries
.Count);
FsPermission childPerm = child.GetFsPermission();
// Copy each default ACL entry from parent to new child's access ACL.
bool parentDefaultIsMinimal = AclUtil.IsMinimalAcl(parentDefaultEntries);
foreach (AclEntry entry in parentDefaultEntries)
{
AclEntryType type = entry.GetType();
string name = entry.GetName();
AclEntry.Builder builder = new AclEntry.Builder().SetScope(AclEntryScope.Access).
SetType(type).SetName(name);
// The child's initial permission bits are treated as the mode parameter,
// which can filter copied permission values for owner, mask and other.
FsAction permission;
if (type == AclEntryType.User && name == null)
{
permission = entry.GetPermission().And(childPerm.GetUserAction());
}
else
{
if (type == AclEntryType.Group && parentDefaultIsMinimal)
{
// This only happens if the default ACL is a minimal ACL: exactly 3
// entries corresponding to owner, group and other. In this case,
// filter the group permissions.
permission = entry.GetPermission().And(childPerm.GetGroupAction());
}
else
{
if (type == AclEntryType.Mask)
{
// Group bits from mode parameter filter permission of mask entry.
permission = entry.GetPermission().And(childPerm.GetGroupAction());
}
else
{
if (type == AclEntryType.Other)
{
permission = entry.GetPermission().And(childPerm.GetOtherAction());
}
else
{
permission = entry.GetPermission();
}
}
}
}
builder.SetPermission(permission);
accessEntries.AddItem(builder.Build());
}
// A new directory also receives a copy of the parent's default ACL.
IList <AclEntry> defaultEntries = child.IsDirectory() ? parentDefaultEntries : Sharpen.Collections
.EmptyList <AclEntry>();
FsPermission newPerm;
if (!AclUtil.IsMinimalAcl(accessEntries) || !defaultEntries.IsEmpty())
{
// Save the new ACL to the child.
child.AddAclFeature(CreateAclFeature(accessEntries, defaultEntries));
newPerm = CreateFsPermissionForExtendedAcl(accessEntries, childPerm);
}
else
{
// The child is receiving a minimal ACL.
newPerm = CreateFsPermissionForMinimalAcl(accessEntries, childPerm);
}
child.SetPermission(newPerm);
}