/// <summary>Create FileStatus with location info by file INode</summary>
/// <exception cref="System.IO.IOException"/>
private static HdfsLocatedFileStatus CreateLocatedFileStatus(FSDirectory fsd, string
fullPath, byte[] path, INode node, byte storagePolicy, int snapshot, bool isRawPath
, INodesInPath iip)
{
System.Diagnostics.Debug.Assert(fsd.HasReadLock());
long size = 0;
// length is zero for directories
short replication = 0;
long blocksize = 0;
LocatedBlocks loc = null;
bool isEncrypted;
FileEncryptionInfo feInfo = isRawPath ? null : fsd.GetFileEncryptionInfo(node, snapshot
, iip);
if (node.IsFile())
{
INodeFile fileNode = node.AsFile();
size = fileNode.ComputeFileSize(snapshot);
replication = fileNode.GetFileReplication(snapshot);
blocksize = fileNode.GetPreferredBlockSize();
bool inSnapshot = snapshot != Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot
.CurrentStateId;
bool isUc = !inSnapshot && fileNode.IsUnderConstruction();
long fileSize = !inSnapshot && isUc?fileNode.ComputeFileSizeNotIncludingLastUcBlock
() : size;
loc = fsd.GetFSNamesystem().GetBlockManager().CreateLocatedBlocks(fileNode.GetBlocks
(snapshot), fileSize, isUc, 0L, size, false, inSnapshot, feInfo);
if (loc == null)
{
loc = new LocatedBlocks();
}
isEncrypted = (feInfo != null) || (isRawPath && fsd.IsInAnEZ(INodesInPath.FromINode
(node)));
}
else
{
isEncrypted = fsd.IsInAnEZ(INodesInPath.FromINode(node));
}
int childrenNum = node.IsDirectory() ? node.AsDirectory().GetChildrenNum(snapshot
) : 0;
INodeAttributes nodeAttrs = fsd.GetAttributes(fullPath, path, node, snapshot);
HdfsLocatedFileStatus status = new HdfsLocatedFileStatus(size, node.IsDirectory()
, replication, blocksize, node.GetModificationTime(snapshot), node.GetAccessTime
(snapshot), GetPermissionForFileStatus(nodeAttrs, isEncrypted), nodeAttrs.GetUserName
(), nodeAttrs.GetGroupName(), node.IsSymlink() ? node.AsSymlink().GetSymlink() :
null, path, node.GetId(), loc, childrenNum, feInfo, storagePolicy);
// Set caching information for the located blocks.
if (loc != null)
{
CacheManager cacheManager = fsd.GetFSNamesystem().GetCacheManager();
foreach (LocatedBlock lb in loc.GetLocatedBlocks())
{
cacheManager.SetCachedLocations(lb);
}
}
return(status);
}
/// <exception cref="Org.Apache.Hadoop.Hdfs.Protocol.QuotaExceededException"/>
internal virtual void UpdateMtimeAndLease(long timestamp)
{
srcParent.UpdateModificationTime(timestamp, srcIIP.GetLatestSnapshotId());
INode dstParent = dstParentIIP.GetLastINode();
dstParent.UpdateModificationTime(timestamp, dstIIP.GetLatestSnapshotId());
// update moved lease with new filename
fsd.GetFSNamesystem().UnprotectedChangeLease(src, dst);
}
/// <summary>
/// Delete a path from the name space
/// Update the count at each ancestor directory with quota
/// <br />
/// Note: This is to be used by
/// <see cref="FSEditLog"/>
/// only.
/// <br />
/// </summary>
/// <param name="src">a string representation of a path to an inode</param>
/// <param name="mtime">the time the inode is removed</param>
/// <exception cref="System.IO.IOException"/>
internal static void DeleteForEditLog(FSDirectory fsd, string src, long mtime)
{
System.Diagnostics.Debug.Assert(fsd.HasWriteLock());
FSNamesystem fsn = fsd.GetFSNamesystem();
INode.BlocksMapUpdateInfo collectedBlocks = new INode.BlocksMapUpdateInfo();
IList <INode> removedINodes = new ChunkedArrayList <INode>();
INodesInPath iip = fsd.GetINodesInPath4Write(FSDirectory.NormalizePath(src), false
);
if (!DeleteAllowed(iip, src))
{
return;
}
IList <INodeDirectory> snapshottableDirs = new AList <INodeDirectory>();
FSDirSnapshotOp.CheckSnapshot(iip.GetLastINode(), snapshottableDirs);
long filesRemoved = UnprotectedDelete(fsd, iip, collectedBlocks, removedINodes, mtime
);
fsn.RemoveSnapshottableDirs(snapshottableDirs);
if (filesRemoved >= 0)
{
fsn.RemoveLeasesAndINodes(src, removedINodes, false);
fsn.RemoveBlocksAndUpdateSafemodeTotal(collectedBlocks);
}
}
/// <summary>Delete the target directory and collect the blocks under it</summary>
/// <param name="iip">the INodesInPath instance containing all the INodes for the path
/// </param>
/// <param name="collectedBlocks">Blocks under the deleted directory</param>
/// <param name="removedINodes">INodes that should be removed from inodeMap</param>
/// <returns>the number of files that have been removed</returns>
/// <exception cref="System.IO.IOException"/>
internal static long Delete(FSDirectory fsd, INodesInPath iip, INode.BlocksMapUpdateInfo
collectedBlocks, IList <INode> removedINodes, long mtime)
{
if (NameNode.stateChangeLog.IsDebugEnabled())
{
NameNode.stateChangeLog.Debug("DIR* FSDirectory.delete: " + iip.GetPath());
}
long filesRemoved;
fsd.WriteLock();
try
{
if (!DeleteAllowed(iip, iip.GetPath()))
{
filesRemoved = -1;
}
else
{
IList <INodeDirectory> snapshottableDirs = new AList <INodeDirectory>();
FSDirSnapshotOp.CheckSnapshot(iip.GetLastINode(), snapshottableDirs);
filesRemoved = UnprotectedDelete(fsd, iip, collectedBlocks, removedINodes, mtime);
fsd.GetFSNamesystem().RemoveSnapshottableDirs(snapshottableDirs);
}
}
finally
{
fsd.WriteUnlock();
}
return(filesRemoved);
}
/// <exception cref="Org.Apache.Hadoop.Hdfs.Protocol.QuotaExceededException"/>
private static bool UnprotectedSetTimes(FSDirectory fsd, INode inode, long mtime,
long atime, bool force, int latest)
{
System.Diagnostics.Debug.Assert(fsd.HasWriteLock());
bool status = false;
if (mtime != -1)
{
inode = inode.SetModificationTime(mtime, latest);
status = true;
}
if (atime != -1)
{
long inodeTime = inode.GetAccessTime();
// if the last access time update was within the last precision interval, then
// no need to store access time
if (atime <= inodeTime + fsd.GetFSNamesystem().GetAccessTimePrecision() && !force)
{
status = false;
}
else
{
inode.SetAccessTime(atime, latest);
status = true;
}
}
return(status);
}
/// <summary>
/// Verify quota for rename operation where srcInodes[srcInodes.length-1] moves
/// dstInodes[dstInodes.length-1]
/// </summary>
/// <exception cref="Org.Apache.Hadoop.Hdfs.Protocol.QuotaExceededException"/>
private static void VerifyQuotaForRename(FSDirectory fsd, INodesInPath src, INodesInPath
dst)
{
if (!fsd.GetFSNamesystem().IsImageLoaded() || fsd.ShouldSkipQuotaChecks())
{
// Do not check quota if edits log is still being processed
return;
}
int i = 0;
while (src.GetINode(i) == dst.GetINode(i))
{
i++;
}
// src[i - 1] is the last common ancestor.
BlockStoragePolicySuite bsps = fsd.GetBlockStoragePolicySuite();
QuotaCounts delta = src.GetLastINode().ComputeQuotaUsage(bsps);
// Reduce the required quota by dst that is being removed
INode dstINode = dst.GetLastINode();
if (dstINode != null)
{
delta.Subtract(dstINode.ComputeQuotaUsage(bsps));
}
FSDirectory.VerifyQuota(dst, dst.Length() - 1, delta, src.GetINode(i - 1));
}
/// <exception cref="Org.Apache.Hadoop.Hdfs.Protocol.QuotaExceededException"/>
private static void VerifyQuota(FSDirectory fsd, INodesInPath targetIIP, QuotaCounts
deltas)
{
if (!fsd.GetFSNamesystem().IsImageLoaded() || fsd.ShouldSkipQuotaChecks())
{
// Do not check quota if editlog is still being processed
return;
}
FSDirectory.VerifyQuota(targetIIP, targetIIP.Length() - 1, deltas, null);
}
/// <summary>Rename src to dst.</summary>
/// <remarks>
/// Rename src to dst.
/// <br />
/// Note: This is to be used by
/// <see cref="FSEditLogLoader"/>
/// only.
/// <br />
/// </remarks>
/// <param name="fsd">FSDirectory</param>
/// <param name="src">source path</param>
/// <param name="dst">destination path</param>
/// <param name="timestamp">modification time</param>
/// <param name="options">Rename options</param>
/// <exception cref="System.IO.IOException"/>
internal static bool RenameForEditLog(FSDirectory fsd, string src, string dst, long
timestamp, params Options.Rename[] options)
{
INode.BlocksMapUpdateInfo collectedBlocks = new INode.BlocksMapUpdateInfo();
INodesInPath srcIIP = fsd.GetINodesInPath4Write(src, false);
INodesInPath dstIIP = fsd.GetINodesInPath4Write(dst, false);
bool ret = UnprotectedRenameTo(fsd, src, dst, srcIIP, dstIIP, timestamp, collectedBlocks
, options);
if (!collectedBlocks.GetToDeleteList().IsEmpty())
{
fsd.GetFSNamesystem().RemoveBlocksAndUpdateSafemodeTotal(collectedBlocks);
}
return(ret);
}
/// <exception cref="System.IO.IOException"/>
private static ContentSummary GetContentSummaryInt(FSDirectory fsd, INodesInPath
iip)
{
fsd.ReadLock();
try
{
INode targetNode = iip.GetLastINode();
if (targetNode == null)
{
throw new FileNotFoundException("File does not exist: " + iip.GetPath());
}
else
{
// Make it relinquish locks everytime contentCountLimit entries are
// processed. 0 means disabled. I.e. blocking for the entire duration.
ContentSummaryComputationContext cscc = new ContentSummaryComputationContext(fsd,
fsd.GetFSNamesystem(), fsd.GetContentCountLimit(), fsd.GetContentSleepMicroSec()
);
ContentSummary cs = targetNode.ComputeAndConvertContentSummary(cscc);
fsd.AddYieldCount(cscc.GetYieldCount());
return(cs);
}
}
finally
{
fsd.ReadUnlock();
}
}
/// <summary>Rename src to dst.</summary>
/// <remarks>
/// Rename src to dst.
/// See
/// <see cref="Org.Apache.Hadoop.Hdfs.DistributedFileSystem.Rename(Org.Apache.Hadoop.FS.Path, Org.Apache.Hadoop.FS.Path, Org.Apache.Hadoop.FS.Options.Rename[])
/// "/>
/// for details related to rename semantics and exceptions.
/// </remarks>
/// <param name="fsd">FSDirectory</param>
/// <param name="src">source path</param>
/// <param name="dst">destination path</param>
/// <param name="timestamp">modification time</param>
/// <param name="collectedBlocks">blocks to be removed</param>
/// <param name="options">Rename options</param>
/// <returns>whether a file/directory gets overwritten in the dst path</returns>
/// <exception cref="System.IO.IOException"/>
internal static bool UnprotectedRenameTo(FSDirectory fsd, string src, string dst,
INodesInPath srcIIP, INodesInPath dstIIP, long timestamp, INode.BlocksMapUpdateInfo
collectedBlocks, params Options.Rename[] options)
{
System.Diagnostics.Debug.Assert(fsd.HasWriteLock());
bool overwrite = options != null && Arrays.AsList(options).Contains(Options.Rename
.Overwrite);
string error;
INode srcInode = srcIIP.GetLastINode();
ValidateRenameSource(srcIIP);
// validate the destination
if (dst.Equals(src))
{
throw new FileAlreadyExistsException("The source " + src + " and destination " +
dst + " are the same");
}
ValidateDestination(src, dst, srcInode);
if (dstIIP.Length() == 1)
{
error = "rename destination cannot be the root";
NameNode.stateChangeLog.Warn("DIR* FSDirectory.unprotectedRenameTo: " + error);
throw new IOException(error);
}
BlockStoragePolicySuite bsps = fsd.GetBlockStoragePolicySuite();
fsd.ezManager.CheckMoveValidity(srcIIP, dstIIP, src);
INode dstInode = dstIIP.GetLastINode();
IList <INodeDirectory> snapshottableDirs = new AList <INodeDirectory>();
if (dstInode != null)
{
// Destination exists
ValidateOverwrite(src, dst, overwrite, srcInode, dstInode);
FSDirSnapshotOp.CheckSnapshot(dstInode, snapshottableDirs);
}
INode dstParent = dstIIP.GetINode(-2);
if (dstParent == null)
{
error = "rename destination parent " + dst + " not found.";
NameNode.stateChangeLog.Warn("DIR* FSDirectory.unprotectedRenameTo: " + error);
throw new FileNotFoundException(error);
}
if (!dstParent.IsDirectory())
{
error = "rename destination parent " + dst + " is a file.";
NameNode.stateChangeLog.Warn("DIR* FSDirectory.unprotectedRenameTo: " + error);
throw new ParentNotDirectoryException(error);
}
// Ensure dst has quota to accommodate rename
VerifyFsLimitsForRename(fsd, srcIIP, dstIIP);
VerifyQuotaForRename(fsd, srcIIP, dstIIP);
FSDirRenameOp.RenameOperation tx = new FSDirRenameOp.RenameOperation(fsd, src, dst
, srcIIP, dstIIP);
bool undoRemoveSrc = true;
tx.RemoveSrc();
bool undoRemoveDst = false;
long removedNum = 0;
try
{
if (dstInode != null)
{
// dst exists, remove it
removedNum = tx.RemoveDst();
if (removedNum != -1)
{
undoRemoveDst = true;
}
}
// add src as dst to complete rename
if (tx.AddSourceToDestination())
{
undoRemoveSrc = false;
if (NameNode.stateChangeLog.IsDebugEnabled())
{
NameNode.stateChangeLog.Debug("DIR* FSDirectory.unprotectedRenameTo: " + src + " is renamed to "
+ dst);
}
tx.UpdateMtimeAndLease(timestamp);
// Collect the blocks and remove the lease for previous dst
bool filesDeleted = false;
if (undoRemoveDst)
{
undoRemoveDst = false;
if (removedNum > 0)
{
filesDeleted = tx.CleanDst(bsps, collectedBlocks);
}
}
if (snapshottableDirs.Count > 0)
{
// There are snapshottable directories (without snapshots) to be
// deleted. Need to update the SnapshotManager.
fsd.GetFSNamesystem().RemoveSnapshottableDirs(snapshottableDirs);
}
tx.UpdateQuotasInSourceTree(bsps);
return(filesDeleted);
}
}
finally
{
if (undoRemoveSrc)
{
tx.RestoreSource();
}
if (undoRemoveDst)
{
// Rename failed - restore dst
tx.RestoreDst(bsps);
}
}
NameNode.stateChangeLog.Warn("DIR* FSDirectory.unprotectedRenameTo: " + "failed to rename "
+ src + " to " + dst);
throw new IOException("rename from " + src + " to " + dst + " failed.");
}
