/// <summary>
/// Check if the given INode (or one of its descendants) is snapshottable and
/// already has snapshots.
/// </summary>
/// <param name="target">The given INode</param>
/// <param name="snapshottableDirs">
/// The list of directories that are snapshottable
/// but do not have snapshots yet
/// </param>
/// <exception cref="Org.Apache.Hadoop.Hdfs.Protocol.SnapshotException"/>
internal static void CheckSnapshot(INode target, IList <INodeDirectory> snapshottableDirs
)
{
if (target.IsDirectory())
{
INodeDirectory targetDir = target.AsDirectory();
DirectorySnapshottableFeature sf = targetDir.GetDirectorySnapshottableFeature();
if (sf != null)
{
if (sf.GetNumSnapshots() > 0)
{
string fullPath = targetDir.GetFullPathName();
throw new SnapshotException("The directory " + fullPath + " cannot be deleted since "
+ fullPath + " is snapshottable and already has snapshots");
}
else
{
if (snapshottableDirs != null)
{
snapshottableDirs.AddItem(targetDir);
}
}
}
foreach (INode child in targetDir.GetChildrenList(Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot
.CurrentStateId))
{
CheckSnapshot(child, snapshottableDirs);
}
}
}
/// <summary>
/// Guarded by
/// <see cref="FSNamesystem.ReadLock()"/>
///
/// </summary>
/// <exception cref="Org.Apache.Hadoop.Security.AccessControlException"/>
private void CheckSubAccess(byte[][] pathByNameArr, int pathIdx, INode inode, int
snapshotId, FsAction access, bool ignoreEmptyDir)
{
if (inode == null || !inode.IsDirectory())
{
return;
}
Stack <INodeDirectory> directories = new Stack <INodeDirectory>();
for (directories.Push(inode.AsDirectory()); !directories.IsEmpty();)
{
INodeDirectory d = directories.Pop();
ReadOnlyList <INode> cList = d.GetChildrenList(snapshotId);
if (!(cList.IsEmpty() && ignoreEmptyDir))
{
//TODO have to figure this out with inodeattribute provider
Check(GetINodeAttrs(pathByNameArr, pathIdx, d, snapshotId), inode.GetFullPathName
(), access);
}
foreach (INode child in cList)
{
if (child.IsDirectory())
{
directories.Push(child.AsDirectory());
}
}
}
}
public virtual void TestReset()
{
Configuration conf = new Configuration();
FSEditLog fsEditLog = Org.Mockito.Mockito.Mock <FSEditLog>();
FSImage fsImage = Org.Mockito.Mockito.Mock <FSImage>();
Org.Mockito.Mockito.When(fsImage.GetEditLog()).ThenReturn(fsEditLog);
FSNamesystem fsn = new FSNamesystem(conf, fsImage);
fsn.ImageLoadComplete();
NUnit.Framework.Assert.IsTrue(fsn.IsImageLoaded());
fsn.Clear();
NUnit.Framework.Assert.IsFalse(fsn.IsImageLoaded());
INodeDirectory root = (INodeDirectory)fsn.GetFSDirectory().GetINode("/");
NUnit.Framework.Assert.IsTrue(root.GetChildrenList(Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot
.CurrentStateId).IsEmpty());
fsn.ImageLoadComplete();
NUnit.Framework.Assert.IsTrue(fsn.IsImageLoaded());
}
public virtual void TestSnapshotOnRoot()
{
Path root = new Path("/");
hdfs.AllowSnapshot(root);
hdfs.CreateSnapshot(root, "s1");
cluster.Shutdown();
cluster = new MiniDFSCluster.Builder(conf).Format(false).NumDataNodes(Replication
).Build();
cluster.WaitActive();
fsn = cluster.GetNamesystem();
hdfs = cluster.GetFileSystem();
// save namespace and restart cluster
hdfs.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter);
hdfs.SaveNamespace();
hdfs.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeLeave);
cluster.Shutdown();
cluster = new MiniDFSCluster.Builder(conf).Format(false).NumDataNodes(Replication
).Build();
cluster.WaitActive();
fsn = cluster.GetNamesystem();
hdfs = cluster.GetFileSystem();
INodeDirectory rootNode = fsn.dir.GetINode4Write(root.ToString()).AsDirectory();
NUnit.Framework.Assert.IsTrue("The children list of root should be empty", rootNode
.GetChildrenList(Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.CurrentStateId
).IsEmpty());
// one snapshot on root: s1
IList <DirectoryWithSnapshotFeature.DirectoryDiff> diffList = rootNode.GetDiffs().
AsList();
NUnit.Framework.Assert.AreEqual(1, diffList.Count);
Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot s1 = rootNode.GetSnapshot
(DFSUtil.String2Bytes("s1"));
NUnit.Framework.Assert.AreEqual(s1.GetId(), diffList[0].GetSnapshotId());
// check SnapshotManager's snapshottable directory list
NUnit.Framework.Assert.AreEqual(1, fsn.GetSnapshotManager().GetNumSnapshottableDirs
());
SnapshottableDirectoryStatus[] sdirs = fsn.GetSnapshotManager().GetSnapshottableDirListing
(null);
NUnit.Framework.Assert.AreEqual(root, sdirs[0].GetFullPath());
// save namespace and restart cluster
hdfs.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter);
hdfs.SaveNamespace();
hdfs.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeLeave);
cluster.Shutdown();
cluster = new MiniDFSCluster.Builder(conf).Format(false).NumDataNodes(Replication
).Build();
cluster.WaitActive();
fsn = cluster.GetNamesystem();
hdfs = cluster.GetFileSystem();
}
/// <summary>
/// Get a partial listing of the indicated directory
/// We will stop when any of the following conditions is met:
/// 1) this.lsLimit files have been added
/// 2) needLocation is true AND enough files have been added such
/// that at least this.lsLimit block locations are in the response
/// </summary>
/// <param name="fsd">FSDirectory</param>
/// <param name="iip">
/// the INodesInPath instance containing all the INodes along the
/// path
/// </param>
/// <param name="src">the directory name</param>
/// <param name="startAfter">the name to start listing after</param>
/// <param name="needLocation">if block locations are returned</param>
/// <returns>a partial listing starting after startAfter</returns>
/// <exception cref="System.IO.IOException"/>
private static DirectoryListing GetListing(FSDirectory fsd, INodesInPath iip, string
src, byte[] startAfter, bool needLocation, bool isSuperUser)
{
string srcs = FSDirectory.NormalizePath(src);
bool isRawPath = FSDirectory.IsReservedRawName(src);
fsd.ReadLock();
try
{
if (srcs.EndsWith(HdfsConstants.SeparatorDotSnapshotDir))
{
return(GetSnapshotsListing(fsd, srcs, startAfter));
}
int snapshot = iip.GetPathSnapshotId();
INode targetNode = iip.GetLastINode();
if (targetNode == null)
{
return(null);
}
byte parentStoragePolicy = isSuperUser ? targetNode.GetStoragePolicyID() : BlockStoragePolicySuite
.IdUnspecified;
if (!targetNode.IsDirectory())
{
return(new DirectoryListing(new HdfsFileStatus[] { CreateFileStatus(fsd, src, HdfsFileStatus
.EmptyName, targetNode, needLocation, parentStoragePolicy, snapshot, isRawPath,
iip) }, 0));
}
INodeDirectory dirInode = targetNode.AsDirectory();
ReadOnlyList <INode> contents = dirInode.GetChildrenList(snapshot);
int startChild = INodeDirectory.NextChild(contents, startAfter);
int totalNumChildren = contents.Size();
int numOfListing = Math.Min(totalNumChildren - startChild, fsd.GetLsLimit());
int locationBudget = fsd.GetLsLimit();
int listingCnt = 0;
HdfsFileStatus[] listing = new HdfsFileStatus[numOfListing];
for (int i = 0; i < numOfListing && locationBudget > 0; i++)
{
INode cur = contents.Get(startChild + i);
byte curPolicy = isSuperUser && !cur.IsSymlink() ? cur.GetLocalStoragePolicyID() :
BlockStoragePolicySuite.IdUnspecified;
listing[i] = CreateFileStatus(fsd, src, cur.GetLocalNameBytes(), cur, needLocation
, GetStoragePolicyID(curPolicy, parentStoragePolicy), snapshot, isRawPath, iip);
listingCnt++;
if (needLocation)
{
// Once we hit lsLimit locations, stop.
// This helps to prevent excessively large response payloads.
// Approximate #locations with locatedBlockCount() * repl_factor
LocatedBlocks blks = ((HdfsLocatedFileStatus)listing[i]).GetBlockLocations();
locationBudget -= (blks == null) ? 0 : blks.LocatedBlockCount() * listing[i].GetReplication
();
}
}
// truncate return array if necessary
if (listingCnt < numOfListing)
{
listing = Arrays.CopyOf(listing, listingCnt);
}
return(new DirectoryListing(listing, totalNumChildren - startChild - listingCnt));
}
finally
{
fsd.ReadUnlock();
}
}