/// <summary>Verify if the namespace quota is violated after applying delta.</summary>
/// <exception cref="Org.Apache.Hadoop.Hdfs.Protocol.NSQuotaExceededException"/>
private void VerifyNamespaceQuota(long delta)
{
if (Quota.IsViolated(quota.GetNameSpace(), usage.GetNameSpace(), delta))
{
throw new NSQuotaExceededException(quota.GetNameSpace(), usage.GetNameSpace() + delta
);
}
}
/// <summary>Test if the quota can be correctly updated for append</summary>
/// <exception cref="System.Exception"/>
public virtual void TestUpdateQuotaForAppend()
{
Path foo = new Path(dir, "foo");
Path bar = new Path(foo, "bar");
long currentFileLen = Blocksize;
DFSTestUtil.CreateFile(dfs, bar, currentFileLen, Replication, seed);
dfs.SetQuota(foo, long.MaxValue - 1, long.MaxValue - 1);
// append half of the block data, the previous file length is at block
// boundary
DFSTestUtil.AppendFile(dfs, bar, Blocksize / 2);
currentFileLen += (Blocksize / 2);
INodeDirectory fooNode = fsdir.GetINode4Write(foo.ToString()).AsDirectory();
NUnit.Framework.Assert.IsTrue(fooNode.IsQuotaSet());
QuotaCounts quota = fooNode.GetDirectoryWithQuotaFeature().GetSpaceConsumed();
long ns = quota.GetNameSpace();
long ds = quota.GetStorageSpace();
NUnit.Framework.Assert.AreEqual(2, ns);
// foo and bar
NUnit.Framework.Assert.AreEqual(currentFileLen * Replication, ds);
ContentSummary c = dfs.GetContentSummary(foo);
NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), ds);
// append another block, the previous file length is not at block boundary
DFSTestUtil.AppendFile(dfs, bar, Blocksize);
currentFileLen += Blocksize;
quota = fooNode.GetDirectoryWithQuotaFeature().GetSpaceConsumed();
ns = quota.GetNameSpace();
ds = quota.GetStorageSpace();
NUnit.Framework.Assert.AreEqual(2, ns);
// foo and bar
NUnit.Framework.Assert.AreEqual(currentFileLen * Replication, ds);
c = dfs.GetContentSummary(foo);
NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), ds);
// append several blocks
DFSTestUtil.AppendFile(dfs, bar, Blocksize * 3 + Blocksize / 8);
currentFileLen += (Blocksize * 3 + Blocksize / 8);
quota = fooNode.GetDirectoryWithQuotaFeature().GetSpaceConsumed();
ns = quota.GetNameSpace();
ds = quota.GetStorageSpace();
NUnit.Framework.Assert.AreEqual(2, ns);
// foo and bar
NUnit.Framework.Assert.AreEqual(currentFileLen * Replication, ds);
c = dfs.GetContentSummary(foo);
NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), ds);
}
private void LoadRootINode(FsImageProto.INodeSection.INode p)
{
INodeDirectory root = LoadINodeDirectory(p, parent.GetLoaderContext());
QuotaCounts q = root.GetQuotaCounts();
long nsQuota = q.GetNameSpace();
long dsQuota = q.GetStorageSpace();
if (nsQuota != -1 || dsQuota != -1)
{
dir.rootDir.GetDirectoryWithQuotaFeature().SetQuota(nsQuota, dsQuota);
}
EnumCounters <StorageType> typeQuotas = q.GetTypeSpaces();
if (typeQuotas.AnyGreaterOrEqual(0))
{
dir.rootDir.GetDirectoryWithQuotaFeature().SetQuota(typeQuotas);
}
dir.rootDir.CloneModificationTime(root);
dir.rootDir.ClonePermissionStatus(root);
AclFeature af = root.GetFeature(typeof(AclFeature));
if (af != null)
{
dir.rootDir.AddAclFeature(af);
}
// root dir supports having extended attributes according to POSIX
XAttrFeature f = root.GetXAttrFeature();
if (f != null)
{
dir.rootDir.AddXAttrFeature(f);
}
dir.AddRootDirToEncryptionZone(f);
}
/// <summary>Set the namespace, storagespace and typespace quota for a directory.</summary>
/// <remarks>
/// Set the namespace, storagespace and typespace quota for a directory.
/// Note: This does not support ".inodes" relative path.
/// </remarks>
/// <exception cref="System.IO.IOException"/>
internal static void SetQuota(FSDirectory fsd, string src, long nsQuota, long ssQuota
, StorageType type)
{
if (fsd.IsPermissionEnabled())
{
FSPermissionChecker pc = fsd.GetPermissionChecker();
pc.CheckSuperuserPrivilege();
}
fsd.WriteLock();
try
{
INodeDirectory changed = UnprotectedSetQuota(fsd, src, nsQuota, ssQuota, type);
if (changed != null)
{
QuotaCounts q = changed.GetQuotaCounts();
if (type == null)
{
fsd.GetEditLog().LogSetQuota(src, q.GetNameSpace(), q.GetStorageSpace());
}
else
{
fsd.GetEditLog().LogSetQuotaByStorageType(src, q.GetTypeSpaces().Get(type), type);
}
}
}
finally
{
fsd.WriteUnlock();
}
}
/// <summary>
/// Compute
/// <see cref="Org.Apache.Hadoop.FS.ContentSummary"/>
/// .
/// </summary>
public ContentSummary ComputeAndConvertContentSummary(ContentSummaryComputationContext
summary)
{
ContentCounts counts = ComputeContentSummary(summary).GetCounts();
QuotaCounts q = GetQuotaCounts();
return(new ContentSummary.Builder().Length(counts.GetLength()).FileCount(counts.GetFileCount
() + counts.GetSymlinkCount()).DirectoryCount(counts.GetDirectoryCount()).Quota(
q.GetNameSpace()).SpaceConsumed(counts.GetStoragespace()).SpaceQuota(q.GetStorageSpace
()).TypeConsumed(counts.GetTypeSpaces()).TypeQuota(q.GetTypeSpaces().AsArray()).
Build());
}
/// <summary>
/// Test if the quota can be correctly updated when file length is updated
/// through fsync
/// </summary>
/// <exception cref="System.Exception"/>
public virtual void TestUpdateQuotaForFSync()
{
Path foo = new Path("/foo");
Path bar = new Path(foo, "bar");
DFSTestUtil.CreateFile(dfs, bar, Blocksize, Replication, 0L);
dfs.SetQuota(foo, long.MaxValue - 1, long.MaxValue - 1);
FSDataOutputStream @out = dfs.Append(bar);
@out.Write(new byte[Blocksize / 4]);
((DFSOutputStream)@out.GetWrappedStream()).Hsync(EnumSet.Of(HdfsDataOutputStream.SyncFlag
.UpdateLength));
INodeDirectory fooNode = fsdir.GetINode4Write(foo.ToString()).AsDirectory();
QuotaCounts quota = fooNode.GetDirectoryWithQuotaFeature().GetSpaceConsumed();
long ns = quota.GetNameSpace();
long ds = quota.GetStorageSpace();
NUnit.Framework.Assert.AreEqual(2, ns);
// foo and bar
NUnit.Framework.Assert.AreEqual(Blocksize * 2 * Replication, ds);
// file is under construction
@out.Write(new byte[Blocksize / 4]);
@out.Close();
fooNode = fsdir.GetINode4Write(foo.ToString()).AsDirectory();
quota = fooNode.GetDirectoryWithQuotaFeature().GetSpaceConsumed();
ns = quota.GetNameSpace();
ds = quota.GetStorageSpace();
NUnit.Framework.Assert.AreEqual(2, ns);
NUnit.Framework.Assert.AreEqual((Blocksize + Blocksize / 2) * Replication, ds);
// append another block
DFSTestUtil.AppendFile(dfs, bar, Blocksize);
quota = fooNode.GetDirectoryWithQuotaFeature().GetSpaceConsumed();
ns = quota.GetNameSpace();
ds = quota.GetStorageSpace();
NUnit.Framework.Assert.AreEqual(2, ns);
// foo and bar
NUnit.Framework.Assert.AreEqual((Blocksize * 2 + Blocksize / 2) * Replication, ds
);
}
/// <summary>
/// Delete a path from the name space
/// Update the count at each ancestor directory with quota
/// </summary>
/// <param name="iip">the inodes resolved from the path</param>
/// <param name="collectedBlocks">blocks collected from the deleted path</param>
/// <param name="removedINodes">inodes that should be removed from inodeMap</param>
/// <param name="mtime">the time the inode is removed</param>
/// <returns>the number of inodes deleted; 0 if no inodes are deleted.</returns>
private static long UnprotectedDelete(FSDirectory fsd, INodesInPath iip, INode.BlocksMapUpdateInfo
collectedBlocks, IList <INode> removedINodes, long mtime)
{
System.Diagnostics.Debug.Assert(fsd.HasWriteLock());
// check if target node exists
INode targetNode = iip.GetLastINode();
if (targetNode == null)
{
return(-1);
}
// record modification
int latestSnapshot = iip.GetLatestSnapshotId();
targetNode.RecordModification(latestSnapshot);
// Remove the node from the namespace
long removed = fsd.RemoveLastINode(iip);
if (removed == -1)
{
return(-1);
}
// set the parent's modification time
INodeDirectory parent = targetNode.GetParent();
parent.UpdateModificationTime(mtime, latestSnapshot);
fsd.UpdateCountForDelete(targetNode, iip);
if (removed == 0)
{
return(0);
}
// collect block and update quota
if (!targetNode.IsInLatestSnapshot(latestSnapshot))
{
targetNode.DestroyAndCollectBlocks(fsd.GetBlockStoragePolicySuite(), collectedBlocks
, removedINodes);
}
else
{
QuotaCounts counts = targetNode.CleanSubtree(fsd.GetBlockStoragePolicySuite(), Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot
.CurrentStateId, latestSnapshot, collectedBlocks, removedINodes);
removed = counts.GetNameSpace();
fsd.UpdateCountNoQuotaCheck(iip, iip.Length() - 1, counts.Negation());
}
if (NameNode.stateChangeLog.IsDebugEnabled())
{
NameNode.stateChangeLog.Debug("DIR* FSDirectory.unprotectedDelete: " + iip.GetPath
() + " is removed");
}
return(removed);
}
/// <summary>Test if the quota can be correctly updated for create file</summary>
/// <exception cref="System.Exception"/>
public virtual void TestQuotaUpdateWithFileCreate()
{
Path foo = new Path(dir, "foo");
Path createdFile = new Path(foo, "created_file.data");
dfs.Mkdirs(foo);
dfs.SetQuota(foo, long.MaxValue - 1, long.MaxValue - 1);
long fileLen = Blocksize * 2 + Blocksize / 2;
DFSTestUtil.CreateFile(dfs, createdFile, Blocksize / 16, fileLen, Blocksize, Replication
, seed);
INode fnode = fsdir.GetINode4Write(foo.ToString());
NUnit.Framework.Assert.IsTrue(fnode.IsDirectory());
NUnit.Framework.Assert.IsTrue(fnode.IsQuotaSet());
QuotaCounts cnt = fnode.AsDirectory().GetDirectoryWithQuotaFeature().GetSpaceConsumed
();
NUnit.Framework.Assert.AreEqual(2, cnt.GetNameSpace());
NUnit.Framework.Assert.AreEqual(fileLen * Replication, cnt.GetStorageSpace());
}
/// <summary>
/// Both traditional space quota and the storage type quota for SSD are set and
/// not exceeded.
/// </summary>
/// <exception cref="System.Exception"/>
public virtual void TestQuotaByStorageTypeWithTraditionalQuota()
{
Path foo = new Path(dir, "foo");
dfs.Mkdirs(foo);
dfs.SetStoragePolicy(foo, HdfsConstants.OnessdStoragePolicyName);
dfs.SetQuotaByStorageType(foo, StorageType.Ssd, Blocksize * 10);
dfs.SetQuota(foo, long.MaxValue - 1, Replication * Blocksize * 10);
INode fnode = fsdir.GetINode4Write(foo.ToString());
NUnit.Framework.Assert.IsTrue(fnode.IsDirectory());
NUnit.Framework.Assert.IsTrue(fnode.IsQuotaSet());
Path createdFile = new Path(foo, "created_file.data");
long fileLen = Blocksize * 2 + Blocksize / 2;
DFSTestUtil.CreateFile(dfs, createdFile, Blocksize / 16, fileLen, Blocksize, Replication
, seed);
QuotaCounts cnt = fnode.AsDirectory().GetDirectoryWithQuotaFeature().GetSpaceConsumed
();
NUnit.Framework.Assert.AreEqual(2, cnt.GetNameSpace());
NUnit.Framework.Assert.AreEqual(fileLen * Replication, cnt.GetStorageSpace());
dfs.Delete(createdFile, true);
QuotaCounts cntAfterDelete = fnode.AsDirectory().GetDirectoryWithQuotaFeature().GetSpaceConsumed
();
NUnit.Framework.Assert.AreEqual(1, cntAfterDelete.GetNameSpace());
NUnit.Framework.Assert.AreEqual(0, cntAfterDelete.GetStorageSpace());
// Validate the computeQuotaUsage()
QuotaCounts counts = new QuotaCounts.Builder().Build();
fnode.ComputeQuotaUsage(fsn.GetBlockManager().GetStoragePolicySuite(), counts, true
);
NUnit.Framework.Assert.AreEqual(fnode.DumpTreeRecursively().ToString(), 1, counts
.GetNameSpace());
NUnit.Framework.Assert.AreEqual(fnode.DumpTreeRecursively().ToString(), 0, counts
.GetStorageSpace());
}
public static FsImageProto.INodeSection.INodeDirectory.Builder BuildINodeDirectory
(INodeDirectoryAttributes dir, FSImageFormatProtobuf.SaverContext state)
{
QuotaCounts quota = dir.GetQuotaCounts();
FsImageProto.INodeSection.INodeDirectory.Builder b = FsImageProto.INodeSection.INodeDirectory
.NewBuilder().SetModificationTime(dir.GetModificationTime()).SetNsQuota(quota.GetNameSpace
()).SetDsQuota(quota.GetStorageSpace()).SetPermission(BuildPermissionStatus(dir,
state.GetStringMap()));
if (quota.GetTypeSpaces().AnyGreaterOrEqual(0))
{
b.SetTypeQuotas(BuildQuotaByStorageTypeEntries(quota));
}
AclFeature f = dir.GetAclFeature();
if (f != null)
{
b.SetAcl(BuildAclEntries(f, state.GetStringMap()));
}
XAttrFeature xAttrFeature = dir.GetXAttrFeature();
if (xAttrFeature != null)
{
b.SetXAttrs(BuildXAttrs(xAttrFeature, state.GetStringMap()));
}
return(b);
}
/// <summary>
/// See
/// <see cref="Org.Apache.Hadoop.Hdfs.Protocol.ClientProtocol.SetQuota(string, long, long, Org.Apache.Hadoop.FS.StorageType)
/// "/>
/// for the contract.
/// Sets quota for for a directory.
/// </summary>
/// <returns>INodeDirectory if any of the quotas have changed. null otherwise.</returns>
/// <exception cref="System.IO.FileNotFoundException">if the path does not exist.</exception>
/// <exception cref="Org.Apache.Hadoop.FS.PathIsNotDirectoryException">if the path is not a directory.
/// </exception>
/// <exception cref="Org.Apache.Hadoop.Hdfs.Protocol.QuotaExceededException">
/// if the directory tree size is
/// greater than the given quota
/// </exception>
/// <exception cref="Org.Apache.Hadoop.FS.UnresolvedLinkException">if a symlink is encountered in src.
/// </exception>
/// <exception cref="Org.Apache.Hadoop.Hdfs.Protocol.SnapshotAccessControlException">if path is in RO snapshot
/// </exception>
/// <exception cref="Org.Apache.Hadoop.Hdfs.Server.Namenode.UnsupportedActionException
/// "/>
internal static INodeDirectory UnprotectedSetQuota(FSDirectory fsd, string src, long
nsQuota, long ssQuota, StorageType type)
{
System.Diagnostics.Debug.Assert(fsd.HasWriteLock());
// sanity check
if ((nsQuota < 0 && nsQuota != HdfsConstants.QuotaDontSet && nsQuota != HdfsConstants
.QuotaReset) || (ssQuota < 0 && ssQuota != HdfsConstants.QuotaDontSet && ssQuota
!= HdfsConstants.QuotaReset))
{
throw new ArgumentException("Illegal value for nsQuota or " + "ssQuota : " + nsQuota
+ " and " + ssQuota);
}
// sanity check for quota by storage type
if ((type != null) && (!fsd.IsQuotaByStorageTypeEnabled() || nsQuota != HdfsConstants
.QuotaDontSet))
{
throw new UnsupportedActionException("Failed to set quota by storage type because either"
+ DFSConfigKeys.DfsQuotaByStoragetypeEnabledKey + " is set to " + fsd.IsQuotaByStorageTypeEnabled
() + " or nsQuota value is illegal " + nsQuota);
}
string srcs = FSDirectory.NormalizePath(src);
INodesInPath iip = fsd.GetINodesInPath4Write(srcs, true);
INodeDirectory dirNode = INodeDirectory.ValueOf(iip.GetLastINode(), srcs);
if (dirNode.IsRoot() && nsQuota == HdfsConstants.QuotaReset)
{
throw new ArgumentException("Cannot clear namespace quota on root.");
}
else
{
// a directory inode
QuotaCounts oldQuota = dirNode.GetQuotaCounts();
long oldNsQuota = oldQuota.GetNameSpace();
long oldSsQuota = oldQuota.GetStorageSpace();
if (nsQuota == HdfsConstants.QuotaDontSet)
{
nsQuota = oldNsQuota;
}
if (ssQuota == HdfsConstants.QuotaDontSet)
{
ssQuota = oldSsQuota;
}
// unchanged space/namespace quota
if (type == null && oldNsQuota == nsQuota && oldSsQuota == ssQuota)
{
return(null);
}
// unchanged type quota
if (type != null)
{
EnumCounters <StorageType> oldTypeQuotas = oldQuota.GetTypeSpaces();
if (oldTypeQuotas != null && oldTypeQuotas.Get(type) == ssQuota)
{
return(null);
}
}
int latest = iip.GetLatestSnapshotId();
dirNode.RecordModification(latest);
dirNode.SetQuota(fsd.GetBlockStoragePolicySuite(), nsQuota, ssQuota, type);
return(dirNode);
}
}
/// <exception cref="System.IO.IOException"/>
private static void WriteQuota(QuotaCounts quota, DataOutput @out)
{
@out.WriteLong(quota.GetNameSpace());
@out.WriteLong(quota.GetStorageSpace());
}
/// <exception cref="Org.Apache.Hadoop.Hdfs.Protocol.QuotaExceededException">
/// if namespace, storagespace or storage type
/// space quota is violated after applying the deltas.
/// </exception>
internal void VerifyQuota(QuotaCounts counts)
{
VerifyNamespaceQuota(counts.GetNameSpace());
VerifyStoragespaceQuota(counts.GetStorageSpace());
VerifyQuotaByStorageType(counts.GetTypeSpaces());
}
internal void SetSpaceConsumed(QuotaCounts c)
{
usage.SetNameSpace(c.GetNameSpace());
usage.SetStorageSpace(c.GetStorageSpace());
usage.SetTypeSpaces(c.GetTypeSpaces());
}