internal virtual void SetTypeSpaces(EnumCounters <StorageType> that) { if (that != null) { this.tsCounts.Set(that); } }
/// <summary> /// Sets namespace and storagespace take by the directory rooted /// at this INode. /// </summary> /// <remarks> /// Sets namespace and storagespace take by the directory rooted /// at this INode. This should be used carefully. It does not check /// for quota violations. /// </remarks> /// <param name="namespace">size of the directory to be set</param> /// <param name="storagespace">storage space take by all the nodes under this directory /// </param> /// <param name="typespaces">counters of storage type usage</param> internal void SetSpaceConsumed(long @namespace, long storagespace, EnumCounters <StorageType > typespaces) { usage.SetNameSpace(@namespace); usage.SetStorageSpace(storagespace); usage.SetTypeSpaces(typespaces); }
private void LoadRootINode(FsImageProto.INodeSection.INode p) { INodeDirectory root = LoadINodeDirectory(p, parent.GetLoaderContext()); QuotaCounts q = root.GetQuotaCounts(); long nsQuota = q.GetNameSpace(); long dsQuota = q.GetStorageSpace(); if (nsQuota != -1 || dsQuota != -1) { dir.rootDir.GetDirectoryWithQuotaFeature().SetQuota(nsQuota, dsQuota); } EnumCounters <StorageType> typeQuotas = q.GetTypeSpaces(); if (typeQuotas.AnyGreaterOrEqual(0)) { dir.rootDir.GetDirectoryWithQuotaFeature().SetQuota(typeQuotas); } dir.rootDir.CloneModificationTime(root); dir.rootDir.ClonePermissionStatus(root); AclFeature af = root.GetFeature(typeof(AclFeature)); if (af != null) { dir.rootDir.AddAclFeature(af); } // root dir supports having extended attributes according to POSIX XAttrFeature f = root.GetXAttrFeature(); if (f != null) { dir.rootDir.AddXAttrFeature(f); } dir.AddRootDirToEncryptionZone(f); }
public Builder() { // Name space and storage space counts (HDFS-7775 refactors the original disk // space count to storage space counts) // Storage type space counts this.nsSsCounts = new EnumCounters <Quota>(typeof(Quota)); this.tsCounts = new EnumCounters <StorageType>(typeof(StorageType)); }
public virtual EnumCounters <StorageType> GetTypeSpaces() { EnumCounters <StorageType> ret = new EnumCounters <StorageType>(typeof(StorageType) ); ret.Set(tsCounts); return(ret); }
public virtual QuotaCounts.Builder TypeSpaces(EnumCounters <StorageType> val) { if (val != null) { this.tsCounts.Set(val); } return(this); }
/// <exception cref="Org.Apache.Hadoop.Hdfs.Protocol.QuotaByStorageTypeExceededException /// "/> private void VerifyQuotaByStorageType(EnumCounters <StorageType> typeDelta) { if (!IsQuotaByStorageTypeSet()) { return; } foreach (StorageType t in StorageType.GetTypesSupportingQuota()) { if (!IsQuotaByStorageTypeSet(t)) { continue; } if (Quota.IsViolated(quota.GetTypeSpace(t), usage.GetTypeSpace(t), typeDelta.Get( t))) { throw new QuotaByStorageTypeExceededException(quota.GetTypeSpace(t), usage.GetTypeSpace (t) + typeDelta.Get(t), t); } } }
/// <summary>Set storage type quota in a batch.</summary> /// <remarks>Set storage type quota in a batch. (Only used by FSImage load)</remarks> /// <param name="tsQuotas">type space counts for all storage types supporting quota</param> internal void SetQuota(EnumCounters <StorageType> tsQuotas) { this.quota.SetTypeSpaces(tsQuotas); }
public static INodeDirectory LoadINodeDirectory(FsImageProto.INodeSection.INode n , FSImageFormatProtobuf.LoaderContext state) { System.Diagnostics.Debug.Assert(n.GetType() == FsImageProto.INodeSection.INode.Type .Directory); FsImageProto.INodeSection.INodeDirectory d = n.GetDirectory(); PermissionStatus permissions = LoadPermission(d.GetPermission(), state.GetStringTable ()); INodeDirectory dir = new INodeDirectory(n.GetId(), n.GetName().ToByteArray(), permissions , d.GetModificationTime()); long nsQuota = d.GetNsQuota(); long dsQuota = d.GetDsQuota(); if (nsQuota >= 0 || dsQuota >= 0) { dir.AddDirectoryWithQuotaFeature(new DirectoryWithQuotaFeature.Builder().NameSpaceQuota (nsQuota).StorageSpaceQuota(dsQuota).Build()); } EnumCounters <StorageType> typeQuotas = null; if (d.HasTypeQuotas()) { ImmutableList <QuotaByStorageTypeEntry> qes = LoadQuotaByStorageTypeEntries(d.GetTypeQuotas ()); typeQuotas = new EnumCounters <StorageType>(typeof(StorageType), HdfsConstants.QuotaReset ); foreach (QuotaByStorageTypeEntry qe in qes) { if (qe.GetQuota() >= 0 && qe.GetStorageType() != null && qe.GetStorageType().SupportTypeQuota ()) { typeQuotas.Set(qe.GetStorageType(), qe.GetQuota()); } } if (typeQuotas.AnyGreaterOrEqual(0)) { DirectoryWithQuotaFeature q = dir.GetDirectoryWithQuotaFeature(); if (q == null) { dir.AddDirectoryWithQuotaFeature(new DirectoryWithQuotaFeature.Builder().TypeQuotas (typeQuotas).Build()); } else { q.SetQuota(typeQuotas); } } } if (d.HasAcl()) { int[] entries = AclEntryStatusFormat.ToInt(LoadAclEntries(d.GetAcl(), state.GetStringTable ())); dir.AddAclFeature(new AclFeature(entries)); } if (d.HasXAttrs()) { dir.AddXAttrFeature(new XAttrFeature(LoadXAttrs(d.GetXAttrs(), state.GetStringTable ()))); } return(dir); }
public virtual QuotaCounts UpdateQuotaAndCollectBlocks(BlockStoragePolicySuite bsps , INodeFile file, FileDiff removed, INode.BlocksMapUpdateInfo collectedBlocks, IList <INode> removedINodes) { long oldStoragespace = file.StoragespaceConsumed(); byte storagePolicyID = file.GetStoragePolicyID(); BlockStoragePolicy bsp = null; EnumCounters <StorageType> typeSpaces = new EnumCounters <StorageType>(typeof(StorageType )); if (storagePolicyID != BlockStoragePolicySuite.IdUnspecified) { bsp = bsps.GetPolicy(file.GetStoragePolicyID()); } if (removed.snapshotINode != null) { short replication = removed.snapshotINode.GetFileReplication(); short currentRepl = file.GetBlockReplication(); if (currentRepl == 0) { long oldFileSizeNoRep = file.ComputeFileSize(true, true); oldStoragespace = oldFileSizeNoRep * replication; if (bsp != null) { IList <StorageType> oldTypeChosen = bsp.ChooseStorageTypes(replication); foreach (StorageType t in oldTypeChosen) { if (t.SupportTypeQuota()) { typeSpaces.Add(t, -oldFileSizeNoRep); } } } } else { if (replication > currentRepl) { long oldFileSizeNoRep = file.StoragespaceConsumedNoReplication(); oldStoragespace = oldFileSizeNoRep * replication; if (bsp != null) { IList <StorageType> oldTypeChosen = bsp.ChooseStorageTypes(replication); foreach (StorageType t in oldTypeChosen) { if (t.SupportTypeQuota()) { typeSpaces.Add(t, -oldFileSizeNoRep); } } IList <StorageType> newTypeChosen = bsp.ChooseStorageTypes(currentRepl); foreach (StorageType t_1 in newTypeChosen) { if (t_1.SupportTypeQuota()) { typeSpaces.Add(t_1, oldFileSizeNoRep); } } } } } AclFeature aclFeature = removed.GetSnapshotINode().GetAclFeature(); if (aclFeature != null) { AclStorage.RemoveAclFeature(aclFeature); } } GetDiffs().CombineAndCollectSnapshotBlocks(bsps, file, removed, collectedBlocks, removedINodes); long ssDelta = oldStoragespace - file.StoragespaceConsumed(); return(new QuotaCounts.Builder().StorageSpace(ssDelta).TypeSpaces(typeSpaces).Build ()); }
/// <summary>Load DirectoryDiff list for a directory with snapshot feature</summary> /// <exception cref="System.IO.IOException"/> private void LoadDirectoryDiffList(InputStream @in, INodeDirectory dir, int size, IList <INodeReference> refList) { if (!dir.IsWithSnapshot()) { dir.AddSnapshotFeature(null); } DirectoryWithSnapshotFeature.DirectoryDiffList diffs = dir.GetDiffs(); FSImageFormatProtobuf.LoaderContext state = parent.GetLoaderContext(); for (int i = 0; i < size; i++) { // load a directory diff FsImageProto.SnapshotDiffSection.DirectoryDiff diffInPb = FsImageProto.SnapshotDiffSection.DirectoryDiff .ParseDelimitedFrom(@in); int snapshotId = diffInPb.GetSnapshotId(); Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot snapshot = snapshotMap[snapshotId ]; int childrenSize = diffInPb.GetChildrenSize(); bool useRoot = diffInPb.GetIsSnapshotRoot(); INodeDirectoryAttributes copy = null; if (useRoot) { copy = snapshot.GetRoot(); } else { if (diffInPb.HasSnapshotCopy()) { FsImageProto.INodeSection.INodeDirectory dirCopyInPb = diffInPb.GetSnapshotCopy(); byte[] name = diffInPb.GetName().ToByteArray(); PermissionStatus permission = FSImageFormatPBINode.Loader.LoadPermission(dirCopyInPb .GetPermission(), state.GetStringTable()); AclFeature acl = null; if (dirCopyInPb.HasAcl()) { int[] entries = AclEntryStatusFormat.ToInt(FSImageFormatPBINode.Loader.LoadAclEntries (dirCopyInPb.GetAcl(), state.GetStringTable())); acl = new AclFeature(entries); } XAttrFeature xAttrs = null; if (dirCopyInPb.HasXAttrs()) { xAttrs = new XAttrFeature(FSImageFormatPBINode.Loader.LoadXAttrs(dirCopyInPb.GetXAttrs (), state.GetStringTable())); } long modTime = dirCopyInPb.GetModificationTime(); bool noQuota = dirCopyInPb.GetNsQuota() == -1 && dirCopyInPb.GetDsQuota() == -1 && (!dirCopyInPb.HasTypeQuotas()); if (noQuota) { copy = new INodeDirectoryAttributes.SnapshotCopy(name, permission, acl, modTime, xAttrs); } else { EnumCounters <StorageType> typeQuotas = null; if (dirCopyInPb.HasTypeQuotas()) { ImmutableList <QuotaByStorageTypeEntry> qes = FSImageFormatPBINode.Loader.LoadQuotaByStorageTypeEntries (dirCopyInPb.GetTypeQuotas()); typeQuotas = new EnumCounters <StorageType>(typeof(StorageType), HdfsConstants.QuotaReset ); foreach (QuotaByStorageTypeEntry qe in qes) { if (qe.GetQuota() >= 0 && qe.GetStorageType() != null && qe.GetStorageType().SupportTypeQuota ()) { typeQuotas.Set(qe.GetStorageType(), qe.GetQuota()); } } } copy = new INodeDirectoryAttributes.CopyWithQuota(name, permission, acl, modTime, dirCopyInPb.GetNsQuota(), dirCopyInPb.GetDsQuota(), typeQuotas, xAttrs); } } } // load created list IList <INode> clist = LoadCreatedList(@in, dir, diffInPb.GetCreatedListSize()); // load deleted list IList <INode> dlist = LoadDeletedList(refList, @in, dir, diffInPb.GetDeletedINodeList (), diffInPb.GetDeletedINodeRefList()); // create the directory diff DirectoryWithSnapshotFeature.DirectoryDiff diff = new DirectoryWithSnapshotFeature.DirectoryDiff (snapshotId, copy, null, childrenSize, clist, dlist, useRoot); diffs.AddFirst(diff); } }
/// <summary> /// See /// <see cref="Org.Apache.Hadoop.Hdfs.Protocol.ClientProtocol.SetQuota(string, long, long, Org.Apache.Hadoop.FS.StorageType) /// "/> /// for the contract. /// Sets quota for for a directory. /// </summary> /// <returns>INodeDirectory if any of the quotas have changed. null otherwise.</returns> /// <exception cref="System.IO.FileNotFoundException">if the path does not exist.</exception> /// <exception cref="Org.Apache.Hadoop.FS.PathIsNotDirectoryException">if the path is not a directory. /// </exception> /// <exception cref="Org.Apache.Hadoop.Hdfs.Protocol.QuotaExceededException"> /// if the directory tree size is /// greater than the given quota /// </exception> /// <exception cref="Org.Apache.Hadoop.FS.UnresolvedLinkException">if a symlink is encountered in src. /// </exception> /// <exception cref="Org.Apache.Hadoop.Hdfs.Protocol.SnapshotAccessControlException">if path is in RO snapshot /// </exception> /// <exception cref="Org.Apache.Hadoop.Hdfs.Server.Namenode.UnsupportedActionException /// "/> internal static INodeDirectory UnprotectedSetQuota(FSDirectory fsd, string src, long nsQuota, long ssQuota, StorageType type) { System.Diagnostics.Debug.Assert(fsd.HasWriteLock()); // sanity check if ((nsQuota < 0 && nsQuota != HdfsConstants.QuotaDontSet && nsQuota != HdfsConstants .QuotaReset) || (ssQuota < 0 && ssQuota != HdfsConstants.QuotaDontSet && ssQuota != HdfsConstants.QuotaReset)) { throw new ArgumentException("Illegal value for nsQuota or " + "ssQuota : " + nsQuota + " and " + ssQuota); } // sanity check for quota by storage type if ((type != null) && (!fsd.IsQuotaByStorageTypeEnabled() || nsQuota != HdfsConstants .QuotaDontSet)) { throw new UnsupportedActionException("Failed to set quota by storage type because either" + DFSConfigKeys.DfsQuotaByStoragetypeEnabledKey + " is set to " + fsd.IsQuotaByStorageTypeEnabled () + " or nsQuota value is illegal " + nsQuota); } string srcs = FSDirectory.NormalizePath(src); INodesInPath iip = fsd.GetINodesInPath4Write(srcs, true); INodeDirectory dirNode = INodeDirectory.ValueOf(iip.GetLastINode(), srcs); if (dirNode.IsRoot() && nsQuota == HdfsConstants.QuotaReset) { throw new ArgumentException("Cannot clear namespace quota on root."); } else { // a directory inode QuotaCounts oldQuota = dirNode.GetQuotaCounts(); long oldNsQuota = oldQuota.GetNameSpace(); long oldSsQuota = oldQuota.GetStorageSpace(); if (nsQuota == HdfsConstants.QuotaDontSet) { nsQuota = oldNsQuota; } if (ssQuota == HdfsConstants.QuotaDontSet) { ssQuota = oldSsQuota; } // unchanged space/namespace quota if (type == null && oldNsQuota == nsQuota && oldSsQuota == ssQuota) { return(null); } // unchanged type quota if (type != null) { EnumCounters <StorageType> oldTypeQuotas = oldQuota.GetTypeSpaces(); if (oldTypeQuotas != null && oldTypeQuotas.Get(type) == ssQuota) { return(null); } } int latest = iip.GetLatestSnapshotId(); dirNode.RecordModification(latest); dirNode.SetQuota(fsd.GetBlockStoragePolicySuite(), nsQuota, ssQuota, type); return(dirNode); } }
public CopyWithQuota(byte[] name, PermissionStatus permissions, AclFeature aclFeature , long modificationTime, long nsQuota, long dsQuota, EnumCounters <StorageType> typeQuotas , XAttrFeature xAttrsFeature) : base(name, permissions, aclFeature, modificationTime, xAttrsFeature) { this.quota = new QuotaCounts.Builder().NameSpace(nsQuota).StorageSpace(dsQuota).TypeSpaces (typeQuotas).Build(); }
private QuotaCounts(QuotaCounts.Builder builder) { this.nsSsCounts = builder.nsSsCounts; this.tsCounts = builder.tsCounts; }
public virtual DirectoryWithQuotaFeature.Builder TypeQuotas(EnumCounters <StorageType > typeQuotas) { this.quota.SetTypeSpaces(typeQuotas); return(this); }
private ContentCounts(EnumCounters <Content> contents, EnumCounters <StorageType> types ) { this.contents = contents; this.types = types; }
public Builder() { // storage spaces used by corresponding storage types contents = new EnumCounters <Content>(typeof(Content)); types = new EnumCounters <StorageType>(typeof(StorageType)); }
public virtual void AddTypeSpaces(EnumCounters <StorageType> that) { this.types.Add(that); }