internal virtual void SetTypeSpaces(EnumCounters <StorageType> that)
{
if (that != null)
{
this.tsCounts.Set(that);
}
}
/// <summary>
/// Sets namespace and storagespace take by the directory rooted
/// at this INode.
/// </summary>
/// <remarks>
/// Sets namespace and storagespace take by the directory rooted
/// at this INode. This should be used carefully. It does not check
/// for quota violations.
/// </remarks>
/// <param name="namespace">size of the directory to be set</param>
/// <param name="storagespace">storage space take by all the nodes under this directory
/// </param>
/// <param name="typespaces">counters of storage type usage</param>
internal void SetSpaceConsumed(long @namespace, long storagespace, EnumCounters <StorageType
> typespaces)
{
usage.SetNameSpace(@namespace);
usage.SetStorageSpace(storagespace);
usage.SetTypeSpaces(typespaces);
}
private void LoadRootINode(FsImageProto.INodeSection.INode p)
{
INodeDirectory root = LoadINodeDirectory(p, parent.GetLoaderContext());
QuotaCounts q = root.GetQuotaCounts();
long nsQuota = q.GetNameSpace();
long dsQuota = q.GetStorageSpace();
if (nsQuota != -1 || dsQuota != -1)
{
dir.rootDir.GetDirectoryWithQuotaFeature().SetQuota(nsQuota, dsQuota);
}
EnumCounters <StorageType> typeQuotas = q.GetTypeSpaces();
if (typeQuotas.AnyGreaterOrEqual(0))
{
dir.rootDir.GetDirectoryWithQuotaFeature().SetQuota(typeQuotas);
}
dir.rootDir.CloneModificationTime(root);
dir.rootDir.ClonePermissionStatus(root);
AclFeature af = root.GetFeature(typeof(AclFeature));
if (af != null)
{
dir.rootDir.AddAclFeature(af);
}
// root dir supports having extended attributes according to POSIX
XAttrFeature f = root.GetXAttrFeature();
if (f != null)
{
dir.rootDir.AddXAttrFeature(f);
}
dir.AddRootDirToEncryptionZone(f);
}
public Builder()
{
// Name space and storage space counts (HDFS-7775 refactors the original disk
// space count to storage space counts)
// Storage type space counts
this.nsSsCounts = new EnumCounters <Quota>(typeof(Quota));
this.tsCounts = new EnumCounters <StorageType>(typeof(StorageType));
}
public virtual EnumCounters <StorageType> GetTypeSpaces()
{
EnumCounters <StorageType> ret = new EnumCounters <StorageType>(typeof(StorageType)
);
ret.Set(tsCounts);
return(ret);
}
public virtual QuotaCounts.Builder TypeSpaces(EnumCounters <StorageType> val)
{
if (val != null)
{
this.tsCounts.Set(val);
}
return(this);
}
/// <exception cref="Org.Apache.Hadoop.Hdfs.Protocol.QuotaByStorageTypeExceededException
/// "/>
private void VerifyQuotaByStorageType(EnumCounters <StorageType> typeDelta)
{
if (!IsQuotaByStorageTypeSet())
{
return;
}
foreach (StorageType t in StorageType.GetTypesSupportingQuota())
{
if (!IsQuotaByStorageTypeSet(t))
{
continue;
}
if (Quota.IsViolated(quota.GetTypeSpace(t), usage.GetTypeSpace(t), typeDelta.Get(
t)))
{
throw new QuotaByStorageTypeExceededException(quota.GetTypeSpace(t), usage.GetTypeSpace
(t) + typeDelta.Get(t), t);
}
}
}
/// <summary>Set storage type quota in a batch.</summary>
/// <remarks>Set storage type quota in a batch. (Only used by FSImage load)</remarks>
/// <param name="tsQuotas">type space counts for all storage types supporting quota</param>
internal void SetQuota(EnumCounters <StorageType> tsQuotas)
{
this.quota.SetTypeSpaces(tsQuotas);
}
public static INodeDirectory LoadINodeDirectory(FsImageProto.INodeSection.INode n
, FSImageFormatProtobuf.LoaderContext state)
{
System.Diagnostics.Debug.Assert(n.GetType() == FsImageProto.INodeSection.INode.Type
.Directory);
FsImageProto.INodeSection.INodeDirectory d = n.GetDirectory();
PermissionStatus permissions = LoadPermission(d.GetPermission(), state.GetStringTable
());
INodeDirectory dir = new INodeDirectory(n.GetId(), n.GetName().ToByteArray(), permissions
, d.GetModificationTime());
long nsQuota = d.GetNsQuota();
long dsQuota = d.GetDsQuota();
if (nsQuota >= 0 || dsQuota >= 0)
{
dir.AddDirectoryWithQuotaFeature(new DirectoryWithQuotaFeature.Builder().NameSpaceQuota
(nsQuota).StorageSpaceQuota(dsQuota).Build());
}
EnumCounters <StorageType> typeQuotas = null;
if (d.HasTypeQuotas())
{
ImmutableList <QuotaByStorageTypeEntry> qes = LoadQuotaByStorageTypeEntries(d.GetTypeQuotas
());
typeQuotas = new EnumCounters <StorageType>(typeof(StorageType), HdfsConstants.QuotaReset
);
foreach (QuotaByStorageTypeEntry qe in qes)
{
if (qe.GetQuota() >= 0 && qe.GetStorageType() != null && qe.GetStorageType().SupportTypeQuota
())
{
typeQuotas.Set(qe.GetStorageType(), qe.GetQuota());
}
}
if (typeQuotas.AnyGreaterOrEqual(0))
{
DirectoryWithQuotaFeature q = dir.GetDirectoryWithQuotaFeature();
if (q == null)
{
dir.AddDirectoryWithQuotaFeature(new DirectoryWithQuotaFeature.Builder().TypeQuotas
(typeQuotas).Build());
}
else
{
q.SetQuota(typeQuotas);
}
}
}
if (d.HasAcl())
{
int[] entries = AclEntryStatusFormat.ToInt(LoadAclEntries(d.GetAcl(), state.GetStringTable
()));
dir.AddAclFeature(new AclFeature(entries));
}
if (d.HasXAttrs())
{
dir.AddXAttrFeature(new XAttrFeature(LoadXAttrs(d.GetXAttrs(), state.GetStringTable
())));
}
return(dir);
}
public virtual QuotaCounts UpdateQuotaAndCollectBlocks(BlockStoragePolicySuite bsps
, INodeFile file, FileDiff removed, INode.BlocksMapUpdateInfo collectedBlocks, IList
<INode> removedINodes)
{
long oldStoragespace = file.StoragespaceConsumed();
byte storagePolicyID = file.GetStoragePolicyID();
BlockStoragePolicy bsp = null;
EnumCounters <StorageType> typeSpaces = new EnumCounters <StorageType>(typeof(StorageType
));
if (storagePolicyID != BlockStoragePolicySuite.IdUnspecified)
{
bsp = bsps.GetPolicy(file.GetStoragePolicyID());
}
if (removed.snapshotINode != null)
{
short replication = removed.snapshotINode.GetFileReplication();
short currentRepl = file.GetBlockReplication();
if (currentRepl == 0)
{
long oldFileSizeNoRep = file.ComputeFileSize(true, true);
oldStoragespace = oldFileSizeNoRep * replication;
if (bsp != null)
{
IList <StorageType> oldTypeChosen = bsp.ChooseStorageTypes(replication);
foreach (StorageType t in oldTypeChosen)
{
if (t.SupportTypeQuota())
{
typeSpaces.Add(t, -oldFileSizeNoRep);
}
}
}
}
else
{
if (replication > currentRepl)
{
long oldFileSizeNoRep = file.StoragespaceConsumedNoReplication();
oldStoragespace = oldFileSizeNoRep * replication;
if (bsp != null)
{
IList <StorageType> oldTypeChosen = bsp.ChooseStorageTypes(replication);
foreach (StorageType t in oldTypeChosen)
{
if (t.SupportTypeQuota())
{
typeSpaces.Add(t, -oldFileSizeNoRep);
}
}
IList <StorageType> newTypeChosen = bsp.ChooseStorageTypes(currentRepl);
foreach (StorageType t_1 in newTypeChosen)
{
if (t_1.SupportTypeQuota())
{
typeSpaces.Add(t_1, oldFileSizeNoRep);
}
}
}
}
}
AclFeature aclFeature = removed.GetSnapshotINode().GetAclFeature();
if (aclFeature != null)
{
AclStorage.RemoveAclFeature(aclFeature);
}
}
GetDiffs().CombineAndCollectSnapshotBlocks(bsps, file, removed, collectedBlocks,
removedINodes);
long ssDelta = oldStoragespace - file.StoragespaceConsumed();
return(new QuotaCounts.Builder().StorageSpace(ssDelta).TypeSpaces(typeSpaces).Build
());
}
/// <summary>Load DirectoryDiff list for a directory with snapshot feature</summary>
/// <exception cref="System.IO.IOException"/>
private void LoadDirectoryDiffList(InputStream @in, INodeDirectory dir, int size,
IList <INodeReference> refList)
{
if (!dir.IsWithSnapshot())
{
dir.AddSnapshotFeature(null);
}
DirectoryWithSnapshotFeature.DirectoryDiffList diffs = dir.GetDiffs();
FSImageFormatProtobuf.LoaderContext state = parent.GetLoaderContext();
for (int i = 0; i < size; i++)
{
// load a directory diff
FsImageProto.SnapshotDiffSection.DirectoryDiff diffInPb = FsImageProto.SnapshotDiffSection.DirectoryDiff
.ParseDelimitedFrom(@in);
int snapshotId = diffInPb.GetSnapshotId();
Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot snapshot = snapshotMap[snapshotId
];
int childrenSize = diffInPb.GetChildrenSize();
bool useRoot = diffInPb.GetIsSnapshotRoot();
INodeDirectoryAttributes copy = null;
if (useRoot)
{
copy = snapshot.GetRoot();
}
else
{
if (diffInPb.HasSnapshotCopy())
{
FsImageProto.INodeSection.INodeDirectory dirCopyInPb = diffInPb.GetSnapshotCopy();
byte[] name = diffInPb.GetName().ToByteArray();
PermissionStatus permission = FSImageFormatPBINode.Loader.LoadPermission(dirCopyInPb
.GetPermission(), state.GetStringTable());
AclFeature acl = null;
if (dirCopyInPb.HasAcl())
{
int[] entries = AclEntryStatusFormat.ToInt(FSImageFormatPBINode.Loader.LoadAclEntries
(dirCopyInPb.GetAcl(), state.GetStringTable()));
acl = new AclFeature(entries);
}
XAttrFeature xAttrs = null;
if (dirCopyInPb.HasXAttrs())
{
xAttrs = new XAttrFeature(FSImageFormatPBINode.Loader.LoadXAttrs(dirCopyInPb.GetXAttrs
(), state.GetStringTable()));
}
long modTime = dirCopyInPb.GetModificationTime();
bool noQuota = dirCopyInPb.GetNsQuota() == -1 && dirCopyInPb.GetDsQuota() == -1 &&
(!dirCopyInPb.HasTypeQuotas());
if (noQuota)
{
copy = new INodeDirectoryAttributes.SnapshotCopy(name, permission, acl, modTime,
xAttrs);
}
else
{
EnumCounters <StorageType> typeQuotas = null;
if (dirCopyInPb.HasTypeQuotas())
{
ImmutableList <QuotaByStorageTypeEntry> qes = FSImageFormatPBINode.Loader.LoadQuotaByStorageTypeEntries
(dirCopyInPb.GetTypeQuotas());
typeQuotas = new EnumCounters <StorageType>(typeof(StorageType), HdfsConstants.QuotaReset
);
foreach (QuotaByStorageTypeEntry qe in qes)
{
if (qe.GetQuota() >= 0 && qe.GetStorageType() != null && qe.GetStorageType().SupportTypeQuota
())
{
typeQuotas.Set(qe.GetStorageType(), qe.GetQuota());
}
}
}
copy = new INodeDirectoryAttributes.CopyWithQuota(name, permission, acl, modTime,
dirCopyInPb.GetNsQuota(), dirCopyInPb.GetDsQuota(), typeQuotas, xAttrs);
}
}
}
// load created list
IList <INode> clist = LoadCreatedList(@in, dir, diffInPb.GetCreatedListSize());
// load deleted list
IList <INode> dlist = LoadDeletedList(refList, @in, dir, diffInPb.GetDeletedINodeList
(), diffInPb.GetDeletedINodeRefList());
// create the directory diff
DirectoryWithSnapshotFeature.DirectoryDiff diff = new DirectoryWithSnapshotFeature.DirectoryDiff
(snapshotId, copy, null, childrenSize, clist, dlist, useRoot);
diffs.AddFirst(diff);
}
}
/// <summary>
/// See
/// <see cref="Org.Apache.Hadoop.Hdfs.Protocol.ClientProtocol.SetQuota(string, long, long, Org.Apache.Hadoop.FS.StorageType)
/// "/>
/// for the contract.
/// Sets quota for for a directory.
/// </summary>
/// <returns>INodeDirectory if any of the quotas have changed. null otherwise.</returns>
/// <exception cref="System.IO.FileNotFoundException">if the path does not exist.</exception>
/// <exception cref="Org.Apache.Hadoop.FS.PathIsNotDirectoryException">if the path is not a directory.
/// </exception>
/// <exception cref="Org.Apache.Hadoop.Hdfs.Protocol.QuotaExceededException">
/// if the directory tree size is
/// greater than the given quota
/// </exception>
/// <exception cref="Org.Apache.Hadoop.FS.UnresolvedLinkException">if a symlink is encountered in src.
/// </exception>
/// <exception cref="Org.Apache.Hadoop.Hdfs.Protocol.SnapshotAccessControlException">if path is in RO snapshot
/// </exception>
/// <exception cref="Org.Apache.Hadoop.Hdfs.Server.Namenode.UnsupportedActionException
/// "/>
internal static INodeDirectory UnprotectedSetQuota(FSDirectory fsd, string src, long
nsQuota, long ssQuota, StorageType type)
{
System.Diagnostics.Debug.Assert(fsd.HasWriteLock());
// sanity check
if ((nsQuota < 0 && nsQuota != HdfsConstants.QuotaDontSet && nsQuota != HdfsConstants
.QuotaReset) || (ssQuota < 0 && ssQuota != HdfsConstants.QuotaDontSet && ssQuota
!= HdfsConstants.QuotaReset))
{
throw new ArgumentException("Illegal value for nsQuota or " + "ssQuota : " + nsQuota
+ " and " + ssQuota);
}
// sanity check for quota by storage type
if ((type != null) && (!fsd.IsQuotaByStorageTypeEnabled() || nsQuota != HdfsConstants
.QuotaDontSet))
{
throw new UnsupportedActionException("Failed to set quota by storage type because either"
+ DFSConfigKeys.DfsQuotaByStoragetypeEnabledKey + " is set to " + fsd.IsQuotaByStorageTypeEnabled
() + " or nsQuota value is illegal " + nsQuota);
}
string srcs = FSDirectory.NormalizePath(src);
INodesInPath iip = fsd.GetINodesInPath4Write(srcs, true);
INodeDirectory dirNode = INodeDirectory.ValueOf(iip.GetLastINode(), srcs);
if (dirNode.IsRoot() && nsQuota == HdfsConstants.QuotaReset)
{
throw new ArgumentException("Cannot clear namespace quota on root.");
}
else
{
// a directory inode
QuotaCounts oldQuota = dirNode.GetQuotaCounts();
long oldNsQuota = oldQuota.GetNameSpace();
long oldSsQuota = oldQuota.GetStorageSpace();
if (nsQuota == HdfsConstants.QuotaDontSet)
{
nsQuota = oldNsQuota;
}
if (ssQuota == HdfsConstants.QuotaDontSet)
{
ssQuota = oldSsQuota;
}
// unchanged space/namespace quota
if (type == null && oldNsQuota == nsQuota && oldSsQuota == ssQuota)
{
return(null);
}
// unchanged type quota
if (type != null)
{
EnumCounters <StorageType> oldTypeQuotas = oldQuota.GetTypeSpaces();
if (oldTypeQuotas != null && oldTypeQuotas.Get(type) == ssQuota)
{
return(null);
}
}
int latest = iip.GetLatestSnapshotId();
dirNode.RecordModification(latest);
dirNode.SetQuota(fsd.GetBlockStoragePolicySuite(), nsQuota, ssQuota, type);
return(dirNode);
}
}
public CopyWithQuota(byte[] name, PermissionStatus permissions, AclFeature aclFeature
, long modificationTime, long nsQuota, long dsQuota, EnumCounters <StorageType> typeQuotas
, XAttrFeature xAttrsFeature)
: base(name, permissions, aclFeature, modificationTime, xAttrsFeature)
{
this.quota = new QuotaCounts.Builder().NameSpace(nsQuota).StorageSpace(dsQuota).TypeSpaces
(typeQuotas).Build();
}
private QuotaCounts(QuotaCounts.Builder builder)
{
this.nsSsCounts = builder.nsSsCounts;
this.tsCounts = builder.tsCounts;
}
public virtual DirectoryWithQuotaFeature.Builder TypeQuotas(EnumCounters <StorageType
> typeQuotas)
{
this.quota.SetTypeSpaces(typeQuotas);
return(this);
}
private ContentCounts(EnumCounters <Content> contents, EnumCounters <StorageType> types
)
{
this.contents = contents;
this.types = types;
}
public Builder()
{
// storage spaces used by corresponding storage types
contents = new EnumCounters <Content>(typeof(Content));
types = new EnumCounters <StorageType>(typeof(StorageType));
}
public virtual void AddTypeSpaces(EnumCounters <StorageType> that)
{
this.types.Add(that);
}
