/// <summary>Test if the quota can be correctly updated for append</summary>
/// <exception cref="System.Exception"/>
public virtual void TestUpdateQuotaForAppend()
{
Path foo = new Path(dir, "foo");
Path bar = new Path(foo, "bar");
long currentFileLen = Blocksize;
DFSTestUtil.CreateFile(dfs, bar, currentFileLen, Replication, seed);
dfs.SetQuota(foo, long.MaxValue - 1, long.MaxValue - 1);
// append half of the block data, the previous file length is at block
// boundary
DFSTestUtil.AppendFile(dfs, bar, Blocksize / 2);
currentFileLen += (Blocksize / 2);
INodeDirectory fooNode = fsdir.GetINode4Write(foo.ToString()).AsDirectory();
NUnit.Framework.Assert.IsTrue(fooNode.IsQuotaSet());
QuotaCounts quota = fooNode.GetDirectoryWithQuotaFeature().GetSpaceConsumed();
long ns = quota.GetNameSpace();
long ds = quota.GetStorageSpace();
NUnit.Framework.Assert.AreEqual(2, ns);
// foo and bar
NUnit.Framework.Assert.AreEqual(currentFileLen * Replication, ds);
ContentSummary c = dfs.GetContentSummary(foo);
NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), ds);
// append another block, the previous file length is not at block boundary
DFSTestUtil.AppendFile(dfs, bar, Blocksize);
currentFileLen += Blocksize;
quota = fooNode.GetDirectoryWithQuotaFeature().GetSpaceConsumed();
ns = quota.GetNameSpace();
ds = quota.GetStorageSpace();
NUnit.Framework.Assert.AreEqual(2, ns);
// foo and bar
NUnit.Framework.Assert.AreEqual(currentFileLen * Replication, ds);
c = dfs.GetContentSummary(foo);
NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), ds);
// append several blocks
DFSTestUtil.AppendFile(dfs, bar, Blocksize * 3 + Blocksize / 8);
currentFileLen += (Blocksize * 3 + Blocksize / 8);
quota = fooNode.GetDirectoryWithQuotaFeature().GetSpaceConsumed();
ns = quota.GetNameSpace();
ds = quota.GetStorageSpace();
NUnit.Framework.Assert.AreEqual(2, ns);
// foo and bar
NUnit.Framework.Assert.AreEqual(currentFileLen * Replication, ds);
c = dfs.GetContentSummary(foo);
NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), ds);
}
/// <summary>
/// Test append over a specific type of storage quota does not mark file as
/// UC or create a lease
/// </summary>
/// <exception cref="System.Exception"/>
public virtual void TestAppendOverTypeQuota()
{
Path dir = new Path("/TestAppendOverTypeQuota");
Path file = new Path(dir, "file");
// create partial block file
dfs.Mkdirs(dir);
// set the storage policy on dir
dfs.SetStoragePolicy(dir, HdfsConstants.OnessdStoragePolicyName);
DFSTestUtil.CreateFile(dfs, file, Blocksize / 2, Replication, seed);
// set quota of SSD to 1L
dfs.SetQuotaByStorageType(dir, StorageType.Ssd, 1L);
INodeDirectory dirNode = fsdir.GetINode4Write(dir.ToString()).AsDirectory();
long spaceUsed = dirNode.GetDirectoryWithQuotaFeature().GetSpaceConsumed().GetStorageSpace
();
try
{
DFSTestUtil.AppendFile(dfs, file, Blocksize);
NUnit.Framework.Assert.Fail("append didn't fail");
}
catch (RemoteException e)
{
NUnit.Framework.Assert.IsTrue(e.GetClassName().Contains("QuotaByStorageTypeExceededException"
));
}
// check that the file exists, isn't UC, and has no dangling lease
INodeFile inode = fsdir.GetINode(file.ToString()).AsFile();
NUnit.Framework.Assert.IsNotNull(inode);
NUnit.Framework.Assert.IsFalse("should not be UC", inode.IsUnderConstruction());
NUnit.Framework.Assert.IsNull("should not have a lease", cluster.GetNamesystem().
GetLeaseManager().GetLeaseByPath(file.ToString()));
// make sure the quota usage is unchanged
long newSpaceUsed = dirNode.GetDirectoryWithQuotaFeature().GetSpaceConsumed().GetStorageSpace
();
NUnit.Framework.Assert.AreEqual(spaceUsed, newSpaceUsed);
// make sure edits aren't corrupted
dfs.RecoverLease(file);
cluster.RestartNameNodes();
}
/// <summary>Test truncate over quota does not mark file as UC or create a lease</summary>
/// <exception cref="System.Exception"/>
public virtual void TestTruncateOverQuota()
{
Path dir = new Path("/TestTruncateOverquota");
Path file = new Path(dir, "file");
// create partial block file
dfs.Mkdirs(dir);
DFSTestUtil.CreateFile(dfs, file, Blocksize / 2, Replication, seed);
// lower quota to cause exception when appending to partial block
dfs.SetQuota(dir, long.MaxValue - 1, 1);
INodeDirectory dirNode = fsdir.GetINode4Write(dir.ToString()).AsDirectory();
long spaceUsed = dirNode.GetDirectoryWithQuotaFeature().GetSpaceConsumed().GetStorageSpace
();
try
{
dfs.Truncate(file, Blocksize / 2 - 1);
NUnit.Framework.Assert.Fail("truncate didn't fail");
}
catch (RemoteException e)
{
NUnit.Framework.Assert.IsTrue(e.GetClassName().Contains("DSQuotaExceededException"
));
}
// check that the file exists, isn't UC, and has no dangling lease
INodeFile inode = fsdir.GetINode(file.ToString()).AsFile();
NUnit.Framework.Assert.IsNotNull(inode);
NUnit.Framework.Assert.IsFalse("should not be UC", inode.IsUnderConstruction());
NUnit.Framework.Assert.IsNull("should not have a lease", cluster.GetNamesystem().
GetLeaseManager().GetLeaseByPath(file.ToString()));
// make sure the quota usage is unchanged
long newSpaceUsed = dirNode.GetDirectoryWithQuotaFeature().GetSpaceConsumed().GetStorageSpace
();
NUnit.Framework.Assert.AreEqual(spaceUsed, newSpaceUsed);
// make sure edits aren't corrupted
dfs.RecoverLease(file);
cluster.RestartNameNodes();
}
/// <summary>
/// Test if the quota can be correctly updated when file length is updated
/// through fsync
/// </summary>
/// <exception cref="System.Exception"/>
public virtual void TestUpdateQuotaForFSync()
{
Path foo = new Path("/foo");
Path bar = new Path(foo, "bar");
DFSTestUtil.CreateFile(dfs, bar, Blocksize, Replication, 0L);
dfs.SetQuota(foo, long.MaxValue - 1, long.MaxValue - 1);
FSDataOutputStream @out = dfs.Append(bar);
@out.Write(new byte[Blocksize / 4]);
((DFSOutputStream)@out.GetWrappedStream()).Hsync(EnumSet.Of(HdfsDataOutputStream.SyncFlag
.UpdateLength));
INodeDirectory fooNode = fsdir.GetINode4Write(foo.ToString()).AsDirectory();
QuotaCounts quota = fooNode.GetDirectoryWithQuotaFeature().GetSpaceConsumed();
long ns = quota.GetNameSpace();
long ds = quota.GetStorageSpace();
NUnit.Framework.Assert.AreEqual(2, ns);
// foo and bar
NUnit.Framework.Assert.AreEqual(Blocksize * 2 * Replication, ds);
// file is under construction
@out.Write(new byte[Blocksize / 4]);
@out.Close();
fooNode = fsdir.GetINode4Write(foo.ToString()).AsDirectory();
quota = fooNode.GetDirectoryWithQuotaFeature().GetSpaceConsumed();
ns = quota.GetNameSpace();
ds = quota.GetStorageSpace();
NUnit.Framework.Assert.AreEqual(2, ns);
NUnit.Framework.Assert.AreEqual((Blocksize + Blocksize / 2) * Replication, ds);
// append another block
DFSTestUtil.AppendFile(dfs, bar, Blocksize);
quota = fooNode.GetDirectoryWithQuotaFeature().GetSpaceConsumed();
ns = quota.GetNameSpace();
ds = quota.GetStorageSpace();
NUnit.Framework.Assert.AreEqual(2, ns);
// foo and bar
NUnit.Framework.Assert.AreEqual((Blocksize * 2 + Blocksize / 2) * Replication, ds
);
}
/// <exception cref="System.Exception"/>
private void TestTruncate(long newLength, long expectedDiff, long expectedUsage)
{
// before doing the real truncation, make sure the computation is correct
INodesInPath iip = fsdir.GetINodesInPath4Write(file.ToString());
INodeFile fileNode = iip.GetLastINode().AsFile();
fileNode.RecordModification(iip.GetLatestSnapshotId(), true);
long diff = fileNode.ComputeQuotaDeltaForTruncate(newLength);
NUnit.Framework.Assert.AreEqual(expectedDiff, diff);
// do the real truncation
dfs.Truncate(file, newLength);
// wait for truncate to finish
TestFileTruncate.CheckBlockRecovery(file, dfs);
INodeDirectory dirNode = fsdir.GetINode4Write(dir.ToString()).AsDirectory();
long spaceUsed = dirNode.GetDirectoryWithQuotaFeature().GetSpaceConsumed().GetStorageSpace
();
long diskUsed = dirNode.GetDirectoryWithQuotaFeature().GetSpaceConsumed().GetTypeSpaces
().Get(StorageType.Disk);
NUnit.Framework.Assert.AreEqual(expectedUsage, spaceUsed);
NUnit.Framework.Assert.AreEqual(expectedUsage, diskUsed);
}
public static INodeDirectory LoadINodeDirectory(FsImageProto.INodeSection.INode n
, FSImageFormatProtobuf.LoaderContext state)
{
System.Diagnostics.Debug.Assert(n.GetType() == FsImageProto.INodeSection.INode.Type
.Directory);
FsImageProto.INodeSection.INodeDirectory d = n.GetDirectory();
PermissionStatus permissions = LoadPermission(d.GetPermission(), state.GetStringTable
());
INodeDirectory dir = new INodeDirectory(n.GetId(), n.GetName().ToByteArray(), permissions
, d.GetModificationTime());
long nsQuota = d.GetNsQuota();
long dsQuota = d.GetDsQuota();
if (nsQuota >= 0 || dsQuota >= 0)
{
dir.AddDirectoryWithQuotaFeature(new DirectoryWithQuotaFeature.Builder().NameSpaceQuota
(nsQuota).StorageSpaceQuota(dsQuota).Build());
}
EnumCounters <StorageType> typeQuotas = null;
if (d.HasTypeQuotas())
{
ImmutableList <QuotaByStorageTypeEntry> qes = LoadQuotaByStorageTypeEntries(d.GetTypeQuotas
());
typeQuotas = new EnumCounters <StorageType>(typeof(StorageType), HdfsConstants.QuotaReset
);
foreach (QuotaByStorageTypeEntry qe in qes)
{
if (qe.GetQuota() >= 0 && qe.GetStorageType() != null && qe.GetStorageType().SupportTypeQuota
())
{
typeQuotas.Set(qe.GetStorageType(), qe.GetQuota());
}
}
if (typeQuotas.AnyGreaterOrEqual(0))
{
DirectoryWithQuotaFeature q = dir.GetDirectoryWithQuotaFeature();
if (q == null)
{
dir.AddDirectoryWithQuotaFeature(new DirectoryWithQuotaFeature.Builder().TypeQuotas
(typeQuotas).Build());
}
else
{
q.SetQuota(typeQuotas);
}
}
}
if (d.HasAcl())
{
int[] entries = AclEntryStatusFormat.ToInt(LoadAclEntries(d.GetAcl(), state.GetStringTable
()));
dir.AddAclFeature(new AclFeature(entries));
}
if (d.HasXAttrs())
{
dir.AddXAttrFeature(new XAttrFeature(LoadXAttrs(d.GetXAttrs(), state.GetStringTable
())));
}
return(dir);
}