/// <exception cref="System.Exception"/>
public virtual void TestContentSummaryWithoutQuotaByStorageType()
{
Path foo = new Path(dir, "foo");
Path createdFile1 = new Path(foo, "created_file1.data");
dfs.Mkdirs(foo);
// set storage policy on directory "foo" to ONESSD
dfs.SetStoragePolicy(foo, HdfsConstants.OnessdStoragePolicyName);
INode fnode = fsdir.GetINode4Write(foo.ToString());
NUnit.Framework.Assert.IsTrue(fnode.IsDirectory());
NUnit.Framework.Assert.IsTrue(!fnode.IsQuotaSet());
// Create file of size 2 * BLOCKSIZE under directory "foo"
long file1Len = Blocksize * 2;
int bufLen = Blocksize / 16;
DFSTestUtil.CreateFile(dfs, createdFile1, bufLen, file1Len, Blocksize, Replication
, seed);
// Verify getContentSummary without any quota set
ContentSummary cs = dfs.GetContentSummary(foo);
NUnit.Framework.Assert.AreEqual(cs.GetSpaceConsumed(), file1Len * Replication);
NUnit.Framework.Assert.AreEqual(cs.GetTypeConsumed(StorageType.Ssd), file1Len);
NUnit.Framework.Assert.AreEqual(cs.GetTypeConsumed(StorageType.Disk), file1Len *
2);
}
/// <exception cref="System.Exception"/>
public virtual void TestQuotaByStorageTypeWithSnapshot()
{
Path sub1 = new Path(dir, "Sub1");
dfs.Mkdirs(sub1);
// Setup ONE_SSD policy and SSD quota of 4 * BLOCKSIZE on sub1
dfs.SetStoragePolicy(sub1, HdfsConstants.OnessdStoragePolicyName);
dfs.SetQuotaByStorageType(sub1, StorageType.Ssd, 4 * Blocksize);
INode sub1Node = fsdir.GetINode4Write(sub1.ToString());
NUnit.Framework.Assert.IsTrue(sub1Node.IsDirectory());
NUnit.Framework.Assert.IsTrue(sub1Node.IsQuotaSet());
// Create file1 of size 2 * BLOCKSIZE under sub1
Path file1 = new Path(sub1, "file1");
long file1Len = 2 * Blocksize;
DFSTestUtil.CreateFile(dfs, file1, file1Len, Replication, seed);
// Create snapshot on sub1 named s1
SnapshotTestHelper.CreateSnapshot(dfs, sub1, "s1");
// Verify sub1 SSD usage is unchanged after creating snapshot s1
long ssdConsumed = sub1Node.AsDirectory().GetDirectoryWithQuotaFeature().GetSpaceConsumed
().GetTypeSpaces().Get(StorageType.Ssd);
NUnit.Framework.Assert.AreEqual(file1Len, ssdConsumed);
// Delete file1
dfs.Delete(file1, false);
// Verify sub1 SSD usage is unchanged due to the existence of snapshot s1
ssdConsumed = sub1Node.AsDirectory().GetDirectoryWithQuotaFeature().GetSpaceConsumed
().GetTypeSpaces().Get(StorageType.Ssd);
NUnit.Framework.Assert.AreEqual(file1Len, ssdConsumed);
QuotaCounts counts1 = new QuotaCounts.Builder().Build();
sub1Node.ComputeQuotaUsage(fsn.GetBlockManager().GetStoragePolicySuite(), counts1
, true);
NUnit.Framework.Assert.AreEqual(sub1Node.DumpTreeRecursively().ToString(), file1Len
, counts1.GetTypeSpaces().Get(StorageType.Ssd));
ContentSummary cs1 = dfs.GetContentSummary(sub1);
NUnit.Framework.Assert.AreEqual(cs1.GetSpaceConsumed(), file1Len * Replication);
NUnit.Framework.Assert.AreEqual(cs1.GetTypeConsumed(StorageType.Ssd), file1Len);
NUnit.Framework.Assert.AreEqual(cs1.GetTypeConsumed(StorageType.Disk), file1Len *
2);
// Delete the snapshot s1
dfs.DeleteSnapshot(sub1, "s1");
// Verify sub1 SSD usage is fully reclaimed and changed to 0
ssdConsumed = sub1Node.AsDirectory().GetDirectoryWithQuotaFeature().GetSpaceConsumed
().GetTypeSpaces().Get(StorageType.Ssd);
NUnit.Framework.Assert.AreEqual(0, ssdConsumed);
QuotaCounts counts2 = new QuotaCounts.Builder().Build();
sub1Node.ComputeQuotaUsage(fsn.GetBlockManager().GetStoragePolicySuite(), counts2
, true);
NUnit.Framework.Assert.AreEqual(sub1Node.DumpTreeRecursively().ToString(), 0, counts2
.GetTypeSpaces().Get(StorageType.Ssd));
ContentSummary cs2 = dfs.GetContentSummary(sub1);
NUnit.Framework.Assert.AreEqual(cs2.GetSpaceConsumed(), 0);
NUnit.Framework.Assert.AreEqual(cs2.GetTypeConsumed(StorageType.Ssd), 0);
NUnit.Framework.Assert.AreEqual(cs2.GetTypeConsumed(StorageType.Disk), 0);
}
/// <exception cref="System.Exception"/>
public virtual void TestQuotaByStorageTypeWithFileCreateRename()
{
Path foo = new Path(dir, "foo");
dfs.Mkdirs(foo);
Path createdFile1foo = new Path(foo, "created_file1.data");
Path bar = new Path(dir, "bar");
dfs.Mkdirs(bar);
Path createdFile1bar = new Path(bar, "created_file1.data");
// set storage policy on directory "foo" and "bar" to ONESSD
dfs.SetStoragePolicy(foo, HdfsConstants.OnessdStoragePolicyName);
dfs.SetStoragePolicy(bar, HdfsConstants.OnessdStoragePolicyName);
// set quota by storage type on directory "foo"
dfs.SetQuotaByStorageType(foo, StorageType.Ssd, Blocksize * 4);
dfs.SetQuotaByStorageType(bar, StorageType.Ssd, Blocksize * 2);
INode fnode = fsdir.GetINode4Write(foo.ToString());
NUnit.Framework.Assert.IsTrue(fnode.IsDirectory());
NUnit.Framework.Assert.IsTrue(fnode.IsQuotaSet());
// Create file of size 3 * BLOCKSIZE under directory "foo"
long file1Len = Blocksize * 3;
int bufLen = Blocksize / 16;
DFSTestUtil.CreateFile(dfs, createdFile1foo, bufLen, file1Len, Blocksize, Replication
, seed);
// Verify space consumed and remaining quota
long ssdConsumed = fnode.AsDirectory().GetDirectoryWithQuotaFeature().GetSpaceConsumed
().GetTypeSpaces().Get(StorageType.Ssd);
NUnit.Framework.Assert.AreEqual(file1Len, ssdConsumed);
// move file from foo to bar
try
{
dfs.Rename(createdFile1foo, createdFile1bar);
NUnit.Framework.Assert.Fail("Should have failed with QuotaByStorageTypeExceededException "
);
}
catch (Exception t)
{
Log.Info("Got expected exception ", t);
}
ContentSummary cs = dfs.GetContentSummary(foo);
NUnit.Framework.Assert.AreEqual(cs.GetSpaceConsumed(), file1Len * Replication);
NUnit.Framework.Assert.AreEqual(cs.GetTypeConsumed(StorageType.Ssd), file1Len);
NUnit.Framework.Assert.AreEqual(cs.GetTypeConsumed(StorageType.Disk), file1Len *
2);
}
/// <exception cref="System.Exception"/>
public virtual void TestContentSummaryWithoutStoragePolicy()
{
Path foo = new Path(dir, "foo");
Path createdFile1 = new Path(foo, "created_file1.data");
dfs.Mkdirs(foo);
INode fnode = fsdir.GetINode4Write(foo.ToString());
NUnit.Framework.Assert.IsTrue(fnode.IsDirectory());
NUnit.Framework.Assert.IsTrue(!fnode.IsQuotaSet());
// Create file of size 2 * BLOCKSIZE under directory "foo"
long file1Len = Blocksize * 2;
int bufLen = Blocksize / 16;
DFSTestUtil.CreateFile(dfs, createdFile1, bufLen, file1Len, Blocksize, Replication
, seed);
// Verify getContentSummary without any quota set
// Expect no type quota and usage information available
ContentSummary cs = dfs.GetContentSummary(foo);
NUnit.Framework.Assert.AreEqual(cs.GetSpaceConsumed(), file1Len * Replication);
foreach (StorageType t in StorageType.Values())
{
NUnit.Framework.Assert.AreEqual(cs.GetTypeConsumed(t), 0);
NUnit.Framework.Assert.AreEqual(cs.GetTypeQuota(t), -1);
}
}
/// <exception cref="System.Exception"/>
public virtual void TestQuotaByStorageTypeWithFileCreateDelete()
{
Path foo = new Path(dir, "foo");
Path createdFile1 = new Path(foo, "created_file1.data");
dfs.Mkdirs(foo);
dfs.SetStoragePolicy(foo, HdfsConstants.OnessdStoragePolicyName);
// set quota by storage type on directory "foo"
dfs.SetQuotaByStorageType(foo, StorageType.Ssd, Blocksize * 10);
INode fnode = fsdir.GetINode4Write(foo.ToString());
NUnit.Framework.Assert.IsTrue(fnode.IsDirectory());
NUnit.Framework.Assert.IsTrue(fnode.IsQuotaSet());
// Create file of size 2.5 * BLOCKSIZE under directory "foo"
long file1Len = Blocksize * 2 + Blocksize / 2;
int bufLen = Blocksize / 16;
DFSTestUtil.CreateFile(dfs, createdFile1, bufLen, file1Len, Blocksize, Replication
, seed);
// Verify space consumed and remaining quota
long storageTypeConsumed = fnode.AsDirectory().GetDirectoryWithQuotaFeature().GetSpaceConsumed
().GetTypeSpaces().Get(StorageType.Ssd);
NUnit.Framework.Assert.AreEqual(file1Len, storageTypeConsumed);
// Delete file and verify the consumed space of the storage type is updated
dfs.Delete(createdFile1, false);
storageTypeConsumed = fnode.AsDirectory().GetDirectoryWithQuotaFeature().GetSpaceConsumed
().GetTypeSpaces().Get(StorageType.Ssd);
NUnit.Framework.Assert.AreEqual(0, storageTypeConsumed);
QuotaCounts counts = new QuotaCounts.Builder().Build();
fnode.ComputeQuotaUsage(fsn.GetBlockManager().GetStoragePolicySuite(), counts, true
);
NUnit.Framework.Assert.AreEqual(fnode.DumpTreeRecursively().ToString(), 0, counts
.GetTypeSpaces().Get(StorageType.Ssd));
ContentSummary cs = dfs.GetContentSummary(foo);
NUnit.Framework.Assert.AreEqual(cs.GetSpaceConsumed(), 0);
NUnit.Framework.Assert.AreEqual(cs.GetTypeConsumed(StorageType.Ssd), 0);
NUnit.Framework.Assert.AreEqual(cs.GetTypeConsumed(StorageType.Disk), 0);
}
/// <exception cref="System.Exception"/>
public virtual void TestQuotaByStorageTypeWithFileCreateTruncate()
{
Path foo = new Path(dir, "foo");
Path createdFile1 = new Path(foo, "created_file1.data");
dfs.Mkdirs(foo);
// set storage policy on directory "foo" to ONESSD
dfs.SetStoragePolicy(foo, HdfsConstants.OnessdStoragePolicyName);
// set quota by storage type on directory "foo"
dfs.SetQuotaByStorageType(foo, StorageType.Ssd, Blocksize * 4);
INode fnode = fsdir.GetINode4Write(foo.ToString());
NUnit.Framework.Assert.IsTrue(fnode.IsDirectory());
NUnit.Framework.Assert.IsTrue(fnode.IsQuotaSet());
// Create file of size 2 * BLOCKSIZE under directory "foo"
long file1Len = Blocksize * 2;
int bufLen = Blocksize / 16;
DFSTestUtil.CreateFile(dfs, createdFile1, bufLen, file1Len, Blocksize, Replication
, seed);
// Verify SSD consumed before truncate
long ssdConsumed = fnode.AsDirectory().GetDirectoryWithQuotaFeature().GetSpaceConsumed
().GetTypeSpaces().Get(StorageType.Ssd);
NUnit.Framework.Assert.AreEqual(file1Len, ssdConsumed);
// Truncate file to 1 * BLOCKSIZE
int newFile1Len = Blocksize * 1;
dfs.Truncate(createdFile1, newFile1Len);
// Verify SSD consumed after truncate
ssdConsumed = fnode.AsDirectory().GetDirectoryWithQuotaFeature().GetSpaceConsumed
().GetTypeSpaces().Get(StorageType.Ssd);
NUnit.Framework.Assert.AreEqual(newFile1Len, ssdConsumed);
ContentSummary cs = dfs.GetContentSummary(foo);
NUnit.Framework.Assert.AreEqual(cs.GetSpaceConsumed(), newFile1Len * Replication);
NUnit.Framework.Assert.AreEqual(cs.GetTypeConsumed(StorageType.Ssd), newFile1Len);
NUnit.Framework.Assert.AreEqual(cs.GetTypeConsumed(StorageType.Disk), newFile1Len
* 2);
}