public virtual void TestQuotaCommands()
{
Configuration conf = new HdfsConfiguration();
// set a smaller block size so that we can test with smaller
// Space quotas
int DefaultBlockSize = 512;
conf.SetLong(DFSConfigKeys.DfsBlockSizeKey, DefaultBlockSize);
// Make it relinquish locks. When run serially, the result should
// be identical.
conf.SetInt(DFSConfigKeys.DfsContentSummaryLimitKey, 2);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(2).Build();
FileSystem fs = cluster.GetFileSystem();
NUnit.Framework.Assert.IsTrue("Not a HDFS: " + fs.GetUri(), fs is DistributedFileSystem
);
DistributedFileSystem dfs = (DistributedFileSystem)fs;
DFSAdmin admin = new DFSAdmin(conf);
try
{
int fileLen = 1024;
short replication = 5;
long spaceQuota = fileLen * replication * 15 / 8;
// 1: create a directory /test and set its quota to be 3
Path parent = new Path("/test");
NUnit.Framework.Assert.IsTrue(dfs.Mkdirs(parent));
string[] args = new string[] { "-setQuota", "3", parent.ToString() };
RunCommand(admin, args, false);
//try setting space quota with a 'binary prefix'
RunCommand(admin, false, "-setSpaceQuota", "2t", parent.ToString());
NUnit.Framework.Assert.AreEqual(2L << 40, dfs.GetContentSummary(parent).GetSpaceQuota
());
// set diskspace quota to 10000
RunCommand(admin, false, "-setSpaceQuota", System.Convert.ToString(spaceQuota), parent
.ToString());
// 2: create directory /test/data0
Path childDir0 = new Path(parent, "data0");
NUnit.Framework.Assert.IsTrue(dfs.Mkdirs(childDir0));
// 3: create a file /test/datafile0
Path childFile0 = new Path(parent, "datafile0");
DFSTestUtil.CreateFile(fs, childFile0, fileLen, replication, 0);
// 4: count -q /test
ContentSummary c = dfs.GetContentSummary(parent);
NUnit.Framework.Assert.AreEqual(c.GetFileCount() + c.GetDirectoryCount(), 3);
NUnit.Framework.Assert.AreEqual(c.GetQuota(), 3);
NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), fileLen * replication);
NUnit.Framework.Assert.AreEqual(c.GetSpaceQuota(), spaceQuota);
// 5: count -q /test/data0
c = dfs.GetContentSummary(childDir0);
NUnit.Framework.Assert.AreEqual(c.GetFileCount() + c.GetDirectoryCount(), 1);
NUnit.Framework.Assert.AreEqual(c.GetQuota(), -1);
// check disk space consumed
c = dfs.GetContentSummary(parent);
NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), fileLen * replication);
// 6: create a directory /test/data1
Path childDir1 = new Path(parent, "data1");
bool hasException = false;
try
{
NUnit.Framework.Assert.IsFalse(dfs.Mkdirs(childDir1));
}
catch (QuotaExceededException)
{
hasException = true;
}
NUnit.Framework.Assert.IsTrue(hasException);
OutputStream fout;
// 7: create a file /test/datafile1
Path childFile1 = new Path(parent, "datafile1");
hasException = false;
try
{
fout = dfs.Create(childFile1);
}
catch (QuotaExceededException)
{
hasException = true;
}
NUnit.Framework.Assert.IsTrue(hasException);
// 8: clear quota /test
RunCommand(admin, new string[] { "-clrQuota", parent.ToString() }, false);
c = dfs.GetContentSummary(parent);
NUnit.Framework.Assert.AreEqual(c.GetQuota(), -1);
NUnit.Framework.Assert.AreEqual(c.GetSpaceQuota(), spaceQuota);
// 9: clear quota /test/data0
RunCommand(admin, new string[] { "-clrQuota", childDir0.ToString() }, false);
c = dfs.GetContentSummary(childDir0);
NUnit.Framework.Assert.AreEqual(c.GetQuota(), -1);
// 10: create a file /test/datafile1
fout = dfs.Create(childFile1, replication);
// 10.s: but writing fileLen bytes should result in an quota exception
try
{
fout.Write(new byte[fileLen]);
fout.Close();
NUnit.Framework.Assert.Fail();
}
catch (QuotaExceededException)
{
IOUtils.CloseStream(fout);
}
//delete the file
dfs.Delete(childFile1, false);
// 9.s: clear diskspace quota
RunCommand(admin, false, "-clrSpaceQuota", parent.ToString());
c = dfs.GetContentSummary(parent);
NUnit.Framework.Assert.AreEqual(c.GetQuota(), -1);
NUnit.Framework.Assert.AreEqual(c.GetSpaceQuota(), -1);
// now creating childFile1 should succeed
DFSTestUtil.CreateFile(dfs, childFile1, fileLen, replication, 0);
// 11: set the quota of /test to be 1
// HADOOP-5872 - we can set quota even if it is immediately violated
args = new string[] { "-setQuota", "1", parent.ToString() };
RunCommand(admin, args, false);
RunCommand(admin, false, "-setSpaceQuota", Sharpen.Extensions.ToString(fileLen),
args[2]);
// for space quota
// 12: set the quota of /test/data0 to be 1
args = new string[] { "-setQuota", "1", childDir0.ToString() };
RunCommand(admin, args, false);
// 13: not able create a directory under data0
hasException = false;
try
{
NUnit.Framework.Assert.IsFalse(dfs.Mkdirs(new Path(childDir0, "in")));
}
catch (QuotaExceededException)
{
hasException = true;
}
NUnit.Framework.Assert.IsTrue(hasException);
c = dfs.GetContentSummary(childDir0);
NUnit.Framework.Assert.AreEqual(c.GetDirectoryCount() + c.GetFileCount(), 1);
NUnit.Framework.Assert.AreEqual(c.GetQuota(), 1);
// 14a: set quota on a non-existent directory
Path nonExistentPath = new Path("/test1");
NUnit.Framework.Assert.IsFalse(dfs.Exists(nonExistentPath));
args = new string[] { "-setQuota", "1", nonExistentPath.ToString() };
RunCommand(admin, args, true);
RunCommand(admin, true, "-setSpaceQuota", "1g", nonExistentPath.ToString());
// for space quota
// 14b: set quota on a file
NUnit.Framework.Assert.IsTrue(dfs.IsFile(childFile0));
args[1] = childFile0.ToString();
RunCommand(admin, args, true);
// same for space quota
RunCommand(admin, true, "-setSpaceQuota", "1t", args[1]);
// 15a: clear quota on a file
args[0] = "-clrQuota";
RunCommand(admin, args, true);
RunCommand(admin, true, "-clrSpaceQuota", args[1]);
// 15b: clear quota on a non-existent directory
args[1] = nonExistentPath.ToString();
RunCommand(admin, args, true);
RunCommand(admin, true, "-clrSpaceQuota", args[1]);
// 16a: set the quota of /test to be 0
args = new string[] { "-setQuota", "0", parent.ToString() };
RunCommand(admin, args, true);
RunCommand(admin, true, "-setSpaceQuota", "0", args[2]);
// 16b: set the quota of /test to be -1
args[1] = "-1";
RunCommand(admin, args, true);
RunCommand(admin, true, "-setSpaceQuota", args[1], args[2]);
// 16c: set the quota of /test to be Long.MAX_VALUE+1
args[1] = (long.MaxValue + 1L).ToString();
RunCommand(admin, args, true);
RunCommand(admin, true, "-setSpaceQuota", args[1], args[2]);
// 16d: set the quota of /test to be a non integer
args[1] = "33aa1.5";
RunCommand(admin, args, true);
RunCommand(admin, true, "-setSpaceQuota", args[1], args[2]);
// 16e: set space quota with a value larger than Long.MAX_VALUE
RunCommand(admin, true, "-setSpaceQuota", (long.MaxValue / 1024 / 1024 + 1024) +
"m", args[2]);
// 17: setQuota by a non-administrator
string username = "******";
UserGroupInformation ugi = UserGroupInformation.CreateUserForTesting(username, new
string[] { "groupyy" });
string[] args2 = args.MemberwiseClone();
// need final ref for doAs block
ugi.DoAs(new _PrivilegedExceptionAction_275(this, username, conf, args2, parent));
// 18: clrQuota by a non-administrator
// 19: clrQuota on the root directory ("/") should fail
RunCommand(admin, true, "-clrQuota", "/");
// 20: setQuota on the root directory ("/") should succeed
RunCommand(admin, false, "-setQuota", "1000000", "/");
RunCommand(admin, true, "-clrQuota", "/");
RunCommand(admin, false, "-clrSpaceQuota", "/");
RunCommand(admin, new string[] { "-clrQuota", parent.ToString() }, false);
RunCommand(admin, false, "-clrSpaceQuota", parent.ToString());
// 2: create directory /test/data2
Path childDir2 = new Path(parent, "data2");
NUnit.Framework.Assert.IsTrue(dfs.Mkdirs(childDir2));
Path childFile2 = new Path(childDir2, "datafile2");
Path childFile3 = new Path(childDir2, "datafile3");
long spaceQuota2 = DefaultBlockSize * replication;
long fileLen2 = DefaultBlockSize;
// set space quota to a real low value
RunCommand(admin, false, "-setSpaceQuota", System.Convert.ToString(spaceQuota2),
childDir2.ToString());
// clear space quota
RunCommand(admin, false, "-clrSpaceQuota", childDir2.ToString());
// create a file that is greater than the size of space quota
DFSTestUtil.CreateFile(fs, childFile2, fileLen2, replication, 0);
// now set space quota again. This should succeed
RunCommand(admin, false, "-setSpaceQuota", System.Convert.ToString(spaceQuota2),
childDir2.ToString());
hasException = false;
try
{
DFSTestUtil.CreateFile(fs, childFile3, fileLen2, replication, 0);
}
catch (DSQuotaExceededException)
{
hasException = true;
}
NUnit.Framework.Assert.IsTrue(hasException);
// now test the same for root
Path childFile4 = new Path("/", "datafile2");
Path childFile5 = new Path("/", "datafile3");
RunCommand(admin, true, "-clrQuota", "/");
RunCommand(admin, false, "-clrSpaceQuota", "/");
// set space quota to a real low value
RunCommand(admin, false, "-setSpaceQuota", System.Convert.ToString(spaceQuota2),
"/");
RunCommand(admin, false, "-clrSpaceQuota", "/");
DFSTestUtil.CreateFile(fs, childFile4, fileLen2, replication, 0);
RunCommand(admin, false, "-setSpaceQuota", System.Convert.ToString(spaceQuota2),
"/");
hasException = false;
try
{
DFSTestUtil.CreateFile(fs, childFile5, fileLen2, replication, 0);
}
catch (DSQuotaExceededException)
{
hasException = true;
}
NUnit.Framework.Assert.IsTrue(hasException);
NUnit.Framework.Assert.AreEqual(4, cluster.GetNamesystem().GetFSDirectory().GetYieldCount
());
}
finally
{
cluster.Shutdown();
}
}