public virtual void TestQuotaCommands() { Configuration conf = new HdfsConfiguration(); // set a smaller block size so that we can test with smaller // Space quotas int DefaultBlockSize = 512; conf.SetLong(DFSConfigKeys.DfsBlockSizeKey, DefaultBlockSize); // Make it relinquish locks. When run serially, the result should // be identical. conf.SetInt(DFSConfigKeys.DfsContentSummaryLimitKey, 2); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(2).Build(); FileSystem fs = cluster.GetFileSystem(); NUnit.Framework.Assert.IsTrue("Not a HDFS: " + fs.GetUri(), fs is DistributedFileSystem ); DistributedFileSystem dfs = (DistributedFileSystem)fs; DFSAdmin admin = new DFSAdmin(conf); try { int fileLen = 1024; short replication = 5; long spaceQuota = fileLen * replication * 15 / 8; // 1: create a directory /test and set its quota to be 3 Path parent = new Path("/test"); NUnit.Framework.Assert.IsTrue(dfs.Mkdirs(parent)); string[] args = new string[] { "-setQuota", "3", parent.ToString() }; RunCommand(admin, args, false); //try setting space quota with a 'binary prefix' RunCommand(admin, false, "-setSpaceQuota", "2t", parent.ToString()); NUnit.Framework.Assert.AreEqual(2L << 40, dfs.GetContentSummary(parent).GetSpaceQuota ()); // set diskspace quota to 10000 RunCommand(admin, false, "-setSpaceQuota", System.Convert.ToString(spaceQuota), parent .ToString()); // 2: create directory /test/data0 Path childDir0 = new Path(parent, "data0"); NUnit.Framework.Assert.IsTrue(dfs.Mkdirs(childDir0)); // 3: create a file /test/datafile0 Path childFile0 = new Path(parent, "datafile0"); DFSTestUtil.CreateFile(fs, childFile0, fileLen, replication, 0); // 4: count -q /test ContentSummary c = dfs.GetContentSummary(parent); NUnit.Framework.Assert.AreEqual(c.GetFileCount() + c.GetDirectoryCount(), 3); NUnit.Framework.Assert.AreEqual(c.GetQuota(), 3); NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), fileLen * replication); NUnit.Framework.Assert.AreEqual(c.GetSpaceQuota(), spaceQuota); // 5: count -q /test/data0 c = dfs.GetContentSummary(childDir0); NUnit.Framework.Assert.AreEqual(c.GetFileCount() + c.GetDirectoryCount(), 1); NUnit.Framework.Assert.AreEqual(c.GetQuota(), -1); // check disk space consumed c = dfs.GetContentSummary(parent); NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), fileLen * replication); // 6: create a directory /test/data1 Path childDir1 = new Path(parent, "data1"); bool hasException = false; try { NUnit.Framework.Assert.IsFalse(dfs.Mkdirs(childDir1)); } catch (QuotaExceededException) { hasException = true; } NUnit.Framework.Assert.IsTrue(hasException); OutputStream fout; // 7: create a file /test/datafile1 Path childFile1 = new Path(parent, "datafile1"); hasException = false; try { fout = dfs.Create(childFile1); } catch (QuotaExceededException) { hasException = true; } NUnit.Framework.Assert.IsTrue(hasException); // 8: clear quota /test RunCommand(admin, new string[] { "-clrQuota", parent.ToString() }, false); c = dfs.GetContentSummary(parent); NUnit.Framework.Assert.AreEqual(c.GetQuota(), -1); NUnit.Framework.Assert.AreEqual(c.GetSpaceQuota(), spaceQuota); // 9: clear quota /test/data0 RunCommand(admin, new string[] { "-clrQuota", childDir0.ToString() }, false); c = dfs.GetContentSummary(childDir0); NUnit.Framework.Assert.AreEqual(c.GetQuota(), -1); // 10: create a file /test/datafile1 fout = dfs.Create(childFile1, replication); // 10.s: but writing fileLen bytes should result in an quota exception try { fout.Write(new byte[fileLen]); fout.Close(); NUnit.Framework.Assert.Fail(); } catch (QuotaExceededException) { IOUtils.CloseStream(fout); } //delete the file dfs.Delete(childFile1, false); // 9.s: clear diskspace quota RunCommand(admin, false, "-clrSpaceQuota", parent.ToString()); c = dfs.GetContentSummary(parent); NUnit.Framework.Assert.AreEqual(c.GetQuota(), -1); NUnit.Framework.Assert.AreEqual(c.GetSpaceQuota(), -1); // now creating childFile1 should succeed DFSTestUtil.CreateFile(dfs, childFile1, fileLen, replication, 0); // 11: set the quota of /test to be 1 // HADOOP-5872 - we can set quota even if it is immediately violated args = new string[] { "-setQuota", "1", parent.ToString() }; RunCommand(admin, args, false); RunCommand(admin, false, "-setSpaceQuota", Sharpen.Extensions.ToString(fileLen), args[2]); // for space quota // 12: set the quota of /test/data0 to be 1 args = new string[] { "-setQuota", "1", childDir0.ToString() }; RunCommand(admin, args, false); // 13: not able create a directory under data0 hasException = false; try { NUnit.Framework.Assert.IsFalse(dfs.Mkdirs(new Path(childDir0, "in"))); } catch (QuotaExceededException) { hasException = true; } NUnit.Framework.Assert.IsTrue(hasException); c = dfs.GetContentSummary(childDir0); NUnit.Framework.Assert.AreEqual(c.GetDirectoryCount() + c.GetFileCount(), 1); NUnit.Framework.Assert.AreEqual(c.GetQuota(), 1); // 14a: set quota on a non-existent directory Path nonExistentPath = new Path("/test1"); NUnit.Framework.Assert.IsFalse(dfs.Exists(nonExistentPath)); args = new string[] { "-setQuota", "1", nonExistentPath.ToString() }; RunCommand(admin, args, true); RunCommand(admin, true, "-setSpaceQuota", "1g", nonExistentPath.ToString()); // for space quota // 14b: set quota on a file NUnit.Framework.Assert.IsTrue(dfs.IsFile(childFile0)); args[1] = childFile0.ToString(); RunCommand(admin, args, true); // same for space quota RunCommand(admin, true, "-setSpaceQuota", "1t", args[1]); // 15a: clear quota on a file args[0] = "-clrQuota"; RunCommand(admin, args, true); RunCommand(admin, true, "-clrSpaceQuota", args[1]); // 15b: clear quota on a non-existent directory args[1] = nonExistentPath.ToString(); RunCommand(admin, args, true); RunCommand(admin, true, "-clrSpaceQuota", args[1]); // 16a: set the quota of /test to be 0 args = new string[] { "-setQuota", "0", parent.ToString() }; RunCommand(admin, args, true); RunCommand(admin, true, "-setSpaceQuota", "0", args[2]); // 16b: set the quota of /test to be -1 args[1] = "-1"; RunCommand(admin, args, true); RunCommand(admin, true, "-setSpaceQuota", args[1], args[2]); // 16c: set the quota of /test to be Long.MAX_VALUE+1 args[1] = (long.MaxValue + 1L).ToString(); RunCommand(admin, args, true); RunCommand(admin, true, "-setSpaceQuota", args[1], args[2]); // 16d: set the quota of /test to be a non integer args[1] = "33aa1.5"; RunCommand(admin, args, true); RunCommand(admin, true, "-setSpaceQuota", args[1], args[2]); // 16e: set space quota with a value larger than Long.MAX_VALUE RunCommand(admin, true, "-setSpaceQuota", (long.MaxValue / 1024 / 1024 + 1024) + "m", args[2]); // 17: setQuota by a non-administrator string username = "******"; UserGroupInformation ugi = UserGroupInformation.CreateUserForTesting(username, new string[] { "groupyy" }); string[] args2 = args.MemberwiseClone(); // need final ref for doAs block ugi.DoAs(new _PrivilegedExceptionAction_275(this, username, conf, args2, parent)); // 18: clrQuota by a non-administrator // 19: clrQuota on the root directory ("/") should fail RunCommand(admin, true, "-clrQuota", "/"); // 20: setQuota on the root directory ("/") should succeed RunCommand(admin, false, "-setQuota", "1000000", "/"); RunCommand(admin, true, "-clrQuota", "/"); RunCommand(admin, false, "-clrSpaceQuota", "/"); RunCommand(admin, new string[] { "-clrQuota", parent.ToString() }, false); RunCommand(admin, false, "-clrSpaceQuota", parent.ToString()); // 2: create directory /test/data2 Path childDir2 = new Path(parent, "data2"); NUnit.Framework.Assert.IsTrue(dfs.Mkdirs(childDir2)); Path childFile2 = new Path(childDir2, "datafile2"); Path childFile3 = new Path(childDir2, "datafile3"); long spaceQuota2 = DefaultBlockSize * replication; long fileLen2 = DefaultBlockSize; // set space quota to a real low value RunCommand(admin, false, "-setSpaceQuota", System.Convert.ToString(spaceQuota2), childDir2.ToString()); // clear space quota RunCommand(admin, false, "-clrSpaceQuota", childDir2.ToString()); // create a file that is greater than the size of space quota DFSTestUtil.CreateFile(fs, childFile2, fileLen2, replication, 0); // now set space quota again. This should succeed RunCommand(admin, false, "-setSpaceQuota", System.Convert.ToString(spaceQuota2), childDir2.ToString()); hasException = false; try { DFSTestUtil.CreateFile(fs, childFile3, fileLen2, replication, 0); } catch (DSQuotaExceededException) { hasException = true; } NUnit.Framework.Assert.IsTrue(hasException); // now test the same for root Path childFile4 = new Path("/", "datafile2"); Path childFile5 = new Path("/", "datafile3"); RunCommand(admin, true, "-clrQuota", "/"); RunCommand(admin, false, "-clrSpaceQuota", "/"); // set space quota to a real low value RunCommand(admin, false, "-setSpaceQuota", System.Convert.ToString(spaceQuota2), "/"); RunCommand(admin, false, "-clrSpaceQuota", "/"); DFSTestUtil.CreateFile(fs, childFile4, fileLen2, replication, 0); RunCommand(admin, false, "-setSpaceQuota", System.Convert.ToString(spaceQuota2), "/"); hasException = false; try { DFSTestUtil.CreateFile(fs, childFile5, fileLen2, replication, 0); } catch (DSQuotaExceededException) { hasException = true; } NUnit.Framework.Assert.IsTrue(hasException); NUnit.Framework.Assert.AreEqual(4, cluster.GetNamesystem().GetFSDirectory().GetYieldCount ()); } finally { cluster.Shutdown(); } }