public virtual void TestConcatWithQuotaDecrease()
{
short srcRepl = 3;
// note this is different with REPL_FACTOR
int srcNum = 10;
Path foo = new Path("/foo");
Path[] srcs = new Path[srcNum];
Path target = new Path(foo, "target");
DFSTestUtil.CreateFile(dfs, target, blockSize, ReplFactor, 0L);
dfs.SetQuota(foo, long.MaxValue - 1, long.MaxValue - 1);
for (int i = 0; i < srcNum; i++)
{
srcs[i] = new Path(foo, "src" + i);
DFSTestUtil.CreateFile(dfs, srcs[i], blockSize * 2, srcRepl, 0L);
}
ContentSummary summary = dfs.GetContentSummary(foo);
NUnit.Framework.Assert.AreEqual(11, summary.GetFileCount());
NUnit.Framework.Assert.AreEqual(blockSize * ReplFactor + blockSize * 2 * srcRepl
* srcNum, summary.GetSpaceConsumed());
dfs.Concat(target, srcs);
summary = dfs.GetContentSummary(foo);
NUnit.Framework.Assert.AreEqual(1, summary.GetFileCount());
NUnit.Framework.Assert.AreEqual(blockSize * ReplFactor + blockSize * 2 * ReplFactor
* srcNum, summary.GetSpaceConsumed());
}
/// <summary>Test if the quota can be correctly updated for append</summary>
/// <exception cref="System.Exception"/>
public virtual void TestUpdateQuotaForAppend()
{
Path foo = new Path(dir, "foo");
Path bar = new Path(foo, "bar");
long currentFileLen = Blocksize;
DFSTestUtil.CreateFile(dfs, bar, currentFileLen, Replication, seed);
dfs.SetQuota(foo, long.MaxValue - 1, long.MaxValue - 1);
// append half of the block data, the previous file length is at block
// boundary
DFSTestUtil.AppendFile(dfs, bar, Blocksize / 2);
currentFileLen += (Blocksize / 2);
INodeDirectory fooNode = fsdir.GetINode4Write(foo.ToString()).AsDirectory();
NUnit.Framework.Assert.IsTrue(fooNode.IsQuotaSet());
QuotaCounts quota = fooNode.GetDirectoryWithQuotaFeature().GetSpaceConsumed();
long ns = quota.GetNameSpace();
long ds = quota.GetStorageSpace();
NUnit.Framework.Assert.AreEqual(2, ns);
// foo and bar
NUnit.Framework.Assert.AreEqual(currentFileLen * Replication, ds);
ContentSummary c = dfs.GetContentSummary(foo);
NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), ds);
// append another block, the previous file length is not at block boundary
DFSTestUtil.AppendFile(dfs, bar, Blocksize);
currentFileLen += Blocksize;
quota = fooNode.GetDirectoryWithQuotaFeature().GetSpaceConsumed();
ns = quota.GetNameSpace();
ds = quota.GetStorageSpace();
NUnit.Framework.Assert.AreEqual(2, ns);
// foo and bar
NUnit.Framework.Assert.AreEqual(currentFileLen * Replication, ds);
c = dfs.GetContentSummary(foo);
NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), ds);
// append several blocks
DFSTestUtil.AppendFile(dfs, bar, Blocksize * 3 + Blocksize / 8);
currentFileLen += (Blocksize * 3 + Blocksize / 8);
quota = fooNode.GetDirectoryWithQuotaFeature().GetSpaceConsumed();
ns = quota.GetNameSpace();
ds = quota.GetStorageSpace();
NUnit.Framework.Assert.AreEqual(2, ns);
// foo and bar
NUnit.Framework.Assert.AreEqual(currentFileLen * Replication, ds);
c = dfs.GetContentSummary(foo);
NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), ds);
}
/// <exception cref="System.Exception"/>
public virtual void TestContentSummaryWithoutQuotaByStorageType()
{
Path foo = new Path(dir, "foo");
Path createdFile1 = new Path(foo, "created_file1.data");
dfs.Mkdirs(foo);
// set storage policy on directory "foo" to ONESSD
dfs.SetStoragePolicy(foo, HdfsConstants.OnessdStoragePolicyName);
INode fnode = fsdir.GetINode4Write(foo.ToString());
NUnit.Framework.Assert.IsTrue(fnode.IsDirectory());
NUnit.Framework.Assert.IsTrue(!fnode.IsQuotaSet());
// Create file of size 2 * BLOCKSIZE under directory "foo"
long file1Len = Blocksize * 2;
int bufLen = Blocksize / 16;
DFSTestUtil.CreateFile(dfs, createdFile1, bufLen, file1Len, Blocksize, Replication
, seed);
// Verify getContentSummary without any quota set
ContentSummary cs = dfs.GetContentSummary(foo);
NUnit.Framework.Assert.AreEqual(cs.GetSpaceConsumed(), file1Len * Replication);
NUnit.Framework.Assert.AreEqual(cs.GetTypeConsumed(StorageType.Ssd), file1Len);
NUnit.Framework.Assert.AreEqual(cs.GetTypeConsumed(StorageType.Disk), file1Len *
2);
}
/// <exception cref="System.Exception"/>
public Void Run()
{
string path = ServletUtil.GetDecodedPath(request, "/contentSummary");
PrintWriter @out = response.GetWriter();
XMLOutputter xml = new XMLOutputter(@out, "UTF-8");
xml.Declaration();
try
{
ClientProtocol nnproxy = this._enclosing.CreateNameNodeProxy();
ContentSummary cs = nnproxy.GetContentSummary(path);
xml.StartTag(typeof(ContentSummary).FullName);
if (cs != null)
{
xml.Attribute("length", string.Empty + cs.GetLength());
xml.Attribute("fileCount", string.Empty + cs.GetFileCount());
xml.Attribute("directoryCount", string.Empty + cs.GetDirectoryCount());
xml.Attribute("quota", string.Empty + cs.GetQuota());
xml.Attribute("spaceConsumed", string.Empty + cs.GetSpaceConsumed());
xml.Attribute("spaceQuota", string.Empty + cs.GetSpaceQuota());
}
xml.EndTag();
}
catch (IOException ioe)
{
this._enclosing.WriteXml(ioe, path, xml);
}
xml.EndDocument();
return(null);
}
/// <exception cref="System.Exception"/>
private void TestContentSummary()
{
FileSystem fs = FileSystem.Get(GetProxiedFSConf());
Path path = new Path(GetProxiedFSTestDir(), "foo.txt");
OutputStream os = fs.Create(path);
os.Write(1);
os.Close();
ContentSummary hdfsContentSummary = fs.GetContentSummary(path);
fs.Close();
fs = GetHttpFSFileSystem();
ContentSummary httpContentSummary = fs.GetContentSummary(path);
fs.Close();
NUnit.Framework.Assert.AreEqual(httpContentSummary.GetDirectoryCount(), hdfsContentSummary
.GetDirectoryCount());
NUnit.Framework.Assert.AreEqual(httpContentSummary.GetFileCount(), hdfsContentSummary
.GetFileCount());
NUnit.Framework.Assert.AreEqual(httpContentSummary.GetLength(), hdfsContentSummary
.GetLength());
NUnit.Framework.Assert.AreEqual(httpContentSummary.GetQuota(), hdfsContentSummary
.GetQuota());
NUnit.Framework.Assert.AreEqual(httpContentSummary.GetSpaceConsumed(), hdfsContentSummary
.GetSpaceConsumed());
NUnit.Framework.Assert.AreEqual(httpContentSummary.GetSpaceQuota(), hdfsContentSummary
.GetSpaceQuota());
}
/// <exception cref="System.Exception"/>
public virtual void TestContentSummaryWithoutStoragePolicy()
{
Path foo = new Path(dir, "foo");
Path createdFile1 = new Path(foo, "created_file1.data");
dfs.Mkdirs(foo);
INode fnode = fsdir.GetINode4Write(foo.ToString());
NUnit.Framework.Assert.IsTrue(fnode.IsDirectory());
NUnit.Framework.Assert.IsTrue(!fnode.IsQuotaSet());
// Create file of size 2 * BLOCKSIZE under directory "foo"
long file1Len = Blocksize * 2;
int bufLen = Blocksize / 16;
DFSTestUtil.CreateFile(dfs, createdFile1, bufLen, file1Len, Blocksize, Replication
, seed);
// Verify getContentSummary without any quota set
// Expect no type quota and usage information available
ContentSummary cs = dfs.GetContentSummary(foo);
NUnit.Framework.Assert.AreEqual(cs.GetSpaceConsumed(), file1Len * Replication);
foreach (StorageType t in StorageType.Values())
{
NUnit.Framework.Assert.AreEqual(cs.GetTypeConsumed(t), 0);
NUnit.Framework.Assert.AreEqual(cs.GetTypeQuota(t), -1);
}
}
/// <exception cref="System.Exception"/>
public virtual void TestQuotaByStorageTypeWithSnapshot()
{
Path sub1 = new Path(dir, "Sub1");
dfs.Mkdirs(sub1);
// Setup ONE_SSD policy and SSD quota of 4 * BLOCKSIZE on sub1
dfs.SetStoragePolicy(sub1, HdfsConstants.OnessdStoragePolicyName);
dfs.SetQuotaByStorageType(sub1, StorageType.Ssd, 4 * Blocksize);
INode sub1Node = fsdir.GetINode4Write(sub1.ToString());
NUnit.Framework.Assert.IsTrue(sub1Node.IsDirectory());
NUnit.Framework.Assert.IsTrue(sub1Node.IsQuotaSet());
// Create file1 of size 2 * BLOCKSIZE under sub1
Path file1 = new Path(sub1, "file1");
long file1Len = 2 * Blocksize;
DFSTestUtil.CreateFile(dfs, file1, file1Len, Replication, seed);
// Create snapshot on sub1 named s1
SnapshotTestHelper.CreateSnapshot(dfs, sub1, "s1");
// Verify sub1 SSD usage is unchanged after creating snapshot s1
long ssdConsumed = sub1Node.AsDirectory().GetDirectoryWithQuotaFeature().GetSpaceConsumed
().GetTypeSpaces().Get(StorageType.Ssd);
NUnit.Framework.Assert.AreEqual(file1Len, ssdConsumed);
// Delete file1
dfs.Delete(file1, false);
// Verify sub1 SSD usage is unchanged due to the existence of snapshot s1
ssdConsumed = sub1Node.AsDirectory().GetDirectoryWithQuotaFeature().GetSpaceConsumed
().GetTypeSpaces().Get(StorageType.Ssd);
NUnit.Framework.Assert.AreEqual(file1Len, ssdConsumed);
QuotaCounts counts1 = new QuotaCounts.Builder().Build();
sub1Node.ComputeQuotaUsage(fsn.GetBlockManager().GetStoragePolicySuite(), counts1
, true);
NUnit.Framework.Assert.AreEqual(sub1Node.DumpTreeRecursively().ToString(), file1Len
, counts1.GetTypeSpaces().Get(StorageType.Ssd));
ContentSummary cs1 = dfs.GetContentSummary(sub1);
NUnit.Framework.Assert.AreEqual(cs1.GetSpaceConsumed(), file1Len * Replication);
NUnit.Framework.Assert.AreEqual(cs1.GetTypeConsumed(StorageType.Ssd), file1Len);
NUnit.Framework.Assert.AreEqual(cs1.GetTypeConsumed(StorageType.Disk), file1Len *
2);
// Delete the snapshot s1
dfs.DeleteSnapshot(sub1, "s1");
// Verify sub1 SSD usage is fully reclaimed and changed to 0
ssdConsumed = sub1Node.AsDirectory().GetDirectoryWithQuotaFeature().GetSpaceConsumed
().GetTypeSpaces().Get(StorageType.Ssd);
NUnit.Framework.Assert.AreEqual(0, ssdConsumed);
QuotaCounts counts2 = new QuotaCounts.Builder().Build();
sub1Node.ComputeQuotaUsage(fsn.GetBlockManager().GetStoragePolicySuite(), counts2
, true);
NUnit.Framework.Assert.AreEqual(sub1Node.DumpTreeRecursively().ToString(), 0, counts2
.GetTypeSpaces().Get(StorageType.Ssd));
ContentSummary cs2 = dfs.GetContentSummary(sub1);
NUnit.Framework.Assert.AreEqual(cs2.GetSpaceConsumed(), 0);
NUnit.Framework.Assert.AreEqual(cs2.GetTypeConsumed(StorageType.Ssd), 0);
NUnit.Framework.Assert.AreEqual(cs2.GetTypeConsumed(StorageType.Disk), 0);
}
/// <summary>
/// Test that quotas are properly tracked by the standby through
/// create, append, delete.
/// </summary>
/// <exception cref="System.Exception"/>
public virtual void TestQuotasTrackedOnStandby()
{
fs.Mkdirs(TestDir);
DistributedFileSystem dfs = (DistributedFileSystem)fs;
dfs.SetQuota(TestDir, NsQuota, DsQuota);
long expectedSize = 3 * BlockSize + BlockSize / 2;
DFSTestUtil.CreateFile(fs, TestFile, expectedSize, (short)1, 1L);
HATestUtil.WaitForStandbyToCatchUp(nn0, nn1);
ContentSummary cs = nn1.GetRpcServer().GetContentSummary(TestDirStr);
NUnit.Framework.Assert.AreEqual(NsQuota, cs.GetQuota());
NUnit.Framework.Assert.AreEqual(DsQuota, cs.GetSpaceQuota());
NUnit.Framework.Assert.AreEqual(expectedSize, cs.GetSpaceConsumed());
NUnit.Framework.Assert.AreEqual(1, cs.GetDirectoryCount());
NUnit.Framework.Assert.AreEqual(1, cs.GetFileCount());
// Append to the file and make sure quota is updated correctly.
FSDataOutputStream stm = fs.Append(TestFile);
try
{
byte[] data = new byte[(int)(BlockSize * 3 / 2)];
stm.Write(data);
expectedSize += data.Length;
}
finally
{
IOUtils.CloseStream(stm);
}
HATestUtil.WaitForStandbyToCatchUp(nn0, nn1);
cs = nn1.GetRpcServer().GetContentSummary(TestDirStr);
NUnit.Framework.Assert.AreEqual(NsQuota, cs.GetQuota());
NUnit.Framework.Assert.AreEqual(DsQuota, cs.GetSpaceQuota());
NUnit.Framework.Assert.AreEqual(expectedSize, cs.GetSpaceConsumed());
NUnit.Framework.Assert.AreEqual(1, cs.GetDirectoryCount());
NUnit.Framework.Assert.AreEqual(1, cs.GetFileCount());
fs.Delete(TestFile, true);
expectedSize = 0;
HATestUtil.WaitForStandbyToCatchUp(nn0, nn1);
cs = nn1.GetRpcServer().GetContentSummary(TestDirStr);
NUnit.Framework.Assert.AreEqual(NsQuota, cs.GetQuota());
NUnit.Framework.Assert.AreEqual(DsQuota, cs.GetSpaceQuota());
NUnit.Framework.Assert.AreEqual(expectedSize, cs.GetSpaceConsumed());
NUnit.Framework.Assert.AreEqual(1, cs.GetDirectoryCount());
NUnit.Framework.Assert.AreEqual(0, cs.GetFileCount());
}
public virtual void TestConcatWithQuotaIncrease()
{
short repl = 3;
int srcNum = 10;
Path foo = new Path("/foo");
Path bar = new Path(foo, "bar");
Path[] srcs = new Path[srcNum];
Path target = new Path(bar, "target");
DFSTestUtil.CreateFile(dfs, target, blockSize, repl, 0L);
long dsQuota = blockSize * repl + blockSize * srcNum * ReplFactor;
dfs.SetQuota(foo, long.MaxValue - 1, dsQuota);
for (int i = 0; i < srcNum; i++)
{
srcs[i] = new Path(bar, "src" + i);
DFSTestUtil.CreateFile(dfs, srcs[i], blockSize, ReplFactor, 0L);
}
ContentSummary summary = dfs.GetContentSummary(bar);
NUnit.Framework.Assert.AreEqual(11, summary.GetFileCount());
NUnit.Framework.Assert.AreEqual(dsQuota, summary.GetSpaceConsumed());
try
{
dfs.Concat(target, srcs);
NUnit.Framework.Assert.Fail("QuotaExceededException expected");
}
catch (RemoteException e)
{
NUnit.Framework.Assert.IsTrue(e.UnwrapRemoteException() is QuotaExceededException
);
}
dfs.SetQuota(foo, long.MaxValue - 1, long.MaxValue - 1);
dfs.Concat(target, srcs);
summary = dfs.GetContentSummary(bar);
NUnit.Framework.Assert.AreEqual(1, summary.GetFileCount());
NUnit.Framework.Assert.AreEqual(blockSize * repl * (srcNum + 1), summary.GetSpaceConsumed
());
}
/// <exception cref="System.Exception"/>
public virtual void TestQuotaByStorageTypeWithFileCreateRename()
{
Path foo = new Path(dir, "foo");
dfs.Mkdirs(foo);
Path createdFile1foo = new Path(foo, "created_file1.data");
Path bar = new Path(dir, "bar");
dfs.Mkdirs(bar);
Path createdFile1bar = new Path(bar, "created_file1.data");
// set storage policy on directory "foo" and "bar" to ONESSD
dfs.SetStoragePolicy(foo, HdfsConstants.OnessdStoragePolicyName);
dfs.SetStoragePolicy(bar, HdfsConstants.OnessdStoragePolicyName);
// set quota by storage type on directory "foo"
dfs.SetQuotaByStorageType(foo, StorageType.Ssd, Blocksize * 4);
dfs.SetQuotaByStorageType(bar, StorageType.Ssd, Blocksize * 2);
INode fnode = fsdir.GetINode4Write(foo.ToString());
NUnit.Framework.Assert.IsTrue(fnode.IsDirectory());
NUnit.Framework.Assert.IsTrue(fnode.IsQuotaSet());
// Create file of size 3 * BLOCKSIZE under directory "foo"
long file1Len = Blocksize * 3;
int bufLen = Blocksize / 16;
DFSTestUtil.CreateFile(dfs, createdFile1foo, bufLen, file1Len, Blocksize, Replication
, seed);
// Verify space consumed and remaining quota
long ssdConsumed = fnode.AsDirectory().GetDirectoryWithQuotaFeature().GetSpaceConsumed
().GetTypeSpaces().Get(StorageType.Ssd);
NUnit.Framework.Assert.AreEqual(file1Len, ssdConsumed);
// move file from foo to bar
try
{
dfs.Rename(createdFile1foo, createdFile1bar);
NUnit.Framework.Assert.Fail("Should have failed with QuotaByStorageTypeExceededException "
);
}
catch (Exception t)
{
Log.Info("Got expected exception ", t);
}
ContentSummary cs = dfs.GetContentSummary(foo);
NUnit.Framework.Assert.AreEqual(cs.GetSpaceConsumed(), file1Len * Replication);
NUnit.Framework.Assert.AreEqual(cs.GetTypeConsumed(StorageType.Ssd), file1Len);
NUnit.Framework.Assert.AreEqual(cs.GetTypeConsumed(StorageType.Disk), file1Len *
2);
}
/// <summary>Convert a ContentSummary to a Json string.</summary>
public static string ToJsonString(ContentSummary contentsummary)
{
if (contentsummary == null)
{
return(null);
}
IDictionary <string, object> m = new SortedDictionary <string, object>();
m["length"] = contentsummary.GetLength();
m["fileCount"] = contentsummary.GetFileCount();
m["directoryCount"] = contentsummary.GetDirectoryCount();
m["quota"] = contentsummary.GetQuota();
m["spaceConsumed"] = contentsummary.GetSpaceConsumed();
m["spaceQuota"] = contentsummary.GetSpaceQuota();
return(ToJsonString(typeof(ContentSummary), m));
}
/// <summary>
/// Converts a <code>ContentSummary</code> object into a JSON array
/// object.
/// </summary>
/// <param name="contentSummary">the content summary</param>
/// <returns>The JSON representation of the content summary.</returns>
private static IDictionary ContentSummaryToJSON(ContentSummary contentSummary)
{
IDictionary json = new LinkedHashMap();
json[HttpFSFileSystem.ContentSummaryDirectoryCountJson] = contentSummary.GetDirectoryCount
();
json[HttpFSFileSystem.ContentSummaryFileCountJson] = contentSummary.GetFileCount(
);
json[HttpFSFileSystem.ContentSummaryLengthJson] = contentSummary.GetLength();
json[HttpFSFileSystem.ContentSummaryQuotaJson] = contentSummary.GetQuota();
json[HttpFSFileSystem.ContentSummarySpaceConsumedJson] = contentSummary.GetSpaceConsumed
();
json[HttpFSFileSystem.ContentSummarySpaceQuotaJson] = contentSummary.GetSpaceQuota
();
IDictionary response = new LinkedHashMap();
response[HttpFSFileSystem.ContentSummaryJson] = json;
return(response);
}
/// <exception cref="System.Exception"/>
public virtual void TestQuotaByStorageTypeWithFileCreateDelete()
{
Path foo = new Path(dir, "foo");
Path createdFile1 = new Path(foo, "created_file1.data");
dfs.Mkdirs(foo);
dfs.SetStoragePolicy(foo, HdfsConstants.OnessdStoragePolicyName);
// set quota by storage type on directory "foo"
dfs.SetQuotaByStorageType(foo, StorageType.Ssd, Blocksize * 10);
INode fnode = fsdir.GetINode4Write(foo.ToString());
NUnit.Framework.Assert.IsTrue(fnode.IsDirectory());
NUnit.Framework.Assert.IsTrue(fnode.IsQuotaSet());
// Create file of size 2.5 * BLOCKSIZE under directory "foo"
long file1Len = Blocksize * 2 + Blocksize / 2;
int bufLen = Blocksize / 16;
DFSTestUtil.CreateFile(dfs, createdFile1, bufLen, file1Len, Blocksize, Replication
, seed);
// Verify space consumed and remaining quota
long storageTypeConsumed = fnode.AsDirectory().GetDirectoryWithQuotaFeature().GetSpaceConsumed
().GetTypeSpaces().Get(StorageType.Ssd);
NUnit.Framework.Assert.AreEqual(file1Len, storageTypeConsumed);
// Delete file and verify the consumed space of the storage type is updated
dfs.Delete(createdFile1, false);
storageTypeConsumed = fnode.AsDirectory().GetDirectoryWithQuotaFeature().GetSpaceConsumed
().GetTypeSpaces().Get(StorageType.Ssd);
NUnit.Framework.Assert.AreEqual(0, storageTypeConsumed);
QuotaCounts counts = new QuotaCounts.Builder().Build();
fnode.ComputeQuotaUsage(fsn.GetBlockManager().GetStoragePolicySuite(), counts, true
);
NUnit.Framework.Assert.AreEqual(fnode.DumpTreeRecursively().ToString(), 0, counts
.GetTypeSpaces().Get(StorageType.Ssd));
ContentSummary cs = dfs.GetContentSummary(foo);
NUnit.Framework.Assert.AreEqual(cs.GetSpaceConsumed(), 0);
NUnit.Framework.Assert.AreEqual(cs.GetTypeConsumed(StorageType.Ssd), 0);
NUnit.Framework.Assert.AreEqual(cs.GetTypeConsumed(StorageType.Disk), 0);
}
/// <exception cref="System.Exception"/>
public virtual void TestQuotaByStorageTypeWithFileCreateTruncate()
{
Path foo = new Path(dir, "foo");
Path createdFile1 = new Path(foo, "created_file1.data");
dfs.Mkdirs(foo);
// set storage policy on directory "foo" to ONESSD
dfs.SetStoragePolicy(foo, HdfsConstants.OnessdStoragePolicyName);
// set quota by storage type on directory "foo"
dfs.SetQuotaByStorageType(foo, StorageType.Ssd, Blocksize * 4);
INode fnode = fsdir.GetINode4Write(foo.ToString());
NUnit.Framework.Assert.IsTrue(fnode.IsDirectory());
NUnit.Framework.Assert.IsTrue(fnode.IsQuotaSet());
// Create file of size 2 * BLOCKSIZE under directory "foo"
long file1Len = Blocksize * 2;
int bufLen = Blocksize / 16;
DFSTestUtil.CreateFile(dfs, createdFile1, bufLen, file1Len, Blocksize, Replication
, seed);
// Verify SSD consumed before truncate
long ssdConsumed = fnode.AsDirectory().GetDirectoryWithQuotaFeature().GetSpaceConsumed
().GetTypeSpaces().Get(StorageType.Ssd);
NUnit.Framework.Assert.AreEqual(file1Len, ssdConsumed);
// Truncate file to 1 * BLOCKSIZE
int newFile1Len = Blocksize * 1;
dfs.Truncate(createdFile1, newFile1Len);
// Verify SSD consumed after truncate
ssdConsumed = fnode.AsDirectory().GetDirectoryWithQuotaFeature().GetSpaceConsumed
().GetTypeSpaces().Get(StorageType.Ssd);
NUnit.Framework.Assert.AreEqual(newFile1Len, ssdConsumed);
ContentSummary cs = dfs.GetContentSummary(foo);
NUnit.Framework.Assert.AreEqual(cs.GetSpaceConsumed(), newFile1Len * Replication);
NUnit.Framework.Assert.AreEqual(cs.GetTypeConsumed(StorageType.Ssd), newFile1Len);
NUnit.Framework.Assert.AreEqual(cs.GetTypeConsumed(StorageType.Disk), newFile1Len
* 2);
}
public virtual void TestTruncate()
{
short repl = 3;
int blockSize = 1024;
int numOfBlocks = 2;
Path dir = GetTestRootPath(fSys, "test/hadoop");
Path file = GetTestRootPath(fSys, "test/hadoop/file");
byte[] data = GetFileData(numOfBlocks, blockSize);
CreateFile(fSys, file, data, blockSize, repl);
int newLength = blockSize;
bool isReady = fSys.Truncate(file, newLength);
NUnit.Framework.Assert.IsTrue("Recovery is not expected.", isReady);
FileStatus fileStatus = fSys.GetFileStatus(file);
NUnit.Framework.Assert.AreEqual(fileStatus.GetLen(), newLength);
AppendTestUtil.CheckFullFile(fSys, file, newLength, data, file.ToString());
ContentSummary cs = fSys.GetContentSummary(dir);
NUnit.Framework.Assert.AreEqual("Bad disk space usage", cs.GetSpaceConsumed(), newLength
* repl);
NUnit.Framework.Assert.IsTrue("Deleted", fSys.Delete(dir, true));
}
public virtual void TestQuotaCommands()
{
Configuration conf = new HdfsConfiguration();
// set a smaller block size so that we can test with smaller
// Space quotas
int DefaultBlockSize = 512;
conf.SetLong(DFSConfigKeys.DfsBlockSizeKey, DefaultBlockSize);
// Make it relinquish locks. When run serially, the result should
// be identical.
conf.SetInt(DFSConfigKeys.DfsContentSummaryLimitKey, 2);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(2).Build();
FileSystem fs = cluster.GetFileSystem();
NUnit.Framework.Assert.IsTrue("Not a HDFS: " + fs.GetUri(), fs is DistributedFileSystem
);
DistributedFileSystem dfs = (DistributedFileSystem)fs;
DFSAdmin admin = new DFSAdmin(conf);
try
{
int fileLen = 1024;
short replication = 5;
long spaceQuota = fileLen * replication * 15 / 8;
// 1: create a directory /test and set its quota to be 3
Path parent = new Path("/test");
NUnit.Framework.Assert.IsTrue(dfs.Mkdirs(parent));
string[] args = new string[] { "-setQuota", "3", parent.ToString() };
RunCommand(admin, args, false);
//try setting space quota with a 'binary prefix'
RunCommand(admin, false, "-setSpaceQuota", "2t", parent.ToString());
NUnit.Framework.Assert.AreEqual(2L << 40, dfs.GetContentSummary(parent).GetSpaceQuota
());
// set diskspace quota to 10000
RunCommand(admin, false, "-setSpaceQuota", System.Convert.ToString(spaceQuota), parent
.ToString());
// 2: create directory /test/data0
Path childDir0 = new Path(parent, "data0");
NUnit.Framework.Assert.IsTrue(dfs.Mkdirs(childDir0));
// 3: create a file /test/datafile0
Path childFile0 = new Path(parent, "datafile0");
DFSTestUtil.CreateFile(fs, childFile0, fileLen, replication, 0);
// 4: count -q /test
ContentSummary c = dfs.GetContentSummary(parent);
NUnit.Framework.Assert.AreEqual(c.GetFileCount() + c.GetDirectoryCount(), 3);
NUnit.Framework.Assert.AreEqual(c.GetQuota(), 3);
NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), fileLen * replication);
NUnit.Framework.Assert.AreEqual(c.GetSpaceQuota(), spaceQuota);
// 5: count -q /test/data0
c = dfs.GetContentSummary(childDir0);
NUnit.Framework.Assert.AreEqual(c.GetFileCount() + c.GetDirectoryCount(), 1);
NUnit.Framework.Assert.AreEqual(c.GetQuota(), -1);
// check disk space consumed
c = dfs.GetContentSummary(parent);
NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), fileLen * replication);
// 6: create a directory /test/data1
Path childDir1 = new Path(parent, "data1");
bool hasException = false;
try
{
NUnit.Framework.Assert.IsFalse(dfs.Mkdirs(childDir1));
}
catch (QuotaExceededException)
{
hasException = true;
}
NUnit.Framework.Assert.IsTrue(hasException);
OutputStream fout;
// 7: create a file /test/datafile1
Path childFile1 = new Path(parent, "datafile1");
hasException = false;
try
{
fout = dfs.Create(childFile1);
}
catch (QuotaExceededException)
{
hasException = true;
}
NUnit.Framework.Assert.IsTrue(hasException);
// 8: clear quota /test
RunCommand(admin, new string[] { "-clrQuota", parent.ToString() }, false);
c = dfs.GetContentSummary(parent);
NUnit.Framework.Assert.AreEqual(c.GetQuota(), -1);
NUnit.Framework.Assert.AreEqual(c.GetSpaceQuota(), spaceQuota);
// 9: clear quota /test/data0
RunCommand(admin, new string[] { "-clrQuota", childDir0.ToString() }, false);
c = dfs.GetContentSummary(childDir0);
NUnit.Framework.Assert.AreEqual(c.GetQuota(), -1);
// 10: create a file /test/datafile1
fout = dfs.Create(childFile1, replication);
// 10.s: but writing fileLen bytes should result in an quota exception
try
{
fout.Write(new byte[fileLen]);
fout.Close();
NUnit.Framework.Assert.Fail();
}
catch (QuotaExceededException)
{
IOUtils.CloseStream(fout);
}
//delete the file
dfs.Delete(childFile1, false);
// 9.s: clear diskspace quota
RunCommand(admin, false, "-clrSpaceQuota", parent.ToString());
c = dfs.GetContentSummary(parent);
NUnit.Framework.Assert.AreEqual(c.GetQuota(), -1);
NUnit.Framework.Assert.AreEqual(c.GetSpaceQuota(), -1);
// now creating childFile1 should succeed
DFSTestUtil.CreateFile(dfs, childFile1, fileLen, replication, 0);
// 11: set the quota of /test to be 1
// HADOOP-5872 - we can set quota even if it is immediately violated
args = new string[] { "-setQuota", "1", parent.ToString() };
RunCommand(admin, args, false);
RunCommand(admin, false, "-setSpaceQuota", Sharpen.Extensions.ToString(fileLen),
args[2]);
// for space quota
// 12: set the quota of /test/data0 to be 1
args = new string[] { "-setQuota", "1", childDir0.ToString() };
RunCommand(admin, args, false);
// 13: not able create a directory under data0
hasException = false;
try
{
NUnit.Framework.Assert.IsFalse(dfs.Mkdirs(new Path(childDir0, "in")));
}
catch (QuotaExceededException)
{
hasException = true;
}
NUnit.Framework.Assert.IsTrue(hasException);
c = dfs.GetContentSummary(childDir0);
NUnit.Framework.Assert.AreEqual(c.GetDirectoryCount() + c.GetFileCount(), 1);
NUnit.Framework.Assert.AreEqual(c.GetQuota(), 1);
// 14a: set quota on a non-existent directory
Path nonExistentPath = new Path("/test1");
NUnit.Framework.Assert.IsFalse(dfs.Exists(nonExistentPath));
args = new string[] { "-setQuota", "1", nonExistentPath.ToString() };
RunCommand(admin, args, true);
RunCommand(admin, true, "-setSpaceQuota", "1g", nonExistentPath.ToString());
// for space quota
// 14b: set quota on a file
NUnit.Framework.Assert.IsTrue(dfs.IsFile(childFile0));
args[1] = childFile0.ToString();
RunCommand(admin, args, true);
// same for space quota
RunCommand(admin, true, "-setSpaceQuota", "1t", args[1]);
// 15a: clear quota on a file
args[0] = "-clrQuota";
RunCommand(admin, args, true);
RunCommand(admin, true, "-clrSpaceQuota", args[1]);
// 15b: clear quota on a non-existent directory
args[1] = nonExistentPath.ToString();
RunCommand(admin, args, true);
RunCommand(admin, true, "-clrSpaceQuota", args[1]);
// 16a: set the quota of /test to be 0
args = new string[] { "-setQuota", "0", parent.ToString() };
RunCommand(admin, args, true);
RunCommand(admin, true, "-setSpaceQuota", "0", args[2]);
// 16b: set the quota of /test to be -1
args[1] = "-1";
RunCommand(admin, args, true);
RunCommand(admin, true, "-setSpaceQuota", args[1], args[2]);
// 16c: set the quota of /test to be Long.MAX_VALUE+1
args[1] = (long.MaxValue + 1L).ToString();
RunCommand(admin, args, true);
RunCommand(admin, true, "-setSpaceQuota", args[1], args[2]);
// 16d: set the quota of /test to be a non integer
args[1] = "33aa1.5";
RunCommand(admin, args, true);
RunCommand(admin, true, "-setSpaceQuota", args[1], args[2]);
// 16e: set space quota with a value larger than Long.MAX_VALUE
RunCommand(admin, true, "-setSpaceQuota", (long.MaxValue / 1024 / 1024 + 1024) +
"m", args[2]);
// 17: setQuota by a non-administrator
string username = "******";
UserGroupInformation ugi = UserGroupInformation.CreateUserForTesting(username, new
string[] { "groupyy" });
string[] args2 = args.MemberwiseClone();
// need final ref for doAs block
ugi.DoAs(new _PrivilegedExceptionAction_275(this, username, conf, args2, parent));
// 18: clrQuota by a non-administrator
// 19: clrQuota on the root directory ("/") should fail
RunCommand(admin, true, "-clrQuota", "/");
// 20: setQuota on the root directory ("/") should succeed
RunCommand(admin, false, "-setQuota", "1000000", "/");
RunCommand(admin, true, "-clrQuota", "/");
RunCommand(admin, false, "-clrSpaceQuota", "/");
RunCommand(admin, new string[] { "-clrQuota", parent.ToString() }, false);
RunCommand(admin, false, "-clrSpaceQuota", parent.ToString());
// 2: create directory /test/data2
Path childDir2 = new Path(parent, "data2");
NUnit.Framework.Assert.IsTrue(dfs.Mkdirs(childDir2));
Path childFile2 = new Path(childDir2, "datafile2");
Path childFile3 = new Path(childDir2, "datafile3");
long spaceQuota2 = DefaultBlockSize * replication;
long fileLen2 = DefaultBlockSize;
// set space quota to a real low value
RunCommand(admin, false, "-setSpaceQuota", System.Convert.ToString(spaceQuota2),
childDir2.ToString());
// clear space quota
RunCommand(admin, false, "-clrSpaceQuota", childDir2.ToString());
// create a file that is greater than the size of space quota
DFSTestUtil.CreateFile(fs, childFile2, fileLen2, replication, 0);
// now set space quota again. This should succeed
RunCommand(admin, false, "-setSpaceQuota", System.Convert.ToString(spaceQuota2),
childDir2.ToString());
hasException = false;
try
{
DFSTestUtil.CreateFile(fs, childFile3, fileLen2, replication, 0);
}
catch (DSQuotaExceededException)
{
hasException = true;
}
NUnit.Framework.Assert.IsTrue(hasException);
// now test the same for root
Path childFile4 = new Path("/", "datafile2");
Path childFile5 = new Path("/", "datafile3");
RunCommand(admin, true, "-clrQuota", "/");
RunCommand(admin, false, "-clrSpaceQuota", "/");
// set space quota to a real low value
RunCommand(admin, false, "-setSpaceQuota", System.Convert.ToString(spaceQuota2),
"/");
RunCommand(admin, false, "-clrSpaceQuota", "/");
DFSTestUtil.CreateFile(fs, childFile4, fileLen2, replication, 0);
RunCommand(admin, false, "-setSpaceQuota", System.Convert.ToString(spaceQuota2),
"/");
hasException = false;
try
{
DFSTestUtil.CreateFile(fs, childFile5, fileLen2, replication, 0);
}
catch (DSQuotaExceededException)
{
hasException = true;
}
NUnit.Framework.Assert.IsTrue(hasException);
NUnit.Framework.Assert.AreEqual(4, cluster.GetNamesystem().GetFSDirectory().GetYieldCount
());
}
finally
{
cluster.Shutdown();
}
}
public virtual void TestSpaceCommands()
{
Configuration conf = new HdfsConfiguration();
// set a smaller block size so that we can test with smaller
// diskspace quotas
conf.Set(DFSConfigKeys.DfsBlockSizeKey, "512");
// Make it relinquish locks. When run serially, the result should
// be identical.
conf.SetInt(DFSConfigKeys.DfsContentSummaryLimitKey, 2);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(2).Build();
FileSystem fs = cluster.GetFileSystem();
NUnit.Framework.Assert.IsTrue("Not a HDFS: " + fs.GetUri(), fs is DistributedFileSystem
);
DistributedFileSystem dfs = (DistributedFileSystem)fs;
try
{
int fileLen = 1024;
short replication = 3;
int fileSpace = fileLen * replication;
// create directory /nqdir0/qdir1/qdir20/nqdir30
NUnit.Framework.Assert.IsTrue(dfs.Mkdirs(new Path("/nqdir0/qdir1/qdir20/nqdir30")
));
// set the quota of /nqdir0/qdir1 to 4 * fileSpace
Path quotaDir1 = new Path("/nqdir0/qdir1");
dfs.SetQuota(quotaDir1, HdfsConstants.QuotaDontSet, 4 * fileSpace);
ContentSummary c = dfs.GetContentSummary(quotaDir1);
NUnit.Framework.Assert.AreEqual(c.GetSpaceQuota(), 4 * fileSpace);
// set the quota of /nqdir0/qdir1/qdir20 to 6 * fileSpace
Path quotaDir20 = new Path("/nqdir0/qdir1/qdir20");
dfs.SetQuota(quotaDir20, HdfsConstants.QuotaDontSet, 6 * fileSpace);
c = dfs.GetContentSummary(quotaDir20);
NUnit.Framework.Assert.AreEqual(c.GetSpaceQuota(), 6 * fileSpace);
// Create /nqdir0/qdir1/qdir21 and set its space quota to 2 * fileSpace
Path quotaDir21 = new Path("/nqdir0/qdir1/qdir21");
NUnit.Framework.Assert.IsTrue(dfs.Mkdirs(quotaDir21));
dfs.SetQuota(quotaDir21, HdfsConstants.QuotaDontSet, 2 * fileSpace);
c = dfs.GetContentSummary(quotaDir21);
NUnit.Framework.Assert.AreEqual(c.GetSpaceQuota(), 2 * fileSpace);
// 5: Create directory /nqdir0/qdir1/qdir21/nqdir32
Path tempPath = new Path(quotaDir21, "nqdir32");
NUnit.Framework.Assert.IsTrue(dfs.Mkdirs(tempPath));
// create a file under nqdir32/fileDir
DFSTestUtil.CreateFile(dfs, new Path(tempPath, "fileDir/file1"), fileLen, replication
, 0);
c = dfs.GetContentSummary(quotaDir21);
NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), fileSpace);
// Create a larger file /nqdir0/qdir1/qdir21/nqdir33/
bool hasException = false;
try
{
DFSTestUtil.CreateFile(dfs, new Path(quotaDir21, "nqdir33/file2"), 2 * fileLen, replication
, 0);
}
catch (DSQuotaExceededException)
{
hasException = true;
}
NUnit.Framework.Assert.IsTrue(hasException);
// delete nqdir33
NUnit.Framework.Assert.IsTrue(dfs.Delete(new Path(quotaDir21, "nqdir33"), true));
c = dfs.GetContentSummary(quotaDir21);
NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), fileSpace);
NUnit.Framework.Assert.AreEqual(c.GetSpaceQuota(), 2 * fileSpace);
// Verify space before the move:
c = dfs.GetContentSummary(quotaDir20);
NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), 0);
// Move /nqdir0/qdir1/qdir21/nqdir32 /nqdir0/qdir1/qdir20/nqdir30
Path dstPath = new Path(quotaDir20, "nqdir30");
Path srcPath = new Path(quotaDir21, "nqdir32");
NUnit.Framework.Assert.IsTrue(dfs.Rename(srcPath, dstPath));
// verify space after the move
c = dfs.GetContentSummary(quotaDir20);
NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), fileSpace);
// verify space for its parent
c = dfs.GetContentSummary(quotaDir1);
NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), fileSpace);
// verify space for source for the move
c = dfs.GetContentSummary(quotaDir21);
NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), 0);
Path file2 = new Path(dstPath, "fileDir/file2");
int file2Len = 2 * fileLen;
// create a larger file under /nqdir0/qdir1/qdir20/nqdir30
DFSTestUtil.CreateFile(dfs, file2, file2Len, replication, 0);
c = dfs.GetContentSummary(quotaDir20);
NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), 3 * fileSpace);
c = dfs.GetContentSummary(quotaDir21);
NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), 0);
// Reverse: Move /nqdir0/qdir1/qdir20/nqdir30 to /nqdir0/qdir1/qdir21/
hasException = false;
try
{
NUnit.Framework.Assert.IsFalse(dfs.Rename(dstPath, srcPath));
}
catch (DSQuotaExceededException)
{
hasException = true;
}
NUnit.Framework.Assert.IsTrue(hasException);
// make sure no intermediate directories left by failed rename
NUnit.Framework.Assert.IsFalse(dfs.Exists(srcPath));
// directory should exist
NUnit.Framework.Assert.IsTrue(dfs.Exists(dstPath));
// verify space after the failed move
c = dfs.GetContentSummary(quotaDir20);
NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), 3 * fileSpace);
c = dfs.GetContentSummary(quotaDir21);
NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), 0);
// Test Append :
// verify space quota
c = dfs.GetContentSummary(quotaDir1);
NUnit.Framework.Assert.AreEqual(c.GetSpaceQuota(), 4 * fileSpace);
// verify space before append;
c = dfs.GetContentSummary(dstPath);
NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), 3 * fileSpace);
OutputStream @out = dfs.Append(file2);
// appending 1 fileLen should succeed
@out.Write(new byte[fileLen]);
@out.Close();
file2Len += fileLen;
// after append
// verify space after append;
c = dfs.GetContentSummary(dstPath);
NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), 4 * fileSpace);
// now increase the quota for quotaDir1
dfs.SetQuota(quotaDir1, HdfsConstants.QuotaDontSet, 5 * fileSpace);
// Now, appending more than 1 fileLen should result in an error
@out = dfs.Append(file2);
hasException = false;
try
{
@out.Write(new byte[fileLen + 1024]);
@out.Flush();
@out.Close();
}
catch (DSQuotaExceededException)
{
hasException = true;
IOUtils.CloseStream(@out);
}
NUnit.Framework.Assert.IsTrue(hasException);
file2Len += fileLen;
// after partial append
// verify space after partial append
c = dfs.GetContentSummary(dstPath);
NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), 5 * fileSpace);
// Test set replication :
// first reduce the replication
dfs.SetReplication(file2, (short)(replication - 1));
// verify that space is reduced by file2Len
c = dfs.GetContentSummary(dstPath);
NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), 5 * fileSpace - file2Len);
// now try to increase the replication and and expect an error.
hasException = false;
try
{
dfs.SetReplication(file2, (short)(replication + 1));
}
catch (DSQuotaExceededException)
{
hasException = true;
}
NUnit.Framework.Assert.IsTrue(hasException);
// verify space consumed remains unchanged.
c = dfs.GetContentSummary(dstPath);
NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), 5 * fileSpace - file2Len);
// now increase the quota for quotaDir1 and quotaDir20
dfs.SetQuota(quotaDir1, HdfsConstants.QuotaDontSet, 10 * fileSpace);
dfs.SetQuota(quotaDir20, HdfsConstants.QuotaDontSet, 10 * fileSpace);
// then increasing replication should be ok.
dfs.SetReplication(file2, (short)(replication + 1));
// verify increase in space
c = dfs.GetContentSummary(dstPath);
NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), 5 * fileSpace + file2Len);
// Test HDFS-2053 :
// Create directory /hdfs-2053
Path quotaDir2053 = new Path("/hdfs-2053");
NUnit.Framework.Assert.IsTrue(dfs.Mkdirs(quotaDir2053));
// Create subdirectories /hdfs-2053/{A,B,C}
Path quotaDir2053_A = new Path(quotaDir2053, "A");
NUnit.Framework.Assert.IsTrue(dfs.Mkdirs(quotaDir2053_A));
Path quotaDir2053_B = new Path(quotaDir2053, "B");
NUnit.Framework.Assert.IsTrue(dfs.Mkdirs(quotaDir2053_B));
Path quotaDir2053_C = new Path(quotaDir2053, "C");
NUnit.Framework.Assert.IsTrue(dfs.Mkdirs(quotaDir2053_C));
// Factors to vary the sizes of test files created in each subdir.
// The actual factors are not really important but they allow us to create
// identifiable file sizes per subdir, which helps during debugging.
int sizeFactorA = 1;
int sizeFactorB = 2;
int sizeFactorC = 4;
// Set space quota for subdirectory C
dfs.SetQuota(quotaDir2053_C, HdfsConstants.QuotaDontSet, (sizeFactorC + 1) * fileSpace
);
c = dfs.GetContentSummary(quotaDir2053_C);
NUnit.Framework.Assert.AreEqual(c.GetSpaceQuota(), (sizeFactorC + 1) * fileSpace);
// Create a file under subdirectory A
DFSTestUtil.CreateFile(dfs, new Path(quotaDir2053_A, "fileA"), sizeFactorA * fileLen
, replication, 0);
c = dfs.GetContentSummary(quotaDir2053_A);
NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), sizeFactorA * fileSpace);
// Create a file under subdirectory B
DFSTestUtil.CreateFile(dfs, new Path(quotaDir2053_B, "fileB"), sizeFactorB * fileLen
, replication, 0);
c = dfs.GetContentSummary(quotaDir2053_B);
NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), sizeFactorB * fileSpace);
// Create a file under subdirectory C (which has a space quota)
DFSTestUtil.CreateFile(dfs, new Path(quotaDir2053_C, "fileC"), sizeFactorC * fileLen
, replication, 0);
c = dfs.GetContentSummary(quotaDir2053_C);
NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), sizeFactorC * fileSpace);
// Check space consumed for /hdfs-2053
c = dfs.GetContentSummary(quotaDir2053);
NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), (sizeFactorA + sizeFactorB
+ sizeFactorC) * fileSpace);
NUnit.Framework.Assert.AreEqual(20, cluster.GetNamesystem().GetFSDirectory().GetYieldCount
());
}
finally
{
cluster.Shutdown();
}
}
