public virtual void TearDown()
{
fs.Delete(Path, false);
if (cluster != null)
{
fs.Close();
cluster.Shutdown();
cluster = null;
}
}
public virtual void TestSkipQuotaCheck()
{
try
{
// set quota. nsQuota of 1 means no files can be created
// under this directory.
hdfs.SetQuota(sub2, 1, long.MaxValue);
// create a file
try
{
// this should fail
DFSTestUtil.CreateFile(hdfs, file6, 1024, Replication, seed);
throw new IOException("The create should have failed.");
}
catch (NSQuotaExceededException)
{
}
// ignored
// disable the quota check and retry. this should succeed.
fsdir.DisableQuotaChecks();
DFSTestUtil.CreateFile(hdfs, file6, 1024, Replication, seed);
// trying again after re-enabling the check.
hdfs.Delete(file6, false);
// cleanup
fsdir.EnableQuotaChecks();
try
{
// this should fail
DFSTestUtil.CreateFile(hdfs, file6, 1024, Replication, seed);
throw new IOException("The create should have failed.");
}
catch (NSQuotaExceededException)
{
}
}
finally
{
// ignored
hdfs.Delete(file6, false);
// cleanup, in case the test failed in the middle.
hdfs.SetQuota(sub2, long.MaxValue, long.MaxValue);
}
}
/// <exception cref="System.Exception"/>
public virtual void TestQuotaByStorageTypeWithFileCreateDelete()
{
Path foo = new Path(dir, "foo");
Path createdFile1 = new Path(foo, "created_file1.data");
dfs.Mkdirs(foo);
dfs.SetStoragePolicy(foo, HdfsConstants.OnessdStoragePolicyName);
// set quota by storage type on directory "foo"
dfs.SetQuotaByStorageType(foo, StorageType.Ssd, Blocksize * 10);
INode fnode = fsdir.GetINode4Write(foo.ToString());
NUnit.Framework.Assert.IsTrue(fnode.IsDirectory());
NUnit.Framework.Assert.IsTrue(fnode.IsQuotaSet());
// Create file of size 2.5 * BLOCKSIZE under directory "foo"
long file1Len = Blocksize * 2 + Blocksize / 2;
int bufLen = Blocksize / 16;
DFSTestUtil.CreateFile(dfs, createdFile1, bufLen, file1Len, Blocksize, Replication
, seed);
// Verify space consumed and remaining quota
long storageTypeConsumed = fnode.AsDirectory().GetDirectoryWithQuotaFeature().GetSpaceConsumed
().GetTypeSpaces().Get(StorageType.Ssd);
NUnit.Framework.Assert.AreEqual(file1Len, storageTypeConsumed);
// Delete file and verify the consumed space of the storage type is updated
dfs.Delete(createdFile1, false);
storageTypeConsumed = fnode.AsDirectory().GetDirectoryWithQuotaFeature().GetSpaceConsumed
().GetTypeSpaces().Get(StorageType.Ssd);
NUnit.Framework.Assert.AreEqual(0, storageTypeConsumed);
QuotaCounts counts = new QuotaCounts.Builder().Build();
fnode.ComputeQuotaUsage(fsn.GetBlockManager().GetStoragePolicySuite(), counts, true
);
NUnit.Framework.Assert.AreEqual(fnode.DumpTreeRecursively().ToString(), 0, counts
.GetTypeSpaces().Get(StorageType.Ssd));
ContentSummary cs = dfs.GetContentSummary(foo);
NUnit.Framework.Assert.AreEqual(cs.GetSpaceConsumed(), 0);
NUnit.Framework.Assert.AreEqual(cs.GetTypeConsumed(StorageType.Ssd), 0);
NUnit.Framework.Assert.AreEqual(cs.GetTypeConsumed(StorageType.Disk), 0);
}
/* Verify mkdir works ok in .reserved/raw directory. */
/// <exception cref="System.Exception"/>
public virtual void TestReservedRawMkdir()
{
Path zone = new Path("zone");
Path slashZone = new Path("/", zone);
fs.Mkdirs(slashZone);
dfsAdmin.CreateEncryptionZone(slashZone, TestKey);
Path rawRoot = new Path("/.reserved/raw");
Path dir1 = new Path("dir1");
Path rawDir1 = new Path(rawRoot, dir1);
fs.Mkdirs(rawDir1);
AssertPathEquals(rawDir1, new Path("/", dir1));
fs.Delete(rawDir1, true);
Path rawZone = new Path(rawRoot, zone);
Path rawDir1EZ = new Path(rawZone, dir1);
fs.Mkdirs(rawDir1EZ);
AssertPathEquals(rawDir1EZ, new Path(slashZone, dir1));
fs.Delete(rawDir1EZ, true);
}
/// <summary>for snapshot file after deleting the original file.</summary>
/// <exception cref="System.Exception"/>
public virtual void TestSnapshotPathINodesAfterDeletion()
{
// Create a snapshot for the dir, and check the inodes for the path
// pointing to a snapshot file
hdfs.AllowSnapshot(sub1);
hdfs.CreateSnapshot(sub1, "s2");
// Delete the original file /TestSnapshot/sub1/file1
hdfs.Delete(file1, false);
Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot snapshot;
{
// Resolve the path for the snapshot file
// /TestSnapshot/sub1/.snapshot/s2/file1
string snapshotPath = sub1.ToString() + "/.snapshot/s2/file1";
string[] names = INode.GetPathNames(snapshotPath);
byte[][] components = INode.GetPathComponents(names);
INodesInPath nodesInPath = INodesInPath.Resolve(fsdir.rootDir, components, false);
// Length of inodes should be (components.length - 1), since we will ignore
// ".snapshot"
NUnit.Framework.Assert.AreEqual(nodesInPath.Length(), components.Length - 1);
// SnapshotRootIndex should be 3: {root, Testsnapshot, sub1, s2, file1}
snapshot = GetSnapshot(nodesInPath, "s2", 3);
AssertSnapshot(nodesInPath, true, snapshot, 3);
// Check the INode for file1 (snapshot file)
INode inode = nodesInPath.GetLastINode();
NUnit.Framework.Assert.AreEqual(file1.GetName(), inode.GetLocalName());
NUnit.Framework.Assert.IsTrue(inode.AsFile().IsWithSnapshot());
}
// Check the INodes for path /TestSnapshot/sub1/file1
string[] names_1 = INode.GetPathNames(file1.ToString());
byte[][] components_1 = INode.GetPathComponents(names_1);
INodesInPath nodesInPath_1 = INodesInPath.Resolve(fsdir.rootDir, components_1, false
);
// The length of inodes should be equal to components.length
NUnit.Framework.Assert.AreEqual(nodesInPath_1.Length(), components_1.Length);
// The number of non-null elements should be components.length - 1 since
// file1 has been deleted
NUnit.Framework.Assert.AreEqual(GetNumNonNull(nodesInPath_1), components_1.Length
- 1);
// The returned nodesInPath should be non-snapshot
AssertSnapshot(nodesInPath_1, false, snapshot, -1);
// The last INode should be null, and the one before should be associated
// with sub1
NUnit.Framework.Assert.IsNull(nodesInPath_1.GetINode(components_1.Length - 1));
NUnit.Framework.Assert.AreEqual(nodesInPath_1.GetINode(components_1.Length - 2).GetFullPathName
(), sub1.ToString());
NUnit.Framework.Assert.AreEqual(nodesInPath_1.GetINode(components_1.Length - 3).GetFullPathName
(), dir.ToString());
hdfs.DeleteSnapshot(sub1, "s2");
hdfs.DisallowSnapshot(sub1);
}
/// <exception cref="System.Exception"/>
public virtual void TestDatanodeReRegistration()
{
// Create a test file
DistributedFileSystem dfs = cluster.GetFileSystem();
Path path = new Path("/testRR");
// Create a file and shutdown the DNs, which populates InvalidateBlocks
DFSTestUtil.CreateFile(dfs, path, dfs.GetDefaultBlockSize(), (short)NumOfDatanodes
, unchecked ((int)(0xED0ED0)));
foreach (DataNode dn in cluster.GetDataNodes())
{
dn.Shutdown();
}
dfs.Delete(path, false);
namesystem.WriteLock();
InvalidateBlocks invalidateBlocks;
int expected = NumOfDatanodes;
try
{
invalidateBlocks = (InvalidateBlocks)Whitebox.GetInternalState(cluster.GetNamesystem
().GetBlockManager(), "invalidateBlocks");
NUnit.Framework.Assert.AreEqual("Expected invalidate blocks to be the number of DNs"
, (long)expected, invalidateBlocks.NumBlocks());
}
finally
{
namesystem.WriteUnlock();
}
// Re-register each DN and see that it wipes the invalidation work
foreach (DataNode dn_1 in cluster.GetDataNodes())
{
DatanodeID did = dn_1.GetDatanodeId();
DatanodeRegistration reg = new DatanodeRegistration(new DatanodeID(UUID.RandomUUID
().ToString(), did), new StorageInfo(HdfsServerConstants.NodeType.DataNode), new
ExportedBlockKeys(), VersionInfo.GetVersion());
namesystem.WriteLock();
try
{
bm.GetDatanodeManager().RegisterDatanode(reg);
expected--;
NUnit.Framework.Assert.AreEqual("Expected number of invalidate blocks to decrease"
, (long)expected, invalidateBlocks.NumBlocks());
}
finally
{
namesystem.WriteUnlock();
}
}
}
public virtual void TearDown()
{
if (fs.Exists(new Path("/sub1")))
{
if (fs.Exists(new Path("/sub1/.snapshot")))
{
foreach (FileStatus st in fs.ListStatus(new Path("/sub1/.snapshot")))
{
fs.DeleteSnapshot(new Path("/sub1"), st.GetPath().GetName());
}
fs.DisallowSnapshot(new Path("/sub1"));
}
fs.Delete(new Path("/sub1"), true);
}
}
public virtual void TestFileAdd()
{
// Add files with 100 blocks
Path file = GetTestPath("testFileAdd");
CreateFile(file, 3200, (short)3);
long blockCount = 32;
int blockCapacity = namesystem.GetBlockCapacity();
UpdateMetrics();
MetricsAsserts.AssertGauge("BlockCapacity", blockCapacity, MetricsAsserts.GetMetrics
(NsMetrics));
MetricsRecordBuilder rb = MetricsAsserts.GetMetrics(NnMetrics);
// File create operations is 1
// Number of files created is depth of <code>file</code> path
MetricsAsserts.AssertCounter("CreateFileOps", 1L, rb);
MetricsAsserts.AssertCounter("FilesCreated", (long)file.Depth(), rb);
UpdateMetrics();
long filesTotal = file.Depth() + 1;
// Add 1 for root
rb = MetricsAsserts.GetMetrics(NsMetrics);
MetricsAsserts.AssertGauge("FilesTotal", filesTotal, rb);
MetricsAsserts.AssertGauge("BlocksTotal", blockCount, rb);
fs.Delete(file, true);
filesTotal--;
// reduce the filecount for deleted file
rb = WaitForDnMetricValue(NsMetrics, "FilesTotal", filesTotal);
MetricsAsserts.AssertGauge("BlocksTotal", 0L, rb);
MetricsAsserts.AssertGauge("PendingDeletionBlocks", 0L, rb);
rb = MetricsAsserts.GetMetrics(NnMetrics);
// Delete file operations and number of files deleted must be 1
MetricsAsserts.AssertCounter("DeleteFileOps", 1L, rb);
MetricsAsserts.AssertCounter("FilesDeleted", 1L, rb);
}
public virtual void TestPendingDeletion()
{
Path foo = new Path("/foo");
DFSTestUtil.CreateFile(dfs, foo, Blocksize, Replication, 0);
// restart NN
cluster.RestartNameNode(true);
dfs.Delete(foo, true);
NUnit.Framework.Assert.AreEqual(0, cluster.GetNamesystem().GetBlocksTotal());
NUnit.Framework.Assert.AreEqual(Replication, cluster.GetNamesystem().GetPendingDeletionBlocks
());
Sharpen.Thread.Sleep(6000);
NUnit.Framework.Assert.AreEqual(0, cluster.GetNamesystem().GetBlocksTotal());
NUnit.Framework.Assert.AreEqual(0, cluster.GetNamesystem().GetPendingDeletionBlocks
());
string nnStartedStr = cluster.GetNamesystem().GetNNStarted();
long nnStarted = new SimpleDateFormat("EEE MMM dd HH:mm:ss zzz yyyy").Parse(nnStartedStr
).GetTime();
long blockDeletionStartTime = cluster.GetNamesystem().GetBlockDeletionStartTime();
NUnit.Framework.Assert.IsTrue(string.Format("Expect blockDeletionStartTime = %d > nnStarted = %d/nnStartedStr = %s."
, blockDeletionStartTime, nnStarted, nnStartedStr), blockDeletionStartTime > nnStarted
);
}
/// <summary>Test unlinking a file whose blocks we are caching in the DFSClient.</summary>
/// <remarks>
/// Test unlinking a file whose blocks we are caching in the DFSClient.
/// The DataNode will notify the DFSClient that the replica is stale via the
/// ShortCircuitShm.
/// </remarks>
/// <exception cref="System.Exception"/>
public virtual void TestUnlinkingReplicasInFileDescriptorCache()
{
BlockReaderTestUtil.EnableShortCircuitShmTracing();
TemporarySocketDirectory sockDir = new TemporarySocketDirectory();
Configuration conf = CreateShortCircuitConf("testUnlinkingReplicasInFileDescriptorCache"
, sockDir);
// We don't want the CacheCleaner to time out short-circuit shared memory
// segments during the test, so set the timeout really high.
conf.SetLong(DFSConfigKeys.DfsClientReadShortcircuitStreamsCacheExpiryMsKey, 1000000000L
);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();
cluster.WaitActive();
DistributedFileSystem fs = cluster.GetFileSystem();
ShortCircuitCache cache = fs.GetClient().GetClientContext().GetShortCircuitCache(
);
cache.GetDfsClientShmManager().Visit(new _Visitor_556());
// The ClientShmManager starts off empty.
Path TestPath = new Path("/test_file");
int TestFileLen = 8193;
int Seed = unchecked ((int)(0xFADE0));
DFSTestUtil.CreateFile(fs, TestPath, TestFileLen, (short)1, Seed);
byte[] contents = DFSTestUtil.ReadFileBuffer(fs, TestPath);
byte[] expected = DFSTestUtil.CalculateFileContentsFromSeed(Seed, TestFileLen);
NUnit.Framework.Assert.IsTrue(Arrays.Equals(contents, expected));
// Loading this file brought the ShortCircuitReplica into our local
// replica cache.
DatanodeInfo datanode = new DatanodeInfo(cluster.GetDataNodes()[0].GetDatanodeId(
));
cache.GetDfsClientShmManager().Visit(new _Visitor_577(datanode));
// Remove the file whose blocks we just read.
fs.Delete(TestPath, false);
// Wait for the replica to be purged from the DFSClient's cache.
GenericTestUtils.WaitFor(new _Supplier_593(this, cache, datanode), 10, 60000);
// Check that all slots have been invalidated.
cluster.Shutdown();
sockDir.Close();
}
public virtual void TestEditsLogRename()
{
DistributedFileSystem fs = cluster.GetFileSystem();
Path src1 = GetTestRootPath(fc, "testEditsLogRename/srcdir/src1");
Path dst1 = GetTestRootPath(fc, "testEditsLogRename/dstdir/dst1");
CreateFile(src1);
fs.Mkdirs(dst1.GetParent());
CreateFile(dst1);
// Set quota so that dst1 parent cannot allow under it new files/directories
fs.SetQuota(dst1.GetParent(), 2, HdfsConstants.QuotaDontSet);
// Free up quota for a subsequent rename
fs.Delete(dst1, true);
Rename(src1, dst1, true, true, false, Options.Rename.Overwrite);
// Restart the cluster and ensure the above operations can be
// loaded from the edits log
RestartCluster();
fs = cluster.GetFileSystem();
src1 = GetTestRootPath(fc, "testEditsLogRename/srcdir/src1");
dst1 = GetTestRootPath(fc, "testEditsLogRename/dstdir/dst1");
NUnit.Framework.Assert.IsFalse(fs.Exists(src1));
// ensure src1 is already renamed
NUnit.Framework.Assert.IsTrue(fs.Exists(dst1));
}
public virtual void TestQuotaCommands()
{
Configuration conf = new HdfsConfiguration();
// set a smaller block size so that we can test with smaller
// Space quotas
int DefaultBlockSize = 512;
conf.SetLong(DFSConfigKeys.DfsBlockSizeKey, DefaultBlockSize);
// Make it relinquish locks. When run serially, the result should
// be identical.
conf.SetInt(DFSConfigKeys.DfsContentSummaryLimitKey, 2);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(2).Build();
FileSystem fs = cluster.GetFileSystem();
NUnit.Framework.Assert.IsTrue("Not a HDFS: " + fs.GetUri(), fs is DistributedFileSystem
);
DistributedFileSystem dfs = (DistributedFileSystem)fs;
DFSAdmin admin = new DFSAdmin(conf);
try
{
int fileLen = 1024;
short replication = 5;
long spaceQuota = fileLen * replication * 15 / 8;
// 1: create a directory /test and set its quota to be 3
Path parent = new Path("/test");
NUnit.Framework.Assert.IsTrue(dfs.Mkdirs(parent));
string[] args = new string[] { "-setQuota", "3", parent.ToString() };
RunCommand(admin, args, false);
//try setting space quota with a 'binary prefix'
RunCommand(admin, false, "-setSpaceQuota", "2t", parent.ToString());
NUnit.Framework.Assert.AreEqual(2L << 40, dfs.GetContentSummary(parent).GetSpaceQuota
());
// set diskspace quota to 10000
RunCommand(admin, false, "-setSpaceQuota", System.Convert.ToString(spaceQuota), parent
.ToString());
// 2: create directory /test/data0
Path childDir0 = new Path(parent, "data0");
NUnit.Framework.Assert.IsTrue(dfs.Mkdirs(childDir0));
// 3: create a file /test/datafile0
Path childFile0 = new Path(parent, "datafile0");
DFSTestUtil.CreateFile(fs, childFile0, fileLen, replication, 0);
// 4: count -q /test
ContentSummary c = dfs.GetContentSummary(parent);
NUnit.Framework.Assert.AreEqual(c.GetFileCount() + c.GetDirectoryCount(), 3);
NUnit.Framework.Assert.AreEqual(c.GetQuota(), 3);
NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), fileLen * replication);
NUnit.Framework.Assert.AreEqual(c.GetSpaceQuota(), spaceQuota);
// 5: count -q /test/data0
c = dfs.GetContentSummary(childDir0);
NUnit.Framework.Assert.AreEqual(c.GetFileCount() + c.GetDirectoryCount(), 1);
NUnit.Framework.Assert.AreEqual(c.GetQuota(), -1);
// check disk space consumed
c = dfs.GetContentSummary(parent);
NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), fileLen * replication);
// 6: create a directory /test/data1
Path childDir1 = new Path(parent, "data1");
bool hasException = false;
try
{
NUnit.Framework.Assert.IsFalse(dfs.Mkdirs(childDir1));
}
catch (QuotaExceededException)
{
hasException = true;
}
NUnit.Framework.Assert.IsTrue(hasException);
OutputStream fout;
// 7: create a file /test/datafile1
Path childFile1 = new Path(parent, "datafile1");
hasException = false;
try
{
fout = dfs.Create(childFile1);
}
catch (QuotaExceededException)
{
hasException = true;
}
NUnit.Framework.Assert.IsTrue(hasException);
// 8: clear quota /test
RunCommand(admin, new string[] { "-clrQuota", parent.ToString() }, false);
c = dfs.GetContentSummary(parent);
NUnit.Framework.Assert.AreEqual(c.GetQuota(), -1);
NUnit.Framework.Assert.AreEqual(c.GetSpaceQuota(), spaceQuota);
// 9: clear quota /test/data0
RunCommand(admin, new string[] { "-clrQuota", childDir0.ToString() }, false);
c = dfs.GetContentSummary(childDir0);
NUnit.Framework.Assert.AreEqual(c.GetQuota(), -1);
// 10: create a file /test/datafile1
fout = dfs.Create(childFile1, replication);
// 10.s: but writing fileLen bytes should result in an quota exception
try
{
fout.Write(new byte[fileLen]);
fout.Close();
NUnit.Framework.Assert.Fail();
}
catch (QuotaExceededException)
{
IOUtils.CloseStream(fout);
}
//delete the file
dfs.Delete(childFile1, false);
// 9.s: clear diskspace quota
RunCommand(admin, false, "-clrSpaceQuota", parent.ToString());
c = dfs.GetContentSummary(parent);
NUnit.Framework.Assert.AreEqual(c.GetQuota(), -1);
NUnit.Framework.Assert.AreEqual(c.GetSpaceQuota(), -1);
// now creating childFile1 should succeed
DFSTestUtil.CreateFile(dfs, childFile1, fileLen, replication, 0);
// 11: set the quota of /test to be 1
// HADOOP-5872 - we can set quota even if it is immediately violated
args = new string[] { "-setQuota", "1", parent.ToString() };
RunCommand(admin, args, false);
RunCommand(admin, false, "-setSpaceQuota", Sharpen.Extensions.ToString(fileLen),
args[2]);
// for space quota
// 12: set the quota of /test/data0 to be 1
args = new string[] { "-setQuota", "1", childDir0.ToString() };
RunCommand(admin, args, false);
// 13: not able create a directory under data0
hasException = false;
try
{
NUnit.Framework.Assert.IsFalse(dfs.Mkdirs(new Path(childDir0, "in")));
}
catch (QuotaExceededException)
{
hasException = true;
}
NUnit.Framework.Assert.IsTrue(hasException);
c = dfs.GetContentSummary(childDir0);
NUnit.Framework.Assert.AreEqual(c.GetDirectoryCount() + c.GetFileCount(), 1);
NUnit.Framework.Assert.AreEqual(c.GetQuota(), 1);
// 14a: set quota on a non-existent directory
Path nonExistentPath = new Path("/test1");
NUnit.Framework.Assert.IsFalse(dfs.Exists(nonExistentPath));
args = new string[] { "-setQuota", "1", nonExistentPath.ToString() };
RunCommand(admin, args, true);
RunCommand(admin, true, "-setSpaceQuota", "1g", nonExistentPath.ToString());
// for space quota
// 14b: set quota on a file
NUnit.Framework.Assert.IsTrue(dfs.IsFile(childFile0));
args[1] = childFile0.ToString();
RunCommand(admin, args, true);
// same for space quota
RunCommand(admin, true, "-setSpaceQuota", "1t", args[1]);
// 15a: clear quota on a file
args[0] = "-clrQuota";
RunCommand(admin, args, true);
RunCommand(admin, true, "-clrSpaceQuota", args[1]);
// 15b: clear quota on a non-existent directory
args[1] = nonExistentPath.ToString();
RunCommand(admin, args, true);
RunCommand(admin, true, "-clrSpaceQuota", args[1]);
// 16a: set the quota of /test to be 0
args = new string[] { "-setQuota", "0", parent.ToString() };
RunCommand(admin, args, true);
RunCommand(admin, true, "-setSpaceQuota", "0", args[2]);
// 16b: set the quota of /test to be -1
args[1] = "-1";
RunCommand(admin, args, true);
RunCommand(admin, true, "-setSpaceQuota", args[1], args[2]);
// 16c: set the quota of /test to be Long.MAX_VALUE+1
args[1] = (long.MaxValue + 1L).ToString();
RunCommand(admin, args, true);
RunCommand(admin, true, "-setSpaceQuota", args[1], args[2]);
// 16d: set the quota of /test to be a non integer
args[1] = "33aa1.5";
RunCommand(admin, args, true);
RunCommand(admin, true, "-setSpaceQuota", args[1], args[2]);
// 16e: set space quota with a value larger than Long.MAX_VALUE
RunCommand(admin, true, "-setSpaceQuota", (long.MaxValue / 1024 / 1024 + 1024) +
"m", args[2]);
// 17: setQuota by a non-administrator
string username = "******";
UserGroupInformation ugi = UserGroupInformation.CreateUserForTesting(username, new
string[] { "groupyy" });
string[] args2 = args.MemberwiseClone();
// need final ref for doAs block
ugi.DoAs(new _PrivilegedExceptionAction_275(this, username, conf, args2, parent));
// 18: clrQuota by a non-administrator
// 19: clrQuota on the root directory ("/") should fail
RunCommand(admin, true, "-clrQuota", "/");
// 20: setQuota on the root directory ("/") should succeed
RunCommand(admin, false, "-setQuota", "1000000", "/");
RunCommand(admin, true, "-clrQuota", "/");
RunCommand(admin, false, "-clrSpaceQuota", "/");
RunCommand(admin, new string[] { "-clrQuota", parent.ToString() }, false);
RunCommand(admin, false, "-clrSpaceQuota", parent.ToString());
// 2: create directory /test/data2
Path childDir2 = new Path(parent, "data2");
NUnit.Framework.Assert.IsTrue(dfs.Mkdirs(childDir2));
Path childFile2 = new Path(childDir2, "datafile2");
Path childFile3 = new Path(childDir2, "datafile3");
long spaceQuota2 = DefaultBlockSize * replication;
long fileLen2 = DefaultBlockSize;
// set space quota to a real low value
RunCommand(admin, false, "-setSpaceQuota", System.Convert.ToString(spaceQuota2),
childDir2.ToString());
// clear space quota
RunCommand(admin, false, "-clrSpaceQuota", childDir2.ToString());
// create a file that is greater than the size of space quota
DFSTestUtil.CreateFile(fs, childFile2, fileLen2, replication, 0);
// now set space quota again. This should succeed
RunCommand(admin, false, "-setSpaceQuota", System.Convert.ToString(spaceQuota2),
childDir2.ToString());
hasException = false;
try
{
DFSTestUtil.CreateFile(fs, childFile3, fileLen2, replication, 0);
}
catch (DSQuotaExceededException)
{
hasException = true;
}
NUnit.Framework.Assert.IsTrue(hasException);
// now test the same for root
Path childFile4 = new Path("/", "datafile2");
Path childFile5 = new Path("/", "datafile3");
RunCommand(admin, true, "-clrQuota", "/");
RunCommand(admin, false, "-clrSpaceQuota", "/");
// set space quota to a real low value
RunCommand(admin, false, "-setSpaceQuota", System.Convert.ToString(spaceQuota2),
"/");
RunCommand(admin, false, "-clrSpaceQuota", "/");
DFSTestUtil.CreateFile(fs, childFile4, fileLen2, replication, 0);
RunCommand(admin, false, "-setSpaceQuota", System.Convert.ToString(spaceQuota2),
"/");
hasException = false;
try
{
DFSTestUtil.CreateFile(fs, childFile5, fileLen2, replication, 0);
}
catch (DSQuotaExceededException)
{
hasException = true;
}
NUnit.Framework.Assert.IsTrue(hasException);
NUnit.Framework.Assert.AreEqual(4, cluster.GetNamesystem().GetFSDirectory().GetYieldCount
());
}
finally
{
cluster.Shutdown();
}
}
/// <summary>Test race between delete operation and commitBlockSynchronization method.
/// </summary>
/// <remarks>
/// Test race between delete operation and commitBlockSynchronization method.
/// See HDFS-6825.
/// </remarks>
/// <param name="hasSnapshot"/>
/// <exception cref="System.Exception"/>
private void TestDeleteAndCommitBlockSynchronizationRace(bool hasSnapshot)
{
Log.Info("Start testing, hasSnapshot: " + hasSnapshot);
AList <AbstractMap.SimpleImmutableEntry <string, bool> > testList = new AList <AbstractMap.SimpleImmutableEntry
<string, bool> >();
testList.AddItem(new AbstractMap.SimpleImmutableEntry <string, bool>("/test-file",
false));
testList.AddItem(new AbstractMap.SimpleImmutableEntry <string, bool>("/test-file1"
, true));
testList.AddItem(new AbstractMap.SimpleImmutableEntry <string, bool>("/testdir/testdir1/test-file"
, false));
testList.AddItem(new AbstractMap.SimpleImmutableEntry <string, bool>("/testdir/testdir1/test-file1"
, true));
Path rootPath = new Path("/");
Configuration conf = new Configuration();
// Disable permissions so that another user can recover the lease.
conf.SetBoolean(DFSConfigKeys.DfsPermissionsEnabledKey, false);
conf.SetInt(DFSConfigKeys.DfsBlockSizeKey, BlockSize);
FSDataOutputStream stm = null;
IDictionary <DataNode, DatanodeProtocolClientSideTranslatorPB> dnMap = new Dictionary
<DataNode, DatanodeProtocolClientSideTranslatorPB>();
try
{
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(3).Build();
cluster.WaitActive();
DistributedFileSystem fs = cluster.GetFileSystem();
int stId = 0;
foreach (AbstractMap.SimpleImmutableEntry <string, bool> stest in testList)
{
string testPath = stest.Key;
bool mkSameDir = stest.Value;
Log.Info("test on " + testPath + " mkSameDir: " + mkSameDir + " snapshot: " + hasSnapshot
);
Path fPath = new Path(testPath);
//find grandest non-root parent
Path grandestNonRootParent = fPath;
while (!grandestNonRootParent.GetParent().Equals(rootPath))
{
grandestNonRootParent = grandestNonRootParent.GetParent();
}
stm = fs.Create(fPath);
Log.Info("test on " + testPath + " created " + fPath);
// write a half block
AppendTestUtil.Write(stm, 0, BlockSize / 2);
stm.Hflush();
if (hasSnapshot)
{
SnapshotTestHelper.CreateSnapshot(fs, rootPath, "st" + stId.ToString());
++stId;
}
// Look into the block manager on the active node for the block
// under construction.
NameNode nn = cluster.GetNameNode();
ExtendedBlock blk = DFSTestUtil.GetFirstBlock(fs, fPath);
DatanodeDescriptor expectedPrimary = DFSTestUtil.GetExpectedPrimaryNode(nn, blk);
Log.Info("Expecting block recovery to be triggered on DN " + expectedPrimary);
// Find the corresponding DN daemon, and spy on its connection to the
// active.
DataNode primaryDN = cluster.GetDataNode(expectedPrimary.GetIpcPort());
DatanodeProtocolClientSideTranslatorPB nnSpy = dnMap[primaryDN];
if (nnSpy == null)
{
nnSpy = DataNodeTestUtils.SpyOnBposToNN(primaryDN, nn);
dnMap[primaryDN] = nnSpy;
}
// Delay the commitBlockSynchronization call
GenericTestUtils.DelayAnswer delayer = new GenericTestUtils.DelayAnswer(Log);
Org.Mockito.Mockito.DoAnswer(delayer).When(nnSpy).CommitBlockSynchronization(Org.Mockito.Mockito
.Eq(blk), Org.Mockito.Mockito.AnyInt(), Org.Mockito.Mockito.AnyLong(), Org.Mockito.Mockito
.Eq(true), Org.Mockito.Mockito.Eq(false), (DatanodeID[])Org.Mockito.Mockito.AnyObject
(), (string[])Org.Mockito.Mockito.AnyObject());
// new genstamp
// new length
// close file
// delete block
// new targets
// new target storages
fs.RecoverLease(fPath);
Log.Info("Waiting for commitBlockSynchronization call from primary");
delayer.WaitForCall();
Log.Info("Deleting recursively " + grandestNonRootParent);
fs.Delete(grandestNonRootParent, true);
if (mkSameDir && !grandestNonRootParent.ToString().Equals(testPath))
{
Log.Info("Recreate dir " + grandestNonRootParent + " testpath: " + testPath);
fs.Mkdirs(grandestNonRootParent);
}
delayer.Proceed();
Log.Info("Now wait for result");
delayer.WaitForResult();
Exception t = delayer.GetThrown();
if (t != null)
{
Log.Info("Result exception (snapshot: " + hasSnapshot + "): " + t);
}
}
// end of loop each fPath
Log.Info("Now check we can restart");
cluster.RestartNameNodes();
Log.Info("Restart finished");
}
finally
{
if (stm != null)
{
IOUtils.CloseStream(stm);
}
if (cluster != null)
{
cluster.Shutdown();
}
}
}
/* Ignored */
/// <exception cref="System.Exception"/>
public virtual void TestDelete()
{
fs.Delete(objInSnapshot, true);
}
public virtual void TestNamespaceCommands()
{
Configuration conf = new HdfsConfiguration();
// Make it relinquish locks. When run serially, the result should
// be identical.
conf.SetInt(DFSConfigKeys.DfsContentSummaryLimitKey, 2);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(2).Build();
DistributedFileSystem dfs = cluster.GetFileSystem();
try
{
// 1: create directory /nqdir0/qdir1/qdir20/nqdir30
NUnit.Framework.Assert.IsTrue(dfs.Mkdirs(new Path("/nqdir0/qdir1/qdir20/nqdir30")
));
// 2: set the quota of /nqdir0/qdir1 to be 6
Path quotaDir1 = new Path("/nqdir0/qdir1");
dfs.SetQuota(quotaDir1, 6, HdfsConstants.QuotaDontSet);
ContentSummary c = dfs.GetContentSummary(quotaDir1);
NUnit.Framework.Assert.AreEqual(c.GetDirectoryCount(), 3);
NUnit.Framework.Assert.AreEqual(c.GetQuota(), 6);
// 3: set the quota of /nqdir0/qdir1/qdir20 to be 7
Path quotaDir2 = new Path("/nqdir0/qdir1/qdir20");
dfs.SetQuota(quotaDir2, 7, HdfsConstants.QuotaDontSet);
c = dfs.GetContentSummary(quotaDir2);
NUnit.Framework.Assert.AreEqual(c.GetDirectoryCount(), 2);
NUnit.Framework.Assert.AreEqual(c.GetQuota(), 7);
// 4: Create directory /nqdir0/qdir1/qdir21 and set its quota to 2
Path quotaDir3 = new Path("/nqdir0/qdir1/qdir21");
NUnit.Framework.Assert.IsTrue(dfs.Mkdirs(quotaDir3));
dfs.SetQuota(quotaDir3, 2, HdfsConstants.QuotaDontSet);
c = dfs.GetContentSummary(quotaDir3);
NUnit.Framework.Assert.AreEqual(c.GetDirectoryCount(), 1);
NUnit.Framework.Assert.AreEqual(c.GetQuota(), 2);
// 5: Create directory /nqdir0/qdir1/qdir21/nqdir32
Path tempPath = new Path(quotaDir3, "nqdir32");
NUnit.Framework.Assert.IsTrue(dfs.Mkdirs(tempPath));
c = dfs.GetContentSummary(quotaDir3);
NUnit.Framework.Assert.AreEqual(c.GetDirectoryCount(), 2);
NUnit.Framework.Assert.AreEqual(c.GetQuota(), 2);
// 6: Create directory /nqdir0/qdir1/qdir21/nqdir33
tempPath = new Path(quotaDir3, "nqdir33");
bool hasException = false;
try
{
NUnit.Framework.Assert.IsFalse(dfs.Mkdirs(tempPath));
}
catch (NSQuotaExceededException)
{
hasException = true;
}
NUnit.Framework.Assert.IsTrue(hasException);
c = dfs.GetContentSummary(quotaDir3);
NUnit.Framework.Assert.AreEqual(c.GetDirectoryCount(), 2);
NUnit.Framework.Assert.AreEqual(c.GetQuota(), 2);
// 7: Create directory /nqdir0/qdir1/qdir20/nqdir31
tempPath = new Path(quotaDir2, "nqdir31");
NUnit.Framework.Assert.IsTrue(dfs.Mkdirs(tempPath));
c = dfs.GetContentSummary(quotaDir2);
NUnit.Framework.Assert.AreEqual(c.GetDirectoryCount(), 3);
NUnit.Framework.Assert.AreEqual(c.GetQuota(), 7);
c = dfs.GetContentSummary(quotaDir1);
NUnit.Framework.Assert.AreEqual(c.GetDirectoryCount(), 6);
NUnit.Framework.Assert.AreEqual(c.GetQuota(), 6);
// 8: Create directory /nqdir0/qdir1/qdir20/nqdir33
tempPath = new Path(quotaDir2, "nqdir33");
hasException = false;
try
{
NUnit.Framework.Assert.IsFalse(dfs.Mkdirs(tempPath));
}
catch (NSQuotaExceededException)
{
hasException = true;
}
NUnit.Framework.Assert.IsTrue(hasException);
// 9: Move /nqdir0/qdir1/qdir21/nqdir32 /nqdir0/qdir1/qdir20/nqdir30
tempPath = new Path(quotaDir2, "nqdir30");
dfs.Rename(new Path(quotaDir3, "nqdir32"), tempPath);
c = dfs.GetContentSummary(quotaDir2);
NUnit.Framework.Assert.AreEqual(c.GetDirectoryCount(), 4);
NUnit.Framework.Assert.AreEqual(c.GetQuota(), 7);
c = dfs.GetContentSummary(quotaDir1);
NUnit.Framework.Assert.AreEqual(c.GetDirectoryCount(), 6);
NUnit.Framework.Assert.AreEqual(c.GetQuota(), 6);
// 10: Move /nqdir0/qdir1/qdir20/nqdir30 to /nqdir0/qdir1/qdir21
hasException = false;
try
{
NUnit.Framework.Assert.IsFalse(dfs.Rename(tempPath, quotaDir3));
}
catch (NSQuotaExceededException)
{
hasException = true;
}
NUnit.Framework.Assert.IsTrue(hasException);
NUnit.Framework.Assert.IsTrue(dfs.Exists(tempPath));
NUnit.Framework.Assert.IsFalse(dfs.Exists(new Path(quotaDir3, "nqdir30")));
// 10.a: Rename /nqdir0/qdir1/qdir20/nqdir30 to /nqdir0/qdir1/qdir21/nqdir32
hasException = false;
try
{
NUnit.Framework.Assert.IsFalse(dfs.Rename(tempPath, new Path(quotaDir3, "nqdir32"
)));
}
catch (QuotaExceededException)
{
hasException = true;
}
NUnit.Framework.Assert.IsTrue(hasException);
NUnit.Framework.Assert.IsTrue(dfs.Exists(tempPath));
NUnit.Framework.Assert.IsFalse(dfs.Exists(new Path(quotaDir3, "nqdir32")));
// 11: Move /nqdir0/qdir1/qdir20/nqdir30 to /nqdir0
NUnit.Framework.Assert.IsTrue(dfs.Rename(tempPath, new Path("/nqdir0")));
c = dfs.GetContentSummary(quotaDir2);
NUnit.Framework.Assert.AreEqual(c.GetDirectoryCount(), 2);
NUnit.Framework.Assert.AreEqual(c.GetQuota(), 7);
c = dfs.GetContentSummary(quotaDir1);
NUnit.Framework.Assert.AreEqual(c.GetDirectoryCount(), 4);
NUnit.Framework.Assert.AreEqual(c.GetQuota(), 6);
// 12: Create directory /nqdir0/nqdir30/nqdir33
NUnit.Framework.Assert.IsTrue(dfs.Mkdirs(new Path("/nqdir0/nqdir30/nqdir33")));
// 13: Move /nqdir0/nqdir30 /nqdir0/qdir1/qdir20/qdir30
hasException = false;
try
{
NUnit.Framework.Assert.IsFalse(dfs.Rename(new Path("/nqdir0/nqdir30"), tempPath));
}
catch (NSQuotaExceededException)
{
hasException = true;
}
NUnit.Framework.Assert.IsTrue(hasException);
// 14: Move /nqdir0/qdir1/qdir21 /nqdir0/qdir1/qdir20
NUnit.Framework.Assert.IsTrue(dfs.Rename(quotaDir3, quotaDir2));
c = dfs.GetContentSummary(quotaDir1);
NUnit.Framework.Assert.AreEqual(c.GetDirectoryCount(), 4);
NUnit.Framework.Assert.AreEqual(c.GetQuota(), 6);
c = dfs.GetContentSummary(quotaDir2);
NUnit.Framework.Assert.AreEqual(c.GetDirectoryCount(), 3);
NUnit.Framework.Assert.AreEqual(c.GetQuota(), 7);
tempPath = new Path(quotaDir2, "qdir21");
c = dfs.GetContentSummary(tempPath);
NUnit.Framework.Assert.AreEqual(c.GetDirectoryCount(), 1);
NUnit.Framework.Assert.AreEqual(c.GetQuota(), 2);
// 15: Delete /nqdir0/qdir1/qdir20/qdir21
dfs.Delete(tempPath, true);
c = dfs.GetContentSummary(quotaDir2);
NUnit.Framework.Assert.AreEqual(c.GetDirectoryCount(), 2);
NUnit.Framework.Assert.AreEqual(c.GetQuota(), 7);
c = dfs.GetContentSummary(quotaDir1);
NUnit.Framework.Assert.AreEqual(c.GetDirectoryCount(), 3);
NUnit.Framework.Assert.AreEqual(c.GetQuota(), 6);
// 16: Move /nqdir0/qdir30 /nqdir0/qdir1/qdir20
NUnit.Framework.Assert.IsTrue(dfs.Rename(new Path("/nqdir0/nqdir30"), quotaDir2));
c = dfs.GetContentSummary(quotaDir2);
NUnit.Framework.Assert.AreEqual(c.GetDirectoryCount(), 5);
NUnit.Framework.Assert.AreEqual(c.GetQuota(), 7);
c = dfs.GetContentSummary(quotaDir1);
NUnit.Framework.Assert.AreEqual(c.GetDirectoryCount(), 6);
NUnit.Framework.Assert.AreEqual(c.GetQuota(), 6);
NUnit.Framework.Assert.AreEqual(14, cluster.GetNamesystem().GetFSDirectory().GetYieldCount
());
}
finally
{
cluster.Shutdown();
}
}
/// <summary>Test deleting a file with snapshots.</summary>
/// <remarks>
/// Test deleting a file with snapshots. Need to check the blocksMap to make
/// sure the corresponding record is updated correctly.
/// </remarks>
/// <exception cref="System.Exception"/>
public virtual void TestDeletionWithSnapshots()
{
Path file0 = new Path(sub1, "file0");
Path file1 = new Path(sub1, "file1");
Path sub2 = new Path(sub1, "sub2");
Path file2 = new Path(sub2, "file2");
Path file3 = new Path(sub1, "file3");
Path file4 = new Path(sub1, "file4");
Path file5 = new Path(sub1, "file5");
// Create file under sub1
DFSTestUtil.CreateFile(hdfs, file0, 4 * Blocksize, Replication, seed);
DFSTestUtil.CreateFile(hdfs, file1, 2 * Blocksize, Replication, seed);
DFSTestUtil.CreateFile(hdfs, file2, 3 * Blocksize, Replication, seed);
{
// Normal deletion
INodeFile f2 = AssertBlockCollection(file2.ToString(), 3, fsdir, blockmanager);
BlockInfoContiguous[] blocks = f2.GetBlocks();
hdfs.Delete(sub2, true);
// The INode should have been removed from the blocksMap
foreach (BlockInfoContiguous b in blocks)
{
NUnit.Framework.Assert.IsNull(blockmanager.GetBlockCollection(b));
}
}
// Create snapshots for sub1
string[] snapshots = new string[] { "s0", "s1", "s2" };
DFSTestUtil.CreateFile(hdfs, file3, 5 * Blocksize, Replication, seed);
SnapshotTestHelper.CreateSnapshot(hdfs, sub1, snapshots[0]);
DFSTestUtil.CreateFile(hdfs, file4, 1 * Blocksize, Replication, seed);
SnapshotTestHelper.CreateSnapshot(hdfs, sub1, snapshots[1]);
DFSTestUtil.CreateFile(hdfs, file5, 7 * Blocksize, Replication, seed);
SnapshotTestHelper.CreateSnapshot(hdfs, sub1, snapshots[2]);
{
// set replication so that the inode should be replaced for snapshots
INodeFile f1 = AssertBlockCollection(file1.ToString(), 2, fsdir, blockmanager);
NUnit.Framework.Assert.AreSame(typeof(INodeFile), f1.GetType());
hdfs.SetReplication(file1, (short)2);
f1 = AssertBlockCollection(file1.ToString(), 2, fsdir, blockmanager);
NUnit.Framework.Assert.IsTrue(f1.IsWithSnapshot());
NUnit.Framework.Assert.IsFalse(f1.IsUnderConstruction());
}
// Check the block information for file0
INodeFile f0 = AssertBlockCollection(file0.ToString(), 4, fsdir, blockmanager);
BlockInfoContiguous[] blocks0 = f0.GetBlocks();
// Also check the block information for snapshot of file0
Path snapshotFile0 = SnapshotTestHelper.GetSnapshotPath(sub1, "s0", file0.GetName
());
AssertBlockCollection(snapshotFile0.ToString(), 4, fsdir, blockmanager);
// Delete file0
hdfs.Delete(file0, true);
// Make sure the blocks of file0 is still in blocksMap
foreach (BlockInfoContiguous b_1 in blocks0)
{
NUnit.Framework.Assert.IsNotNull(blockmanager.GetBlockCollection(b_1));
}
AssertBlockCollection(snapshotFile0.ToString(), 4, fsdir, blockmanager);
// Compare the INode in the blocksMap with INodes for snapshots
string s1f0 = SnapshotTestHelper.GetSnapshotPath(sub1, "s1", file0.GetName()).ToString
();
AssertBlockCollection(s1f0, 4, fsdir, blockmanager);
// Delete snapshot s1
hdfs.DeleteSnapshot(sub1, "s1");
// Make sure the first block of file0 is still in blocksMap
foreach (BlockInfoContiguous b_2 in blocks0)
{
NUnit.Framework.Assert.IsNotNull(blockmanager.GetBlockCollection(b_2));
}
AssertBlockCollection(snapshotFile0.ToString(), 4, fsdir, blockmanager);
try
{
INodeFile.ValueOf(fsdir.GetINode(s1f0), s1f0);
NUnit.Framework.Assert.Fail("Expect FileNotFoundException when identifying the INode in a deleted Snapshot"
);
}
catch (IOException e)
{
GenericTestUtils.AssertExceptionContains("File does not exist: " + s1f0, e);
}
}
public virtual void TestSpaceCommands()
{
Configuration conf = new HdfsConfiguration();
// set a smaller block size so that we can test with smaller
// diskspace quotas
conf.Set(DFSConfigKeys.DfsBlockSizeKey, "512");
// Make it relinquish locks. When run serially, the result should
// be identical.
conf.SetInt(DFSConfigKeys.DfsContentSummaryLimitKey, 2);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(2).Build();
FileSystem fs = cluster.GetFileSystem();
NUnit.Framework.Assert.IsTrue("Not a HDFS: " + fs.GetUri(), fs is DistributedFileSystem
);
DistributedFileSystem dfs = (DistributedFileSystem)fs;
try
{
int fileLen = 1024;
short replication = 3;
int fileSpace = fileLen * replication;
// create directory /nqdir0/qdir1/qdir20/nqdir30
NUnit.Framework.Assert.IsTrue(dfs.Mkdirs(new Path("/nqdir0/qdir1/qdir20/nqdir30")
));
// set the quota of /nqdir0/qdir1 to 4 * fileSpace
Path quotaDir1 = new Path("/nqdir0/qdir1");
dfs.SetQuota(quotaDir1, HdfsConstants.QuotaDontSet, 4 * fileSpace);
ContentSummary c = dfs.GetContentSummary(quotaDir1);
NUnit.Framework.Assert.AreEqual(c.GetSpaceQuota(), 4 * fileSpace);
// set the quota of /nqdir0/qdir1/qdir20 to 6 * fileSpace
Path quotaDir20 = new Path("/nqdir0/qdir1/qdir20");
dfs.SetQuota(quotaDir20, HdfsConstants.QuotaDontSet, 6 * fileSpace);
c = dfs.GetContentSummary(quotaDir20);
NUnit.Framework.Assert.AreEqual(c.GetSpaceQuota(), 6 * fileSpace);
// Create /nqdir0/qdir1/qdir21 and set its space quota to 2 * fileSpace
Path quotaDir21 = new Path("/nqdir0/qdir1/qdir21");
NUnit.Framework.Assert.IsTrue(dfs.Mkdirs(quotaDir21));
dfs.SetQuota(quotaDir21, HdfsConstants.QuotaDontSet, 2 * fileSpace);
c = dfs.GetContentSummary(quotaDir21);
NUnit.Framework.Assert.AreEqual(c.GetSpaceQuota(), 2 * fileSpace);
// 5: Create directory /nqdir0/qdir1/qdir21/nqdir32
Path tempPath = new Path(quotaDir21, "nqdir32");
NUnit.Framework.Assert.IsTrue(dfs.Mkdirs(tempPath));
// create a file under nqdir32/fileDir
DFSTestUtil.CreateFile(dfs, new Path(tempPath, "fileDir/file1"), fileLen, replication
, 0);
c = dfs.GetContentSummary(quotaDir21);
NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), fileSpace);
// Create a larger file /nqdir0/qdir1/qdir21/nqdir33/
bool hasException = false;
try
{
DFSTestUtil.CreateFile(dfs, new Path(quotaDir21, "nqdir33/file2"), 2 * fileLen, replication
, 0);
}
catch (DSQuotaExceededException)
{
hasException = true;
}
NUnit.Framework.Assert.IsTrue(hasException);
// delete nqdir33
NUnit.Framework.Assert.IsTrue(dfs.Delete(new Path(quotaDir21, "nqdir33"), true));
c = dfs.GetContentSummary(quotaDir21);
NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), fileSpace);
NUnit.Framework.Assert.AreEqual(c.GetSpaceQuota(), 2 * fileSpace);
// Verify space before the move:
c = dfs.GetContentSummary(quotaDir20);
NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), 0);
// Move /nqdir0/qdir1/qdir21/nqdir32 /nqdir0/qdir1/qdir20/nqdir30
Path dstPath = new Path(quotaDir20, "nqdir30");
Path srcPath = new Path(quotaDir21, "nqdir32");
NUnit.Framework.Assert.IsTrue(dfs.Rename(srcPath, dstPath));
// verify space after the move
c = dfs.GetContentSummary(quotaDir20);
NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), fileSpace);
// verify space for its parent
c = dfs.GetContentSummary(quotaDir1);
NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), fileSpace);
// verify space for source for the move
c = dfs.GetContentSummary(quotaDir21);
NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), 0);
Path file2 = new Path(dstPath, "fileDir/file2");
int file2Len = 2 * fileLen;
// create a larger file under /nqdir0/qdir1/qdir20/nqdir30
DFSTestUtil.CreateFile(dfs, file2, file2Len, replication, 0);
c = dfs.GetContentSummary(quotaDir20);
NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), 3 * fileSpace);
c = dfs.GetContentSummary(quotaDir21);
NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), 0);
// Reverse: Move /nqdir0/qdir1/qdir20/nqdir30 to /nqdir0/qdir1/qdir21/
hasException = false;
try
{
NUnit.Framework.Assert.IsFalse(dfs.Rename(dstPath, srcPath));
}
catch (DSQuotaExceededException)
{
hasException = true;
}
NUnit.Framework.Assert.IsTrue(hasException);
// make sure no intermediate directories left by failed rename
NUnit.Framework.Assert.IsFalse(dfs.Exists(srcPath));
// directory should exist
NUnit.Framework.Assert.IsTrue(dfs.Exists(dstPath));
// verify space after the failed move
c = dfs.GetContentSummary(quotaDir20);
NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), 3 * fileSpace);
c = dfs.GetContentSummary(quotaDir21);
NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), 0);
// Test Append :
// verify space quota
c = dfs.GetContentSummary(quotaDir1);
NUnit.Framework.Assert.AreEqual(c.GetSpaceQuota(), 4 * fileSpace);
// verify space before append;
c = dfs.GetContentSummary(dstPath);
NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), 3 * fileSpace);
OutputStream @out = dfs.Append(file2);
// appending 1 fileLen should succeed
@out.Write(new byte[fileLen]);
@out.Close();
file2Len += fileLen;
// after append
// verify space after append;
c = dfs.GetContentSummary(dstPath);
NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), 4 * fileSpace);
// now increase the quota for quotaDir1
dfs.SetQuota(quotaDir1, HdfsConstants.QuotaDontSet, 5 * fileSpace);
// Now, appending more than 1 fileLen should result in an error
@out = dfs.Append(file2);
hasException = false;
try
{
@out.Write(new byte[fileLen + 1024]);
@out.Flush();
@out.Close();
}
catch (DSQuotaExceededException)
{
hasException = true;
IOUtils.CloseStream(@out);
}
NUnit.Framework.Assert.IsTrue(hasException);
file2Len += fileLen;
// after partial append
// verify space after partial append
c = dfs.GetContentSummary(dstPath);
NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), 5 * fileSpace);
// Test set replication :
// first reduce the replication
dfs.SetReplication(file2, (short)(replication - 1));
// verify that space is reduced by file2Len
c = dfs.GetContentSummary(dstPath);
NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), 5 * fileSpace - file2Len);
// now try to increase the replication and and expect an error.
hasException = false;
try
{
dfs.SetReplication(file2, (short)(replication + 1));
}
catch (DSQuotaExceededException)
{
hasException = true;
}
NUnit.Framework.Assert.IsTrue(hasException);
// verify space consumed remains unchanged.
c = dfs.GetContentSummary(dstPath);
NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), 5 * fileSpace - file2Len);
// now increase the quota for quotaDir1 and quotaDir20
dfs.SetQuota(quotaDir1, HdfsConstants.QuotaDontSet, 10 * fileSpace);
dfs.SetQuota(quotaDir20, HdfsConstants.QuotaDontSet, 10 * fileSpace);
// then increasing replication should be ok.
dfs.SetReplication(file2, (short)(replication + 1));
// verify increase in space
c = dfs.GetContentSummary(dstPath);
NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), 5 * fileSpace + file2Len);
// Test HDFS-2053 :
// Create directory /hdfs-2053
Path quotaDir2053 = new Path("/hdfs-2053");
NUnit.Framework.Assert.IsTrue(dfs.Mkdirs(quotaDir2053));
// Create subdirectories /hdfs-2053/{A,B,C}
Path quotaDir2053_A = new Path(quotaDir2053, "A");
NUnit.Framework.Assert.IsTrue(dfs.Mkdirs(quotaDir2053_A));
Path quotaDir2053_B = new Path(quotaDir2053, "B");
NUnit.Framework.Assert.IsTrue(dfs.Mkdirs(quotaDir2053_B));
Path quotaDir2053_C = new Path(quotaDir2053, "C");
NUnit.Framework.Assert.IsTrue(dfs.Mkdirs(quotaDir2053_C));
// Factors to vary the sizes of test files created in each subdir.
// The actual factors are not really important but they allow us to create
// identifiable file sizes per subdir, which helps during debugging.
int sizeFactorA = 1;
int sizeFactorB = 2;
int sizeFactorC = 4;
// Set space quota for subdirectory C
dfs.SetQuota(quotaDir2053_C, HdfsConstants.QuotaDontSet, (sizeFactorC + 1) * fileSpace
);
c = dfs.GetContentSummary(quotaDir2053_C);
NUnit.Framework.Assert.AreEqual(c.GetSpaceQuota(), (sizeFactorC + 1) * fileSpace);
// Create a file under subdirectory A
DFSTestUtil.CreateFile(dfs, new Path(quotaDir2053_A, "fileA"), sizeFactorA * fileLen
, replication, 0);
c = dfs.GetContentSummary(quotaDir2053_A);
NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), sizeFactorA * fileSpace);
// Create a file under subdirectory B
DFSTestUtil.CreateFile(dfs, new Path(quotaDir2053_B, "fileB"), sizeFactorB * fileLen
, replication, 0);
c = dfs.GetContentSummary(quotaDir2053_B);
NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), sizeFactorB * fileSpace);
// Create a file under subdirectory C (which has a space quota)
DFSTestUtil.CreateFile(dfs, new Path(quotaDir2053_C, "fileC"), sizeFactorC * fileLen
, replication, 0);
c = dfs.GetContentSummary(quotaDir2053_C);
NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), sizeFactorC * fileSpace);
// Check space consumed for /hdfs-2053
c = dfs.GetContentSummary(quotaDir2053);
NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), (sizeFactorA + sizeFactorB
+ sizeFactorC) * fileSpace);
NUnit.Framework.Assert.AreEqual(20, cluster.GetNamesystem().GetFSDirectory().GetYieldCount
());
}
finally
{
cluster.Shutdown();
}
}
public virtual void CreateFiles()
{
hdfs.Delete(new Path(testdir), true);
hdfs.Mkdirs(new Path(testdir));
DFSTestUtil.CreateFile(hdfs, new Path(testdir + "/f1"), 0, (short)1, 0);
}
public virtual void TestPendingAndInvalidate()
{
Configuration Conf = new HdfsConfiguration();
Conf.SetLong(DFSConfigKeys.DfsBlockSizeKey, 1024);
Conf.SetLong(DFSConfigKeys.DfsHeartbeatIntervalKey, DfsReplicationInterval);
Conf.SetInt(DFSConfigKeys.DfsNamenodeReplicationIntervalKey, DfsReplicationInterval
);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(Conf).NumDataNodes(DatanodeCount
).Build();
cluster.WaitActive();
FSNamesystem namesystem = cluster.GetNamesystem();
BlockManager bm = namesystem.GetBlockManager();
DistributedFileSystem fs = cluster.GetFileSystem();
try
{
// 1. create a file
Path filePath = new Path("/tmp.txt");
DFSTestUtil.CreateFile(fs, filePath, 1024, (short)3, 0L);
// 2. disable the heartbeats
foreach (DataNode dn in cluster.GetDataNodes())
{
DataNodeTestUtils.SetHeartbeatsDisabledForTests(dn, true);
}
// 3. mark a couple of blocks as corrupt
LocatedBlock block = NameNodeAdapter.GetBlockLocations(cluster.GetNameNode(), filePath
.ToString(), 0, 1).Get(0);
cluster.GetNamesystem().WriteLock();
try
{
bm.FindAndMarkBlockAsCorrupt(block.GetBlock(), block.GetLocations()[0], "STORAGE_ID"
, "TEST");
bm.FindAndMarkBlockAsCorrupt(block.GetBlock(), block.GetLocations()[1], "STORAGE_ID"
, "TEST");
}
finally
{
cluster.GetNamesystem().WriteUnlock();
}
BlockManagerTestUtil.ComputeAllPendingWork(bm);
BlockManagerTestUtil.UpdateState(bm);
NUnit.Framework.Assert.AreEqual(bm.GetPendingReplicationBlocksCount(), 1L);
NUnit.Framework.Assert.AreEqual(bm.pendingReplications.GetNumReplicas(block.GetBlock
().GetLocalBlock()), 2);
// 4. delete the file
fs.Delete(filePath, true);
// retry at most 10 times, each time sleep for 1s. Note that 10s is much
// less than the default pending record timeout (5~10min)
int retries = 10;
long pendingNum = bm.GetPendingReplicationBlocksCount();
while (pendingNum != 0 && retries-- > 0)
{
Sharpen.Thread.Sleep(1000);
// let NN do the deletion
BlockManagerTestUtil.UpdateState(bm);
pendingNum = bm.GetPendingReplicationBlocksCount();
}
NUnit.Framework.Assert.AreEqual(pendingNum, 0L);
}
finally
{
cluster.Shutdown();
}
}
/// <summary>Test listing all the snapshottable directories</summary>
/// <exception cref="System.Exception"/>
public virtual void TestListSnapshottableDir()
{
cluster.GetNamesystem().GetSnapshotManager().SetAllowNestedSnapshots(true);
// Initially there is no snapshottable directories in the system
SnapshottableDirectoryStatus[] dirs = hdfs.GetSnapshottableDirListing();
NUnit.Framework.Assert.IsNull(dirs);
// Make root as snapshottable
Path root = new Path("/");
hdfs.AllowSnapshot(root);
dirs = hdfs.GetSnapshottableDirListing();
NUnit.Framework.Assert.AreEqual(1, dirs.Length);
NUnit.Framework.Assert.AreEqual(string.Empty, dirs[0].GetDirStatus().GetLocalName
());
NUnit.Framework.Assert.AreEqual(root, dirs[0].GetFullPath());
// Make root non-snaphsottable
hdfs.DisallowSnapshot(root);
dirs = hdfs.GetSnapshottableDirListing();
NUnit.Framework.Assert.IsNull(dirs);
// Make dir1 as snapshottable
hdfs.AllowSnapshot(dir1);
dirs = hdfs.GetSnapshottableDirListing();
NUnit.Framework.Assert.AreEqual(1, dirs.Length);
NUnit.Framework.Assert.AreEqual(dir1.GetName(), dirs[0].GetDirStatus().GetLocalName
());
NUnit.Framework.Assert.AreEqual(dir1, dirs[0].GetFullPath());
// There is no snapshot for dir1 yet
NUnit.Framework.Assert.AreEqual(0, dirs[0].GetSnapshotNumber());
// Make dir2 as snapshottable
hdfs.AllowSnapshot(dir2);
dirs = hdfs.GetSnapshottableDirListing();
NUnit.Framework.Assert.AreEqual(2, dirs.Length);
NUnit.Framework.Assert.AreEqual(dir1.GetName(), dirs[0].GetDirStatus().GetLocalName
());
NUnit.Framework.Assert.AreEqual(dir1, dirs[0].GetFullPath());
NUnit.Framework.Assert.AreEqual(dir2.GetName(), dirs[1].GetDirStatus().GetLocalName
());
NUnit.Framework.Assert.AreEqual(dir2, dirs[1].GetFullPath());
// There is no snapshot for dir2 yet
NUnit.Framework.Assert.AreEqual(0, dirs[1].GetSnapshotNumber());
// Create dir3
Path dir3 = new Path("/TestSnapshot3");
hdfs.Mkdirs(dir3);
// Rename dir3 to dir2
hdfs.Rename(dir3, dir2, Options.Rename.Overwrite);
// Now we only have one snapshottable dir: dir1
dirs = hdfs.GetSnapshottableDirListing();
NUnit.Framework.Assert.AreEqual(1, dirs.Length);
NUnit.Framework.Assert.AreEqual(dir1, dirs[0].GetFullPath());
// Make dir2 snapshottable again
hdfs.AllowSnapshot(dir2);
// Create a snapshot for dir2
hdfs.CreateSnapshot(dir2, "s1");
hdfs.CreateSnapshot(dir2, "s2");
dirs = hdfs.GetSnapshottableDirListing();
// There are now 2 snapshots for dir2
NUnit.Framework.Assert.AreEqual(dir2, dirs[1].GetFullPath());
NUnit.Framework.Assert.AreEqual(2, dirs[1].GetSnapshotNumber());
// Create sub-dirs under dir1
Path sub1 = new Path(dir1, "sub1");
Path file1 = new Path(sub1, "file1");
Path sub2 = new Path(dir1, "sub2");
Path file2 = new Path(sub2, "file2");
DFSTestUtil.CreateFile(hdfs, file1, Blocksize, Replication, seed);
DFSTestUtil.CreateFile(hdfs, file2, Blocksize, Replication, seed);
// Make sub1 and sub2 snapshottable
hdfs.AllowSnapshot(sub1);
hdfs.AllowSnapshot(sub2);
dirs = hdfs.GetSnapshottableDirListing();
NUnit.Framework.Assert.AreEqual(4, dirs.Length);
NUnit.Framework.Assert.AreEqual(dir1, dirs[0].GetFullPath());
NUnit.Framework.Assert.AreEqual(dir2, dirs[1].GetFullPath());
NUnit.Framework.Assert.AreEqual(sub1, dirs[2].GetFullPath());
NUnit.Framework.Assert.AreEqual(sub2, dirs[3].GetFullPath());
// reset sub1
hdfs.DisallowSnapshot(sub1);
dirs = hdfs.GetSnapshottableDirListing();
NUnit.Framework.Assert.AreEqual(3, dirs.Length);
NUnit.Framework.Assert.AreEqual(dir1, dirs[0].GetFullPath());
NUnit.Framework.Assert.AreEqual(dir2, dirs[1].GetFullPath());
NUnit.Framework.Assert.AreEqual(sub2, dirs[2].GetFullPath());
// Remove dir1, both dir1 and sub2 will be removed
hdfs.Delete(dir1, true);
dirs = hdfs.GetSnapshottableDirListing();
NUnit.Framework.Assert.AreEqual(1, dirs.Length);
NUnit.Framework.Assert.AreEqual(dir2.GetName(), dirs[0].GetDirStatus().GetLocalName
());
NUnit.Framework.Assert.AreEqual(dir2, dirs[0].GetFullPath());
}
public static void CreateOriginalFSImage()
{
MiniDFSCluster cluster = null;
try
{
Configuration conf = new Configuration();
conf.SetLong(DFSConfigKeys.DfsNamenodeDelegationTokenMaxLifetimeKey, 10000);
conf.SetLong(DFSConfigKeys.DfsNamenodeDelegationTokenRenewIntervalKey, 5000);
conf.SetBoolean(DFSConfigKeys.DfsNamenodeDelegationTokenAlwaysUseKey, true);
conf.Set(CommonConfigurationKeysPublic.HadoopSecurityAuthToLocal, "RULE:[2:$1@$0](JobTracker@.*FOO.COM)s/@.*//"
+ "DEFAULT");
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();
cluster.WaitActive();
DistributedFileSystem hdfs = cluster.GetFileSystem();
// Create a reasonable namespace
for (int i = 0; i < NumDirs; i++)
{
Path dir = new Path("/dir" + i);
hdfs.Mkdirs(dir);
writtenFiles[dir.ToString()] = PathToFileEntry(hdfs, dir.ToString());
for (int j = 0; j < FilesPerDir; j++)
{
Path file = new Path(dir, "file" + j);
FSDataOutputStream o = hdfs.Create(file);
o.Write(23);
o.Close();
writtenFiles[file.ToString()] = PathToFileEntry(hdfs, file.ToString());
}
}
// Create an empty directory
Path emptydir = new Path("/emptydir");
hdfs.Mkdirs(emptydir);
writtenFiles[emptydir.ToString()] = hdfs.GetFileStatus(emptydir);
//Create a directory whose name should be escaped in XML
Path invalidXMLDir = new Path("/dirContainingInvalidXMLChar\u0000here");
hdfs.Mkdirs(invalidXMLDir);
// Get delegation tokens so we log the delegation token op
Org.Apache.Hadoop.Security.Token.Token <object>[] delegationTokens = hdfs.AddDelegationTokens
(TestRenewer, null);
foreach (Org.Apache.Hadoop.Security.Token.Token <object> t in delegationTokens)
{
Log.Debug("got token " + t);
}
Path snapshot = new Path("/snapshot");
hdfs.Mkdirs(snapshot);
hdfs.AllowSnapshot(snapshot);
hdfs.Mkdirs(new Path("/snapshot/1"));
hdfs.Delete(snapshot, true);
// Set XAttrs so the fsimage contains XAttr ops
Path xattr = new Path("/xattr");
hdfs.Mkdirs(xattr);
hdfs.SetXAttr(xattr, "user.a1", new byte[] { unchecked ((int)(0x31)), unchecked ((int
)(0x32)), unchecked ((int)(0x33)) });
hdfs.SetXAttr(xattr, "user.a2", new byte[] { unchecked ((int)(0x37)), unchecked ((int
)(0x38)), unchecked ((int)(0x39)) });
// OIV should be able to handle empty value XAttrs
hdfs.SetXAttr(xattr, "user.a3", null);
writtenFiles[xattr.ToString()] = hdfs.GetFileStatus(xattr);
// Write results to the fsimage file
hdfs.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter, false);
hdfs.SaveNamespace();
// Determine location of fsimage file
originalFsimage = FSImageTestUtil.FindLatestImageFile(FSImageTestUtil.GetFSImage(
cluster.GetNameNode()).GetStorage().GetStorageDir(0));
if (originalFsimage == null)
{
throw new RuntimeException("Didn't generate or can't find fsimage");
}
Log.Debug("original FS image file is " + originalFsimage);
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
public virtual void TestMissingBlocksAlert()
{
MiniDFSCluster cluster = null;
try
{
Configuration conf = new HdfsConfiguration();
//minimize test delay
conf.SetInt(DFSConfigKeys.DfsNamenodeReplicationIntervalKey, 0);
conf.SetInt(DFSConfigKeys.DfsClientRetryWindowBase, 10);
int fileLen = 10 * 1024;
conf.SetInt(DFSConfigKeys.DfsBlockSizeKey, fileLen / 2);
//start a cluster with single datanode
cluster = new MiniDFSCluster.Builder(conf).Build();
cluster.WaitActive();
BlockManager bm = cluster.GetNamesystem().GetBlockManager();
DistributedFileSystem dfs = cluster.GetFileSystem();
// create a normal file
DFSTestUtil.CreateFile(dfs, new Path("/testMissingBlocksAlert/file1"), fileLen, (
short)3, 0);
Path corruptFile = new Path("/testMissingBlocks/corruptFile");
DFSTestUtil.CreateFile(dfs, corruptFile, fileLen, (short)3, 0);
// Corrupt the block
ExtendedBlock block = DFSTestUtil.GetFirstBlock(dfs, corruptFile);
NUnit.Framework.Assert.IsTrue(cluster.CorruptReplica(0, block));
// read the file so that the corrupt block is reported to NN
FSDataInputStream @in = dfs.Open(corruptFile);
try
{
@in.ReadFully(new byte[fileLen]);
}
catch (ChecksumException)
{
}
// checksum error is expected.
@in.Close();
Log.Info("Waiting for missing blocks count to increase...");
while (dfs.GetMissingBlocksCount() <= 0)
{
Sharpen.Thread.Sleep(100);
}
NUnit.Framework.Assert.IsTrue(dfs.GetMissingBlocksCount() == 1);
NUnit.Framework.Assert.AreEqual(4, dfs.GetUnderReplicatedBlocksCount());
NUnit.Framework.Assert.AreEqual(3, bm.GetUnderReplicatedNotMissingBlocks());
MBeanServer mbs = ManagementFactory.GetPlatformMBeanServer();
ObjectName mxbeanName = new ObjectName("Hadoop:service=NameNode,name=NameNodeInfo"
);
NUnit.Framework.Assert.AreEqual(1, (long)(long)mbs.GetAttribute(mxbeanName, "NumberOfMissingBlocks"
));
// now do the reverse : remove the file expect the number of missing
// blocks to go to zero
dfs.Delete(corruptFile, true);
Log.Info("Waiting for missing blocks count to be zero...");
while (dfs.GetMissingBlocksCount() > 0)
{
Sharpen.Thread.Sleep(100);
}
NUnit.Framework.Assert.AreEqual(2, dfs.GetUnderReplicatedBlocksCount());
NUnit.Framework.Assert.AreEqual(2, bm.GetUnderReplicatedNotMissingBlocks());
NUnit.Framework.Assert.AreEqual(0, (long)(long)mbs.GetAttribute(mxbeanName, "NumberOfMissingBlocks"
));
Path replOneFile = new Path("/testMissingBlocks/replOneFile");
DFSTestUtil.CreateFile(dfs, replOneFile, fileLen, (short)1, 0);
ExtendedBlock replOneBlock = DFSTestUtil.GetFirstBlock(dfs, replOneFile);
NUnit.Framework.Assert.IsTrue(cluster.CorruptReplica(0, replOneBlock));
// read the file so that the corrupt block is reported to NN
@in = dfs.Open(replOneFile);
try
{
@in.ReadFully(new byte[fileLen]);
}
catch (ChecksumException)
{
}
// checksum error is expected.
@in.Close();
NUnit.Framework.Assert.AreEqual(1, dfs.GetMissingReplOneBlocksCount());
NUnit.Framework.Assert.AreEqual(1, (long)(long)mbs.GetAttribute(mxbeanName, "NumberOfMissingBlocksWithReplicationFactorOne"
));
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
