/// <exception cref="System.Exception"/>
internal static INodeFile AssertBlockCollection(string path, int numBlocks, FSDirectory
dir, BlockManager blkManager)
{
INodeFile file = INodeFile.ValueOf(dir.GetINode(path), path);
NUnit.Framework.Assert.AreEqual(numBlocks, file.GetBlocks().Length);
foreach (BlockInfoContiguous b in file.GetBlocks())
{
AssertBlockCollection(blkManager, file, b);
}
return(file);
}
/// <exception cref="System.Exception"/>
internal virtual INodeFile GetINodeFile(Path p)
{
string s = p.ToString();
return(INodeFile.ValueOf(fsdir.GetINode(s), s));
}
/// <summary>Test deleting a file with snapshots.</summary>
/// <remarks>
/// Test deleting a file with snapshots. Need to check the blocksMap to make
/// sure the corresponding record is updated correctly.
/// </remarks>
/// <exception cref="System.Exception"/>
public virtual void TestDeletionWithSnapshots()
{
Path file0 = new Path(sub1, "file0");
Path file1 = new Path(sub1, "file1");
Path sub2 = new Path(sub1, "sub2");
Path file2 = new Path(sub2, "file2");
Path file3 = new Path(sub1, "file3");
Path file4 = new Path(sub1, "file4");
Path file5 = new Path(sub1, "file5");
// Create file under sub1
DFSTestUtil.CreateFile(hdfs, file0, 4 * Blocksize, Replication, seed);
DFSTestUtil.CreateFile(hdfs, file1, 2 * Blocksize, Replication, seed);
DFSTestUtil.CreateFile(hdfs, file2, 3 * Blocksize, Replication, seed);
{
// Normal deletion
INodeFile f2 = AssertBlockCollection(file2.ToString(), 3, fsdir, blockmanager);
BlockInfoContiguous[] blocks = f2.GetBlocks();
hdfs.Delete(sub2, true);
// The INode should have been removed from the blocksMap
foreach (BlockInfoContiguous b in blocks)
{
NUnit.Framework.Assert.IsNull(blockmanager.GetBlockCollection(b));
}
}
// Create snapshots for sub1
string[] snapshots = new string[] { "s0", "s1", "s2" };
DFSTestUtil.CreateFile(hdfs, file3, 5 * Blocksize, Replication, seed);
SnapshotTestHelper.CreateSnapshot(hdfs, sub1, snapshots[0]);
DFSTestUtil.CreateFile(hdfs, file4, 1 * Blocksize, Replication, seed);
SnapshotTestHelper.CreateSnapshot(hdfs, sub1, snapshots[1]);
DFSTestUtil.CreateFile(hdfs, file5, 7 * Blocksize, Replication, seed);
SnapshotTestHelper.CreateSnapshot(hdfs, sub1, snapshots[2]);
{
// set replication so that the inode should be replaced for snapshots
INodeFile f1 = AssertBlockCollection(file1.ToString(), 2, fsdir, blockmanager);
NUnit.Framework.Assert.AreSame(typeof(INodeFile), f1.GetType());
hdfs.SetReplication(file1, (short)2);
f1 = AssertBlockCollection(file1.ToString(), 2, fsdir, blockmanager);
NUnit.Framework.Assert.IsTrue(f1.IsWithSnapshot());
NUnit.Framework.Assert.IsFalse(f1.IsUnderConstruction());
}
// Check the block information for file0
INodeFile f0 = AssertBlockCollection(file0.ToString(), 4, fsdir, blockmanager);
BlockInfoContiguous[] blocks0 = f0.GetBlocks();
// Also check the block information for snapshot of file0
Path snapshotFile0 = SnapshotTestHelper.GetSnapshotPath(sub1, "s0", file0.GetName
());
AssertBlockCollection(snapshotFile0.ToString(), 4, fsdir, blockmanager);
// Delete file0
hdfs.Delete(file0, true);
// Make sure the blocks of file0 is still in blocksMap
foreach (BlockInfoContiguous b_1 in blocks0)
{
NUnit.Framework.Assert.IsNotNull(blockmanager.GetBlockCollection(b_1));
}
AssertBlockCollection(snapshotFile0.ToString(), 4, fsdir, blockmanager);
// Compare the INode in the blocksMap with INodes for snapshots
string s1f0 = SnapshotTestHelper.GetSnapshotPath(sub1, "s1", file0.GetName()).ToString
();
AssertBlockCollection(s1f0, 4, fsdir, blockmanager);
// Delete snapshot s1
hdfs.DeleteSnapshot(sub1, "s1");
// Make sure the first block of file0 is still in blocksMap
foreach (BlockInfoContiguous b_2 in blocks0)
{
NUnit.Framework.Assert.IsNotNull(blockmanager.GetBlockCollection(b_2));
}
AssertBlockCollection(snapshotFile0.ToString(), 4, fsdir, blockmanager);
try
{
INodeFile.ValueOf(fsdir.GetINode(s1f0), s1f0);
NUnit.Framework.Assert.Fail("Expect FileNotFoundException when identifying the INode in a deleted Snapshot"
);
}
catch (IOException e)
{
GenericTestUtils.AssertExceptionContains("File does not exist: " + s1f0, e);
}
}
/// <summary>
/// Test that an append with no locations fails with an exception
/// showing insufficient locations.
/// </summary>
/// <exception cref="System.Exception"/>
public virtual void TestAppendInsufficientLocations()
{
Configuration conf = new Configuration();
// lower heartbeat interval for fast recognition of DN
conf.SetInt(DFSConfigKeys.DfsNamenodeHeartbeatRecheckIntervalKey, 1000);
conf.SetInt(DFSConfigKeys.DfsHeartbeatIntervalKey, 1);
conf.SetInt(DFSConfigKeys.DfsClientSocketTimeoutKey, 3000);
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(4).Build();
DistributedFileSystem fileSystem = null;
try
{
// create a file with replication 3
fileSystem = cluster.GetFileSystem();
Path f = new Path("/testAppend");
FSDataOutputStream create = fileSystem.Create(f, (short)2);
create.Write(Sharpen.Runtime.GetBytesForString("/testAppend"));
create.Close();
// Check for replications
DFSTestUtil.WaitReplication(fileSystem, f, (short)2);
// Shut down all DNs that have the last block location for the file
LocatedBlocks lbs = fileSystem.dfs.GetNamenode().GetBlockLocations("/testAppend",
0, long.MaxValue);
IList <DataNode> dnsOfCluster = cluster.GetDataNodes();
DatanodeInfo[] dnsWithLocations = lbs.GetLastLocatedBlock().GetLocations();
foreach (DataNode dn in dnsOfCluster)
{
foreach (DatanodeInfo loc in dnsWithLocations)
{
if (dn.GetDatanodeId().Equals(loc))
{
dn.Shutdown();
DFSTestUtil.WaitForDatanodeDeath(dn);
}
}
}
// Wait till 0 replication is recognized
DFSTestUtil.WaitReplication(fileSystem, f, (short)0);
// Append to the file, at this state there are 3 live DNs but none of them
// have the block.
try
{
fileSystem.Append(f);
NUnit.Framework.Assert.Fail("Append should fail because insufficient locations");
}
catch (IOException e)
{
Log.Info("Expected exception: ", e);
}
FSDirectory dir = cluster.GetNamesystem().GetFSDirectory();
INodeFile inode = INodeFile.ValueOf(dir.GetINode("/testAppend"), "/testAppend");
NUnit.Framework.Assert.IsTrue("File should remain closed", !inode.IsUnderConstruction
());
}
finally
{
if (null != fileSystem)
{
fileSystem.Close();
}
cluster.Shutdown();
}
}
