public virtual void TestDeletionWithZeroSizeBlock2()
{
Path foo = new Path("/foo");
Path subDir = new Path(foo, "sub");
Path bar = new Path(subDir, "bar");
DFSTestUtil.CreateFile(hdfs, bar, Blocksize, Replication, 0L);
hdfs.Append(bar);
INodeFile barNode = fsdir.GetINode4Write(bar.ToString()).AsFile();
BlockInfoContiguous[] blks = barNode.GetBlocks();
NUnit.Framework.Assert.AreEqual(1, blks.Length);
ExtendedBlock previous = new ExtendedBlock(fsn.GetBlockPoolId(), blks[0]);
cluster.GetNameNodeRpc().AddBlock(bar.ToString(), hdfs.GetClient().GetClientName(
), previous, null, barNode.GetId(), null);
SnapshotTestHelper.CreateSnapshot(hdfs, foo, "s1");
barNode = fsdir.GetINode4Write(bar.ToString()).AsFile();
blks = barNode.GetBlocks();
NUnit.Framework.Assert.AreEqual(2, blks.Length);
NUnit.Framework.Assert.AreEqual(Blocksize, blks[0].GetNumBytes());
NUnit.Framework.Assert.AreEqual(0, blks[1].GetNumBytes());
hdfs.Delete(subDir, true);
Path sbar = SnapshotTestHelper.GetSnapshotPath(foo, "s1", "sub/bar");
barNode = fsdir.GetINode(sbar.ToString()).AsFile();
blks = barNode.GetBlocks();
NUnit.Framework.Assert.AreEqual(1, blks.Length);
NUnit.Framework.Assert.AreEqual(Blocksize, blks[0].GetNumBytes());
}
/// <exception cref="System.Exception"/>
internal virtual void AssertINodeNullInSnapshots(Path path, params string[] snapshots
)
{
foreach (string s in snapshots)
{
AssertINodeNull(SnapshotTestHelper.GetSnapshotPath(path.GetParent(), s, path.GetName
()).ToString());
}
}
/// <summary>
/// Adding as part of jira HDFS-5343
/// Test for checking the cat command on snapshot path it
/// cannot read a file beyond snapshot file length
/// </summary>
/// <exception cref="System.Exception"/>
public virtual void TestSnapshotFileLengthWithCatCommand()
{
FSDataInputStream fis = null;
FileStatus fileStatus = null;
int bytesRead;
byte[] buffer = new byte[Blocksize * 8];
hdfs.Mkdirs(sub);
Path file1 = new Path(sub, file1Name);
DFSTestUtil.CreateFile(hdfs, file1, Blocksize, Replication, Seed);
hdfs.AllowSnapshot(sub);
hdfs.CreateSnapshot(sub, snapshot1);
DFSTestUtil.AppendFile(hdfs, file1, Blocksize);
// Make sure we can read the entire file via its non-snapshot path.
fileStatus = hdfs.GetFileStatus(file1);
NUnit.Framework.Assert.AreEqual("Unexpected file length", Blocksize * 2, fileStatus
.GetLen());
fis = hdfs.Open(file1);
bytesRead = fis.Read(buffer, 0, buffer.Length);
NUnit.Framework.Assert.AreEqual("Unexpected # bytes read", Blocksize * 2, bytesRead
);
fis.Close();
Path file1snap1 = SnapshotTestHelper.GetSnapshotPath(sub, snapshot1, file1Name);
fis = hdfs.Open(file1snap1);
fileStatus = hdfs.GetFileStatus(file1snap1);
NUnit.Framework.Assert.AreEqual(fileStatus.GetLen(), Blocksize);
// Make sure we can only read up to the snapshot length.
bytesRead = fis.Read(buffer, 0, buffer.Length);
NUnit.Framework.Assert.AreEqual("Unexpected # bytes read", Blocksize, bytesRead);
fis.Close();
TextWriter outBackup = System.Console.Out;
TextWriter errBackup = System.Console.Error;
ByteArrayOutputStream bao = new ByteArrayOutputStream();
Runtime.SetOut(new TextWriter(bao));
Runtime.SetErr(new TextWriter(bao));
// Make sure we can cat the file upto to snapshot length
FsShell shell = new FsShell();
try
{
ToolRunner.Run(conf, shell, new string[] { "-cat", "/TestSnapshotFileLength/sub1/.snapshot/snapshot1/file1" });
NUnit.Framework.Assert.AreEqual("Unexpected # bytes from -cat", Blocksize, bao.Size
());
}
finally
{
Runtime.SetOut(outBackup);
Runtime.SetErr(errBackup);
}
}
public static void SetUp()
{
conf = new Configuration();
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();
cluster.WaitActive();
fsn = cluster.GetNamesystem();
fs = cluster.GetFileSystem();
Path path1 = new Path(sub1, "dir1");
NUnit.Framework.Assert.IsTrue(fs.Mkdirs(path1));
Path path2 = new Path(sub2, "dir2");
NUnit.Framework.Assert.IsTrue(fs.Mkdirs(path2));
SnapshotTestHelper.CreateSnapshot(fs, sub1, "testSnapshot");
objInSnapshot = SnapshotTestHelper.GetSnapshotPath(sub1, "testSnapshot", "dir1");
}
/// <summary>Test the snapshot limit of a single snapshottable directory.</summary>
/// <exception cref="System.Exception"/>
public virtual void TestSnapshotLimit()
{
int step = 1000;
string dirStr = "/testSnapshotLimit/dir";
Path dir = new Path(dirStr);
hdfs.Mkdirs(dir, new FsPermission((short)0x1ff));
hdfs.AllowSnapshot(dir);
int s = 0;
for (; s < DirectorySnapshottableFeature.SnapshotLimit; s++)
{
string snapshotName = "s" + s;
hdfs.CreateSnapshot(dir, snapshotName);
//create a file occasionally
if (s % step == 0)
{
Path file = new Path(dirStr, "f" + s);
DFSTestUtil.CreateFile(hdfs, file, Blocksize, Replication, Seed);
}
}
try
{
hdfs.CreateSnapshot(dir, "s" + s);
NUnit.Framework.Assert.Fail("Expected to fail to create snapshot, but didn't.");
}
catch (IOException ioe)
{
SnapshotTestHelper.Log.Info("The exception is expected.", ioe);
}
for (int f = 0; f < DirectorySnapshottableFeature.SnapshotLimit; f += step)
{
string file = "f" + f;
s = Random.Next(step);
for (; s < DirectorySnapshottableFeature.SnapshotLimit; s += Random.Next(step))
{
Path p = SnapshotTestHelper.GetSnapshotPath(dir, "s" + s, file);
//the file #f exists in snapshot #s iff s > f.
NUnit.Framework.Assert.AreEqual(s > f, hdfs.Exists(p));
}
}
}
public virtual void TestGetBlockLocations()
{
Path root = new Path("/");
Path file = new Path("/file");
DFSTestUtil.CreateFile(hdfs, file, Blocksize, Replication, seed);
// take a snapshot on root
SnapshotTestHelper.CreateSnapshot(hdfs, root, "s1");
Path fileInSnapshot = SnapshotTestHelper.GetSnapshotPath(root, "s1", file.GetName
());
FileStatus status = hdfs.GetFileStatus(fileInSnapshot);
// make sure we record the size for the file
NUnit.Framework.Assert.AreEqual(Blocksize, status.GetLen());
// append data to file
DFSTestUtil.AppendFile(hdfs, file, Blocksize - 1);
status = hdfs.GetFileStatus(fileInSnapshot);
// the size of snapshot file should still be BLOCKSIZE
NUnit.Framework.Assert.AreEqual(Blocksize, status.GetLen());
// the size of the file should be (2 * BLOCKSIZE - 1)
status = hdfs.GetFileStatus(file);
NUnit.Framework.Assert.AreEqual(Blocksize * 2 - 1, status.GetLen());
// call DFSClient#callGetBlockLocations for the file in snapshot
LocatedBlocks blocks = DFSClientAdapter.CallGetBlockLocations(cluster.GetNameNodeRpc
(), fileInSnapshot.ToString(), 0, long.MaxValue);
IList <LocatedBlock> blockList = blocks.GetLocatedBlocks();
// should be only one block
NUnit.Framework.Assert.AreEqual(Blocksize, blocks.GetFileLength());
NUnit.Framework.Assert.AreEqual(1, blockList.Count);
// check the last block
LocatedBlock lastBlock = blocks.GetLastLocatedBlock();
NUnit.Framework.Assert.AreEqual(0, lastBlock.GetStartOffset());
NUnit.Framework.Assert.AreEqual(Blocksize, lastBlock.GetBlockSize());
// take another snapshot
SnapshotTestHelper.CreateSnapshot(hdfs, root, "s2");
Path fileInSnapshot2 = SnapshotTestHelper.GetSnapshotPath(root, "s2", file.GetName
());
// append data to file without closing
HdfsDataOutputStream @out = AppendFileWithoutClosing(file, Blocksize);
@out.Hsync(EnumSet.Of(HdfsDataOutputStream.SyncFlag.UpdateLength));
status = hdfs.GetFileStatus(fileInSnapshot2);
// the size of snapshot file should be BLOCKSIZE*2-1
NUnit.Framework.Assert.AreEqual(Blocksize * 2 - 1, status.GetLen());
// the size of the file should be (3 * BLOCKSIZE - 1)
status = hdfs.GetFileStatus(file);
NUnit.Framework.Assert.AreEqual(Blocksize * 3 - 1, status.GetLen());
blocks = DFSClientAdapter.CallGetBlockLocations(cluster.GetNameNodeRpc(), fileInSnapshot2
.ToString(), 0, long.MaxValue);
NUnit.Framework.Assert.IsFalse(blocks.IsUnderConstruction());
NUnit.Framework.Assert.IsTrue(blocks.IsLastBlockComplete());
blockList = blocks.GetLocatedBlocks();
// should be 2 blocks
NUnit.Framework.Assert.AreEqual(Blocksize * 2 - 1, blocks.GetFileLength());
NUnit.Framework.Assert.AreEqual(2, blockList.Count);
// check the last block
lastBlock = blocks.GetLastLocatedBlock();
NUnit.Framework.Assert.AreEqual(Blocksize, lastBlock.GetStartOffset());
NUnit.Framework.Assert.AreEqual(Blocksize, lastBlock.GetBlockSize());
blocks = DFSClientAdapter.CallGetBlockLocations(cluster.GetNameNodeRpc(), fileInSnapshot2
.ToString(), Blocksize, 0);
blockList = blocks.GetLocatedBlocks();
NUnit.Framework.Assert.AreEqual(1, blockList.Count);
// check blocks for file being written
blocks = DFSClientAdapter.CallGetBlockLocations(cluster.GetNameNodeRpc(), file.ToString
(), 0, long.MaxValue);
blockList = blocks.GetLocatedBlocks();
NUnit.Framework.Assert.AreEqual(3, blockList.Count);
NUnit.Framework.Assert.IsTrue(blocks.IsUnderConstruction());
NUnit.Framework.Assert.IsFalse(blocks.IsLastBlockComplete());
lastBlock = blocks.GetLastLocatedBlock();
NUnit.Framework.Assert.AreEqual(Blocksize * 2, lastBlock.GetStartOffset());
NUnit.Framework.Assert.AreEqual(Blocksize - 1, lastBlock.GetBlockSize());
@out.Close();
}
/// <summary>Test deleting a file with snapshots.</summary>
/// <remarks>
/// Test deleting a file with snapshots. Need to check the blocksMap to make
/// sure the corresponding record is updated correctly.
/// </remarks>
/// <exception cref="System.Exception"/>
public virtual void TestDeletionWithSnapshots()
{
Path file0 = new Path(sub1, "file0");
Path file1 = new Path(sub1, "file1");
Path sub2 = new Path(sub1, "sub2");
Path file2 = new Path(sub2, "file2");
Path file3 = new Path(sub1, "file3");
Path file4 = new Path(sub1, "file4");
Path file5 = new Path(sub1, "file5");
// Create file under sub1
DFSTestUtil.CreateFile(hdfs, file0, 4 * Blocksize, Replication, seed);
DFSTestUtil.CreateFile(hdfs, file1, 2 * Blocksize, Replication, seed);
DFSTestUtil.CreateFile(hdfs, file2, 3 * Blocksize, Replication, seed);
{
// Normal deletion
INodeFile f2 = AssertBlockCollection(file2.ToString(), 3, fsdir, blockmanager);
BlockInfoContiguous[] blocks = f2.GetBlocks();
hdfs.Delete(sub2, true);
// The INode should have been removed from the blocksMap
foreach (BlockInfoContiguous b in blocks)
{
NUnit.Framework.Assert.IsNull(blockmanager.GetBlockCollection(b));
}
}
// Create snapshots for sub1
string[] snapshots = new string[] { "s0", "s1", "s2" };
DFSTestUtil.CreateFile(hdfs, file3, 5 * Blocksize, Replication, seed);
SnapshotTestHelper.CreateSnapshot(hdfs, sub1, snapshots[0]);
DFSTestUtil.CreateFile(hdfs, file4, 1 * Blocksize, Replication, seed);
SnapshotTestHelper.CreateSnapshot(hdfs, sub1, snapshots[1]);
DFSTestUtil.CreateFile(hdfs, file5, 7 * Blocksize, Replication, seed);
SnapshotTestHelper.CreateSnapshot(hdfs, sub1, snapshots[2]);
{
// set replication so that the inode should be replaced for snapshots
INodeFile f1 = AssertBlockCollection(file1.ToString(), 2, fsdir, blockmanager);
NUnit.Framework.Assert.AreSame(typeof(INodeFile), f1.GetType());
hdfs.SetReplication(file1, (short)2);
f1 = AssertBlockCollection(file1.ToString(), 2, fsdir, blockmanager);
NUnit.Framework.Assert.IsTrue(f1.IsWithSnapshot());
NUnit.Framework.Assert.IsFalse(f1.IsUnderConstruction());
}
// Check the block information for file0
INodeFile f0 = AssertBlockCollection(file0.ToString(), 4, fsdir, blockmanager);
BlockInfoContiguous[] blocks0 = f0.GetBlocks();
// Also check the block information for snapshot of file0
Path snapshotFile0 = SnapshotTestHelper.GetSnapshotPath(sub1, "s0", file0.GetName
());
AssertBlockCollection(snapshotFile0.ToString(), 4, fsdir, blockmanager);
// Delete file0
hdfs.Delete(file0, true);
// Make sure the blocks of file0 is still in blocksMap
foreach (BlockInfoContiguous b_1 in blocks0)
{
NUnit.Framework.Assert.IsNotNull(blockmanager.GetBlockCollection(b_1));
}
AssertBlockCollection(snapshotFile0.ToString(), 4, fsdir, blockmanager);
// Compare the INode in the blocksMap with INodes for snapshots
string s1f0 = SnapshotTestHelper.GetSnapshotPath(sub1, "s1", file0.GetName()).ToString
();
AssertBlockCollection(s1f0, 4, fsdir, blockmanager);
// Delete snapshot s1
hdfs.DeleteSnapshot(sub1, "s1");
// Make sure the first block of file0 is still in blocksMap
foreach (BlockInfoContiguous b_2 in blocks0)
{
NUnit.Framework.Assert.IsNotNull(blockmanager.GetBlockCollection(b_2));
}
AssertBlockCollection(snapshotFile0.ToString(), 4, fsdir, blockmanager);
try
{
INodeFile.ValueOf(fsdir.GetINode(s1f0), s1f0);
NUnit.Framework.Assert.Fail("Expect FileNotFoundException when identifying the INode in a deleted Snapshot"
);
}
catch (IOException e)
{
GenericTestUtils.AssertExceptionContains("File does not exist: " + s1f0, e);
}
}
/// <summary>
/// Test that we cannot read a file beyond its snapshot length
/// when accessing it via a snapshot path.
/// </summary>
/// <exception cref="System.Exception"/>
public virtual void TestSnapshotfileLength()
{
hdfs.Mkdirs(sub);
int bytesRead;
byte[] buffer = new byte[Blocksize * 8];
int origLen = Blocksize + 1;
int toAppend = Blocksize;
FSDataInputStream fis = null;
FileStatus fileStatus = null;
// Create and write a file.
Path file1 = new Path(sub, file1Name);
DFSTestUtil.CreateFile(hdfs, file1, Blocksize, 0, Blocksize, Replication, Seed);
DFSTestUtil.AppendFile(hdfs, file1, origLen);
// Create a snapshot on the parent directory.
hdfs.AllowSnapshot(sub);
hdfs.CreateSnapshot(sub, snapshot1);
Path file1snap1 = SnapshotTestHelper.GetSnapshotPath(sub, snapshot1, file1Name);
FileChecksum snapChksum1 = hdfs.GetFileChecksum(file1snap1);
Assert.AssertThat("file and snapshot file checksums are not equal", hdfs.GetFileChecksum
(file1), CoreMatchers.Is(snapChksum1));
// Append to the file.
FSDataOutputStream @out = hdfs.Append(file1);
// Nothing has been appended yet. All checksums should still be equal.
Assert.AssertThat("file and snapshot checksums (open for append) are not equal",
hdfs.GetFileChecksum(file1), CoreMatchers.Is(snapChksum1));
Assert.AssertThat("snapshot checksum (post-open for append) has changed", hdfs.GetFileChecksum
(file1snap1), CoreMatchers.Is(snapChksum1));
try
{
AppendTestUtil.Write(@out, 0, toAppend);
// Test reading from snapshot of file that is open for append
byte[] dataFromSnapshot = DFSTestUtil.ReadFileBuffer(hdfs, file1snap1);
Assert.AssertThat("Wrong data size in snapshot.", dataFromSnapshot.Length, CoreMatchers.Is
(origLen));
// Verify that checksum didn't change
Assert.AssertThat("snapshot file checksum (pre-close) has changed", hdfs.GetFileChecksum
(file1), CoreMatchers.Is(snapChksum1));
Assert.AssertThat("snapshot checksum (post-append) has changed", hdfs.GetFileChecksum
(file1snap1), CoreMatchers.Is(snapChksum1));
}
finally
{
@out.Close();
}
Assert.AssertThat("file and snapshot file checksums (post-close) are equal", hdfs
.GetFileChecksum(file1), CoreMatchers.Not(snapChksum1));
Assert.AssertThat("snapshot file checksum (post-close) has changed", hdfs.GetFileChecksum
(file1snap1), CoreMatchers.Is(snapChksum1));
// Make sure we can read the entire file via its non-snapshot path.
fileStatus = hdfs.GetFileStatus(file1);
Assert.AssertThat(fileStatus.GetLen(), CoreMatchers.Is((long)origLen + toAppend));
fis = hdfs.Open(file1);
bytesRead = fis.Read(0, buffer, 0, buffer.Length);
Assert.AssertThat(bytesRead, CoreMatchers.Is(origLen + toAppend));
fis.Close();
// Try to open the file via its snapshot path.
fis = hdfs.Open(file1snap1);
fileStatus = hdfs.GetFileStatus(file1snap1);
Assert.AssertThat(fileStatus.GetLen(), CoreMatchers.Is((long)origLen));
// Make sure we can only read up to the snapshot length.
bytesRead = fis.Read(0, buffer, 0, buffer.Length);
Assert.AssertThat(bytesRead, CoreMatchers.Is(origLen));
fis.Close();
byte[] dataFromSnapshot_1 = DFSTestUtil.ReadFileBuffer(hdfs, file1snap1);
Assert.AssertThat("Wrong data size in snapshot.", dataFromSnapshot_1.Length, CoreMatchers.Is
(origLen));
}