/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
/// <exception cref="Sharpen.TimeoutException"/>
/// <exception cref="Org.Apache.Hadoop.Conf.ReconfigurationException"/>
public virtual void TestAddVolumesDuringWrite()
{
StartDFSCluster(1, 1);
string bpid = cluster.GetNamesystem().GetBlockPoolId();
Path testFile = new Path("/test");
CreateFile(testFile, 4);
// Each volume has 2 blocks.
AddVolumes(2);
// Continue to write the same file, thus the new volumes will have blocks.
DFSTestUtil.AppendFile(cluster.GetFileSystem(), testFile, BlockSize * 8);
VerifyFileLength(cluster.GetFileSystem(), testFile, 8 + 4);
// After appending data, there should be [2, 2, 4, 4] blocks in each volume
// respectively.
IList <int> expectedNumBlocks = Arrays.AsList(2, 2, 4, 4);
IList <IDictionary <DatanodeStorage, BlockListAsLongs> > blockReports = cluster.GetAllBlockReports
(bpid);
NUnit.Framework.Assert.AreEqual(1, blockReports.Count);
// 1 DataNode
NUnit.Framework.Assert.AreEqual(4, blockReports[0].Count);
// 4 volumes
IDictionary <DatanodeStorage, BlockListAsLongs> dnReport = blockReports[0];
IList <int> actualNumBlocks = new AList <int>();
foreach (BlockListAsLongs blockList in dnReport.Values)
{
actualNumBlocks.AddItem(blockList.GetNumberOfBlocks());
}
actualNumBlocks.Sort();
NUnit.Framework.Assert.AreEqual(expectedNumBlocks, actualNumBlocks);
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Sharpen.TimeoutException"/>
/// <exception cref="System.Exception"/>
/// <exception cref="Org.Apache.Hadoop.Conf.ReconfigurationException"/>
public virtual void TestAddVolumesToFederationNN()
{
// Starts a Cluster with 2 NameNode and 3 DataNodes. Each DataNode has 2
// volumes.
int numNameNodes = 2;
int numDataNodes = 1;
StartDFSCluster(numNameNodes, numDataNodes);
Path testFile = new Path("/test");
// Create a file on the first namespace with 4 blocks.
CreateFile(0, testFile, 4);
// Create a file on the second namespace with 4 blocks.
CreateFile(1, testFile, 4);
// Add 2 volumes to the first DataNode.
int numNewVolumes = 2;
AddVolumes(numNewVolumes);
// Append to the file on the first namespace.
DFSTestUtil.AppendFile(cluster.GetFileSystem(0), testFile, BlockSize * 8);
IList <IList <int> > actualNumBlocks = GetNumBlocksReport(0);
NUnit.Framework.Assert.AreEqual(cluster.GetDataNodes().Count, actualNumBlocks.Count
);
IList <int> blocksOnFirstDN = actualNumBlocks[0];
blocksOnFirstDN.Sort();
NUnit.Framework.Assert.AreEqual(Arrays.AsList(2, 2, 4, 4), blocksOnFirstDN);
// Verify the second namespace also has the new volumes and they are empty.
actualNumBlocks = GetNumBlocksReport(1);
NUnit.Framework.Assert.AreEqual(4, actualNumBlocks[0].Count);
NUnit.Framework.Assert.AreEqual(numNewVolumes, Sharpen.Collections.Frequency(actualNumBlocks
[0], 0));
}
/// <summary>Test snapshot after file appending</summary>
/// <exception cref="System.Exception"/>
public virtual void TestSnapshotAfterAppending()
{
Path file = new Path(dir, "file");
// 1. create snapshot --> create file --> append
SnapshotTestHelper.CreateSnapshot(hdfs, dir, "s0");
DFSTestUtil.CreateFile(hdfs, file, Blocksize, Replication, seed);
DFSTestUtil.AppendFile(hdfs, file, Blocksize);
INodeFile fileNode = (INodeFile)fsdir.GetINode(file.ToString());
// 2. create snapshot --> modify the file --> append
hdfs.CreateSnapshot(dir, "s1");
hdfs.SetReplication(file, (short)(Replication - 1));
DFSTestUtil.AppendFile(hdfs, file, Blocksize);
// check corresponding inodes
fileNode = (INodeFile)fsdir.GetINode(file.ToString());
NUnit.Framework.Assert.AreEqual(Replication - 1, fileNode.GetFileReplication());
NUnit.Framework.Assert.AreEqual(Blocksize * 3, fileNode.ComputeFileSize());
// 3. create snapshot --> append
hdfs.CreateSnapshot(dir, "s2");
DFSTestUtil.AppendFile(hdfs, file, Blocksize);
// check corresponding inodes
fileNode = (INodeFile)fsdir.GetINode(file.ToString());
NUnit.Framework.Assert.AreEqual(Replication - 1, fileNode.GetFileReplication());
NUnit.Framework.Assert.AreEqual(Blocksize * 4, fileNode.ComputeFileSize());
}
/// <summary>
/// Adding as part of jira HDFS-5343
/// Test for checking the cat command on snapshot path it
/// cannot read a file beyond snapshot file length
/// </summary>
/// <exception cref="System.Exception"/>
public virtual void TestSnapshotFileLengthWithCatCommand()
{
FSDataInputStream fis = null;
FileStatus fileStatus = null;
int bytesRead;
byte[] buffer = new byte[Blocksize * 8];
hdfs.Mkdirs(sub);
Path file1 = new Path(sub, file1Name);
DFSTestUtil.CreateFile(hdfs, file1, Blocksize, Replication, Seed);
hdfs.AllowSnapshot(sub);
hdfs.CreateSnapshot(sub, snapshot1);
DFSTestUtil.AppendFile(hdfs, file1, Blocksize);
// Make sure we can read the entire file via its non-snapshot path.
fileStatus = hdfs.GetFileStatus(file1);
NUnit.Framework.Assert.AreEqual("Unexpected file length", Blocksize * 2, fileStatus
.GetLen());
fis = hdfs.Open(file1);
bytesRead = fis.Read(buffer, 0, buffer.Length);
NUnit.Framework.Assert.AreEqual("Unexpected # bytes read", Blocksize * 2, bytesRead
);
fis.Close();
Path file1snap1 = SnapshotTestHelper.GetSnapshotPath(sub, snapshot1, file1Name);
fis = hdfs.Open(file1snap1);
fileStatus = hdfs.GetFileStatus(file1snap1);
NUnit.Framework.Assert.AreEqual(fileStatus.GetLen(), Blocksize);
// Make sure we can only read up to the snapshot length.
bytesRead = fis.Read(buffer, 0, buffer.Length);
NUnit.Framework.Assert.AreEqual("Unexpected # bytes read", Blocksize, bytesRead);
fis.Close();
TextWriter outBackup = System.Console.Out;
TextWriter errBackup = System.Console.Error;
ByteArrayOutputStream bao = new ByteArrayOutputStream();
Runtime.SetOut(new TextWriter(bao));
Runtime.SetErr(new TextWriter(bao));
// Make sure we can cat the file upto to snapshot length
FsShell shell = new FsShell();
try
{
ToolRunner.Run(conf, shell, new string[] { "-cat", "/TestSnapshotFileLength/sub1/.snapshot/snapshot1/file1" });
NUnit.Framework.Assert.AreEqual("Unexpected # bytes from -cat", Blocksize, bao.Size
());
}
finally
{
Runtime.SetOut(outBackup);
Runtime.SetErr(errBackup);
}
}
/// <summary>Test the fsimage saving/loading while file appending.</summary>
/// <exception cref="System.Exception"/>
public virtual void TestSaveLoadImageWithAppending()
{
Path sub1 = new Path(dir, "sub1");
Path sub1file1 = new Path(sub1, "sub1file1");
Path sub1file2 = new Path(sub1, "sub1file2");
DFSTestUtil.CreateFile(hdfs, sub1file1, Blocksize, Replication, seed);
DFSTestUtil.CreateFile(hdfs, sub1file2, Blocksize, Replication, seed);
// 1. create snapshot s0
hdfs.AllowSnapshot(dir);
hdfs.CreateSnapshot(dir, "s0");
// 2. create snapshot s1 before appending sub1file1 finishes
HdfsDataOutputStream @out = AppendFileWithoutClosing(sub1file1, Blocksize);
@out.Hsync(EnumSet.Of(HdfsDataOutputStream.SyncFlag.UpdateLength));
// also append sub1file2
DFSTestUtil.AppendFile(hdfs, sub1file2, Blocksize);
hdfs.CreateSnapshot(dir, "s1");
@out.Close();
// 3. create snapshot s2 before appending finishes
@out = AppendFileWithoutClosing(sub1file1, Blocksize);
@out.Hsync(EnumSet.Of(HdfsDataOutputStream.SyncFlag.UpdateLength));
hdfs.CreateSnapshot(dir, "s2");
@out.Close();
// 4. save fsimage before appending finishes
@out = AppendFileWithoutClosing(sub1file1, Blocksize);
@out.Hsync(EnumSet.Of(HdfsDataOutputStream.SyncFlag.UpdateLength));
// dump fsdir
FilePath fsnBefore = DumpTree2File("before");
// save the namesystem to a temp file
FilePath imageFile = SaveFSImageToTempFile();
// 5. load fsimage and compare
// first restart the cluster, and format the cluster
@out.Close();
cluster.Shutdown();
cluster = new MiniDFSCluster.Builder(conf).Format(true).NumDataNodes(Replication)
.Build();
cluster.WaitActive();
fsn = cluster.GetNamesystem();
hdfs = cluster.GetFileSystem();
// then load the fsimage
LoadFSImageFromTempFile(imageFile);
// dump the fsdir tree again
FilePath fsnAfter = DumpTree2File("after");
// compare two dumped tree
SnapshotTestHelper.CompareDumpedTreeInFile(fsnBefore, fsnAfter, true);
}
/// <summary>Test if the quota can be correctly updated for append</summary>
/// <exception cref="System.Exception"/>
public virtual void TestUpdateQuotaForAppend()
{
Path foo = new Path(dir, "foo");
Path bar = new Path(foo, "bar");
long currentFileLen = Blocksize;
DFSTestUtil.CreateFile(dfs, bar, currentFileLen, Replication, seed);
dfs.SetQuota(foo, long.MaxValue - 1, long.MaxValue - 1);
// append half of the block data, the previous file length is at block
// boundary
DFSTestUtil.AppendFile(dfs, bar, Blocksize / 2);
currentFileLen += (Blocksize / 2);
INodeDirectory fooNode = fsdir.GetINode4Write(foo.ToString()).AsDirectory();
NUnit.Framework.Assert.IsTrue(fooNode.IsQuotaSet());
QuotaCounts quota = fooNode.GetDirectoryWithQuotaFeature().GetSpaceConsumed();
long ns = quota.GetNameSpace();
long ds = quota.GetStorageSpace();
NUnit.Framework.Assert.AreEqual(2, ns);
// foo and bar
NUnit.Framework.Assert.AreEqual(currentFileLen * Replication, ds);
ContentSummary c = dfs.GetContentSummary(foo);
NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), ds);
// append another block, the previous file length is not at block boundary
DFSTestUtil.AppendFile(dfs, bar, Blocksize);
currentFileLen += Blocksize;
quota = fooNode.GetDirectoryWithQuotaFeature().GetSpaceConsumed();
ns = quota.GetNameSpace();
ds = quota.GetStorageSpace();
NUnit.Framework.Assert.AreEqual(2, ns);
// foo and bar
NUnit.Framework.Assert.AreEqual(currentFileLen * Replication, ds);
c = dfs.GetContentSummary(foo);
NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), ds);
// append several blocks
DFSTestUtil.AppendFile(dfs, bar, Blocksize * 3 + Blocksize / 8);
currentFileLen += (Blocksize * 3 + Blocksize / 8);
quota = fooNode.GetDirectoryWithQuotaFeature().GetSpaceConsumed();
ns = quota.GetNameSpace();
ds = quota.GetStorageSpace();
NUnit.Framework.Assert.AreEqual(2, ns);
// foo and bar
NUnit.Framework.Assert.AreEqual(currentFileLen * Replication, ds);
c = dfs.GetContentSummary(foo);
NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), ds);
}
/// <exception cref="System.Exception"/>
public virtual void Prepare()
{
// original size: 2.5 blocks
DFSTestUtil.CreateFile(this._enclosing.dfs, TestTruncateQuotaUpdate.file, TestTruncateQuotaUpdate
.Blocksize * 2 + TestTruncateQuotaUpdate.Blocksize / 2, TestTruncateQuotaUpdate.
Replication, 0L);
SnapshotTestHelper.CreateSnapshot(this._enclosing.dfs, TestTruncateQuotaUpdate.dir
, "s1");
// truncate to 1.5 block
this._enclosing.dfs.Truncate(TestTruncateQuotaUpdate.file, TestTruncateQuotaUpdate
.Blocksize + TestTruncateQuotaUpdate.Blocksize / 2);
TestFileTruncate.CheckBlockRecovery(TestTruncateQuotaUpdate.file, this._enclosing
.dfs);
// append another 1 BLOCK
DFSTestUtil.AppendFile(this._enclosing.dfs, TestTruncateQuotaUpdate.file, TestTruncateQuotaUpdate
.Blocksize);
}
/// <exception cref="System.Exception"/>
public virtual void TestGetFileChecksum(Path foo, int appendLength)
{
int appendRounds = 16;
FileChecksum[] fc = new FileChecksum[appendRounds + 1];
DFSTestUtil.CreateFile(dfs, foo, appendLength, Replication, 0L);
fc[0] = dfs.GetFileChecksum(foo);
for (int i = 0; i < appendRounds; i++)
{
DFSTestUtil.AppendFile(dfs, foo, appendLength);
fc[i + 1] = dfs.GetFileChecksum(foo);
}
for (int i_1 = 0; i_1 < appendRounds + 1; i_1++)
{
FileChecksum checksum = dfs.GetFileChecksum(foo, appendLength * (i_1 + 1));
NUnit.Framework.Assert.IsTrue(checksum.Equals(fc[i_1]));
}
}
/// <summary>
/// Test append over a specific type of storage quota does not mark file as
/// UC or create a lease
/// </summary>
/// <exception cref="System.Exception"/>
public virtual void TestAppendOverTypeQuota()
{
Path dir = new Path("/TestAppendOverTypeQuota");
Path file = new Path(dir, "file");
// create partial block file
dfs.Mkdirs(dir);
// set the storage policy on dir
dfs.SetStoragePolicy(dir, HdfsConstants.OnessdStoragePolicyName);
DFSTestUtil.CreateFile(dfs, file, Blocksize / 2, Replication, seed);
// set quota of SSD to 1L
dfs.SetQuotaByStorageType(dir, StorageType.Ssd, 1L);
INodeDirectory dirNode = fsdir.GetINode4Write(dir.ToString()).AsDirectory();
long spaceUsed = dirNode.GetDirectoryWithQuotaFeature().GetSpaceConsumed().GetStorageSpace
();
try
{
DFSTestUtil.AppendFile(dfs, file, Blocksize);
NUnit.Framework.Assert.Fail("append didn't fail");
}
catch (RemoteException e)
{
NUnit.Framework.Assert.IsTrue(e.GetClassName().Contains("QuotaByStorageTypeExceededException"
));
}
// check that the file exists, isn't UC, and has no dangling lease
INodeFile inode = fsdir.GetINode(file.ToString()).AsFile();
NUnit.Framework.Assert.IsNotNull(inode);
NUnit.Framework.Assert.IsFalse("should not be UC", inode.IsUnderConstruction());
NUnit.Framework.Assert.IsNull("should not have a lease", cluster.GetNamesystem().
GetLeaseManager().GetLeaseByPath(file.ToString()));
// make sure the quota usage is unchanged
long newSpaceUsed = dirNode.GetDirectoryWithQuotaFeature().GetSpaceConsumed().GetStorageSpace
();
NUnit.Framework.Assert.AreEqual(spaceUsed, newSpaceUsed);
// make sure edits aren't corrupted
dfs.RecoverLease(file);
cluster.RestartNameNodes();
}
public virtual void TestDiffReportWithRenameAndAppend()
{
Path root = new Path("/");
Path foo = new Path(root, "foo");
DFSTestUtil.CreateFile(hdfs, foo, Blocksize, Replication, seed);
SnapshotTestHelper.CreateSnapshot(hdfs, root, "s0");
Path bar = new Path(root, "bar");
hdfs.Rename(foo, bar);
DFSTestUtil.AppendFile(hdfs, bar, 10);
// append 10 bytes
SnapshotTestHelper.CreateSnapshot(hdfs, root, "s1");
// we always put modification on the file before rename
VerifyDiffReport(root, "s0", "s1", new SnapshotDiffReport.DiffReportEntry(SnapshotDiffReport.DiffType
.Modify, DFSUtil.String2Bytes(string.Empty)), new SnapshotDiffReport.DiffReportEntry
(SnapshotDiffReport.DiffType.Modify, DFSUtil.String2Bytes("foo")), new SnapshotDiffReport.DiffReportEntry
(SnapshotDiffReport.DiffType.Rename, DFSUtil.String2Bytes("foo"), DFSUtil.String2Bytes
("bar")));
}
/// <exception cref="System.Exception"/>
public virtual void TestQuotaByStorageTypeWithFileCreateAppend()
{
Path foo = new Path(dir, "foo");
Path createdFile1 = new Path(foo, "created_file1.data");
dfs.Mkdirs(foo);
// set storage policy on directory "foo" to ONESSD
dfs.SetStoragePolicy(foo, HdfsConstants.OnessdStoragePolicyName);
// set quota by storage type on directory "foo"
dfs.SetQuotaByStorageType(foo, StorageType.Ssd, Blocksize * 4);
INode fnode = fsdir.GetINode4Write(foo.ToString());
NUnit.Framework.Assert.IsTrue(fnode.IsDirectory());
NUnit.Framework.Assert.IsTrue(fnode.IsQuotaSet());
// Create file of size 2 * BLOCKSIZE under directory "foo"
long file1Len = Blocksize * 2;
int bufLen = Blocksize / 16;
DFSTestUtil.CreateFile(dfs, createdFile1, bufLen, file1Len, Blocksize, Replication
, seed);
// Verify space consumed and remaining quota
long ssdConsumed = fnode.AsDirectory().GetDirectoryWithQuotaFeature().GetSpaceConsumed
().GetTypeSpaces().Get(StorageType.Ssd);
NUnit.Framework.Assert.AreEqual(file1Len, ssdConsumed);
// append several blocks
int appendLen = Blocksize * 2;
DFSTestUtil.AppendFile(dfs, createdFile1, appendLen);
file1Len += appendLen;
ssdConsumed = fnode.AsDirectory().GetDirectoryWithQuotaFeature().GetSpaceConsumed
().GetTypeSpaces().Get(StorageType.Ssd);
NUnit.Framework.Assert.AreEqual(file1Len, ssdConsumed);
ContentSummary cs = dfs.GetContentSummary(foo);
NUnit.Framework.Assert.AreEqual(cs.GetSpaceConsumed(), file1Len * Replication);
NUnit.Framework.Assert.AreEqual(cs.GetTypeConsumed(StorageType.Ssd), file1Len);
NUnit.Framework.Assert.AreEqual(cs.GetTypeConsumed(StorageType.Disk), file1Len *
2);
}
/// <summary>Test append over storage quota does not mark file as UC or create lease</summary>
/// <exception cref="System.Exception"/>
public virtual void TestAppendOverStorageQuota()
{
Path dir = new Path("/TestAppendOverQuota");
Path file = new Path(dir, "file");
// create partial block file
dfs.Mkdirs(dir);
DFSTestUtil.CreateFile(dfs, file, Blocksize / 2, Replication, seed);
// lower quota to cause exception when appending to partial block
dfs.SetQuota(dir, long.MaxValue - 1, 1);
INodeDirectory dirNode = fsdir.GetINode4Write(dir.ToString()).AsDirectory();
long spaceUsed = dirNode.GetDirectoryWithQuotaFeature().GetSpaceConsumed().GetStorageSpace
();
try
{
DFSTestUtil.AppendFile(dfs, file, Blocksize);
NUnit.Framework.Assert.Fail("append didn't fail");
}
catch (DSQuotaExceededException)
{
}
// ignore
// check that the file exists, isn't UC, and has no dangling lease
INodeFile inode = fsdir.GetINode(file.ToString()).AsFile();
NUnit.Framework.Assert.IsNotNull(inode);
NUnit.Framework.Assert.IsFalse("should not be UC", inode.IsUnderConstruction());
NUnit.Framework.Assert.IsNull("should not have a lease", cluster.GetNamesystem().
GetLeaseManager().GetLeaseByPath(file.ToString()));
// make sure the quota usage is unchanged
long newSpaceUsed = dirNode.GetDirectoryWithQuotaFeature().GetSpaceConsumed().GetStorageSpace
();
NUnit.Framework.Assert.AreEqual(spaceUsed, newSpaceUsed);
// make sure edits aren't corrupted
dfs.RecoverLease(file);
cluster.RestartNameNodes();
}
/// <summary>
/// Test if the quota can be correctly updated when file length is updated
/// through fsync
/// </summary>
/// <exception cref="System.Exception"/>
public virtual void TestUpdateQuotaForFSync()
{
Path foo = new Path("/foo");
Path bar = new Path(foo, "bar");
DFSTestUtil.CreateFile(dfs, bar, Blocksize, Replication, 0L);
dfs.SetQuota(foo, long.MaxValue - 1, long.MaxValue - 1);
FSDataOutputStream @out = dfs.Append(bar);
@out.Write(new byte[Blocksize / 4]);
((DFSOutputStream)@out.GetWrappedStream()).Hsync(EnumSet.Of(HdfsDataOutputStream.SyncFlag
.UpdateLength));
INodeDirectory fooNode = fsdir.GetINode4Write(foo.ToString()).AsDirectory();
QuotaCounts quota = fooNode.GetDirectoryWithQuotaFeature().GetSpaceConsumed();
long ns = quota.GetNameSpace();
long ds = quota.GetStorageSpace();
NUnit.Framework.Assert.AreEqual(2, ns);
// foo and bar
NUnit.Framework.Assert.AreEqual(Blocksize * 2 * Replication, ds);
// file is under construction
@out.Write(new byte[Blocksize / 4]);
@out.Close();
fooNode = fsdir.GetINode4Write(foo.ToString()).AsDirectory();
quota = fooNode.GetDirectoryWithQuotaFeature().GetSpaceConsumed();
ns = quota.GetNameSpace();
ds = quota.GetStorageSpace();
NUnit.Framework.Assert.AreEqual(2, ns);
NUnit.Framework.Assert.AreEqual((Blocksize + Blocksize / 2) * Replication, ds);
// append another block
DFSTestUtil.AppendFile(dfs, bar, Blocksize);
quota = fooNode.GetDirectoryWithQuotaFeature().GetSpaceConsumed();
ns = quota.GetNameSpace();
ds = quota.GetStorageSpace();
NUnit.Framework.Assert.AreEqual(2, ns);
// foo and bar
NUnit.Framework.Assert.AreEqual((Blocksize * 2 + Blocksize / 2) * Replication, ds
);
}
public virtual void TestGetBlockLocations()
{
Path root = new Path("/");
Path file = new Path("/file");
DFSTestUtil.CreateFile(hdfs, file, Blocksize, Replication, seed);
// take a snapshot on root
SnapshotTestHelper.CreateSnapshot(hdfs, root, "s1");
Path fileInSnapshot = SnapshotTestHelper.GetSnapshotPath(root, "s1", file.GetName
());
FileStatus status = hdfs.GetFileStatus(fileInSnapshot);
// make sure we record the size for the file
NUnit.Framework.Assert.AreEqual(Blocksize, status.GetLen());
// append data to file
DFSTestUtil.AppendFile(hdfs, file, Blocksize - 1);
status = hdfs.GetFileStatus(fileInSnapshot);
// the size of snapshot file should still be BLOCKSIZE
NUnit.Framework.Assert.AreEqual(Blocksize, status.GetLen());
// the size of the file should be (2 * BLOCKSIZE - 1)
status = hdfs.GetFileStatus(file);
NUnit.Framework.Assert.AreEqual(Blocksize * 2 - 1, status.GetLen());
// call DFSClient#callGetBlockLocations for the file in snapshot
LocatedBlocks blocks = DFSClientAdapter.CallGetBlockLocations(cluster.GetNameNodeRpc
(), fileInSnapshot.ToString(), 0, long.MaxValue);
IList <LocatedBlock> blockList = blocks.GetLocatedBlocks();
// should be only one block
NUnit.Framework.Assert.AreEqual(Blocksize, blocks.GetFileLength());
NUnit.Framework.Assert.AreEqual(1, blockList.Count);
// check the last block
LocatedBlock lastBlock = blocks.GetLastLocatedBlock();
NUnit.Framework.Assert.AreEqual(0, lastBlock.GetStartOffset());
NUnit.Framework.Assert.AreEqual(Blocksize, lastBlock.GetBlockSize());
// take another snapshot
SnapshotTestHelper.CreateSnapshot(hdfs, root, "s2");
Path fileInSnapshot2 = SnapshotTestHelper.GetSnapshotPath(root, "s2", file.GetName
());
// append data to file without closing
HdfsDataOutputStream @out = AppendFileWithoutClosing(file, Blocksize);
@out.Hsync(EnumSet.Of(HdfsDataOutputStream.SyncFlag.UpdateLength));
status = hdfs.GetFileStatus(fileInSnapshot2);
// the size of snapshot file should be BLOCKSIZE*2-1
NUnit.Framework.Assert.AreEqual(Blocksize * 2 - 1, status.GetLen());
// the size of the file should be (3 * BLOCKSIZE - 1)
status = hdfs.GetFileStatus(file);
NUnit.Framework.Assert.AreEqual(Blocksize * 3 - 1, status.GetLen());
blocks = DFSClientAdapter.CallGetBlockLocations(cluster.GetNameNodeRpc(), fileInSnapshot2
.ToString(), 0, long.MaxValue);
NUnit.Framework.Assert.IsFalse(blocks.IsUnderConstruction());
NUnit.Framework.Assert.IsTrue(blocks.IsLastBlockComplete());
blockList = blocks.GetLocatedBlocks();
// should be 2 blocks
NUnit.Framework.Assert.AreEqual(Blocksize * 2 - 1, blocks.GetFileLength());
NUnit.Framework.Assert.AreEqual(2, blockList.Count);
// check the last block
lastBlock = blocks.GetLastLocatedBlock();
NUnit.Framework.Assert.AreEqual(Blocksize, lastBlock.GetStartOffset());
NUnit.Framework.Assert.AreEqual(Blocksize, lastBlock.GetBlockSize());
blocks = DFSClientAdapter.CallGetBlockLocations(cluster.GetNameNodeRpc(), fileInSnapshot2
.ToString(), Blocksize, 0);
blockList = blocks.GetLocatedBlocks();
NUnit.Framework.Assert.AreEqual(1, blockList.Count);
// check blocks for file being written
blocks = DFSClientAdapter.CallGetBlockLocations(cluster.GetNameNodeRpc(), file.ToString
(), 0, long.MaxValue);
blockList = blocks.GetLocatedBlocks();
NUnit.Framework.Assert.AreEqual(3, blockList.Count);
NUnit.Framework.Assert.IsTrue(blocks.IsUnderConstruction());
NUnit.Framework.Assert.IsFalse(blocks.IsLastBlockComplete());
lastBlock = blocks.GetLastLocatedBlock();
NUnit.Framework.Assert.AreEqual(Blocksize * 2, lastBlock.GetStartOffset());
NUnit.Framework.Assert.AreEqual(Blocksize - 1, lastBlock.GetBlockSize());
@out.Close();
}
/// <summary>for snapshot file while modifying file after snapshot.</summary>
/// <exception cref="System.Exception"/>
public virtual void TestSnapshotPathINodesAfterModification()
{
// First check the INode for /TestSnapshot/sub1/file1
string[] names = INode.GetPathNames(file1.ToString());
byte[][] components = INode.GetPathComponents(names);
INodesInPath nodesInPath = INodesInPath.Resolve(fsdir.rootDir, components, false);
// The number of inodes should be equal to components.length
NUnit.Framework.Assert.AreEqual(nodesInPath.Length(), components.Length);
// The last INode should be associated with file1
NUnit.Framework.Assert.AreEqual(nodesInPath.GetINode(components.Length - 1).GetFullPathName
(), file1.ToString());
// record the modification time of the inode
long modTime = nodesInPath.GetINode(nodesInPath.Length() - 1).GetModificationTime
();
// Create a snapshot for the dir, and check the inodes for the path
// pointing to a snapshot file
hdfs.AllowSnapshot(sub1);
hdfs.CreateSnapshot(sub1, "s3");
// Modify file1
DFSTestUtil.AppendFile(hdfs, file1, "the content for appending");
// Check the INodes for snapshot of file1
string snapshotPath = sub1.ToString() + "/.snapshot/s3/file1";
names = INode.GetPathNames(snapshotPath);
components = INode.GetPathComponents(names);
INodesInPath ssNodesInPath = INodesInPath.Resolve(fsdir.rootDir, components, false
);
// Length of ssInodes should be (components.length - 1), since we will
// ignore ".snapshot"
NUnit.Framework.Assert.AreEqual(ssNodesInPath.Length(), components.Length - 1);
Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot s3 = GetSnapshot(ssNodesInPath
, "s3", 3);
AssertSnapshot(ssNodesInPath, true, s3, 3);
// Check the INode for snapshot of file1
INode snapshotFileNode = ssNodesInPath.GetLastINode();
NUnit.Framework.Assert.AreEqual(snapshotFileNode.GetLocalName(), file1.GetName());
NUnit.Framework.Assert.IsTrue(snapshotFileNode.AsFile().IsWithSnapshot());
// The modification time of the snapshot INode should be the same with the
// original INode before modification
NUnit.Framework.Assert.AreEqual(modTime, snapshotFileNode.GetModificationTime(ssNodesInPath
.GetPathSnapshotId()));
// Check the INode for /TestSnapshot/sub1/file1 again
names = INode.GetPathNames(file1.ToString());
components = INode.GetPathComponents(names);
INodesInPath newNodesInPath = INodesInPath.Resolve(fsdir.rootDir, components, false
);
AssertSnapshot(newNodesInPath, false, s3, -1);
// The number of inodes should be equal to components.length
NUnit.Framework.Assert.AreEqual(newNodesInPath.Length(), components.Length);
// The last INode should be associated with file1
int last = components.Length - 1;
NUnit.Framework.Assert.AreEqual(newNodesInPath.GetINode(last).GetFullPathName(),
file1.ToString());
// The modification time of the INode for file3 should have been changed
NUnit.Framework.Assert.IsFalse(modTime == newNodesInPath.GetINode(last).GetModificationTime
());
hdfs.DeleteSnapshot(sub1, "s3");
hdfs.DisallowSnapshot(sub1);
}
/// <exception cref="System.IO.IOException"/>
public void Run(FileSystem fs)
{
DFSTestUtil.AppendFile(fs, file1, "new bytes");
}
/// <summary>
/// Test that we cannot read a file beyond its snapshot length
/// when accessing it via a snapshot path.
/// </summary>
/// <exception cref="System.Exception"/>
public virtual void TestSnapshotfileLength()
{
hdfs.Mkdirs(sub);
int bytesRead;
byte[] buffer = new byte[Blocksize * 8];
int origLen = Blocksize + 1;
int toAppend = Blocksize;
FSDataInputStream fis = null;
FileStatus fileStatus = null;
// Create and write a file.
Path file1 = new Path(sub, file1Name);
DFSTestUtil.CreateFile(hdfs, file1, Blocksize, 0, Blocksize, Replication, Seed);
DFSTestUtil.AppendFile(hdfs, file1, origLen);
// Create a snapshot on the parent directory.
hdfs.AllowSnapshot(sub);
hdfs.CreateSnapshot(sub, snapshot1);
Path file1snap1 = SnapshotTestHelper.GetSnapshotPath(sub, snapshot1, file1Name);
FileChecksum snapChksum1 = hdfs.GetFileChecksum(file1snap1);
Assert.AssertThat("file and snapshot file checksums are not equal", hdfs.GetFileChecksum
(file1), CoreMatchers.Is(snapChksum1));
// Append to the file.
FSDataOutputStream @out = hdfs.Append(file1);
// Nothing has been appended yet. All checksums should still be equal.
Assert.AssertThat("file and snapshot checksums (open for append) are not equal",
hdfs.GetFileChecksum(file1), CoreMatchers.Is(snapChksum1));
Assert.AssertThat("snapshot checksum (post-open for append) has changed", hdfs.GetFileChecksum
(file1snap1), CoreMatchers.Is(snapChksum1));
try
{
AppendTestUtil.Write(@out, 0, toAppend);
// Test reading from snapshot of file that is open for append
byte[] dataFromSnapshot = DFSTestUtil.ReadFileBuffer(hdfs, file1snap1);
Assert.AssertThat("Wrong data size in snapshot.", dataFromSnapshot.Length, CoreMatchers.Is
(origLen));
// Verify that checksum didn't change
Assert.AssertThat("snapshot file checksum (pre-close) has changed", hdfs.GetFileChecksum
(file1), CoreMatchers.Is(snapChksum1));
Assert.AssertThat("snapshot checksum (post-append) has changed", hdfs.GetFileChecksum
(file1snap1), CoreMatchers.Is(snapChksum1));
}
finally
{
@out.Close();
}
Assert.AssertThat("file and snapshot file checksums (post-close) are equal", hdfs
.GetFileChecksum(file1), CoreMatchers.Not(snapChksum1));
Assert.AssertThat("snapshot file checksum (post-close) has changed", hdfs.GetFileChecksum
(file1snap1), CoreMatchers.Is(snapChksum1));
// Make sure we can read the entire file via its non-snapshot path.
fileStatus = hdfs.GetFileStatus(file1);
Assert.AssertThat(fileStatus.GetLen(), CoreMatchers.Is((long)origLen + toAppend));
fis = hdfs.Open(file1);
bytesRead = fis.Read(0, buffer, 0, buffer.Length);
Assert.AssertThat(bytesRead, CoreMatchers.Is(origLen + toAppend));
fis.Close();
// Try to open the file via its snapshot path.
fis = hdfs.Open(file1snap1);
fileStatus = hdfs.GetFileStatus(file1snap1);
Assert.AssertThat(fileStatus.GetLen(), CoreMatchers.Is((long)origLen));
// Make sure we can only read up to the snapshot length.
bytesRead = fis.Read(0, buffer, 0, buffer.Length);
Assert.AssertThat(bytesRead, CoreMatchers.Is(origLen));
fis.Close();
byte[] dataFromSnapshot_1 = DFSTestUtil.ReadFileBuffer(hdfs, file1snap1);
Assert.AssertThat("Wrong data size in snapshot.", dataFromSnapshot_1.Length, CoreMatchers.Is
(origLen));
}
