public virtual void TestFileStatusSerialziation()
{
long len = fileSystemTestHelper.CreateFile(fHdfs, testfilename);
FileStatus stat = vfs.GetFileStatus(new Path(testfilename));
NUnit.Framework.Assert.AreEqual(len, stat.GetLen());
// check serialization/deserialization
DataOutputBuffer dob = new DataOutputBuffer();
stat.Write(dob);
DataInputBuffer dib = new DataInputBuffer();
dib.Reset(dob.GetData(), 0, dob.GetLength());
FileStatus deSer = new FileStatus();
deSer.ReadFields(dib);
NUnit.Framework.Assert.AreEqual(len, deSer.GetLen());
}
/// <summary>Tests DataTransferProtocol with the given client configuration.</summary>
/// <param name="conf">client configuration</param>
/// <exception cref="System.IO.IOException">if there is an I/O error</exception>
private void DoTest(HdfsConfiguration conf)
{
fs = FileSystem.Get(cluster.GetURI(), conf);
FileSystemTestHelper.CreateFile(fs, Path, NumBlocks, BlockSize);
Assert.AssertArrayEquals(FileSystemTestHelper.GetFileData(NumBlocks, BlockSize),
Sharpen.Runtime.GetBytesForString(DFSTestUtil.ReadFile(fs, Path), "UTF-8"));
BlockLocation[] blockLocations = fs.GetFileBlockLocations(Path, 0, long.MaxValue);
NUnit.Framework.Assert.IsNotNull(blockLocations);
NUnit.Framework.Assert.AreEqual(NumBlocks, blockLocations.Length);
foreach (BlockLocation blockLocation in blockLocations)
{
NUnit.Framework.Assert.IsNotNull(blockLocation.GetHosts());
NUnit.Framework.Assert.AreEqual(3, blockLocation.GetHosts().Length);
}
}
public virtual void SetUp()
{
InitializeTargetTestRoot();
// Make user and data dirs - we creates links to them in the mount table
fsTarget.Mkdirs(new Path(targetTestRoot, "user"));
fsTarget.Mkdirs(new Path(targetTestRoot, "data"));
fsTarget.Mkdirs(new Path(targetTestRoot, "dir2"));
fsTarget.Mkdirs(new Path(targetTestRoot, "dir3"));
FileSystemTestHelper.CreateFile(fsTarget, new Path(targetTestRoot, "aFile"));
// Now we use the mount fs to set links to user and dir
// in the test root
// Set up the defaultMT in the config with our mount point links
conf = ViewFileSystemTestSetup.CreateConfig();
SetupMountPoints();
fsView = FileSystem.Get(FsConstants.ViewfsUri, conf);
}
/// <exception cref="System.IO.IOException"/>
private void CreateFiles(FileSystem fs, string topdir, TestFsShellPermission.FileEntry
[] entries)
{
foreach (TestFsShellPermission.FileEntry entry in entries)
{
string newPathStr = topdir + "/" + entry.GetPath();
Path newPath = new Path(newPathStr);
if (entry.IsDirectory())
{
fs.Mkdirs(newPath);
}
else
{
FileSystemTestHelper.CreateFile(fs, newPath);
}
fs.SetPermission(newPath, new FsPermission(entry.GetPermission()));
fs.SetOwner(newPath, entry.GetOwner(), entry.GetGroup());
}
}
/// <exception cref="System.Exception"/>
private void TestTruncate()
{
if (!IsLocalFS())
{
short repl = 3;
int blockSize = 1024;
int numOfBlocks = 2;
FileSystem fs = FileSystem.Get(GetProxiedFSConf());
fs.Mkdirs(GetProxiedFSTestDir());
Path file = new Path(GetProxiedFSTestDir(), "foo.txt");
byte[] data = FileSystemTestHelper.GetFileData(numOfBlocks, blockSize);
FileSystemTestHelper.CreateFile(fs, file, data, blockSize, repl);
int newLength = blockSize;
bool isReady = fs.Truncate(file, newLength);
NUnit.Framework.Assert.IsTrue("Recovery is not expected.", isReady);
FileStatus fileStatus = fs.GetFileStatus(file);
NUnit.Framework.Assert.AreEqual(fileStatus.GetLen(), newLength);
AppendTestUtil.CheckFullFile(fs, file, newLength, data, file.ToString());
fs.Close();
}
}
public virtual void TestGetBlockLocations()
{
Path targetFilePath = new Path(targetTestRoot, "data/largeFile");
FileSystemTestHelper.CreateFile(fsTarget, targetFilePath, 10, 1024);
Path viewFilePath = new Path("/data/largeFile");
Assert.True("Created File should be type File", fsView.IsFile(viewFilePath
));
BlockLocation[] viewBL = fsView.GetFileBlockLocations(fsView.GetFileStatus(viewFilePath
), 0, 10240 + 100);
Assert.Equal(SupportsBlocks ? 10 : 1, viewBL.Length);
BlockLocation[] targetBL = fsTarget.GetFileBlockLocations(fsTarget.GetFileStatus(
targetFilePath), 0, 10240 + 100);
CompareBLs(viewBL, targetBL);
// Same test but now get it via the FileStatus Parameter
fsView.GetFileBlockLocations(fsView.GetFileStatus(viewFilePath), 0, 10240 + 100);
targetBL = fsTarget.GetFileBlockLocations(fsTarget.GetFileStatus(targetFilePath),
0, 10240 + 100);
CompareBLs(viewBL, targetBL);
}
public virtual void TestGetFileChecksum()
{
// Create two different files in HDFS
fileSystemTestHelper.CreateFile(fHdfs, someFile);
FileSystemTestHelper.CreateFile(fHdfs, fileSystemTestHelper.GetTestRootPath(fHdfs
, someFile + "other"), 1, 512);
// Get checksum through ViewFS
FileChecksum viewFSCheckSum = vfs.GetFileChecksum(new Path("/vfstmp/someFileForTestGetFileChecksum"
));
// Get checksum through HDFS.
FileChecksum hdfsCheckSum = fHdfs.GetFileChecksum(new Path(someFile));
// Get checksum of different file in HDFS
FileChecksum otherHdfsFileCheckSum = fHdfs.GetFileChecksum(new Path(someFile + "other"
));
// Checksums of the same file (got through HDFS and ViewFS should be same)
NUnit.Framework.Assert.AreEqual("HDFS and ViewFS checksums were not the same", viewFSCheckSum
, hdfsCheckSum);
// Checksum of different files should be different.
NUnit.Framework.Assert.IsFalse("Some other HDFS file which should not have had the same "
+ "checksum as viewFS did!", viewFSCheckSum.Equals(otherHdfsFileCheckSum));
}
/// <summary>
/// Test modify operations (create, mkdir, delete, etc)
/// on the mount file system where the pathname references through
/// the mount points.
/// </summary>
/// <remarks>
/// Test modify operations (create, mkdir, delete, etc)
/// on the mount file system where the pathname references through
/// the mount points. Hence these operation will modify the target
/// file system.
/// Verify the operation via mountfs (ie fSys) and *also* via the
/// target file system (ie fSysLocal) that the mount link points-to.
/// </remarks>
/// <exception cref="System.IO.IOException"/>
private void TestOperationsThroughMountLinksInternal(bool located)
{
// Create file
fileSystemTestHelper.CreateFile(fsView, "/user/foo");
Assert.True("Created file should be type file", fsView.IsFile(new
Path("/user/foo")));
Assert.True("Target of created file should be type file", fsTarget
.IsFile(new Path(targetTestRoot, "user/foo")));
// Delete the created file
Assert.True("Delete should suceed", fsView.Delete(new Path("/user/foo"
), false));
NUnit.Framework.Assert.IsFalse("File should not exist after delete", fsView.Exists
(new Path("/user/foo")));
NUnit.Framework.Assert.IsFalse("Target File should not exist after delete", fsTarget
.Exists(new Path(targetTestRoot, "user/foo")));
// Create file with a 2 component dirs
fileSystemTestHelper.CreateFile(fsView, "/internalDir/linkToDir2/foo");
Assert.True("Created file should be type file", fsView.IsFile(new
Path("/internalDir/linkToDir2/foo")));
Assert.True("Target of created file should be type file", fsTarget
.IsFile(new Path(targetTestRoot, "dir2/foo")));
// Delete the created file
Assert.True("Delete should suceed", fsView.Delete(new Path("/internalDir/linkToDir2/foo"
), false));
NUnit.Framework.Assert.IsFalse("File should not exist after delete", fsView.Exists
(new Path("/internalDir/linkToDir2/foo")));
NUnit.Framework.Assert.IsFalse("Target File should not exist after delete", fsTarget
.Exists(new Path(targetTestRoot, "dir2/foo")));
// Create file with a 3 component dirs
fileSystemTestHelper.CreateFile(fsView, "/internalDir/internalDir2/linkToDir3/foo"
);
Assert.True("Created file should be type file", fsView.IsFile(new
Path("/internalDir/internalDir2/linkToDir3/foo")));
Assert.True("Target of created file should be type file", fsTarget
.IsFile(new Path(targetTestRoot, "dir3/foo")));
// Recursive Create file with missing dirs
fileSystemTestHelper.CreateFile(fsView, "/internalDir/linkToDir2/missingDir/miss2/foo"
);
Assert.True("Created file should be type file", fsView.IsFile(new
Path("/internalDir/linkToDir2/missingDir/miss2/foo")));
Assert.True("Target of created file should be type file", fsTarget
.IsFile(new Path(targetTestRoot, "dir2/missingDir/miss2/foo")));
// Delete the created file
Assert.True("Delete should succeed", fsView.Delete(new Path("/internalDir/internalDir2/linkToDir3/foo"
), false));
NUnit.Framework.Assert.IsFalse("File should not exist after delete", fsView.Exists
(new Path("/internalDir/internalDir2/linkToDir3/foo")));
NUnit.Framework.Assert.IsFalse("Target File should not exist after delete", fsTarget
.Exists(new Path(targetTestRoot, "dir3/foo")));
// mkdir
fsView.Mkdirs(fileSystemTestHelper.GetTestRootPath(fsView, "/user/dirX"));
Assert.True("New dir should be type dir", fsView.IsDirectory(new
Path("/user/dirX")));
Assert.True("Target of new dir should be of type dir", fsTarget
.IsDirectory(new Path(targetTestRoot, "user/dirX")));
fsView.Mkdirs(fileSystemTestHelper.GetTestRootPath(fsView, "/user/dirX/dirY"));
Assert.True("New dir should be type dir", fsView.IsDirectory(new
Path("/user/dirX/dirY")));
Assert.True("Target of new dir should be of type dir", fsTarget
.IsDirectory(new Path(targetTestRoot, "user/dirX/dirY")));
// Delete the created dir
Assert.True("Delete should succeed", fsView.Delete(new Path("/user/dirX/dirY"
), false));
NUnit.Framework.Assert.IsFalse("File should not exist after delete", fsView.Exists
(new Path("/user/dirX/dirY")));
NUnit.Framework.Assert.IsFalse("Target File should not exist after delete", fsTarget
.Exists(new Path(targetTestRoot, "user/dirX/dirY")));
Assert.True("Delete should succeed", fsView.Delete(new Path("/user/dirX"
), false));
NUnit.Framework.Assert.IsFalse("File should not exist after delete", fsView.Exists
(new Path("/user/dirX")));
NUnit.Framework.Assert.IsFalse(fsTarget.Exists(new Path(targetTestRoot, "user/dirX"
)));
// Rename a file
fileSystemTestHelper.CreateFile(fsView, "/user/foo");
fsView.Rename(new Path("/user/foo"), new Path("/user/fooBar"));
NUnit.Framework.Assert.IsFalse("Renamed src should not exist", fsView.Exists(new
Path("/user/foo")));
NUnit.Framework.Assert.IsFalse("Renamed src should not exist in target", fsTarget
.Exists(new Path(targetTestRoot, "user/foo")));
Assert.True("Renamed dest should exist as file", fsView.IsFile
(fileSystemTestHelper.GetTestRootPath(fsView, "/user/fooBar")));
Assert.True("Renamed dest should exist as file in target", fsTarget
.IsFile(new Path(targetTestRoot, "user/fooBar")));
fsView.Mkdirs(new Path("/user/dirFoo"));
fsView.Rename(new Path("/user/dirFoo"), new Path("/user/dirFooBar"));
NUnit.Framework.Assert.IsFalse("Renamed src should not exist", fsView.Exists(new
Path("/user/dirFoo")));
NUnit.Framework.Assert.IsFalse("Renamed src should not exist in target", fsTarget
.Exists(new Path(targetTestRoot, "user/dirFoo")));
Assert.True("Renamed dest should exist as dir", fsView.IsDirectory
(fileSystemTestHelper.GetTestRootPath(fsView, "/user/dirFooBar")));
Assert.True("Renamed dest should exist as dir in target", fsTarget
.IsDirectory(new Path(targetTestRoot, "user/dirFooBar")));
// Make a directory under a directory that's mounted from the root of another FS
fsView.Mkdirs(new Path("/targetRoot/dirFoo"));
Assert.True(fsView.Exists(new Path("/targetRoot/dirFoo")));
bool dirFooPresent = false;
foreach (FileStatus fileStatus in ListStatusInternal(located, new Path("/targetRoot/"
)))
{
if (fileStatus.GetPath().GetName().Equals("dirFoo"))
{
dirFooPresent = true;
}
}
Assert.True(dirFooPresent);
}
