/// <param name="fsTarget">- the target fs of the view fs.</param>
/// <returns>return the ViewFS File context to be used for tests</returns>
/// <exception cref="System.Exception"/>
public static FileSystem SetupForViewFileSystem(Configuration conf, FileSystemTestHelper
fileSystemTestHelper, FileSystem fsTarget)
{
Path targetOfTests = fileSystemTestHelper.GetTestRootPath(fsTarget);
// In case previous test was killed before cleanup
fsTarget.Delete(targetOfTests, true);
fsTarget.Mkdirs(targetOfTests);
// Set up viewfs link for test dir as described above
string testDir = fileSystemTestHelper.GetTestRootPath(fsTarget).ToUri().GetPath();
LinkUpFirstComponents(conf, testDir, fsTarget, "test dir");
// Set up viewfs link for home dir as described above
SetUpHomeDir(conf, fsTarget);
// the test path may be relative to working dir - we need to make that work:
// Set up viewfs link for wd as described above
string wdDir = fsTarget.GetWorkingDirectory().ToUri().GetPath();
LinkUpFirstComponents(conf, wdDir, fsTarget, "working dir");
FileSystem fsView = FileSystem.Get(FsConstants.ViewfsUri, conf);
fsView.SetWorkingDirectory(new Path(wdDir));
// in case testdir relative to wd.
Org.Mortbay.Log.Log.Info("Working dir is: " + fsView.GetWorkingDirectory());
return(fsView);
}
public void ShouldCopyContents()
{
FileSystemTestHelper.RecreateDirectory("testfrom");
FileSystemTestHelper.RecreateDirectory("testto");
Touch(@"testfrom\one.txt", "one");
Touch(@"testfrom\two.txt", "two");
Touch(@"testfrom\subdir\three.txt", "three");
Touch(@"testfrom\subdir\exclude.txt", "three");
var includes = new string[0];
var excludes = new string[0];
var filterFactory = new Mock <IFileNameFilterFactory>();
var filter = new Mock <IFileNameFilter>();
filterFactory.Setup(ff => ff.CreateFileNameFilter(excludes, includes)).Returns(filter.Object);
filter.Setup(f => f.IncludeFile(@"one.txt")).Returns(true);
filter.Setup(f => f.IncludeFile(@"two.txt")).Returns(true);
filter.Setup(f => f.IncludeFile(@"subdir\three.txt")).Returns(true);
filter.Setup(f => f.IncludeFile(@"subdir\exclude.txt")).Returns(false);
filter.Setup(f => f.IncludeFile(@"subdir\")).Returns(true);
new DirectoryUtils(filterFactory.Object).CopyDirectory("testfrom", "testto", excludes, includes);
AssertFileContains(@"testto\one.txt", "one");
AssertFileContains(@"testto\two.txt", "two");
AssertFileContains(@"testto\subdir\three.txt", "three");
Assert.That(File.Exists(@"testto\subdir\exclude.txt"), Is.False);
}
public virtual void TestList()
{
FileStatus fs = fSys.GetFileStatus(new Path("/"));
Assert.True(fs.IsDirectory());
// should return the full path not the chrooted path
Assert.Equal(fs.GetPath(), chrootedTo);
// list on Slash
FileStatus[] dirPaths = fSys.ListStatus(new Path("/"));
Assert.Equal(0, dirPaths.Length);
fileSystemTestHelper.CreateFile(fSys, "/foo");
fileSystemTestHelper.CreateFile(fSys, "/bar");
fSys.Mkdirs(new Path("/dirX"));
fSys.Mkdirs(fileSystemTestHelper.GetTestRootPath(fSys, "/dirY"));
fSys.Mkdirs(new Path("/dirX/dirXX"));
dirPaths = fSys.ListStatus(new Path("/"));
Assert.Equal(4, dirPaths.Length);
// note 2 crc files
// Note the the file status paths are the full paths on target
fs = FileSystemTestHelper.ContainsPath(new Path(chrootedTo, "foo"), dirPaths);
NUnit.Framework.Assert.IsNotNull(fs);
Assert.True(fs.IsFile());
fs = FileSystemTestHelper.ContainsPath(new Path(chrootedTo, "bar"), dirPaths);
NUnit.Framework.Assert.IsNotNull(fs);
Assert.True(fs.IsFile());
fs = FileSystemTestHelper.ContainsPath(new Path(chrootedTo, "dirX"), dirPaths);
NUnit.Framework.Assert.IsNotNull(fs);
Assert.True(fs.IsDirectory());
fs = FileSystemTestHelper.ContainsPath(new Path(chrootedTo, "dirY"), dirPaths);
NUnit.Framework.Assert.IsNotNull(fs);
Assert.True(fs.IsDirectory());
}
public virtual void Setup()
{
fsHelper = new FileSystemTestHelper();
string testRoot = fsHelper.GetTestRootDir();
testRootDir = new FilePath(testRoot).GetAbsoluteFile();
}
public virtual void Init()
{
dfs = Org.Mockito.Mockito.Mock <DistributedFileSystem>();
conf = new Configuration();
uri = new URI("hdfs://" + ServiceValue);
FileSystemTestHelper.AddFileSystemForTesting(uri, conf, dfs);
}
public void ShouldRunBatchFileBeforeRunningTargets()
{
UnzipSolution();
FileSystemTestHelper.RecreateDirectory(@"BeforeBounceFeature\bounce");
//as there is a circular dependancy between BeforeBounceFeature.sln and the main bounce dll's
//this needs to be run and built under the same framework version
#if (FRAMEWORKV35)
File.WriteAllText(@"BeforeBounceFeature\bounce\beforebounce.bat", @"%SystemRoot%\Microsoft.NET\Framework\v3.5\msbuild.exe BeforeBounceFeature.sln /p:Configuration=Debug_3_5");
#else
File.WriteAllText(@"BeforeBounceFeature\bounce\beforebounce.bat", @"%SystemRoot%\Microsoft.NET\Framework\v4.0.30319\msbuild.exe BeforeBounceFeature.sln");
#endif
var shell = new Shell(new FakeLog());
ProcessOutput output = null;
Pushd(@"BeforeBounceFeature", () => {
output = shell.Execute(@"..\bounce.exe", "BeforeBounceFeature");
System.Console.WriteLine(output.ErrorAndOutput);
});
Assert.That(output, Is.Not.Null);
Assert.That(output.ExitCode, Is.EqualTo(0));
Assert.That(output.Error.Trim(), Is.EqualTo(""));
Assert.That(output.Output, Is.StringContaining("building before bounce feature"));
}
public virtual void TestSecureHAToken()
{
Configuration conf = DFSTestUtil.NewHAConfiguration(LogicalName);
conf.SetBoolean(DFSConfigKeys.DfsNamenodeDelegationTokenAlwaysUseKey, true);
MiniDFSCluster cluster = null;
WebHdfsFileSystem fs = null;
try
{
cluster = new MiniDFSCluster.Builder(conf).NnTopology(topo).NumDataNodes(0).Build
();
HATestUtil.SetFailoverConfigurations(cluster, conf, LogicalName);
cluster.WaitActive();
fs = Org.Mockito.Mockito.Spy((WebHdfsFileSystem)FileSystem.Get(WebhdfsUri, conf));
FileSystemTestHelper.AddFileSystemForTesting(WebhdfsUri, conf, fs);
cluster.TransitionToActive(0);
Org.Apache.Hadoop.Security.Token.Token <object> token = ((Org.Apache.Hadoop.Security.Token.Token
<DelegationTokenIdentifier>)fs.GetDelegationToken(null));
cluster.ShutdownNameNode(0);
cluster.TransitionToActive(1);
token.Renew(conf);
token.Cancel(conf);
Org.Mockito.Mockito.Verify(fs).RenewDelegationToken(token);
Org.Mockito.Mockito.Verify(fs).CancelDelegationToken(token);
}
finally
{
IOUtils.Cleanup(null, fs);
if (cluster != null)
{
cluster.Shutdown();
}
}
}
/// <summary>delete the test directory in the target fs</summary>
/// <exception cref="System.Exception"/>
public static void TearDown(FileSystemTestHelper fileSystemTestHelper, FileSystem
fsTarget)
{
Path targetOfTests = fileSystemTestHelper.GetTestRootPath(fsTarget);
fsTarget.Delete(targetOfTests, true);
}
public virtual void SetupCluster()
{
conf = new Configuration();
conf.SetInt(DFSConfigKeys.DfsHaTaileditsPeriodKey, 1);
HAUtil.SetAllowStandbyReads(conf, true);
fsHelper = new FileSystemTestHelper();
string testRoot = fsHelper.GetTestRootDir();
testRootDir = new FilePath(testRoot).GetAbsoluteFile();
conf.Set(DFSConfigKeys.DfsEncryptionKeyProviderUri, JavaKeyStoreProvider.SchemeName
+ "://file" + new Path(testRootDir.ToString(), "test.jks").ToUri());
cluster = new MiniDFSCluster.Builder(conf).NnTopology(MiniDFSNNTopology.SimpleHATopology
()).NumDataNodes(1).Build();
cluster.WaitActive();
cluster.TransitionToActive(0);
fs = (DistributedFileSystem)HATestUtil.ConfigureFailoverFs(cluster, conf);
DFSTestUtil.CreateKey(TestKey, cluster, 0, conf);
DFSTestUtil.CreateKey(TestKey, cluster, 1, conf);
nn0 = cluster.GetNameNode(0);
nn1 = cluster.GetNameNode(1);
dfsAdmin0 = new HdfsAdmin(cluster.GetURI(0), conf);
dfsAdmin1 = new HdfsAdmin(cluster.GetURI(1), conf);
KeyProviderCryptoExtension nn0Provider = cluster.GetNameNode(0).GetNamesystem().GetProvider
();
fs.GetClient().SetKeyProvider(nn0Provider);
}
public virtual void SetUp()
{
// create the test root on local_fs
Configuration conf = new Configuration();
fSysTarget = FileSystem.GetLocal(conf);
fileSystemTestHelper = new FileSystemTestHelper();
chrootedTo = fileSystemTestHelper.GetAbsoluteTestRootPath(fSysTarget);
// In case previous test was killed before cleanup
fSysTarget.Delete(chrootedTo, true);
fSysTarget.Mkdirs(chrootedTo);
// ChRoot to the root of the testDirectory
fSys = new ChRootedFileSystem(chrootedTo.ToUri(), conf);
}
/// <exception cref="System.Exception"/>
internal static TestViewFileSystemDelegation.FakeFileSystem SetupFileSystem(URI uri
, Type clazz)
{
string scheme = uri.GetScheme();
conf.Set("fs." + scheme + ".impl", clazz.FullName);
TestViewFileSystemDelegation.FakeFileSystem fs = (TestViewFileSystemDelegation.FakeFileSystem
)FileSystem.Get(uri, conf);
Assert.Equal(uri, fs.GetUri());
Path targetPath = new FileSystemTestHelper().GetAbsoluteTestRootPath(fs);
ConfigUtil.AddLink(conf, "/mounts/" + scheme, targetPath.ToUri());
return(fs);
}
/// <summary>Tests DataTransferProtocol with the given client configuration.</summary>
/// <param name="conf">client configuration</param>
/// <exception cref="System.IO.IOException">if there is an I/O error</exception>
private void DoTest(HdfsConfiguration conf)
{
fs = FileSystem.Get(cluster.GetURI(), conf);
FileSystemTestHelper.CreateFile(fs, Path, NumBlocks, BlockSize);
Assert.AssertArrayEquals(FileSystemTestHelper.GetFileData(NumBlocks, BlockSize),
Sharpen.Runtime.GetBytesForString(DFSTestUtil.ReadFile(fs, Path), "UTF-8"));
BlockLocation[] blockLocations = fs.GetFileBlockLocations(Path, 0, long.MaxValue);
NUnit.Framework.Assert.IsNotNull(blockLocations);
NUnit.Framework.Assert.AreEqual(NumBlocks, blockLocations.Length);
foreach (BlockLocation blockLocation in blockLocations)
{
NUnit.Framework.Assert.IsNotNull(blockLocation.GetHosts());
NUnit.Framework.Assert.AreEqual(3, blockLocation.GetHosts().Length);
}
}
public virtual void SetUp()
{
InitializeTargetTestRoot();
// Make user and data dirs - we creates links to them in the mount table
fsTarget.Mkdirs(new Path(targetTestRoot, "user"));
fsTarget.Mkdirs(new Path(targetTestRoot, "data"));
fsTarget.Mkdirs(new Path(targetTestRoot, "dir2"));
fsTarget.Mkdirs(new Path(targetTestRoot, "dir3"));
FileSystemTestHelper.CreateFile(fsTarget, new Path(targetTestRoot, "aFile"));
// Now we use the mount fs to set links to user and dir
// in the test root
// Set up the defaultMT in the config with our mount point links
conf = ViewFileSystemTestSetup.CreateConfig();
SetupMountPoints();
fsView = FileSystem.Get(FsConstants.ViewfsUri, conf);
}
public void ShouldPrintOutputOfScriptIfItFailed()
{
UnzipSolution();
FileSystemTestHelper.RecreateDirectory(@"BeforeBounceFeature\BeforeBounceFeature\bounce");
File.WriteAllText(@"BeforeBounceFeature\BeforeBounceFeature\bounce\beforebounce.bat", @"c:\Windows\Microsoft.NET\Framework\v3.5\msbuild.exe NonExtantSolution.sln");
var shell = new Shell(new FakeLog());
ProcessOutput output = null;
Pushd(@"BeforeBounceFeature\BeforeBounceFeature", () => shell.Exec(@"..\..\bounce.exe", "BeforeBounceFeature"));
Assert.That(output, Is.Not.Null);
Assert.That(output.ExitCode, Is.EqualTo(1));
Assert.That(output.Error, Is.StringContaining("MSBUILD : error MSB1009: Project file does not exist."));
}
public void ShouldDeleteDirectory()
{
string dir = "test";
FileSystemTestHelper.RecreateDirectory(dir);
Assert.That(Directory.Exists(dir));
Touch(@"test\one.txt", "one");
var directoryUtils = new DirectoryUtils();
directoryUtils.DeleteDirectory(dir);
Assert.That(!Directory.Exists(dir));
directoryUtils.DeleteDirectory(dir);
Assert.That(!Directory.Exists(dir));
}
public virtual void TestFileStatusOnMountLink()
{
Assert.True(fsView.GetFileStatus(new Path("/")).IsDirectory());
FileSystemTestHelper.CheckFileStatus(fsView, "/", FileSystemTestHelper.FileType.isDir
);
FileSystemTestHelper.CheckFileStatus(fsView, "/user", FileSystemTestHelper.FileType
.isDir);
// link followed => dir
FileSystemTestHelper.CheckFileStatus(fsView, "/data", FileSystemTestHelper.FileType
.isDir);
FileSystemTestHelper.CheckFileStatus(fsView, "/internalDir", FileSystemTestHelper.FileType
.isDir);
FileSystemTestHelper.CheckFileStatus(fsView, "/internalDir/linkToDir2", FileSystemTestHelper.FileType
.isDir);
FileSystemTestHelper.CheckFileStatus(fsView, "/internalDir/internalDir2/linkToDir3"
, FileSystemTestHelper.FileType.isDir);
FileSystemTestHelper.CheckFileStatus(fsView, "/linkToAFile", FileSystemTestHelper.FileType
.isFile);
}
/// <exception cref="System.IO.IOException"/>
private void CreateFiles(FileSystem fs, string topdir, TestFsShellPermission.FileEntry
[] entries)
{
foreach (TestFsShellPermission.FileEntry entry in entries)
{
string newPathStr = topdir + "/" + entry.GetPath();
Path newPath = new Path(newPathStr);
if (entry.IsDirectory())
{
fs.Mkdirs(newPath);
}
else
{
FileSystemTestHelper.CreateFile(fs, newPath);
}
fs.SetPermission(newPath, new FsPermission(entry.GetPermission()));
fs.SetOwner(newPath, entry.GetOwner(), entry.GetGroup());
}
}
public static void Setup()
{
string currentUser = Runtime.GetProperty("user.name");
config.Set("fs.permissions.umask-mode", "u=rwx,g=,o=");
config.Set(DefaultImpersonationProvider.GetTestProvider().GetProxySuperuserGroupConfKey
(currentUser), "*");
config.Set(DefaultImpersonationProvider.GetTestProvider().GetProxySuperuserIpConfKey
(currentUser), "*");
fsHelper = new FileSystemTestHelper();
// Set up java key store
string testRoot = fsHelper.GetTestRootDir();
testRootDir = new FilePath(testRoot).GetAbsoluteFile();
Path jksPath = new Path(testRootDir.ToString(), "test.jks");
config.Set(DFSConfigKeys.DfsEncryptionKeyProviderUri, JavaKeyStoreProvider.SchemeName
+ "://file" + jksPath.ToUri());
ProxyUsers.RefreshSuperUserGroupsConfiguration(config);
cluster = new MiniDFSCluster.Builder(config).NumDataNodes(1).Build();
cluster.WaitActive();
hdfs = cluster.GetFileSystem();
nn = cluster.GetNameNode();
dfsAdmin = new HdfsAdmin(cluster.GetURI(), config);
// Use ephemeral ports in case tests are running in parallel
config.SetInt("nfs3.mountd.port", 0);
config.SetInt("nfs3.server.port", 0);
// Start NFS with allowed.hosts set to "* rw"
config.Set("dfs.nfs.exports.allowed.hosts", "* rw");
nfs = new Org.Apache.Hadoop.Hdfs.Nfs.Nfs3.Nfs3(config);
nfs.StartServiceInternal(false);
nfsd = (RpcProgramNfs3)nfs.GetRpcProgram();
hdfs.GetClient().SetKeyProvider(nn.GetNamesystem().GetProvider());
DFSTestUtil.CreateKey(TestKey, cluster, config);
// Mock SecurityHandler which returns system user.name
securityHandler = Org.Mockito.Mockito.Mock <SecurityHandler>();
Org.Mockito.Mockito.When(securityHandler.GetUser()).ThenReturn(currentUser);
// Mock SecurityHandler which returns a dummy username "harry"
securityHandlerUnpriviledged = Org.Mockito.Mockito.Mock <SecurityHandler>();
Org.Mockito.Mockito.When(securityHandlerUnpriviledged.GetUser()).ThenReturn("harry"
);
}
public virtual void TestGetBlockLocations()
{
Path targetFilePath = new Path(targetTestRoot, "data/largeFile");
FileSystemTestHelper.CreateFile(fsTarget, targetFilePath, 10, 1024);
Path viewFilePath = new Path("/data/largeFile");
Assert.True("Created File should be type File", fsView.IsFile(viewFilePath
));
BlockLocation[] viewBL = fsView.GetFileBlockLocations(fsView.GetFileStatus(viewFilePath
), 0, 10240 + 100);
Assert.Equal(SupportsBlocks ? 10 : 1, viewBL.Length);
BlockLocation[] targetBL = fsTarget.GetFileBlockLocations(fsTarget.GetFileStatus(
targetFilePath), 0, 10240 + 100);
CompareBLs(viewBL, targetBL);
// Same test but now get it via the FileStatus Parameter
fsView.GetFileBlockLocations(fsView.GetFileStatus(viewFilePath), 0, 10240 + 100);
targetBL = fsTarget.GetFileBlockLocations(fsTarget.GetFileStatus(targetFilePath),
0, 10240 + 100);
CompareBLs(viewBL, targetBL);
}
/// <exception cref="System.Exception"/>
private void TestTruncate()
{
if (!IsLocalFS())
{
short repl = 3;
int blockSize = 1024;
int numOfBlocks = 2;
FileSystem fs = FileSystem.Get(GetProxiedFSConf());
fs.Mkdirs(GetProxiedFSTestDir());
Path file = new Path(GetProxiedFSTestDir(), "foo.txt");
byte[] data = FileSystemTestHelper.GetFileData(numOfBlocks, blockSize);
FileSystemTestHelper.CreateFile(fs, file, data, blockSize, repl);
int newLength = blockSize;
bool isReady = fs.Truncate(file, newLength);
NUnit.Framework.Assert.IsTrue("Recovery is not expected.", isReady);
FileStatus fileStatus = fs.GetFileStatus(file);
NUnit.Framework.Assert.AreEqual(fileStatus.GetLen(), newLength);
AppendTestUtil.CheckFullFile(fs, file, newLength, data, file.ToString());
fs.Close();
}
}
public virtual void TestGetFileChecksum()
{
// Create two different files in HDFS
fileSystemTestHelper.CreateFile(fHdfs, someFile);
FileSystemTestHelper.CreateFile(fHdfs, fileSystemTestHelper.GetTestRootPath(fHdfs
, someFile + "other"), 1, 512);
// Get checksum through ViewFS
FileChecksum viewFSCheckSum = vfs.GetFileChecksum(new Path("/vfstmp/someFileForTestGetFileChecksum"
));
// Get checksum through HDFS.
FileChecksum hdfsCheckSum = fHdfs.GetFileChecksum(new Path(someFile));
// Get checksum of different file in HDFS
FileChecksum otherHdfsFileCheckSum = fHdfs.GetFileChecksum(new Path(someFile + "other"
));
// Checksums of the same file (got through HDFS and ViewFS should be same)
NUnit.Framework.Assert.AreEqual("HDFS and ViewFS checksums were not the same", viewFSCheckSum
, hdfsCheckSum);
// Checksum of different files should be different.
NUnit.Framework.Assert.IsFalse("Some other HDFS file which should not have had the same "
+ "checksum as viewFS did!", viewFSCheckSum.Equals(otherHdfsFileCheckSum));
}
public virtual void Setup()
{
conf = new HdfsConfiguration();
fsHelper = new FileSystemTestHelper();
// Set up java key store
string testRoot = fsHelper.GetTestRootDir();
FilePath testRootDir = new FilePath(testRoot).GetAbsoluteFile();
Path jksPath = new Path(testRootDir.ToString(), "test.jks");
conf.Set(DFSConfigKeys.DfsEncryptionKeyProviderUri, JavaKeyStoreProvider.SchemeName
+ "://file" + jksPath.ToUri());
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();
Logger.GetLogger(typeof(EncryptionZoneManager)).SetLevel(Level.Trace);
fs = cluster.GetFileSystem();
fsWrapper = new FileSystemTestWrapper(cluster.GetFileSystem());
fcWrapper = new FileContextTestWrapper(FileContext.GetFileContext(cluster.GetURI(
), conf));
dfsAdmin = new HdfsAdmin(cluster.GetURI(), conf);
// Need to set the client's KeyProvider to the NN's for JKS,
// else the updates do not get flushed properly
fs.GetClient().SetKeyProvider(cluster.GetNameNode().GetNamesystem().GetProvider()
);
DFSTestUtil.CreateKey(TestKey, cluster, conf);
}
public virtual void TestRenameAcrossFs()
{
fSys.Mkdirs(new Path("/newDir/dirFoo"));
fSys.Rename(new Path("/newDir/dirFoo"), new Path("file:///tmp/dirFooBar"));
FileSystemTestHelper.IsDir(fSys, new Path("/tmp/dirFooBar"));
}
private void UnzipSolution()
{
FileSystemTestHelper.RecreateDirectory(@"BeforeBounceFeature");
new DirectoryUtils().CopyDirectory(@"..\..\BeforeBounceFeature", @"BeforeBounceFeature", new string [0], new string [0]);
}
public ViewFileSystemBaseTest()
{
// the view file system - the mounts are here
// the target file system - the mount will point here
this.fileSystemTestHelper = CreateFileSystemHelper();
}
private void UnzipSolution()
{
FileSystemTestHelper.RecreateDirectory(@"BeforeBounceFeature");
new FS.FileSystem().Copy(@"..\..\BeforeBounceFeature", @"BeforeBounceFeature");
}
