/// <param name="fsTarget">- the target fs of the view fs.</param> /// <returns>return the ViewFS File context to be used for tests</returns> /// <exception cref="System.Exception"/> public static FileSystem SetupForViewFileSystem(Configuration conf, FileSystemTestHelper fileSystemTestHelper, FileSystem fsTarget) { Path targetOfTests = fileSystemTestHelper.GetTestRootPath(fsTarget); // In case previous test was killed before cleanup fsTarget.Delete(targetOfTests, true); fsTarget.Mkdirs(targetOfTests); // Set up viewfs link for test dir as described above string testDir = fileSystemTestHelper.GetTestRootPath(fsTarget).ToUri().GetPath(); LinkUpFirstComponents(conf, testDir, fsTarget, "test dir"); // Set up viewfs link for home dir as described above SetUpHomeDir(conf, fsTarget); // the test path may be relative to working dir - we need to make that work: // Set up viewfs link for wd as described above string wdDir = fsTarget.GetWorkingDirectory().ToUri().GetPath(); LinkUpFirstComponents(conf, wdDir, fsTarget, "working dir"); FileSystem fsView = FileSystem.Get(FsConstants.ViewfsUri, conf); fsView.SetWorkingDirectory(new Path(wdDir)); // in case testdir relative to wd. Org.Mortbay.Log.Log.Info("Working dir is: " + fsView.GetWorkingDirectory()); return(fsView); }
public void ShouldCopyContents() { FileSystemTestHelper.RecreateDirectory("testfrom"); FileSystemTestHelper.RecreateDirectory("testto"); Touch(@"testfrom\one.txt", "one"); Touch(@"testfrom\two.txt", "two"); Touch(@"testfrom\subdir\three.txt", "three"); Touch(@"testfrom\subdir\exclude.txt", "three"); var includes = new string[0]; var excludes = new string[0]; var filterFactory = new Mock <IFileNameFilterFactory>(); var filter = new Mock <IFileNameFilter>(); filterFactory.Setup(ff => ff.CreateFileNameFilter(excludes, includes)).Returns(filter.Object); filter.Setup(f => f.IncludeFile(@"one.txt")).Returns(true); filter.Setup(f => f.IncludeFile(@"two.txt")).Returns(true); filter.Setup(f => f.IncludeFile(@"subdir\three.txt")).Returns(true); filter.Setup(f => f.IncludeFile(@"subdir\exclude.txt")).Returns(false); filter.Setup(f => f.IncludeFile(@"subdir\")).Returns(true); new DirectoryUtils(filterFactory.Object).CopyDirectory("testfrom", "testto", excludes, includes); AssertFileContains(@"testto\one.txt", "one"); AssertFileContains(@"testto\two.txt", "two"); AssertFileContains(@"testto\subdir\three.txt", "three"); Assert.That(File.Exists(@"testto\subdir\exclude.txt"), Is.False); }
public virtual void TestList() { FileStatus fs = fSys.GetFileStatus(new Path("/")); Assert.True(fs.IsDirectory()); // should return the full path not the chrooted path Assert.Equal(fs.GetPath(), chrootedTo); // list on Slash FileStatus[] dirPaths = fSys.ListStatus(new Path("/")); Assert.Equal(0, dirPaths.Length); fileSystemTestHelper.CreateFile(fSys, "/foo"); fileSystemTestHelper.CreateFile(fSys, "/bar"); fSys.Mkdirs(new Path("/dirX")); fSys.Mkdirs(fileSystemTestHelper.GetTestRootPath(fSys, "/dirY")); fSys.Mkdirs(new Path("/dirX/dirXX")); dirPaths = fSys.ListStatus(new Path("/")); Assert.Equal(4, dirPaths.Length); // note 2 crc files // Note the the file status paths are the full paths on target fs = FileSystemTestHelper.ContainsPath(new Path(chrootedTo, "foo"), dirPaths); NUnit.Framework.Assert.IsNotNull(fs); Assert.True(fs.IsFile()); fs = FileSystemTestHelper.ContainsPath(new Path(chrootedTo, "bar"), dirPaths); NUnit.Framework.Assert.IsNotNull(fs); Assert.True(fs.IsFile()); fs = FileSystemTestHelper.ContainsPath(new Path(chrootedTo, "dirX"), dirPaths); NUnit.Framework.Assert.IsNotNull(fs); Assert.True(fs.IsDirectory()); fs = FileSystemTestHelper.ContainsPath(new Path(chrootedTo, "dirY"), dirPaths); NUnit.Framework.Assert.IsNotNull(fs); Assert.True(fs.IsDirectory()); }
public virtual void Setup() { fsHelper = new FileSystemTestHelper(); string testRoot = fsHelper.GetTestRootDir(); testRootDir = new FilePath(testRoot).GetAbsoluteFile(); }
public virtual void Init() { dfs = Org.Mockito.Mockito.Mock <DistributedFileSystem>(); conf = new Configuration(); uri = new URI("hdfs://" + ServiceValue); FileSystemTestHelper.AddFileSystemForTesting(uri, conf, dfs); }
public void ShouldRunBatchFileBeforeRunningTargets() { UnzipSolution(); FileSystemTestHelper.RecreateDirectory(@"BeforeBounceFeature\bounce"); //as there is a circular dependancy between BeforeBounceFeature.sln and the main bounce dll's //this needs to be run and built under the same framework version #if (FRAMEWORKV35) File.WriteAllText(@"BeforeBounceFeature\bounce\beforebounce.bat", @"%SystemRoot%\Microsoft.NET\Framework\v3.5\msbuild.exe BeforeBounceFeature.sln /p:Configuration=Debug_3_5"); #else File.WriteAllText(@"BeforeBounceFeature\bounce\beforebounce.bat", @"%SystemRoot%\Microsoft.NET\Framework\v4.0.30319\msbuild.exe BeforeBounceFeature.sln"); #endif var shell = new Shell(new FakeLog()); ProcessOutput output = null; Pushd(@"BeforeBounceFeature", () => { output = shell.Execute(@"..\bounce.exe", "BeforeBounceFeature"); System.Console.WriteLine(output.ErrorAndOutput); }); Assert.That(output, Is.Not.Null); Assert.That(output.ExitCode, Is.EqualTo(0)); Assert.That(output.Error.Trim(), Is.EqualTo("")); Assert.That(output.Output, Is.StringContaining("building before bounce feature")); }
public virtual void TestSecureHAToken() { Configuration conf = DFSTestUtil.NewHAConfiguration(LogicalName); conf.SetBoolean(DFSConfigKeys.DfsNamenodeDelegationTokenAlwaysUseKey, true); MiniDFSCluster cluster = null; WebHdfsFileSystem fs = null; try { cluster = new MiniDFSCluster.Builder(conf).NnTopology(topo).NumDataNodes(0).Build (); HATestUtil.SetFailoverConfigurations(cluster, conf, LogicalName); cluster.WaitActive(); fs = Org.Mockito.Mockito.Spy((WebHdfsFileSystem)FileSystem.Get(WebhdfsUri, conf)); FileSystemTestHelper.AddFileSystemForTesting(WebhdfsUri, conf, fs); cluster.TransitionToActive(0); Org.Apache.Hadoop.Security.Token.Token <object> token = ((Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier>)fs.GetDelegationToken(null)); cluster.ShutdownNameNode(0); cluster.TransitionToActive(1); token.Renew(conf); token.Cancel(conf); Org.Mockito.Mockito.Verify(fs).RenewDelegationToken(token); Org.Mockito.Mockito.Verify(fs).CancelDelegationToken(token); } finally { IOUtils.Cleanup(null, fs); if (cluster != null) { cluster.Shutdown(); } } }
/// <summary>delete the test directory in the target fs</summary> /// <exception cref="System.Exception"/> public static void TearDown(FileSystemTestHelper fileSystemTestHelper, FileSystem fsTarget) { Path targetOfTests = fileSystemTestHelper.GetTestRootPath(fsTarget); fsTarget.Delete(targetOfTests, true); }
public virtual void SetupCluster() { conf = new Configuration(); conf.SetInt(DFSConfigKeys.DfsHaTaileditsPeriodKey, 1); HAUtil.SetAllowStandbyReads(conf, true); fsHelper = new FileSystemTestHelper(); string testRoot = fsHelper.GetTestRootDir(); testRootDir = new FilePath(testRoot).GetAbsoluteFile(); conf.Set(DFSConfigKeys.DfsEncryptionKeyProviderUri, JavaKeyStoreProvider.SchemeName + "://file" + new Path(testRootDir.ToString(), "test.jks").ToUri()); cluster = new MiniDFSCluster.Builder(conf).NnTopology(MiniDFSNNTopology.SimpleHATopology ()).NumDataNodes(1).Build(); cluster.WaitActive(); cluster.TransitionToActive(0); fs = (DistributedFileSystem)HATestUtil.ConfigureFailoverFs(cluster, conf); DFSTestUtil.CreateKey(TestKey, cluster, 0, conf); DFSTestUtil.CreateKey(TestKey, cluster, 1, conf); nn0 = cluster.GetNameNode(0); nn1 = cluster.GetNameNode(1); dfsAdmin0 = new HdfsAdmin(cluster.GetURI(0), conf); dfsAdmin1 = new HdfsAdmin(cluster.GetURI(1), conf); KeyProviderCryptoExtension nn0Provider = cluster.GetNameNode(0).GetNamesystem().GetProvider (); fs.GetClient().SetKeyProvider(nn0Provider); }
public virtual void SetUp() { // create the test root on local_fs Configuration conf = new Configuration(); fSysTarget = FileSystem.GetLocal(conf); fileSystemTestHelper = new FileSystemTestHelper(); chrootedTo = fileSystemTestHelper.GetAbsoluteTestRootPath(fSysTarget); // In case previous test was killed before cleanup fSysTarget.Delete(chrootedTo, true); fSysTarget.Mkdirs(chrootedTo); // ChRoot to the root of the testDirectory fSys = new ChRootedFileSystem(chrootedTo.ToUri(), conf); }
/// <exception cref="System.Exception"/> internal static TestViewFileSystemDelegation.FakeFileSystem SetupFileSystem(URI uri , Type clazz) { string scheme = uri.GetScheme(); conf.Set("fs." + scheme + ".impl", clazz.FullName); TestViewFileSystemDelegation.FakeFileSystem fs = (TestViewFileSystemDelegation.FakeFileSystem )FileSystem.Get(uri, conf); Assert.Equal(uri, fs.GetUri()); Path targetPath = new FileSystemTestHelper().GetAbsoluteTestRootPath(fs); ConfigUtil.AddLink(conf, "/mounts/" + scheme, targetPath.ToUri()); return(fs); }
/// <summary>Tests DataTransferProtocol with the given client configuration.</summary> /// <param name="conf">client configuration</param> /// <exception cref="System.IO.IOException">if there is an I/O error</exception> private void DoTest(HdfsConfiguration conf) { fs = FileSystem.Get(cluster.GetURI(), conf); FileSystemTestHelper.CreateFile(fs, Path, NumBlocks, BlockSize); Assert.AssertArrayEquals(FileSystemTestHelper.GetFileData(NumBlocks, BlockSize), Sharpen.Runtime.GetBytesForString(DFSTestUtil.ReadFile(fs, Path), "UTF-8")); BlockLocation[] blockLocations = fs.GetFileBlockLocations(Path, 0, long.MaxValue); NUnit.Framework.Assert.IsNotNull(blockLocations); NUnit.Framework.Assert.AreEqual(NumBlocks, blockLocations.Length); foreach (BlockLocation blockLocation in blockLocations) { NUnit.Framework.Assert.IsNotNull(blockLocation.GetHosts()); NUnit.Framework.Assert.AreEqual(3, blockLocation.GetHosts().Length); } }
public virtual void SetUp() { InitializeTargetTestRoot(); // Make user and data dirs - we creates links to them in the mount table fsTarget.Mkdirs(new Path(targetTestRoot, "user")); fsTarget.Mkdirs(new Path(targetTestRoot, "data")); fsTarget.Mkdirs(new Path(targetTestRoot, "dir2")); fsTarget.Mkdirs(new Path(targetTestRoot, "dir3")); FileSystemTestHelper.CreateFile(fsTarget, new Path(targetTestRoot, "aFile")); // Now we use the mount fs to set links to user and dir // in the test root // Set up the defaultMT in the config with our mount point links conf = ViewFileSystemTestSetup.CreateConfig(); SetupMountPoints(); fsView = FileSystem.Get(FsConstants.ViewfsUri, conf); }
public void ShouldPrintOutputOfScriptIfItFailed() { UnzipSolution(); FileSystemTestHelper.RecreateDirectory(@"BeforeBounceFeature\BeforeBounceFeature\bounce"); File.WriteAllText(@"BeforeBounceFeature\BeforeBounceFeature\bounce\beforebounce.bat", @"c:\Windows\Microsoft.NET\Framework\v3.5\msbuild.exe NonExtantSolution.sln"); var shell = new Shell(new FakeLog()); ProcessOutput output = null; Pushd(@"BeforeBounceFeature\BeforeBounceFeature", () => shell.Exec(@"..\..\bounce.exe", "BeforeBounceFeature")); Assert.That(output, Is.Not.Null); Assert.That(output.ExitCode, Is.EqualTo(1)); Assert.That(output.Error, Is.StringContaining("MSBUILD : error MSB1009: Project file does not exist.")); }
public void ShouldDeleteDirectory() { string dir = "test"; FileSystemTestHelper.RecreateDirectory(dir); Assert.That(Directory.Exists(dir)); Touch(@"test\one.txt", "one"); var directoryUtils = new DirectoryUtils(); directoryUtils.DeleteDirectory(dir); Assert.That(!Directory.Exists(dir)); directoryUtils.DeleteDirectory(dir); Assert.That(!Directory.Exists(dir)); }
public virtual void TestFileStatusOnMountLink() { Assert.True(fsView.GetFileStatus(new Path("/")).IsDirectory()); FileSystemTestHelper.CheckFileStatus(fsView, "/", FileSystemTestHelper.FileType.isDir ); FileSystemTestHelper.CheckFileStatus(fsView, "/user", FileSystemTestHelper.FileType .isDir); // link followed => dir FileSystemTestHelper.CheckFileStatus(fsView, "/data", FileSystemTestHelper.FileType .isDir); FileSystemTestHelper.CheckFileStatus(fsView, "/internalDir", FileSystemTestHelper.FileType .isDir); FileSystemTestHelper.CheckFileStatus(fsView, "/internalDir/linkToDir2", FileSystemTestHelper.FileType .isDir); FileSystemTestHelper.CheckFileStatus(fsView, "/internalDir/internalDir2/linkToDir3" , FileSystemTestHelper.FileType.isDir); FileSystemTestHelper.CheckFileStatus(fsView, "/linkToAFile", FileSystemTestHelper.FileType .isFile); }
/// <exception cref="System.IO.IOException"/> private void CreateFiles(FileSystem fs, string topdir, TestFsShellPermission.FileEntry [] entries) { foreach (TestFsShellPermission.FileEntry entry in entries) { string newPathStr = topdir + "/" + entry.GetPath(); Path newPath = new Path(newPathStr); if (entry.IsDirectory()) { fs.Mkdirs(newPath); } else { FileSystemTestHelper.CreateFile(fs, newPath); } fs.SetPermission(newPath, new FsPermission(entry.GetPermission())); fs.SetOwner(newPath, entry.GetOwner(), entry.GetGroup()); } }
public static void Setup() { string currentUser = Runtime.GetProperty("user.name"); config.Set("fs.permissions.umask-mode", "u=rwx,g=,o="); config.Set(DefaultImpersonationProvider.GetTestProvider().GetProxySuperuserGroupConfKey (currentUser), "*"); config.Set(DefaultImpersonationProvider.GetTestProvider().GetProxySuperuserIpConfKey (currentUser), "*"); fsHelper = new FileSystemTestHelper(); // Set up java key store string testRoot = fsHelper.GetTestRootDir(); testRootDir = new FilePath(testRoot).GetAbsoluteFile(); Path jksPath = new Path(testRootDir.ToString(), "test.jks"); config.Set(DFSConfigKeys.DfsEncryptionKeyProviderUri, JavaKeyStoreProvider.SchemeName + "://file" + jksPath.ToUri()); ProxyUsers.RefreshSuperUserGroupsConfiguration(config); cluster = new MiniDFSCluster.Builder(config).NumDataNodes(1).Build(); cluster.WaitActive(); hdfs = cluster.GetFileSystem(); nn = cluster.GetNameNode(); dfsAdmin = new HdfsAdmin(cluster.GetURI(), config); // Use ephemeral ports in case tests are running in parallel config.SetInt("nfs3.mountd.port", 0); config.SetInt("nfs3.server.port", 0); // Start NFS with allowed.hosts set to "* rw" config.Set("dfs.nfs.exports.allowed.hosts", "* rw"); nfs = new Org.Apache.Hadoop.Hdfs.Nfs.Nfs3.Nfs3(config); nfs.StartServiceInternal(false); nfsd = (RpcProgramNfs3)nfs.GetRpcProgram(); hdfs.GetClient().SetKeyProvider(nn.GetNamesystem().GetProvider()); DFSTestUtil.CreateKey(TestKey, cluster, config); // Mock SecurityHandler which returns system user.name securityHandler = Org.Mockito.Mockito.Mock <SecurityHandler>(); Org.Mockito.Mockito.When(securityHandler.GetUser()).ThenReturn(currentUser); // Mock SecurityHandler which returns a dummy username "harry" securityHandlerUnpriviledged = Org.Mockito.Mockito.Mock <SecurityHandler>(); Org.Mockito.Mockito.When(securityHandlerUnpriviledged.GetUser()).ThenReturn("harry" ); }
public virtual void TestGetBlockLocations() { Path targetFilePath = new Path(targetTestRoot, "data/largeFile"); FileSystemTestHelper.CreateFile(fsTarget, targetFilePath, 10, 1024); Path viewFilePath = new Path("/data/largeFile"); Assert.True("Created File should be type File", fsView.IsFile(viewFilePath )); BlockLocation[] viewBL = fsView.GetFileBlockLocations(fsView.GetFileStatus(viewFilePath ), 0, 10240 + 100); Assert.Equal(SupportsBlocks ? 10 : 1, viewBL.Length); BlockLocation[] targetBL = fsTarget.GetFileBlockLocations(fsTarget.GetFileStatus( targetFilePath), 0, 10240 + 100); CompareBLs(viewBL, targetBL); // Same test but now get it via the FileStatus Parameter fsView.GetFileBlockLocations(fsView.GetFileStatus(viewFilePath), 0, 10240 + 100); targetBL = fsTarget.GetFileBlockLocations(fsTarget.GetFileStatus(targetFilePath), 0, 10240 + 100); CompareBLs(viewBL, targetBL); }
/// <exception cref="System.Exception"/> private void TestTruncate() { if (!IsLocalFS()) { short repl = 3; int blockSize = 1024; int numOfBlocks = 2; FileSystem fs = FileSystem.Get(GetProxiedFSConf()); fs.Mkdirs(GetProxiedFSTestDir()); Path file = new Path(GetProxiedFSTestDir(), "foo.txt"); byte[] data = FileSystemTestHelper.GetFileData(numOfBlocks, blockSize); FileSystemTestHelper.CreateFile(fs, file, data, blockSize, repl); int newLength = blockSize; bool isReady = fs.Truncate(file, newLength); NUnit.Framework.Assert.IsTrue("Recovery is not expected.", isReady); FileStatus fileStatus = fs.GetFileStatus(file); NUnit.Framework.Assert.AreEqual(fileStatus.GetLen(), newLength); AppendTestUtil.CheckFullFile(fs, file, newLength, data, file.ToString()); fs.Close(); } }
public virtual void TestGetFileChecksum() { // Create two different files in HDFS fileSystemTestHelper.CreateFile(fHdfs, someFile); FileSystemTestHelper.CreateFile(fHdfs, fileSystemTestHelper.GetTestRootPath(fHdfs , someFile + "other"), 1, 512); // Get checksum through ViewFS FileChecksum viewFSCheckSum = vfs.GetFileChecksum(new Path("/vfstmp/someFileForTestGetFileChecksum" )); // Get checksum through HDFS. FileChecksum hdfsCheckSum = fHdfs.GetFileChecksum(new Path(someFile)); // Get checksum of different file in HDFS FileChecksum otherHdfsFileCheckSum = fHdfs.GetFileChecksum(new Path(someFile + "other" )); // Checksums of the same file (got through HDFS and ViewFS should be same) NUnit.Framework.Assert.AreEqual("HDFS and ViewFS checksums were not the same", viewFSCheckSum , hdfsCheckSum); // Checksum of different files should be different. NUnit.Framework.Assert.IsFalse("Some other HDFS file which should not have had the same " + "checksum as viewFS did!", viewFSCheckSum.Equals(otherHdfsFileCheckSum)); }
public virtual void Setup() { conf = new HdfsConfiguration(); fsHelper = new FileSystemTestHelper(); // Set up java key store string testRoot = fsHelper.GetTestRootDir(); FilePath testRootDir = new FilePath(testRoot).GetAbsoluteFile(); Path jksPath = new Path(testRootDir.ToString(), "test.jks"); conf.Set(DFSConfigKeys.DfsEncryptionKeyProviderUri, JavaKeyStoreProvider.SchemeName + "://file" + jksPath.ToUri()); cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build(); Logger.GetLogger(typeof(EncryptionZoneManager)).SetLevel(Level.Trace); fs = cluster.GetFileSystem(); fsWrapper = new FileSystemTestWrapper(cluster.GetFileSystem()); fcWrapper = new FileContextTestWrapper(FileContext.GetFileContext(cluster.GetURI( ), conf)); dfsAdmin = new HdfsAdmin(cluster.GetURI(), conf); // Need to set the client's KeyProvider to the NN's for JKS, // else the updates do not get flushed properly fs.GetClient().SetKeyProvider(cluster.GetNameNode().GetNamesystem().GetProvider() ); DFSTestUtil.CreateKey(TestKey, cluster, conf); }
public virtual void TestRenameAcrossFs() { fSys.Mkdirs(new Path("/newDir/dirFoo")); fSys.Rename(new Path("/newDir/dirFoo"), new Path("file:///tmp/dirFooBar")); FileSystemTestHelper.IsDir(fSys, new Path("/tmp/dirFooBar")); }
private void UnzipSolution() { FileSystemTestHelper.RecreateDirectory(@"BeforeBounceFeature"); new DirectoryUtils().CopyDirectory(@"..\..\BeforeBounceFeature", @"BeforeBounceFeature", new string [0], new string [0]); }
public ViewFileSystemBaseTest() { // the view file system - the mounts are here // the target file system - the mount will point here this.fileSystemTestHelper = CreateFileSystemHelper(); }
private void UnzipSolution() { FileSystemTestHelper.RecreateDirectory(@"BeforeBounceFeature"); new FS.FileSystem().Copy(@"..\..\BeforeBounceFeature", @"BeforeBounceFeature"); }