public virtual void TestWebHdfsRenameSnapshot() { MiniDFSCluster cluster = null; Configuration conf = WebHdfsTestUtil.CreateConf(); try { cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(0).Build(); cluster.WaitActive(); DistributedFileSystem dfs = cluster.GetFileSystem(); FileSystem webHdfs = WebHdfsTestUtil.GetWebHdfsFileSystem(conf, WebHdfsFileSystem .Scheme); Path foo = new Path("/foo"); dfs.Mkdirs(foo); dfs.AllowSnapshot(foo); webHdfs.CreateSnapshot(foo, "s1"); Path s1path = SnapshotTestHelper.GetSnapshotRoot(foo, "s1"); NUnit.Framework.Assert.IsTrue(webHdfs.Exists(s1path)); // rename s1 to s2 webHdfs.RenameSnapshot(foo, "s1", "s2"); NUnit.Framework.Assert.IsFalse(webHdfs.Exists(s1path)); Path s2path = SnapshotTestHelper.GetSnapshotRoot(foo, "s2"); NUnit.Framework.Assert.IsTrue(webHdfs.Exists(s2path)); webHdfs.DeleteSnapshot(foo, "s2"); NUnit.Framework.Assert.IsFalse(webHdfs.Exists(s2path)); } finally { if (cluster != null) { cluster.Shutdown(); } } }
public virtual void TestRaceWhileNNStartup() { MiniDFSCluster cluster = null; Configuration conf = WebHdfsTestUtil.CreateConf(); try { cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(0).Build(); cluster.WaitActive(); NameNode namenode = cluster.GetNameNode(); NamenodeProtocols rpcServer = namenode.GetRpcServer(); Whitebox.SetInternalState(namenode, "rpcServer", null); Path foo = new Path("/foo"); FileSystem webHdfs = WebHdfsTestUtil.GetWebHdfsFileSystem(conf, WebHdfsFileSystem .Scheme); try { webHdfs.Mkdirs(foo); NUnit.Framework.Assert.Fail("Expected RetriableException"); } catch (RetriableException e) { GenericTestUtils.AssertExceptionContains("Namenode is in startup mode", e); } Whitebox.SetInternalState(namenode, "rpcServer", rpcServer); } finally { if (cluster != null) { cluster.Shutdown(); } } }
/// <exception cref="System.Exception"/> public virtual void TestLargeDirectory() { Configuration conf = WebHdfsTestUtil.CreateConf(); int listLimit = 2; // force small chunking of directory listing conf.SetInt(DFSConfigKeys.DfsListLimit, listLimit); // force paths to be only owner-accessible to ensure ugi isn't changing // during listStatus FsPermission.SetUMask(conf, new FsPermission((short)0x3f)); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(3).Build(); try { cluster.WaitActive(); WebHdfsTestUtil.GetWebHdfsFileSystem(conf, WebHdfsFileSystem.Scheme).SetPermission (new Path("/"), new FsPermission(FsAction.All, FsAction.All, FsAction.All)); // trick the NN into not believing it's not the superuser so we can // tell if the correct user is used by listStatus UserGroupInformation.SetLoginUser(UserGroupInformation.CreateUserForTesting("not-superuser" , new string[] { "not-supergroup" })); UserGroupInformation.CreateUserForTesting("me", new string[] { "my-group" }).DoAs (new _PrivilegedExceptionAction_263(conf, listLimit)); } finally { cluster.Shutdown(); } }
public virtual void TestDTInInsecureCluster() { MiniDFSCluster cluster = null; Configuration conf = WebHdfsTestUtil.CreateConf(); try { cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(0).Build(); FileSystem webHdfs = WebHdfsTestUtil.GetWebHdfsFileSystem(conf, WebHdfsFileSystem .Scheme); webHdfs.GetDelegationToken(null); NUnit.Framework.Assert.Fail("No exception is thrown."); } catch (AccessControlException ace) { NUnit.Framework.Assert.IsTrue(ace.Message.StartsWith(WebHdfsFileSystem.CantFallbackToInsecureMsg )); } finally { if (cluster != null) { cluster.Shutdown(); } } }
/// <summary> /// Test for catching "no datanode" IOException, when to create a file /// but datanode is not running for some reason. /// </summary> /// <exception cref="System.Exception"/> public virtual void TestCreateWithNoDN() { MiniDFSCluster cluster = null; Configuration conf = WebHdfsTestUtil.CreateConf(); try { cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(0).Build(); conf.SetInt(DFSConfigKeys.DfsReplicationKey, 1); cluster.WaitActive(); FileSystem fs = WebHdfsTestUtil.GetWebHdfsFileSystem(conf, WebHdfsFileSystem.Scheme ); fs.Create(new Path("/testnodatanode")); NUnit.Framework.Assert.Fail("No exception was thrown"); } catch (IOException ex) { GenericTestUtils.AssertExceptionContains("Failed to find datanode", ex); } finally { if (cluster != null) { cluster.Shutdown(); } } }
/// <summary>Test client retry with namenode restarting.</summary> /// <exception cref="System.Exception"/> public virtual void TestNamenodeRestart() { ((Log4JLogger)NamenodeWebHdfsMethods.Log).GetLogger().SetLevel(Level.All); Configuration conf = WebHdfsTestUtil.CreateConf(); TestDFSClientRetries.NamenodeRestartTest(conf, true); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="Sharpen.URISyntaxException"/> public Void Run() { FileSystem fs = WebHdfsTestUtil.GetWebHdfsFileSystem(conf, WebHdfsFileSystem.Scheme ); Path d = new Path("/my-dir"); NUnit.Framework.Assert.IsTrue(fs.Mkdirs(d)); return(null); }
/// <exception cref="System.Exception"/> protected override void SetUp() { //get file system as a non-superuser UserGroupInformation current = UserGroupInformation.GetCurrentUser(); ugi = UserGroupInformation.CreateUserForTesting(current.GetShortUserName() + "x", new string[] { "user" }); fs = WebHdfsTestUtil.GetWebHdfsFileSystemAs(ugi, conf, WebHdfsFileSystem.Scheme); defaultWorkingDirectory = fs.GetWorkingDirectory().ToUri().GetPath(); }
public virtual void SetUp() { Configuration conf = WebHdfsTestUtil.CreateConf(); serverSocket = Sharpen.Extensions.CreateServerSocket(0, ConnectionBacklog); nnHttpAddress = new IPEndPoint("localhost", serverSocket.GetLocalPort()); conf.Set(DFSConfigKeys.DfsNamenodeHttpAddressKey, "localhost:" + serverSocket.GetLocalPort ()); fs = WebHdfsTestUtil.GetWebHdfsFileSystem(conf, WebHdfsFileSystem.Scheme); fs.connectionFactory = connectionFactory; clients = new AList <SocketChannel>(); serverThread = null; }
//200MB file length /// <summary>Test read and write large files.</summary> /// <exception cref="System.Exception"/> internal static void LargeFileTest(long fileLength) { Configuration conf = WebHdfsTestUtil.CreateConf(); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(3).Build(); try { cluster.WaitActive(); FileSystem fs = WebHdfsTestUtil.GetWebHdfsFileSystem(conf, WebHdfsFileSystem.Scheme ); Path dir = new Path("/test/largeFile"); NUnit.Framework.Assert.IsTrue(fs.Mkdirs(dir)); byte[] data = new byte[1 << 20]; Random.NextBytes(data); byte[] expected = new byte[2 * data.Length]; System.Array.Copy(data, 0, expected, 0, data.Length); System.Array.Copy(data, 0, expected, data.Length, data.Length); Path p = new Path(dir, "file"); TestWebHDFS.Ticker t = new TestWebHDFS.Ticker("WRITE", "fileLength=" + fileLength ); FSDataOutputStream @out = fs.Create(p); try { long remaining = fileLength; for (; remaining > 0;) { t.Tick(fileLength - remaining, "remaining=%d", remaining); int n = (int)Math.Min(remaining, data.Length); @out.Write(data, 0, n); remaining -= n; } } finally { @out.Close(); } t.End(fileLength); NUnit.Framework.Assert.AreEqual(fileLength, fs.GetFileStatus(p).GetLen()); long smallOffset = Random.Next(1 << 20) + (1 << 20); long largeOffset = fileLength - smallOffset; byte[] buf = new byte[data.Length]; VerifySeek(fs, p, largeOffset, fileLength, buf, expected); VerifySeek(fs, p, smallOffset, fileLength, buf, expected); VerifyPread(fs, p, largeOffset, fileLength, buf, expected); } finally { cluster.Shutdown(); } }
/// <exception cref="System.IO.IOException"/> /// <exception cref="Sharpen.URISyntaxException"/> public Void Run() { FileSystem fs = WebHdfsTestUtil.GetWebHdfsFileSystem(conf, WebHdfsFileSystem.Scheme ); Path d = new Path("/my-dir"); NUnit.Framework.Assert.IsTrue(fs.Mkdirs(d)); for (int i = 0; i < listLimit * 3; i++) { Path p = new Path(d, "file-" + i); NUnit.Framework.Assert.IsTrue(fs.CreateNewFile(p)); } NUnit.Framework.Assert.AreEqual(listLimit * 3, fs.ListStatus(d).Length); return(null); }
public virtual void TestSWebHdfsFileSystem() { FileSystem fs = WebHdfsTestUtil.GetWebHdfsFileSystem(conf, "swebhdfs"); Path f = new Path("/testswebhdfs"); FSDataOutputStream os = fs.Create(f); os.Write(23); os.Close(); NUnit.Framework.Assert.IsTrue(fs.Exists(f)); InputStream @is = fs.Open(f); NUnit.Framework.Assert.AreEqual(23, @is.Read()); @is.Close(); fs.Close(); }
public virtual void TestWebHdfsCreateSnapshot() { MiniDFSCluster cluster = null; Configuration conf = WebHdfsTestUtil.CreateConf(); try { cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(0).Build(); cluster.WaitActive(); DistributedFileSystem dfs = cluster.GetFileSystem(); FileSystem webHdfs = WebHdfsTestUtil.GetWebHdfsFileSystem(conf, WebHdfsFileSystem .Scheme); Path foo = new Path("/foo"); dfs.Mkdirs(foo); try { webHdfs.CreateSnapshot(foo); NUnit.Framework.Assert.Fail("Cannot create snapshot on a non-snapshottable directory" ); } catch (Exception e) { GenericTestUtils.AssertExceptionContains("Directory is not a snapshottable directory" , e); } // allow snapshots on /foo dfs.AllowSnapshot(foo); // create snapshots on foo using WebHdfs webHdfs.CreateSnapshot(foo, "s1"); // create snapshot without specifying name Path spath = webHdfs.CreateSnapshot(foo, null); NUnit.Framework.Assert.IsTrue(webHdfs.Exists(spath)); Path s1path = SnapshotTestHelper.GetSnapshotRoot(foo, "s1"); NUnit.Framework.Assert.IsTrue(webHdfs.Exists(s1path)); } finally { if (cluster != null) { cluster.Shutdown(); } } }
/// <exception cref="System.Exception"/> public virtual void TestNumericalUserName() { Configuration conf = WebHdfsTestUtil.CreateConf(); conf.Set(DFSConfigKeys.DfsWebhdfsUserPatternKey, "^[A-Za-z0-9_][A-Za-z0-9._-]*[$]?$" ); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build(); try { cluster.WaitActive(); WebHdfsTestUtil.GetWebHdfsFileSystem(conf, WebHdfsFileSystem.Scheme).SetPermission (new Path("/"), new FsPermission(FsAction.All, FsAction.All, FsAction.All)); UserGroupInformation.CreateUserForTesting("123", new string[] { "my-group" }).DoAs (new _PrivilegedExceptionAction_296(conf)); } finally { cluster.Shutdown(); } }
public virtual void TestSetTokenServiceAndKind() { MiniDFSCluster cluster = null; try { Configuration clusterConf = new HdfsConfiguration(conf); SecurityUtil.SetAuthenticationMethod(UserGroupInformation.AuthenticationMethod.Simple , clusterConf); clusterConf.SetBoolean(DFSConfigKeys.DfsNamenodeDelegationTokenAlwaysUseKey, true ); // trick the NN into thinking s[ecurity is enabled w/o it trying // to login from a keytab UserGroupInformation.SetConfiguration(clusterConf); cluster = new MiniDFSCluster.Builder(clusterConf).NumDataNodes(0).Build(); cluster.WaitActive(); SecurityUtil.SetAuthenticationMethod(UserGroupInformation.AuthenticationMethod.Kerberos , clusterConf); WebHdfsFileSystem fs = WebHdfsTestUtil.GetWebHdfsFileSystem(clusterConf, "webhdfs" ); Whitebox.SetInternalState(fs, "canRefreshDelegationToken", true); URLConnectionFactory factory = new _URLConnectionFactory_268(new _ConnectionConfigurator_262 ()); Whitebox.SetInternalState(fs, "connectionFactory", factory); Org.Apache.Hadoop.Security.Token.Token <object> token1 = fs.GetDelegationToken(); NUnit.Framework.Assert.AreEqual(new Text("bar"), token1.GetKind()); HttpOpParam.OP op = GetOpParam.OP.Getdelegationtoken; Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier> token2 = new _FsPathResponseRunner_281 (op, null, new RenewerParam(null)).Run(); NUnit.Framework.Assert.AreEqual(new Text("bar"), token2.GetKind()); NUnit.Framework.Assert.AreEqual(new Text("foo"), token2.GetService()); } finally { if (cluster != null) { cluster.Shutdown(); } } }
public virtual void TestWebHdfsOffsetAndLength() { MiniDFSCluster cluster = null; Configuration conf = WebHdfsTestUtil.CreateConf(); int Offset = 42; int Length = 512; string Path = "/foo"; byte[] Contents = new byte[1024]; Random.NextBytes(Contents); try { cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build(); WebHdfsFileSystem fs = WebHdfsTestUtil.GetWebHdfsFileSystem(conf, WebHdfsFileSystem .Scheme); using (OutputStream os = fs.Create(new Path(Path))) { os.Write(Contents); } IPEndPoint addr = cluster.GetNameNode().GetHttpAddress(); Uri url = new Uri("http", addr.GetHostString(), addr.Port, WebHdfsFileSystem.PathPrefix + Path + "?op=OPEN" + Param.ToSortedString("&", new OffsetParam((long)Offset), new LengthParam((long)Length))); HttpURLConnection conn = (HttpURLConnection)url.OpenConnection(); conn.SetInstanceFollowRedirects(true); NUnit.Framework.Assert.AreEqual(Length, conn.GetContentLength()); byte[] subContents = new byte[Length]; byte[] realContents = new byte[Length]; System.Array.Copy(Contents, Offset, subContents, 0, Length); IOUtils.ReadFully(conn.GetInputStream(), realContents); Assert.AssertArrayEquals(subContents, realContents); } finally { if (cluster != null) { cluster.Shutdown(); } } }
public virtual void TestDTInInsecureClusterWithFallback() { MiniDFSCluster cluster = null; Configuration conf = WebHdfsTestUtil.CreateConf(); conf.SetBoolean(CommonConfigurationKeys.IpcClientFallbackToSimpleAuthAllowedKey, true); try { cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(0).Build(); FileSystem webHdfs = WebHdfsTestUtil.GetWebHdfsFileSystem(conf, WebHdfsFileSystem .Scheme); NUnit.Framework.Assert.IsNull(webHdfs.GetDelegationToken(null)); } finally { if (cluster != null) { cluster.Shutdown(); } } }
public virtual void TestAccess() { Path p1 = new Path("/pathX"); try { UserGroupInformation ugi = UserGroupInformation.CreateUserForTesting("alpha", new string[] { "beta" }); WebHdfsFileSystem fs = WebHdfsTestUtil.GetWebHdfsFileSystemAs(ugi, conf, WebHdfsFileSystem .Scheme); fs.Mkdirs(p1); fs.SetPermission(p1, new FsPermission((short)0x124)); fs.Access(p1, FsAction.Read); try { fs.Access(p1, FsAction.Write); Fail("The access call should have failed."); } catch (AccessControlException) { } // expected Path badPath = new Path("/bad"); try { fs.Access(badPath, FsAction.Read); Fail("The access call should have failed"); } catch (FileNotFoundException) { } } finally { // expected fs.Delete(p1, true); } }
public static void Init() { conf = WebHdfsTestUtil.CreateConf(); StartCluster(); }
/// <exception cref="System.IO.IOException"/> public virtual void TestResponseCode() { WebHdfsFileSystem webhdfs = (WebHdfsFileSystem)fs; Path root = new Path("/"); Path dir = new Path("/test/testUrl"); NUnit.Framework.Assert.IsTrue(webhdfs.Mkdirs(dir)); Path file = new Path("/test/file"); FSDataOutputStream @out = webhdfs.Create(file); @out.Write(1); @out.Close(); { //test GETHOMEDIRECTORY Uri url = webhdfs.ToUrl(GetOpParam.OP.Gethomedirectory, root); HttpURLConnection conn = (HttpURLConnection)url.OpenConnection(); IDictionary <object, object> m = WebHdfsTestUtil.ConnectAndGetJson(conn, HttpServletResponse .ScOk); NUnit.Framework.Assert.AreEqual(WebHdfsFileSystem.GetHomeDirectoryString(ugi), m[ typeof(Path).Name]); conn.Disconnect(); } { //test GETHOMEDIRECTORY with unauthorized doAs Uri url = webhdfs.ToUrl(GetOpParam.OP.Gethomedirectory, root, new DoAsParam(ugi.GetShortUserName () + "proxy")); HttpURLConnection conn = (HttpURLConnection)url.OpenConnection(); conn.Connect(); NUnit.Framework.Assert.AreEqual(HttpServletResponse.ScForbidden, conn.GetResponseCode ()); conn.Disconnect(); } { //test set owner with empty parameters Uri url = webhdfs.ToUrl(PutOpParam.OP.Setowner, dir); HttpURLConnection conn = (HttpURLConnection)url.OpenConnection(); conn.Connect(); NUnit.Framework.Assert.AreEqual(HttpServletResponse.ScBadRequest, conn.GetResponseCode ()); conn.Disconnect(); } { //test set replication on a directory HttpOpParam.OP op = PutOpParam.OP.Setreplication; Uri url = webhdfs.ToUrl(op, dir); HttpURLConnection conn = (HttpURLConnection)url.OpenConnection(); conn.SetRequestMethod(op.GetType().ToString()); conn.Connect(); NUnit.Framework.Assert.AreEqual(HttpServletResponse.ScOk, conn.GetResponseCode()); NUnit.Framework.Assert.IsFalse(webhdfs.SetReplication(dir, (short)1)); conn.Disconnect(); } { //test get file status for a non-exist file. Path p = new Path(dir, "non-exist"); Uri url = webhdfs.ToUrl(GetOpParam.OP.Getfilestatus, p); HttpURLConnection conn = (HttpURLConnection)url.OpenConnection(); conn.Connect(); NUnit.Framework.Assert.AreEqual(HttpServletResponse.ScNotFound, conn.GetResponseCode ()); conn.Disconnect(); } { //test set permission with empty parameters HttpOpParam.OP op = PutOpParam.OP.Setpermission; Uri url = webhdfs.ToUrl(op, dir); HttpURLConnection conn = (HttpURLConnection)url.OpenConnection(); conn.SetRequestMethod(op.GetType().ToString()); conn.Connect(); NUnit.Framework.Assert.AreEqual(HttpServletResponse.ScOk, conn.GetResponseCode()); NUnit.Framework.Assert.AreEqual(0, conn.GetContentLength()); NUnit.Framework.Assert.AreEqual(MediaType.ApplicationOctetStream, conn.GetContentType ()); NUnit.Framework.Assert.AreEqual((short)0x1ed, webhdfs.GetFileStatus(dir).GetPermission ().ToShort()); conn.Disconnect(); } { //test append. AppendTestUtil.TestAppend(fs, new Path(dir, "append")); } { //test NamenodeAddressParam not set. HttpOpParam.OP op = PutOpParam.OP.Create; Uri url = webhdfs.ToUrl(op, dir); HttpURLConnection conn = (HttpURLConnection)url.OpenConnection(); conn.SetRequestMethod(op.GetType().ToString()); conn.SetDoOutput(false); conn.SetInstanceFollowRedirects(false); conn.Connect(); string redirect = conn.GetHeaderField("Location"); conn.Disconnect(); //remove NamenodeAddressParam WebHdfsFileSystem.Log.Info("redirect = " + redirect); int i = redirect.IndexOf(NamenodeAddressParam.Name); int j = redirect.IndexOf("&", i); string modified = Sharpen.Runtime.Substring(redirect, 0, i - 1) + Sharpen.Runtime.Substring (redirect, j); WebHdfsFileSystem.Log.Info("modified = " + modified); //connect to datanode conn = (HttpURLConnection) new Uri(modified).OpenConnection(); conn.SetRequestMethod(op.GetType().ToString()); conn.SetDoOutput(op.GetDoOutput()); conn.Connect(); NUnit.Framework.Assert.AreEqual(HttpServletResponse.ScBadRequest, conn.GetResponseCode ()); } { //test jsonParse with non-json type. HttpOpParam.OP op = GetOpParam.OP.Open; Uri url = webhdfs.ToUrl(op, file); HttpURLConnection conn = (HttpURLConnection)url.OpenConnection(); conn.SetRequestMethod(op.GetType().ToString()); conn.Connect(); try { WebHdfsFileSystem.JsonParse(conn, false); Fail(); } catch (IOException ioe) { WebHdfsFileSystem.Log.Info("GOOD", ioe); } conn.Disconnect(); } { //test create with path containing spaces HttpOpParam.OP op = PutOpParam.OP.Create; Path path = new Path("/test/path with spaces"); Uri url = webhdfs.ToUrl(op, path); HttpURLConnection conn = (HttpURLConnection)url.OpenConnection(); conn.SetRequestMethod(op.GetType().ToString()); conn.SetDoOutput(false); conn.SetInstanceFollowRedirects(false); string redirect; try { conn.Connect(); NUnit.Framework.Assert.AreEqual(HttpServletResponse.ScTemporaryRedirect, conn.GetResponseCode ()); redirect = conn.GetHeaderField("Location"); } finally { conn.Disconnect(); } conn = (HttpURLConnection) new Uri(redirect).OpenConnection(); conn.SetRequestMethod(op.GetType().ToString()); conn.SetDoOutput(op.GetDoOutput()); try { conn.Connect(); NUnit.Framework.Assert.AreEqual(HttpServletResponse.ScCreated, conn.GetResponseCode ()); } finally { conn.Disconnect(); } } }
/// <summary>Overridden to provide a WebHdfsFileSystem wrapper for the super-user.</summary> /// <returns>WebHdfsFileSystem for super-user</returns> /// <exception cref="System.Exception">if creation fails</exception> protected internal override FileSystem CreateFileSystem() { return(WebHdfsTestUtil.GetWebHdfsFileSystem(conf, WebHdfsFileSystem.Scheme)); }
/// <summary>Overridden to provide a WebHdfsFileSystem wrapper for a specific user.</summary> /// <param name="user">UserGroupInformation specific user</param> /// <returns>WebHdfsFileSystem for specific user</returns> /// <exception cref="System.Exception">if creation fails</exception> protected internal override FileSystem CreateFileSystem(UserGroupInformation user ) { return(WebHdfsTestUtil.GetWebHdfsFileSystemAs(user, conf, WebHdfsFileSystem.Scheme )); }
/// <exception cref="System.Exception"/> public WebHdfsFileSystem Run() { return(WebHdfsTestUtil.GetWebHdfsFileSystem(conf, WebHdfsFileSystem.Scheme)); }