//Do Nothing /// <exception cref="System.Exception"/> public virtual void TestWebHdfsDoAs() { WebHdfsTestUtil.Log.Info("START: testWebHdfsDoAs()"); WebHdfsTestUtil.Log.Info("ugi.getShortUserName()=" + ugi.GetShortUserName()); WebHdfsFileSystem webhdfs = WebHdfsTestUtil.GetWebHdfsFileSystemAs(ugi, config, WebHdfsFileSystem .Scheme); Path root = new Path("/"); cluster.GetFileSystem().SetPermission(root, new FsPermission((short)0x1ff)); Whitebox.SetInternalState(webhdfs, "ugi", proxyUgi); { Path responsePath = webhdfs.GetHomeDirectory(); WebHdfsTestUtil.Log.Info("responsePath=" + responsePath); NUnit.Framework.Assert.AreEqual(webhdfs.GetUri() + "/user/" + ProxyUser, responsePath .ToString()); } Path f = new Path("/testWebHdfsDoAs/a.txt"); { FSDataOutputStream @out = webhdfs.Create(f); @out.Write(Sharpen.Runtime.GetBytesForString("Hello, webhdfs user!")); @out.Close(); FileStatus status = webhdfs.GetFileStatus(f); WebHdfsTestUtil.Log.Info("status.getOwner()=" + status.GetOwner()); NUnit.Framework.Assert.AreEqual(ProxyUser, status.GetOwner()); } { FSDataOutputStream @out = webhdfs.Append(f); @out.Write(Sharpen.Runtime.GetBytesForString("\nHello again!")); @out.Close(); FileStatus status = webhdfs.GetFileStatus(f); WebHdfsTestUtil.Log.Info("status.getOwner()=" + status.GetOwner()); WebHdfsTestUtil.Log.Info("status.getLen() =" + status.GetLen()); NUnit.Framework.Assert.AreEqual(ProxyUser, status.GetOwner()); } }
public virtual void TestWebImageViewer() { WebImageViewer viewer = new WebImageViewer(NetUtils.CreateSocketAddr("localhost:0" )); try { viewer.InitServer(originalFsimage.GetAbsolutePath()); int port = viewer.GetPort(); // create a WebHdfsFileSystem instance URI uri = new URI("webhdfs://localhost:" + port.ToString()); Configuration conf = new Configuration(); WebHdfsFileSystem webhdfs = (WebHdfsFileSystem)FileSystem.Get(uri, conf); // verify the number of directories FileStatus[] statuses = webhdfs.ListStatus(new Path("/")); NUnit.Framework.Assert.AreEqual(NumDirs + 3, statuses.Length); // contains empty and xattr directory // verify the number of files in the directory statuses = webhdfs.ListStatus(new Path("/dir0")); NUnit.Framework.Assert.AreEqual(FilesPerDir, statuses.Length); // compare a file FileStatus status = webhdfs.ListStatus(new Path("/dir0/file0"))[0]; FileStatus expected = writtenFiles["/dir0/file0"]; CompareFile(expected, status); // LISTSTATUS operation to an empty directory statuses = webhdfs.ListStatus(new Path("/emptydir")); NUnit.Framework.Assert.AreEqual(0, statuses.Length); // LISTSTATUS operation to a invalid path Uri url = new Uri("http://localhost:" + port + "/webhdfs/v1/invalid/?op=LISTSTATUS" ); VerifyHttpResponseCode(HttpURLConnection.HttpNotFound, url); // LISTSTATUS operation to a invalid prefix url = new Uri("http://localhost:" + port + "/foo"); VerifyHttpResponseCode(HttpURLConnection.HttpNotFound, url); // GETFILESTATUS operation status = webhdfs.GetFileStatus(new Path("/dir0/file0")); CompareFile(expected, status); // GETFILESTATUS operation to a invalid path url = new Uri("http://localhost:" + port + "/webhdfs/v1/invalid/?op=GETFILESTATUS" ); VerifyHttpResponseCode(HttpURLConnection.HttpNotFound, url); // invalid operation url = new Uri("http://localhost:" + port + "/webhdfs/v1/?op=INVALID"); VerifyHttpResponseCode(HttpURLConnection.HttpBadRequest, url); // invalid method url = new Uri("http://localhost:" + port + "/webhdfs/v1/?op=LISTSTATUS"); HttpURLConnection connection = (HttpURLConnection)url.OpenConnection(); connection.SetRequestMethod("POST"); connection.Connect(); NUnit.Framework.Assert.AreEqual(HttpURLConnection.HttpBadMethod, connection.GetResponseCode ()); } finally { // shutdown the viewer viewer.Close(); } }
public virtual void TestAuditWebHdfsStat() { Path file = new Path(fnames[0]); fs.SetPermission(file, new FsPermission((short)0x1a4)); fs.SetOwner(file, "root", null); SetupAuditLogs(); WebHdfsFileSystem webfs = WebHdfsTestUtil.GetWebHdfsFileSystemAs(userGroupInfo, conf , WebHdfsFileSystem.Scheme); FileStatus st = webfs.GetFileStatus(file); VerifyAuditLogs(true); NUnit.Framework.Assert.IsTrue("failed to stat file", st != null && st.IsFile()); }
/// <exception cref="System.Exception"/> private void ValidateLazyTokenFetch(Configuration clusterConf) { string testUser = "******"; UserGroupInformation ugi = UserGroupInformation.CreateUserForTesting(testUser, new string[] { "supergroup" }); WebHdfsFileSystem fs = ugi.DoAs(new _PrivilegedExceptionAction_304(this, clusterConf )); // verify token ops don't get a token NUnit.Framework.Assert.IsNull(fs.GetRenewToken()); Org.Apache.Hadoop.Security.Token.Token <object> token = ((Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier>)fs.GetDelegationToken(null)); fs.RenewDelegationToken(token); fs.CancelDelegationToken(token); Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Never()).GetDelegationToken(); Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Never()).ReplaceExpiredDelegationToken (); Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Never()).SetDelegationToken(Matchers.Any <Org.Apache.Hadoop.Security.Token.Token>()); NUnit.Framework.Assert.IsNull(fs.GetRenewToken()); Org.Mockito.Mockito.Reset(fs); // verify first non-token op gets a token Path p = new Path("/f"); fs.Create(p, (short)1).Close(); Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Times(1)).GetDelegationToken(); Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Never()).ReplaceExpiredDelegationToken (); Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Times(1)).GetDelegationToken(Matchers.AnyString ()); Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Times(1)).SetDelegationToken(Matchers.Any <Org.Apache.Hadoop.Security.Token.Token>()); token = fs.GetRenewToken(); NUnit.Framework.Assert.IsNotNull(token); NUnit.Framework.Assert.AreEqual(testUser, GetTokenOwner(token)); NUnit.Framework.Assert.AreEqual(fs.GetTokenKind(), token.GetKind()); Org.Mockito.Mockito.Reset(fs); // verify prior token is reused fs.GetFileStatus(p); Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Times(1)).GetDelegationToken(); Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Never()).ReplaceExpiredDelegationToken (); Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Never()).GetDelegationToken(Matchers.AnyString ()); Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Never()).SetDelegationToken(Matchers.Any <Org.Apache.Hadoop.Security.Token.Token>()); Org.Apache.Hadoop.Security.Token.Token <object> token2 = fs.GetRenewToken(); NUnit.Framework.Assert.IsNotNull(token2); NUnit.Framework.Assert.AreEqual(fs.GetTokenKind(), token.GetKind()); NUnit.Framework.Assert.AreSame(token, token2); Org.Mockito.Mockito.Reset(fs); // verify renew of expired token fails w/o getting a new token token = fs.GetRenewToken(); fs.CancelDelegationToken(token); try { fs.RenewDelegationToken(token); NUnit.Framework.Assert.Fail("should have failed"); } catch (SecretManager.InvalidToken) { } catch (Exception ex) { NUnit.Framework.Assert.Fail("wrong exception:" + ex); } Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Never()).GetDelegationToken(); Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Never()).ReplaceExpiredDelegationToken (); Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Never()).GetDelegationToken(Matchers.AnyString ()); Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Never()).SetDelegationToken(Matchers.Any <Org.Apache.Hadoop.Security.Token.Token>()); token2 = fs.GetRenewToken(); NUnit.Framework.Assert.IsNotNull(token2); NUnit.Framework.Assert.AreEqual(fs.GetTokenKind(), token.GetKind()); NUnit.Framework.Assert.AreSame(token, token2); Org.Mockito.Mockito.Reset(fs); // verify cancel of expired token fails w/o getting a new token try { fs.CancelDelegationToken(token); NUnit.Framework.Assert.Fail("should have failed"); } catch (SecretManager.InvalidToken) { } catch (Exception ex) { NUnit.Framework.Assert.Fail("wrong exception:" + ex); } Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Never()).GetDelegationToken(); Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Never()).ReplaceExpiredDelegationToken (); Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Never()).GetDelegationToken(Matchers.AnyString ()); Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Never()).SetDelegationToken(Matchers.Any <Org.Apache.Hadoop.Security.Token.Token>()); token2 = fs.GetRenewToken(); NUnit.Framework.Assert.IsNotNull(token2); NUnit.Framework.Assert.AreEqual(fs.GetTokenKind(), token.GetKind()); NUnit.Framework.Assert.AreSame(token, token2); Org.Mockito.Mockito.Reset(fs); // verify an expired token is replaced with a new token fs.Open(p).Close(); Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Times(2)).GetDelegationToken(); // first bad, then good Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Times(1)).ReplaceExpiredDelegationToken (); Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Times(1)).GetDelegationToken(null ); Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Times(1)).SetDelegationToken(Matchers.Any <Org.Apache.Hadoop.Security.Token.Token>()); token2 = fs.GetRenewToken(); NUnit.Framework.Assert.IsNotNull(token2); NUnit.Framework.Assert.AreEqual(fs.GetTokenKind(), token.GetKind()); NUnit.Framework.Assert.AreNotSame(token, token2); NUnit.Framework.Assert.AreEqual(testUser, GetTokenOwner(token2)); Org.Mockito.Mockito.Reset(fs); // verify with open because it's a little different in how it // opens connections fs.CancelDelegationToken(fs.GetRenewToken()); InputStream @is = fs.Open(p); @is.Read(); @is.Close(); Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Times(2)).GetDelegationToken(); // first bad, then good Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Times(1)).ReplaceExpiredDelegationToken (); Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Times(1)).GetDelegationToken(null ); Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Times(1)).SetDelegationToken(Matchers.Any <Org.Apache.Hadoop.Security.Token.Token>()); token2 = fs.GetRenewToken(); NUnit.Framework.Assert.IsNotNull(token2); NUnit.Framework.Assert.AreEqual(fs.GetTokenKind(), token.GetKind()); NUnit.Framework.Assert.AreNotSame(token, token2); NUnit.Framework.Assert.AreEqual(testUser, GetTokenOwner(token2)); Org.Mockito.Mockito.Reset(fs); // verify fs close cancels the token fs.Close(); Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Never()).GetDelegationToken(); Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Never()).ReplaceExpiredDelegationToken (); Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Never()).GetDelegationToken(Matchers.AnyString ()); Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Never()).SetDelegationToken(Matchers.Any <Org.Apache.Hadoop.Security.Token.Token>()); Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Times(1)).CancelDelegationToken (Matchers.Eq(token2)); // add a token to ugi for a new fs, verify it uses that token token = ((Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier>)fs.GetDelegationToken (null)); ugi.AddToken(token); fs = ugi.DoAs(new _PrivilegedExceptionAction_426(this, clusterConf)); NUnit.Framework.Assert.IsNull(fs.GetRenewToken()); fs.GetFileStatus(new Path("/")); Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Times(1)).GetDelegationToken(); Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Never()).ReplaceExpiredDelegationToken (); Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Never()).GetDelegationToken(Matchers.AnyString ()); Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Times(1)).SetDelegationToken(Matchers.Eq (token)); token2 = fs.GetRenewToken(); NUnit.Framework.Assert.IsNotNull(token2); NUnit.Framework.Assert.AreEqual(fs.GetTokenKind(), token.GetKind()); NUnit.Framework.Assert.AreSame(token, token2); Org.Mockito.Mockito.Reset(fs); // verify it reuses the prior ugi token fs.GetFileStatus(new Path("/")); Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Times(1)).GetDelegationToken(); Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Never()).ReplaceExpiredDelegationToken (); Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Never()).GetDelegationToken(Matchers.AnyString ()); Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Never()).SetDelegationToken(Matchers.Any <Org.Apache.Hadoop.Security.Token.Token>()); token2 = fs.GetRenewToken(); NUnit.Framework.Assert.IsNotNull(token2); NUnit.Framework.Assert.AreEqual(fs.GetTokenKind(), token.GetKind()); NUnit.Framework.Assert.AreSame(token, token2); Org.Mockito.Mockito.Reset(fs); // verify an expired ugi token is NOT replaced with a new token fs.CancelDelegationToken(token); for (int i = 0; i < 2; i++) { try { fs.GetFileStatus(new Path("/")); NUnit.Framework.Assert.Fail("didn't fail"); } catch (SecretManager.InvalidToken) { } catch (Exception ex) { NUnit.Framework.Assert.Fail("wrong exception:" + ex); } Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Times(1)).GetDelegationToken(); Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Times(1)).ReplaceExpiredDelegationToken (); Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Never()).GetDelegationToken(Matchers.AnyString ()); Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Never()).SetDelegationToken(Matchers.Any <Org.Apache.Hadoop.Security.Token.Token>()); token2 = fs.GetRenewToken(); NUnit.Framework.Assert.IsNotNull(token2); NUnit.Framework.Assert.AreEqual(fs.GetTokenKind(), token.GetKind()); NUnit.Framework.Assert.AreSame(token, token2); Org.Mockito.Mockito.Reset(fs); } // verify fs close does NOT cancel the ugi token fs.Close(); Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Never()).GetDelegationToken(); Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Never()).ReplaceExpiredDelegationToken (); Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Never()).GetDelegationToken(Matchers.AnyString ()); Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Never()).SetDelegationToken(Matchers.Any <Org.Apache.Hadoop.Security.Token.Token>()); Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Never()).CancelDelegationToken (Matchers.Any <Org.Apache.Hadoop.Security.Token.Token>()); }
/// <exception cref="System.IO.IOException"/> public virtual void TestResponseCode() { WebHdfsFileSystem webhdfs = (WebHdfsFileSystem)fs; Path root = new Path("/"); Path dir = new Path("/test/testUrl"); NUnit.Framework.Assert.IsTrue(webhdfs.Mkdirs(dir)); Path file = new Path("/test/file"); FSDataOutputStream @out = webhdfs.Create(file); @out.Write(1); @out.Close(); { //test GETHOMEDIRECTORY Uri url = webhdfs.ToUrl(GetOpParam.OP.Gethomedirectory, root); HttpURLConnection conn = (HttpURLConnection)url.OpenConnection(); IDictionary <object, object> m = WebHdfsTestUtil.ConnectAndGetJson(conn, HttpServletResponse .ScOk); NUnit.Framework.Assert.AreEqual(WebHdfsFileSystem.GetHomeDirectoryString(ugi), m[ typeof(Path).Name]); conn.Disconnect(); } { //test GETHOMEDIRECTORY with unauthorized doAs Uri url = webhdfs.ToUrl(GetOpParam.OP.Gethomedirectory, root, new DoAsParam(ugi.GetShortUserName () + "proxy")); HttpURLConnection conn = (HttpURLConnection)url.OpenConnection(); conn.Connect(); NUnit.Framework.Assert.AreEqual(HttpServletResponse.ScForbidden, conn.GetResponseCode ()); conn.Disconnect(); } { //test set owner with empty parameters Uri url = webhdfs.ToUrl(PutOpParam.OP.Setowner, dir); HttpURLConnection conn = (HttpURLConnection)url.OpenConnection(); conn.Connect(); NUnit.Framework.Assert.AreEqual(HttpServletResponse.ScBadRequest, conn.GetResponseCode ()); conn.Disconnect(); } { //test set replication on a directory HttpOpParam.OP op = PutOpParam.OP.Setreplication; Uri url = webhdfs.ToUrl(op, dir); HttpURLConnection conn = (HttpURLConnection)url.OpenConnection(); conn.SetRequestMethod(op.GetType().ToString()); conn.Connect(); NUnit.Framework.Assert.AreEqual(HttpServletResponse.ScOk, conn.GetResponseCode()); NUnit.Framework.Assert.IsFalse(webhdfs.SetReplication(dir, (short)1)); conn.Disconnect(); } { //test get file status for a non-exist file. Path p = new Path(dir, "non-exist"); Uri url = webhdfs.ToUrl(GetOpParam.OP.Getfilestatus, p); HttpURLConnection conn = (HttpURLConnection)url.OpenConnection(); conn.Connect(); NUnit.Framework.Assert.AreEqual(HttpServletResponse.ScNotFound, conn.GetResponseCode ()); conn.Disconnect(); } { //test set permission with empty parameters HttpOpParam.OP op = PutOpParam.OP.Setpermission; Uri url = webhdfs.ToUrl(op, dir); HttpURLConnection conn = (HttpURLConnection)url.OpenConnection(); conn.SetRequestMethod(op.GetType().ToString()); conn.Connect(); NUnit.Framework.Assert.AreEqual(HttpServletResponse.ScOk, conn.GetResponseCode()); NUnit.Framework.Assert.AreEqual(0, conn.GetContentLength()); NUnit.Framework.Assert.AreEqual(MediaType.ApplicationOctetStream, conn.GetContentType ()); NUnit.Framework.Assert.AreEqual((short)0x1ed, webhdfs.GetFileStatus(dir).GetPermission ().ToShort()); conn.Disconnect(); } { //test append. AppendTestUtil.TestAppend(fs, new Path(dir, "append")); } { //test NamenodeAddressParam not set. HttpOpParam.OP op = PutOpParam.OP.Create; Uri url = webhdfs.ToUrl(op, dir); HttpURLConnection conn = (HttpURLConnection)url.OpenConnection(); conn.SetRequestMethod(op.GetType().ToString()); conn.SetDoOutput(false); conn.SetInstanceFollowRedirects(false); conn.Connect(); string redirect = conn.GetHeaderField("Location"); conn.Disconnect(); //remove NamenodeAddressParam WebHdfsFileSystem.Log.Info("redirect = " + redirect); int i = redirect.IndexOf(NamenodeAddressParam.Name); int j = redirect.IndexOf("&", i); string modified = Sharpen.Runtime.Substring(redirect, 0, i - 1) + Sharpen.Runtime.Substring (redirect, j); WebHdfsFileSystem.Log.Info("modified = " + modified); //connect to datanode conn = (HttpURLConnection) new Uri(modified).OpenConnection(); conn.SetRequestMethod(op.GetType().ToString()); conn.SetDoOutput(op.GetDoOutput()); conn.Connect(); NUnit.Framework.Assert.AreEqual(HttpServletResponse.ScBadRequest, conn.GetResponseCode ()); } { //test jsonParse with non-json type. HttpOpParam.OP op = GetOpParam.OP.Open; Uri url = webhdfs.ToUrl(op, file); HttpURLConnection conn = (HttpURLConnection)url.OpenConnection(); conn.SetRequestMethod(op.GetType().ToString()); conn.Connect(); try { WebHdfsFileSystem.JsonParse(conn, false); Fail(); } catch (IOException ioe) { WebHdfsFileSystem.Log.Info("GOOD", ioe); } conn.Disconnect(); } { //test create with path containing spaces HttpOpParam.OP op = PutOpParam.OP.Create; Path path = new Path("/test/path with spaces"); Uri url = webhdfs.ToUrl(op, path); HttpURLConnection conn = (HttpURLConnection)url.OpenConnection(); conn.SetRequestMethod(op.GetType().ToString()); conn.SetDoOutput(false); conn.SetInstanceFollowRedirects(false); string redirect; try { conn.Connect(); NUnit.Framework.Assert.AreEqual(HttpServletResponse.ScTemporaryRedirect, conn.GetResponseCode ()); redirect = conn.GetHeaderField("Location"); } finally { conn.Disconnect(); } conn = (HttpURLConnection) new Uri(redirect).OpenConnection(); conn.SetRequestMethod(op.GetType().ToString()); conn.SetDoOutput(op.GetDoOutput()); try { conn.Connect(); NUnit.Framework.Assert.AreEqual(HttpServletResponse.ScCreated, conn.GetResponseCode ()); } finally { conn.Disconnect(); } } }