public virtual void TestJsonParseClosesInputStream()
{
WebHdfsFileSystem webhdfs = (WebHdfsFileSystem)fileSystem;
Path file = GetTestRootPath(fSys, "test/hadoop/file");
CreateFile(file);
HttpOpParam.OP op = GetOpParam.OP.Gethomedirectory;
Uri url = webhdfs.ToUrl(op, file);
HttpURLConnection conn = (HttpURLConnection)url.OpenConnection();
conn.SetRequestMethod(op.GetType().ToString());
conn.Connect();
InputStream myIn = new _InputStream_184(this, conn);
HttpURLConnection spyConn = Org.Mockito.Mockito.Spy(conn);
Org.Mockito.Mockito.DoReturn(myIn).When(spyConn).GetInputStream();
try
{
NUnit.Framework.Assert.IsFalse(closedInputStream);
WebHdfsFileSystem.JsonParse(spyConn, false);
NUnit.Framework.Assert.IsTrue(closedInputStream);
}
catch (IOException)
{
TestCase.Fail();
}
conn.Disconnect();
}
/// <exception cref="System.IO.IOException"/>
public static Uri ToUrl(WebHdfsFileSystem webhdfs, HttpOpParam.OP op, Path fspath
, params Param <object, object>[] parameters)
{
Uri url = webhdfs.ToUrl(op, fspath, parameters);
WebHdfsTestUtil.Log.Info("url=" + url);
return(url);
}
/// <summary>Get an object for the given op.</summary>
public static HttpOpParam.TemporaryRedirectOp ValueOf(HttpOpParam.OP op)
{
foreach (HttpOpParam.TemporaryRedirectOp t in values)
{
if (op == t.op)
{
return(t);
}
}
throw new ArgumentException(op + " not found.");
}
/// <summary>
/// Test get with offset and length parameters that combine to request a length
/// greater than actual file length.
/// </summary>
/// <exception cref="System.IO.IOException"/>
public virtual void TestOffsetPlusLengthParamsLongerThanFile()
{
WebHdfsFileSystem webhdfs = (WebHdfsFileSystem)fs;
Path dir = new Path("/test");
NUnit.Framework.Assert.IsTrue(webhdfs.Mkdirs(dir));
// Create a file with some content.
Path testFile = new Path("/test/testOffsetPlusLengthParamsLongerThanFile");
string content = "testOffsetPlusLengthParamsLongerThanFile";
FSDataOutputStream testFileOut = webhdfs.Create(testFile);
try
{
testFileOut.Write(Sharpen.Runtime.GetBytesForString(content, "US-ASCII"));
}
finally
{
IOUtils.CloseStream(testFileOut);
}
// Open the file, but request offset starting at 1 and length equal to file
// length. Considering the offset, this is longer than the actual content.
HttpOpParam.OP op = GetOpParam.OP.Open;
Uri url = webhdfs.ToUrl(op, testFile, new LengthParam(Sharpen.Extensions.ValueOf(
content.Length)), new OffsetParam(1L));
HttpURLConnection conn = null;
InputStream @is = null;
try
{
conn = (HttpURLConnection)url.OpenConnection();
conn.SetRequestMethod(op.GetType().ToString());
conn.SetDoOutput(op.GetDoOutput());
conn.SetInstanceFollowRedirects(true);
// Expect OK response and Content-Length header equal to actual length.
NUnit.Framework.Assert.AreEqual(HttpServletResponse.ScOk, conn.GetResponseCode());
NUnit.Framework.Assert.AreEqual((content.Length - 1).ToString(), conn.GetHeaderField
("Content-Length"));
// Check content matches.
byte[] respBody = new byte[content.Length - 1];
@is = conn.GetInputStream();
IOUtils.ReadFully(@is, respBody, 0, content.Length - 1);
NUnit.Framework.Assert.AreEqual(Sharpen.Runtime.Substring(content, 1), Sharpen.Runtime.GetStringForBytes
(respBody, "US-ASCII"));
}
finally
{
IOUtils.CloseStream(@is);
if (conn != null)
{
conn.Disconnect();
}
}
}
/// <exception cref="System.IO.IOException"/>
private void CheckNoTokenForOperation(HttpOpParam.OP op)
{
WebHdfsFileSystem fs = SpyWebhdfsInSecureSetup();
Org.Mockito.Mockito.DoReturn(null).When(fs).GetDelegationToken(null);
fs.Initialize(URI.Create("webhdfs://127.0.0.1:0"), conf);
// do not get a token!
fs.ToUrl(op, null);
Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Never()).GetDelegationToken();
Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Never()).GetDelegationToken(null
);
Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Never()).SetDelegationToken((Org.Apache.Hadoop.Security.Token.Token
<object>)Matchers.Any <Org.Apache.Hadoop.Security.Token.Token>());
}
public virtual void TestSetTokenServiceAndKind()
{
MiniDFSCluster cluster = null;
try
{
Configuration clusterConf = new HdfsConfiguration(conf);
SecurityUtil.SetAuthenticationMethod(UserGroupInformation.AuthenticationMethod.Simple
, clusterConf);
clusterConf.SetBoolean(DFSConfigKeys.DfsNamenodeDelegationTokenAlwaysUseKey, true
);
// trick the NN into thinking s[ecurity is enabled w/o it trying
// to login from a keytab
UserGroupInformation.SetConfiguration(clusterConf);
cluster = new MiniDFSCluster.Builder(clusterConf).NumDataNodes(0).Build();
cluster.WaitActive();
SecurityUtil.SetAuthenticationMethod(UserGroupInformation.AuthenticationMethod.Kerberos
, clusterConf);
WebHdfsFileSystem fs = WebHdfsTestUtil.GetWebHdfsFileSystem(clusterConf, "webhdfs"
);
Whitebox.SetInternalState(fs, "canRefreshDelegationToken", true);
URLConnectionFactory factory = new _URLConnectionFactory_268(new _ConnectionConfigurator_262
());
Whitebox.SetInternalState(fs, "connectionFactory", factory);
Org.Apache.Hadoop.Security.Token.Token <object> token1 = fs.GetDelegationToken();
NUnit.Framework.Assert.AreEqual(new Text("bar"), token1.GetKind());
HttpOpParam.OP op = GetOpParam.OP.Getdelegationtoken;
Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier> token2 = new _FsPathResponseRunner_281
(op, null, new RenewerParam(null)).Run();
NUnit.Framework.Assert.AreEqual(new Text("bar"), token2.GetKind());
NUnit.Framework.Assert.AreEqual(new Text("foo"), token2.GetService());
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
private TemporaryRedirectOp(HttpOpParam.OP op)
{
this.op = op;
}
public _FsPathResponseRunner_281(HttpOpParam.OP baseArg1, Path baseArg2, Param <object
, object>[] baseArg3)
: base(baseArg1, baseArg2, baseArg3)
{
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestResponseCode()
{
WebHdfsFileSystem webhdfs = (WebHdfsFileSystem)fs;
Path root = new Path("/");
Path dir = new Path("/test/testUrl");
NUnit.Framework.Assert.IsTrue(webhdfs.Mkdirs(dir));
Path file = new Path("/test/file");
FSDataOutputStream @out = webhdfs.Create(file);
@out.Write(1);
@out.Close();
{
//test GETHOMEDIRECTORY
Uri url = webhdfs.ToUrl(GetOpParam.OP.Gethomedirectory, root);
HttpURLConnection conn = (HttpURLConnection)url.OpenConnection();
IDictionary <object, object> m = WebHdfsTestUtil.ConnectAndGetJson(conn, HttpServletResponse
.ScOk);
NUnit.Framework.Assert.AreEqual(WebHdfsFileSystem.GetHomeDirectoryString(ugi), m[
typeof(Path).Name]);
conn.Disconnect();
}
{
//test GETHOMEDIRECTORY with unauthorized doAs
Uri url = webhdfs.ToUrl(GetOpParam.OP.Gethomedirectory, root, new DoAsParam(ugi.GetShortUserName
() + "proxy"));
HttpURLConnection conn = (HttpURLConnection)url.OpenConnection();
conn.Connect();
NUnit.Framework.Assert.AreEqual(HttpServletResponse.ScForbidden, conn.GetResponseCode
());
conn.Disconnect();
}
{
//test set owner with empty parameters
Uri url = webhdfs.ToUrl(PutOpParam.OP.Setowner, dir);
HttpURLConnection conn = (HttpURLConnection)url.OpenConnection();
conn.Connect();
NUnit.Framework.Assert.AreEqual(HttpServletResponse.ScBadRequest, conn.GetResponseCode
());
conn.Disconnect();
}
{
//test set replication on a directory
HttpOpParam.OP op = PutOpParam.OP.Setreplication;
Uri url = webhdfs.ToUrl(op, dir);
HttpURLConnection conn = (HttpURLConnection)url.OpenConnection();
conn.SetRequestMethod(op.GetType().ToString());
conn.Connect();
NUnit.Framework.Assert.AreEqual(HttpServletResponse.ScOk, conn.GetResponseCode());
NUnit.Framework.Assert.IsFalse(webhdfs.SetReplication(dir, (short)1));
conn.Disconnect();
}
{
//test get file status for a non-exist file.
Path p = new Path(dir, "non-exist");
Uri url = webhdfs.ToUrl(GetOpParam.OP.Getfilestatus, p);
HttpURLConnection conn = (HttpURLConnection)url.OpenConnection();
conn.Connect();
NUnit.Framework.Assert.AreEqual(HttpServletResponse.ScNotFound, conn.GetResponseCode
());
conn.Disconnect();
}
{
//test set permission with empty parameters
HttpOpParam.OP op = PutOpParam.OP.Setpermission;
Uri url = webhdfs.ToUrl(op, dir);
HttpURLConnection conn = (HttpURLConnection)url.OpenConnection();
conn.SetRequestMethod(op.GetType().ToString());
conn.Connect();
NUnit.Framework.Assert.AreEqual(HttpServletResponse.ScOk, conn.GetResponseCode());
NUnit.Framework.Assert.AreEqual(0, conn.GetContentLength());
NUnit.Framework.Assert.AreEqual(MediaType.ApplicationOctetStream, conn.GetContentType
());
NUnit.Framework.Assert.AreEqual((short)0x1ed, webhdfs.GetFileStatus(dir).GetPermission
().ToShort());
conn.Disconnect();
}
{
//test append.
AppendTestUtil.TestAppend(fs, new Path(dir, "append"));
}
{
//test NamenodeAddressParam not set.
HttpOpParam.OP op = PutOpParam.OP.Create;
Uri url = webhdfs.ToUrl(op, dir);
HttpURLConnection conn = (HttpURLConnection)url.OpenConnection();
conn.SetRequestMethod(op.GetType().ToString());
conn.SetDoOutput(false);
conn.SetInstanceFollowRedirects(false);
conn.Connect();
string redirect = conn.GetHeaderField("Location");
conn.Disconnect();
//remove NamenodeAddressParam
WebHdfsFileSystem.Log.Info("redirect = " + redirect);
int i = redirect.IndexOf(NamenodeAddressParam.Name);
int j = redirect.IndexOf("&", i);
string modified = Sharpen.Runtime.Substring(redirect, 0, i - 1) + Sharpen.Runtime.Substring
(redirect, j);
WebHdfsFileSystem.Log.Info("modified = " + modified);
//connect to datanode
conn = (HttpURLConnection) new Uri(modified).OpenConnection();
conn.SetRequestMethod(op.GetType().ToString());
conn.SetDoOutput(op.GetDoOutput());
conn.Connect();
NUnit.Framework.Assert.AreEqual(HttpServletResponse.ScBadRequest, conn.GetResponseCode
());
}
{
//test jsonParse with non-json type.
HttpOpParam.OP op = GetOpParam.OP.Open;
Uri url = webhdfs.ToUrl(op, file);
HttpURLConnection conn = (HttpURLConnection)url.OpenConnection();
conn.SetRequestMethod(op.GetType().ToString());
conn.Connect();
try
{
WebHdfsFileSystem.JsonParse(conn, false);
Fail();
}
catch (IOException ioe)
{
WebHdfsFileSystem.Log.Info("GOOD", ioe);
}
conn.Disconnect();
}
{
//test create with path containing spaces
HttpOpParam.OP op = PutOpParam.OP.Create;
Path path = new Path("/test/path with spaces");
Uri url = webhdfs.ToUrl(op, path);
HttpURLConnection conn = (HttpURLConnection)url.OpenConnection();
conn.SetRequestMethod(op.GetType().ToString());
conn.SetDoOutput(false);
conn.SetInstanceFollowRedirects(false);
string redirect;
try
{
conn.Connect();
NUnit.Framework.Assert.AreEqual(HttpServletResponse.ScTemporaryRedirect, conn.GetResponseCode
());
redirect = conn.GetHeaderField("Location");
}
finally
{
conn.Disconnect();
}
conn = (HttpURLConnection) new Uri(redirect).OpenConnection();
conn.SetRequestMethod(op.GetType().ToString());
conn.SetDoOutput(op.GetDoOutput());
try
{
conn.Connect();
NUnit.Framework.Assert.AreEqual(HttpServletResponse.ScCreated, conn.GetResponseCode
());
}
finally
{
conn.Disconnect();
}
}
}
