public virtual void TestWebHdfsRenameSnapshot()
{
MiniDFSCluster cluster = null;
Configuration conf = WebHdfsTestUtil.CreateConf();
try
{
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(0).Build();
cluster.WaitActive();
DistributedFileSystem dfs = cluster.GetFileSystem();
FileSystem webHdfs = WebHdfsTestUtil.GetWebHdfsFileSystem(conf, WebHdfsFileSystem
.Scheme);
Path foo = new Path("/foo");
dfs.Mkdirs(foo);
dfs.AllowSnapshot(foo);
webHdfs.CreateSnapshot(foo, "s1");
Path s1path = SnapshotTestHelper.GetSnapshotRoot(foo, "s1");
NUnit.Framework.Assert.IsTrue(webHdfs.Exists(s1path));
// rename s1 to s2
webHdfs.RenameSnapshot(foo, "s1", "s2");
NUnit.Framework.Assert.IsFalse(webHdfs.Exists(s1path));
Path s2path = SnapshotTestHelper.GetSnapshotRoot(foo, "s2");
NUnit.Framework.Assert.IsTrue(webHdfs.Exists(s2path));
webHdfs.DeleteSnapshot(foo, "s2");
NUnit.Framework.Assert.IsFalse(webHdfs.Exists(s2path));
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
public virtual void TestDTInInsecureCluster()
{
MiniDFSCluster cluster = null;
Configuration conf = WebHdfsTestUtil.CreateConf();
try
{
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(0).Build();
FileSystem webHdfs = WebHdfsTestUtil.GetWebHdfsFileSystem(conf, WebHdfsFileSystem
.Scheme);
webHdfs.GetDelegationToken(null);
NUnit.Framework.Assert.Fail("No exception is thrown.");
}
catch (AccessControlException ace)
{
NUnit.Framework.Assert.IsTrue(ace.Message.StartsWith(WebHdfsFileSystem.CantFallbackToInsecureMsg
));
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
public virtual void TestRaceWhileNNStartup()
{
MiniDFSCluster cluster = null;
Configuration conf = WebHdfsTestUtil.CreateConf();
try
{
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(0).Build();
cluster.WaitActive();
NameNode namenode = cluster.GetNameNode();
NamenodeProtocols rpcServer = namenode.GetRpcServer();
Whitebox.SetInternalState(namenode, "rpcServer", null);
Path foo = new Path("/foo");
FileSystem webHdfs = WebHdfsTestUtil.GetWebHdfsFileSystem(conf, WebHdfsFileSystem
.Scheme);
try
{
webHdfs.Mkdirs(foo);
NUnit.Framework.Assert.Fail("Expected RetriableException");
}
catch (RetriableException e)
{
GenericTestUtils.AssertExceptionContains("Namenode is in startup mode", e);
}
Whitebox.SetInternalState(namenode, "rpcServer", rpcServer);
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
/// <summary>
/// Test for catching "no datanode" IOException, when to create a file
/// but datanode is not running for some reason.
/// </summary>
/// <exception cref="System.Exception"/>
public virtual void TestCreateWithNoDN()
{
MiniDFSCluster cluster = null;
Configuration conf = WebHdfsTestUtil.CreateConf();
try
{
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(0).Build();
conf.SetInt(DFSConfigKeys.DfsReplicationKey, 1);
cluster.WaitActive();
FileSystem fs = WebHdfsTestUtil.GetWebHdfsFileSystem(conf, WebHdfsFileSystem.Scheme
);
fs.Create(new Path("/testnodatanode"));
NUnit.Framework.Assert.Fail("No exception was thrown");
}
catch (IOException ex)
{
GenericTestUtils.AssertExceptionContains("Failed to find datanode", ex);
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
/// <exception cref="System.Exception"/>
public virtual void TestLargeDirectory()
{
Configuration conf = WebHdfsTestUtil.CreateConf();
int listLimit = 2;
// force small chunking of directory listing
conf.SetInt(DFSConfigKeys.DfsListLimit, listLimit);
// force paths to be only owner-accessible to ensure ugi isn't changing
// during listStatus
FsPermission.SetUMask(conf, new FsPermission((short)0x3f));
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(3).Build();
try
{
cluster.WaitActive();
WebHdfsTestUtil.GetWebHdfsFileSystem(conf, WebHdfsFileSystem.Scheme).SetPermission
(new Path("/"), new FsPermission(FsAction.All, FsAction.All, FsAction.All));
// trick the NN into not believing it's not the superuser so we can
// tell if the correct user is used by listStatus
UserGroupInformation.SetLoginUser(UserGroupInformation.CreateUserForTesting("not-superuser"
, new string[] { "not-supergroup" }));
UserGroupInformation.CreateUserForTesting("me", new string[] { "my-group" }).DoAs
(new _PrivilegedExceptionAction_263(conf, listLimit));
}
finally
{
cluster.Shutdown();
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Sharpen.URISyntaxException"/>
public Void Run()
{
FileSystem fs = WebHdfsTestUtil.GetWebHdfsFileSystem(conf, WebHdfsFileSystem.Scheme
);
Path d = new Path("/my-dir");
NUnit.Framework.Assert.IsTrue(fs.Mkdirs(d));
return(null);
}
public virtual void SetUp()
{
Configuration conf = WebHdfsTestUtil.CreateConf();
serverSocket = Sharpen.Extensions.CreateServerSocket(0, ConnectionBacklog);
nnHttpAddress = new IPEndPoint("localhost", serverSocket.GetLocalPort());
conf.Set(DFSConfigKeys.DfsNamenodeHttpAddressKey, "localhost:" + serverSocket.GetLocalPort
());
fs = WebHdfsTestUtil.GetWebHdfsFileSystem(conf, WebHdfsFileSystem.Scheme);
fs.connectionFactory = connectionFactory;
clients = new AList <SocketChannel>();
serverThread = null;
}
//200MB file length
/// <summary>Test read and write large files.</summary>
/// <exception cref="System.Exception"/>
internal static void LargeFileTest(long fileLength)
{
Configuration conf = WebHdfsTestUtil.CreateConf();
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(3).Build();
try
{
cluster.WaitActive();
FileSystem fs = WebHdfsTestUtil.GetWebHdfsFileSystem(conf, WebHdfsFileSystem.Scheme
);
Path dir = new Path("/test/largeFile");
NUnit.Framework.Assert.IsTrue(fs.Mkdirs(dir));
byte[] data = new byte[1 << 20];
Random.NextBytes(data);
byte[] expected = new byte[2 * data.Length];
System.Array.Copy(data, 0, expected, 0, data.Length);
System.Array.Copy(data, 0, expected, data.Length, data.Length);
Path p = new Path(dir, "file");
TestWebHDFS.Ticker t = new TestWebHDFS.Ticker("WRITE", "fileLength=" + fileLength
);
FSDataOutputStream @out = fs.Create(p);
try
{
long remaining = fileLength;
for (; remaining > 0;)
{
t.Tick(fileLength - remaining, "remaining=%d", remaining);
int n = (int)Math.Min(remaining, data.Length);
@out.Write(data, 0, n);
remaining -= n;
}
}
finally
{
@out.Close();
}
t.End(fileLength);
NUnit.Framework.Assert.AreEqual(fileLength, fs.GetFileStatus(p).GetLen());
long smallOffset = Random.Next(1 << 20) + (1 << 20);
long largeOffset = fileLength - smallOffset;
byte[] buf = new byte[data.Length];
VerifySeek(fs, p, largeOffset, fileLength, buf, expected);
VerifySeek(fs, p, smallOffset, fileLength, buf, expected);
VerifyPread(fs, p, largeOffset, fileLength, buf, expected);
}
finally
{
cluster.Shutdown();
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Sharpen.URISyntaxException"/>
public Void Run()
{
FileSystem fs = WebHdfsTestUtil.GetWebHdfsFileSystem(conf, WebHdfsFileSystem.Scheme
);
Path d = new Path("/my-dir");
NUnit.Framework.Assert.IsTrue(fs.Mkdirs(d));
for (int i = 0; i < listLimit * 3; i++)
{
Path p = new Path(d, "file-" + i);
NUnit.Framework.Assert.IsTrue(fs.CreateNewFile(p));
}
NUnit.Framework.Assert.AreEqual(listLimit * 3, fs.ListStatus(d).Length);
return(null);
}
public virtual void TestSWebHdfsFileSystem()
{
FileSystem fs = WebHdfsTestUtil.GetWebHdfsFileSystem(conf, "swebhdfs");
Path f = new Path("/testswebhdfs");
FSDataOutputStream os = fs.Create(f);
os.Write(23);
os.Close();
NUnit.Framework.Assert.IsTrue(fs.Exists(f));
InputStream @is = fs.Open(f);
NUnit.Framework.Assert.AreEqual(23, @is.Read());
@is.Close();
fs.Close();
}
public virtual void TestWebHdfsCreateSnapshot()
{
MiniDFSCluster cluster = null;
Configuration conf = WebHdfsTestUtil.CreateConf();
try
{
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(0).Build();
cluster.WaitActive();
DistributedFileSystem dfs = cluster.GetFileSystem();
FileSystem webHdfs = WebHdfsTestUtil.GetWebHdfsFileSystem(conf, WebHdfsFileSystem
.Scheme);
Path foo = new Path("/foo");
dfs.Mkdirs(foo);
try
{
webHdfs.CreateSnapshot(foo);
NUnit.Framework.Assert.Fail("Cannot create snapshot on a non-snapshottable directory"
);
}
catch (Exception e)
{
GenericTestUtils.AssertExceptionContains("Directory is not a snapshottable directory"
, e);
}
// allow snapshots on /foo
dfs.AllowSnapshot(foo);
// create snapshots on foo using WebHdfs
webHdfs.CreateSnapshot(foo, "s1");
// create snapshot without specifying name
Path spath = webHdfs.CreateSnapshot(foo, null);
NUnit.Framework.Assert.IsTrue(webHdfs.Exists(spath));
Path s1path = SnapshotTestHelper.GetSnapshotRoot(foo, "s1");
NUnit.Framework.Assert.IsTrue(webHdfs.Exists(s1path));
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
/// <exception cref="System.Exception"/>
public virtual void TestNumericalUserName()
{
Configuration conf = WebHdfsTestUtil.CreateConf();
conf.Set(DFSConfigKeys.DfsWebhdfsUserPatternKey, "^[A-Za-z0-9_][A-Za-z0-9._-]*[$]?$"
);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();
try
{
cluster.WaitActive();
WebHdfsTestUtil.GetWebHdfsFileSystem(conf, WebHdfsFileSystem.Scheme).SetPermission
(new Path("/"), new FsPermission(FsAction.All, FsAction.All, FsAction.All));
UserGroupInformation.CreateUserForTesting("123", new string[] { "my-group" }).DoAs
(new _PrivilegedExceptionAction_296(conf));
}
finally
{
cluster.Shutdown();
}
}
public virtual void TestWebHdfsOffsetAndLength()
{
MiniDFSCluster cluster = null;
Configuration conf = WebHdfsTestUtil.CreateConf();
int Offset = 42;
int Length = 512;
string Path = "/foo";
byte[] Contents = new byte[1024];
Random.NextBytes(Contents);
try
{
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();
WebHdfsFileSystem fs = WebHdfsTestUtil.GetWebHdfsFileSystem(conf, WebHdfsFileSystem
.Scheme);
using (OutputStream os = fs.Create(new Path(Path)))
{
os.Write(Contents);
}
IPEndPoint addr = cluster.GetNameNode().GetHttpAddress();
Uri url = new Uri("http", addr.GetHostString(), addr.Port, WebHdfsFileSystem.PathPrefix
+ Path + "?op=OPEN" + Param.ToSortedString("&", new OffsetParam((long)Offset),
new LengthParam((long)Length)));
HttpURLConnection conn = (HttpURLConnection)url.OpenConnection();
conn.SetInstanceFollowRedirects(true);
NUnit.Framework.Assert.AreEqual(Length, conn.GetContentLength());
byte[] subContents = new byte[Length];
byte[] realContents = new byte[Length];
System.Array.Copy(Contents, Offset, subContents, 0, Length);
IOUtils.ReadFully(conn.GetInputStream(), realContents);
Assert.AssertArrayEquals(subContents, realContents);
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
public virtual void TestSetTokenServiceAndKind()
{
MiniDFSCluster cluster = null;
try
{
Configuration clusterConf = new HdfsConfiguration(conf);
SecurityUtil.SetAuthenticationMethod(UserGroupInformation.AuthenticationMethod.Simple
, clusterConf);
clusterConf.SetBoolean(DFSConfigKeys.DfsNamenodeDelegationTokenAlwaysUseKey, true
);
// trick the NN into thinking s[ecurity is enabled w/o it trying
// to login from a keytab
UserGroupInformation.SetConfiguration(clusterConf);
cluster = new MiniDFSCluster.Builder(clusterConf).NumDataNodes(0).Build();
cluster.WaitActive();
SecurityUtil.SetAuthenticationMethod(UserGroupInformation.AuthenticationMethod.Kerberos
, clusterConf);
WebHdfsFileSystem fs = WebHdfsTestUtil.GetWebHdfsFileSystem(clusterConf, "webhdfs"
);
Whitebox.SetInternalState(fs, "canRefreshDelegationToken", true);
URLConnectionFactory factory = new _URLConnectionFactory_268(new _ConnectionConfigurator_262
());
Whitebox.SetInternalState(fs, "connectionFactory", factory);
Org.Apache.Hadoop.Security.Token.Token <object> token1 = fs.GetDelegationToken();
NUnit.Framework.Assert.AreEqual(new Text("bar"), token1.GetKind());
HttpOpParam.OP op = GetOpParam.OP.Getdelegationtoken;
Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier> token2 = new _FsPathResponseRunner_281
(op, null, new RenewerParam(null)).Run();
NUnit.Framework.Assert.AreEqual(new Text("bar"), token2.GetKind());
NUnit.Framework.Assert.AreEqual(new Text("foo"), token2.GetService());
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
public virtual void TestDTInInsecureClusterWithFallback()
{
MiniDFSCluster cluster = null;
Configuration conf = WebHdfsTestUtil.CreateConf();
conf.SetBoolean(CommonConfigurationKeys.IpcClientFallbackToSimpleAuthAllowedKey,
true);
try
{
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(0).Build();
FileSystem webHdfs = WebHdfsTestUtil.GetWebHdfsFileSystem(conf, WebHdfsFileSystem
.Scheme);
NUnit.Framework.Assert.IsNull(webHdfs.GetDelegationToken(null));
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
/// <summary>Overridden to provide a WebHdfsFileSystem wrapper for the super-user.</summary>
/// <returns>WebHdfsFileSystem for super-user</returns>
/// <exception cref="System.Exception">if creation fails</exception>
protected internal override FileSystem CreateFileSystem()
{
return(WebHdfsTestUtil.GetWebHdfsFileSystem(conf, WebHdfsFileSystem.Scheme));
}
/// <exception cref="System.Exception"/>
public WebHdfsFileSystem Run()
{
return(WebHdfsTestUtil.GetWebHdfsFileSystem(conf, WebHdfsFileSystem.Scheme));
}