public virtual void FileSystemCache()
{
string dir = TestDirHelper.GetTestDir().GetAbsolutePath();
string services = StringUtils.Join(",", Arrays.AsList(typeof(InstrumentationService
).FullName, typeof(SchedulerService).FullName, typeof(FileSystemAccessService).FullName
));
Configuration hadoopConf = new Configuration(false);
hadoopConf.Set(CommonConfigurationKeysPublic.FsDefaultNameKey, TestHdfsHelper.GetHdfsConf
().Get(CommonConfigurationKeysPublic.FsDefaultNameKey));
CreateHadoopConf(hadoopConf);
Configuration conf = new Configuration(false);
conf.Set("server.services", services);
conf.Set("server.hadoop.filesystem.cache.purge.frequency", "1");
conf.Set("server.hadoop.filesystem.cache.purge.timeout", "1");
Org.Apache.Hadoop.Lib.Server.Server server = new Org.Apache.Hadoop.Lib.Server.Server
("server", dir, dir, dir, dir, conf);
try
{
server.Init();
FileSystemAccess hadoop = server.Get <FileSystemAccess>();
FileSystem fs1 = hadoop.CreateFileSystem("u", hadoop.GetFileSystemConfiguration()
);
NUnit.Framework.Assert.IsNotNull(fs1);
fs1.Mkdirs(new Path("/tmp/foo1"));
hadoop.ReleaseFileSystem(fs1);
//still around because of caching
fs1.Mkdirs(new Path("/tmp/foo2"));
FileSystem fs2 = hadoop.CreateFileSystem("u", hadoop.GetFileSystemConfiguration()
);
//should be same instance because of caching
NUnit.Framework.Assert.AreEqual(fs1, fs2);
Sharpen.Thread.Sleep(4 * 1000);
//still around because of lease count is 1 (fs2 is out)
fs1.Mkdirs(new Path("/tmp/foo2"));
Sharpen.Thread.Sleep(4 * 1000);
//still around because of lease count is 1 (fs2 is out)
fs2.Mkdirs(new Path("/tmp/foo"));
hadoop.ReleaseFileSystem(fs2);
Sharpen.Thread.Sleep(4 * 1000);
//should not be around as lease count is 0
try
{
fs2.Mkdirs(new Path("/tmp/foo"));
NUnit.Framework.Assert.Fail();
}
catch (IOException)
{
}
catch (Exception)
{
NUnit.Framework.Assert.Fail();
}
}
finally
{
server.Destroy();
}
}
public virtual void FileSystemExecutorException()
{
string dir = TestDirHelper.GetTestDir().GetAbsolutePath();
string services = StringUtils.Join(",", Arrays.AsList(typeof(InstrumentationService
).FullName, typeof(SchedulerService).FullName, typeof(FileSystemAccessService).FullName
));
Configuration hadoopConf = new Configuration(false);
hadoopConf.Set(CommonConfigurationKeysPublic.FsDefaultNameKey, TestHdfsHelper.GetHdfsConf
().Get(CommonConfigurationKeysPublic.FsDefaultNameKey));
CreateHadoopConf(hadoopConf);
Configuration conf = new Configuration(false);
conf.Set("server.services", services);
conf.Set("server.hadoop.filesystem.cache.purge.timeout", "0");
Org.Apache.Hadoop.Lib.Server.Server server = new Org.Apache.Hadoop.Lib.Server.Server
("server", dir, dir, dir, dir, conf);
server.Init();
FileSystemAccess hadoop = server.Get <FileSystemAccess>();
FileSystem[] fsa = new FileSystem[1];
try
{
hadoop.Execute("u", hadoop.GetFileSystemConfiguration(), new _FileSystemExecutor_377
(fsa));
NUnit.Framework.Assert.Fail();
}
catch (FileSystemAccessException ex)
{
NUnit.Framework.Assert.AreEqual(ex.GetError(), FileSystemAccessException.ERROR.H03
);
}
catch (Exception)
{
NUnit.Framework.Assert.Fail();
}
try
{
fsa[0].Mkdirs(new Path("/tmp/foo"));
NUnit.Framework.Assert.Fail();
}
catch (IOException)
{
}
catch (Exception)
{
NUnit.Framework.Assert.Fail();
}
server.Destroy();
}
/// <exception cref="System.Exception"/>
private void CreateHttpFSServer()
{
FilePath homeDir = TestDirHelper.GetTestDir();
NUnit.Framework.Assert.IsTrue(new FilePath(homeDir, "conf").Mkdir());
NUnit.Framework.Assert.IsTrue(new FilePath(homeDir, "log").Mkdir());
NUnit.Framework.Assert.IsTrue(new FilePath(homeDir, "temp").Mkdir());
HttpFSServerWebApp.SetHomeDirForCurrentThread(homeDir.GetAbsolutePath());
FilePath secretFile = new FilePath(new FilePath(homeDir, "conf"), "secret");
TextWriter w = new FileWriter(secretFile);
w.Write("secret");
w.Close();
//HDFS configuration
FilePath hadoopConfDir = new FilePath(new FilePath(homeDir, "conf"), "hadoop-conf"
);
hadoopConfDir.Mkdirs();
string fsDefaultName = TestHdfsHelper.GetHdfsConf().Get(CommonConfigurationKeysPublic
.FsDefaultNameKey);
Configuration conf = new Configuration(false);
conf.Set(CommonConfigurationKeysPublic.FsDefaultNameKey, fsDefaultName);
FilePath hdfsSite = new FilePath(hadoopConfDir, "hdfs-site.xml");
OutputStream os = new FileOutputStream(hdfsSite);
conf.WriteXml(os);
os.Close();
conf = new Configuration(false);
conf.Set("httpfs.proxyuser.client.hosts", "*");
conf.Set("httpfs.proxyuser.client.groups", "*");
conf.Set("httpfs.authentication.type", "kerberos");
conf.Set("httpfs.authentication.signature.secret.file", secretFile.GetAbsolutePath
());
FilePath httpfsSite = new FilePath(new FilePath(homeDir, "conf"), "httpfs-site.xml"
);
os = new FileOutputStream(httpfsSite);
conf.WriteXml(os);
os.Close();
ClassLoader cl = Sharpen.Thread.CurrentThread().GetContextClassLoader();
Uri url = cl.GetResource("webapp");
WebAppContext context = new WebAppContext(url.AbsolutePath, "/webhdfs");
Org.Mortbay.Jetty.Server server = TestJettyHelper.GetJettyServer();
server.AddHandler(context);
server.Start();
HttpFSServerWebApp.Get().SetAuthority(TestJettyHelper.GetAuthority());
}
public virtual void TestGlobFilter()
{
CreateHttpFSServer(false);
FileSystem fs = FileSystem.Get(TestHdfsHelper.GetHdfsConf());
fs.Mkdirs(new Path("/tmp"));
fs.Create(new Path("/tmp/foo.txt")).Close();
string user = HadoopUsersConfTestHelper.GetHadoopUsers()[0];
Uri url = new Uri(TestJettyHelper.GetJettyURL(), MessageFormat.Format("/webhdfs/v1/tmp?user.name={0}&op=liststatus&filter=f*"
, user));
HttpURLConnection conn = (HttpURLConnection)url.OpenConnection();
NUnit.Framework.Assert.AreEqual(conn.GetResponseCode(), HttpURLConnection.HttpOk);
BufferedReader reader = new BufferedReader(new InputStreamReader(conn.GetInputStream
()));
reader.ReadLine();
reader.Close();
}
public virtual void TestXAttrs()
{
string name1 = "user.a1";
byte[] value1 = new byte[] { unchecked ((int)(0x31)), unchecked ((int)(0x32)), unchecked (
(int)(0x33)) };
string name2 = "user.a2";
byte[] value2 = new byte[] { unchecked ((int)(0x41)), unchecked ((int)(0x42)), unchecked (
(int)(0x43)) };
string dir = "/xattrTest";
string path = dir + "/file";
CreateHttpFSServer(false);
FileSystem fs = FileSystem.Get(TestHdfsHelper.GetHdfsConf());
fs.Mkdirs(new Path(dir));
CreateWithHttp(path, null);
string statusJson = GetStatus(path, "GETXATTRS");
IDictionary <string, byte[]> xAttrs = GetXAttrs(statusJson);
NUnit.Framework.Assert.AreEqual(0, xAttrs.Count);
// Set two xattrs
PutCmd(path, "SETXATTR", SetXAttrParam(name1, value1));
PutCmd(path, "SETXATTR", SetXAttrParam(name2, value2));
statusJson = GetStatus(path, "GETXATTRS");
xAttrs = GetXAttrs(statusJson);
NUnit.Framework.Assert.AreEqual(2, xAttrs.Count);
Assert.AssertArrayEquals(value1, xAttrs[name1]);
Assert.AssertArrayEquals(value2, xAttrs[name2]);
// Remove one xattr
PutCmd(path, "REMOVEXATTR", "xattr.name=" + name1);
statusJson = GetStatus(path, "GETXATTRS");
xAttrs = GetXAttrs(statusJson);
NUnit.Framework.Assert.AreEqual(1, xAttrs.Count);
Assert.AssertArrayEquals(value2, xAttrs[name2]);
// Remove another xattr, then there is no xattr
PutCmd(path, "REMOVEXATTR", "xattr.name=" + name2);
statusJson = GetStatus(path, "GETXATTRS");
xAttrs = GetXAttrs(statusJson);
NUnit.Framework.Assert.AreEqual(0, xAttrs.Count);
}
public virtual void TestPerms()
{
CreateHttpFSServer(false);
FileSystem fs = FileSystem.Get(TestHdfsHelper.GetHdfsConf());
fs.Mkdirs(new Path("/perm"));
CreateWithHttp("/perm/none", null);
string statusJson = GetStatus("/perm/none", "GETFILESTATUS");
NUnit.Framework.Assert.IsTrue("755".Equals(GetPerms(statusJson)));
CreateWithHttp("/perm/p-777", "777");
statusJson = GetStatus("/perm/p-777", "GETFILESTATUS");
NUnit.Framework.Assert.IsTrue("777".Equals(GetPerms(statusJson)));
CreateWithHttp("/perm/p-654", "654");
statusJson = GetStatus("/perm/p-654", "GETFILESTATUS");
NUnit.Framework.Assert.IsTrue("654".Equals(GetPerms(statusJson)));
CreateWithHttp("/perm/p-321", "321");
statusJson = GetStatus("/perm/p-321", "GETFILESTATUS");
NUnit.Framework.Assert.IsTrue("321".Equals(GetPerms(statusJson)));
}
public virtual void CreateFileSystem()
{
string dir = TestDirHelper.GetTestDir().GetAbsolutePath();
string services = StringUtils.Join(",", Arrays.AsList(typeof(InstrumentationService
).FullName, typeof(SchedulerService).FullName, typeof(FileSystemAccessService).FullName
));
Configuration hadoopConf = new Configuration(false);
hadoopConf.Set(CommonConfigurationKeysPublic.FsDefaultNameKey, TestHdfsHelper.GetHdfsConf
().Get(CommonConfigurationKeysPublic.FsDefaultNameKey));
CreateHadoopConf(hadoopConf);
Configuration conf = new Configuration(false);
conf.Set("server.services", services);
conf.Set("server.hadoop.filesystem.cache.purge.timeout", "0");
Org.Apache.Hadoop.Lib.Server.Server server = new Org.Apache.Hadoop.Lib.Server.Server
("server", dir, dir, dir, dir, conf);
server.Init();
FileSystemAccess hadoop = server.Get <FileSystemAccess>();
FileSystem fs = hadoop.CreateFileSystem("u", hadoop.GetFileSystemConfiguration());
NUnit.Framework.Assert.IsNotNull(fs);
fs.Mkdirs(new Path("/tmp/foo"));
hadoop.ReleaseFileSystem(fs);
try
{
fs.Mkdirs(new Path("/tmp/foo"));
NUnit.Framework.Assert.Fail();
}
catch (IOException)
{
}
catch (Exception)
{
NUnit.Framework.Assert.Fail();
}
server.Destroy();
}
public virtual void TestOpenOffsetLength()
{
CreateHttpFSServer(false);
byte[] array = new byte[] { 0, 1, 2, 3 };
FileSystem fs = FileSystem.Get(TestHdfsHelper.GetHdfsConf());
fs.Mkdirs(new Path("/tmp"));
OutputStream os = fs.Create(new Path("/tmp/foo"));
os.Write(array);
os.Close();
string user = HadoopUsersConfTestHelper.GetHadoopUsers()[0];
Uri url = new Uri(TestJettyHelper.GetJettyURL(), MessageFormat.Format("/webhdfs/v1/tmp/foo?user.name={0}&op=open&offset=1&length=2"
, user));
HttpURLConnection conn = (HttpURLConnection)url.OpenConnection();
NUnit.Framework.Assert.AreEqual(HttpURLConnection.HttpOk, conn.GetResponseCode());
InputStream @is = conn.GetInputStream();
NUnit.Framework.Assert.AreEqual(1, @is.Read());
NUnit.Framework.Assert.AreEqual(2, @is.Read());
NUnit.Framework.Assert.AreEqual(-1, @is.Read());
}
public virtual void FileSystemExecutorNoNameNode()
{
string dir = TestDirHelper.GetTestDir().GetAbsolutePath();
string services = StringUtils.Join(",", Arrays.AsList(typeof(InstrumentationService
).FullName, typeof(SchedulerService).FullName, typeof(FileSystemAccessService).FullName
));
Configuration hadoopConf = new Configuration(false);
hadoopConf.Set(CommonConfigurationKeysPublic.FsDefaultNameKey, TestHdfsHelper.GetHdfsConf
().Get(CommonConfigurationKeysPublic.FsDefaultNameKey));
CreateHadoopConf(hadoopConf);
Configuration conf = new Configuration(false);
conf.Set("server.services", services);
Org.Apache.Hadoop.Lib.Server.Server server = new Org.Apache.Hadoop.Lib.Server.Server
("server", dir, dir, dir, dir, conf);
server.Init();
FileSystemAccess fsAccess = server.Get <FileSystemAccess>();
Configuration hdfsConf = fsAccess.GetFileSystemConfiguration();
hdfsConf.Set(CommonConfigurationKeysPublic.FsDefaultNameKey, string.Empty);
fsAccess.Execute("u", hdfsConf, new _FileSystemExecutor_346());
}
public virtual void TestDirAcls()
{
string defUser1 = "default:user:glarch:r-x";
string defSpec1 = "aclspec=" + defUser1;
string dir = "/aclDirTest";
string statusJson;
IList <string> aclEntries;
CreateHttpFSServer(false);
FileSystem fs = FileSystem.Get(TestHdfsHelper.GetHdfsConf());
fs.Mkdirs(new Path(dir));
/* getfilestatus and liststatus don't have 'aclBit' in their reply */
statusJson = GetStatus(dir, "GETFILESTATUS");
NUnit.Framework.Assert.AreEqual(-1, statusJson.IndexOf("aclBit"));
/* No ACLs, either */
statusJson = GetStatus(dir, "GETACLSTATUS");
aclEntries = GetAclEntries(statusJson);
NUnit.Framework.Assert.IsTrue(aclEntries.Count == 0);
/* Give it a default ACL and verify */
PutCmd(dir, "SETACL", defSpec1);
statusJson = GetStatus(dir, "GETFILESTATUS");
Assert.AssertNotEquals(-1, statusJson.IndexOf("aclBit"));
statusJson = GetStatus(dir, "GETACLSTATUS");
aclEntries = GetAclEntries(statusJson);
NUnit.Framework.Assert.IsTrue(aclEntries.Count == 5);
/* 4 Entries are default:(user|group|mask|other):perm */
NUnit.Framework.Assert.IsTrue(aclEntries.Contains(defUser1));
/* Remove the default ACL and re-verify */
PutCmd(dir, "REMOVEDEFAULTACL", null);
statusJson = GetStatus(dir, "GETFILESTATUS");
NUnit.Framework.Assert.AreEqual(-1, statusJson.IndexOf("aclBit"));
statusJson = GetStatus(dir, "GETACLSTATUS");
aclEntries = GetAclEntries(statusJson);
NUnit.Framework.Assert.IsTrue(aclEntries.Count == 0);
}
/// <exception cref="System.Exception"/>
private void CreateHttpFSServer(bool addDelegationTokenAuthHandler)
{
FilePath homeDir = TestDirHelper.GetTestDir();
NUnit.Framework.Assert.IsTrue(new FilePath(homeDir, "conf").Mkdir());
NUnit.Framework.Assert.IsTrue(new FilePath(homeDir, "log").Mkdir());
NUnit.Framework.Assert.IsTrue(new FilePath(homeDir, "temp").Mkdir());
HttpFSServerWebApp.SetHomeDirForCurrentThread(homeDir.GetAbsolutePath());
FilePath secretFile = new FilePath(new FilePath(homeDir, "conf"), "secret");
TextWriter w = new FileWriter(secretFile);
w.Write("secret");
w.Close();
//HDFS configuration
FilePath hadoopConfDir = new FilePath(new FilePath(homeDir, "conf"), "hadoop-conf"
);
hadoopConfDir.Mkdirs();
string fsDefaultName = TestHdfsHelper.GetHdfsConf().Get(CommonConfigurationKeysPublic
.FsDefaultNameKey);
Configuration conf = new Configuration(false);
conf.Set(CommonConfigurationKeysPublic.FsDefaultNameKey, fsDefaultName);
conf.SetBoolean(DFSConfigKeys.DfsNamenodeAclsEnabledKey, true);
conf.SetBoolean(DFSConfigKeys.DfsNamenodeXattrsEnabledKey, true);
FilePath hdfsSite = new FilePath(hadoopConfDir, "hdfs-site.xml");
OutputStream os = new FileOutputStream(hdfsSite);
conf.WriteXml(os);
os.Close();
//HTTPFS configuration
conf = new Configuration(false);
if (addDelegationTokenAuthHandler)
{
conf.Set("httpfs.authentication.type", typeof(HttpFSKerberosAuthenticationHandlerForTesting
).FullName);
}
conf.Set("httpfs.services.ext", typeof(TestHttpFSServer.MockGroups).FullName);
conf.Set("httpfs.admin.group", HadoopUsersConfTestHelper.GetHadoopUserGroups(HadoopUsersConfTestHelper
.GetHadoopUsers()[0])[0]);
conf.Set("httpfs.proxyuser." + HadoopUsersConfTestHelper.GetHadoopProxyUser() + ".groups"
, HadoopUsersConfTestHelper.GetHadoopProxyUserGroups());
conf.Set("httpfs.proxyuser." + HadoopUsersConfTestHelper.GetHadoopProxyUser() + ".hosts"
, HadoopUsersConfTestHelper.GetHadoopProxyUserHosts());
conf.Set("httpfs.authentication.signature.secret.file", secretFile.GetAbsolutePath
());
conf.Set("httpfs.hadoop.config.dir", hadoopConfDir.ToString());
FilePath httpfsSite = new FilePath(new FilePath(homeDir, "conf"), "httpfs-site.xml"
);
os = new FileOutputStream(httpfsSite);
conf.WriteXml(os);
os.Close();
ClassLoader cl = Sharpen.Thread.CurrentThread().GetContextClassLoader();
Uri url = cl.GetResource("webapp");
WebAppContext context = new WebAppContext(url.AbsolutePath, "/webhdfs");
Org.Mortbay.Jetty.Server server = TestJettyHelper.GetJettyServer();
server.AddHandler(context);
server.Start();
if (addDelegationTokenAuthHandler)
{
HttpFSServerWebApp.Get().SetAuthority(TestJettyHelper.GetAuthority());
}
}
public virtual void TestFileAcls()
{
string aclUser1 = "user:foo:rw-";
string aclUser2 = "user:bar:r--";
string aclGroup1 = "group::r--";
string aclSpec = "aclspec=user::rwx," + aclUser1 + "," + aclGroup1 + ",other::---";
string modAclSpec = "aclspec=" + aclUser2;
string remAclSpec = "aclspec=" + aclUser1;
string dir = "/aclFileTest";
string path = dir + "/test";
string statusJson;
IList <string> aclEntries;
CreateHttpFSServer(false);
FileSystem fs = FileSystem.Get(TestHdfsHelper.GetHdfsConf());
fs.Mkdirs(new Path(dir));
CreateWithHttp(path, null);
/* getfilestatus and liststatus don't have 'aclBit' in their reply */
statusJson = GetStatus(path, "GETFILESTATUS");
NUnit.Framework.Assert.AreEqual(-1, statusJson.IndexOf("aclBit"));
statusJson = GetStatus(dir, "LISTSTATUS");
NUnit.Framework.Assert.AreEqual(-1, statusJson.IndexOf("aclBit"));
/* getaclstatus works and returns no entries */
statusJson = GetStatus(path, "GETACLSTATUS");
aclEntries = GetAclEntries(statusJson);
NUnit.Framework.Assert.IsTrue(aclEntries.Count == 0);
/*
* Now set an ACL on the file. (getfile|list)status have aclBit,
* and aclstatus has entries that looks familiar.
*/
PutCmd(path, "SETACL", aclSpec);
statusJson = GetStatus(path, "GETFILESTATUS");
Assert.AssertNotEquals(-1, statusJson.IndexOf("aclBit"));
statusJson = GetStatus(dir, "LISTSTATUS");
Assert.AssertNotEquals(-1, statusJson.IndexOf("aclBit"));
statusJson = GetStatus(path, "GETACLSTATUS");
aclEntries = GetAclEntries(statusJson);
NUnit.Framework.Assert.IsTrue(aclEntries.Count == 2);
NUnit.Framework.Assert.IsTrue(aclEntries.Contains(aclUser1));
NUnit.Framework.Assert.IsTrue(aclEntries.Contains(aclGroup1));
/* Modify acl entries to add another user acl */
PutCmd(path, "MODIFYACLENTRIES", modAclSpec);
statusJson = GetStatus(path, "GETACLSTATUS");
aclEntries = GetAclEntries(statusJson);
NUnit.Framework.Assert.IsTrue(aclEntries.Count == 3);
NUnit.Framework.Assert.IsTrue(aclEntries.Contains(aclUser1));
NUnit.Framework.Assert.IsTrue(aclEntries.Contains(aclUser2));
NUnit.Framework.Assert.IsTrue(aclEntries.Contains(aclGroup1));
/* Remove the first user acl entry and verify */
PutCmd(path, "REMOVEACLENTRIES", remAclSpec);
statusJson = GetStatus(path, "GETACLSTATUS");
aclEntries = GetAclEntries(statusJson);
NUnit.Framework.Assert.IsTrue(aclEntries.Count == 2);
NUnit.Framework.Assert.IsTrue(aclEntries.Contains(aclUser2));
NUnit.Framework.Assert.IsTrue(aclEntries.Contains(aclGroup1));
/* Remove all acls and verify */
PutCmd(path, "REMOVEACL", null);
statusJson = GetStatus(path, "GETACLSTATUS");
aclEntries = GetAclEntries(statusJson);
NUnit.Framework.Assert.IsTrue(aclEntries.Count == 0);
statusJson = GetStatus(path, "GETFILESTATUS");
NUnit.Framework.Assert.AreEqual(-1, statusJson.IndexOf("aclBit"));
statusJson = GetStatus(dir, "LISTSTATUS");
NUnit.Framework.Assert.AreEqual(-1, statusJson.IndexOf("aclBit"));
}
protected internal override Configuration GetProxiedFSConf()
{
return(TestHdfsHelper.GetHdfsConf());
}
protected internal override string GetProxiedFSURI()
{
return(TestHdfsHelper.GetHdfsConf().Get(CommonConfigurationKeysPublic.FsDefaultNameKey
));
}
protected internal override Path GetProxiedFSTestDir()
{
return(TestHdfsHelper.GetHdfsTestDir());
}
