public virtual void FileSystemCache() { string dir = TestDirHelper.GetTestDir().GetAbsolutePath(); string services = StringUtils.Join(",", Arrays.AsList(typeof(InstrumentationService ).FullName, typeof(SchedulerService).FullName, typeof(FileSystemAccessService).FullName )); Configuration hadoopConf = new Configuration(false); hadoopConf.Set(CommonConfigurationKeysPublic.FsDefaultNameKey, TestHdfsHelper.GetHdfsConf ().Get(CommonConfigurationKeysPublic.FsDefaultNameKey)); CreateHadoopConf(hadoopConf); Configuration conf = new Configuration(false); conf.Set("server.services", services); conf.Set("server.hadoop.filesystem.cache.purge.frequency", "1"); conf.Set("server.hadoop.filesystem.cache.purge.timeout", "1"); Org.Apache.Hadoop.Lib.Server.Server server = new Org.Apache.Hadoop.Lib.Server.Server ("server", dir, dir, dir, dir, conf); try { server.Init(); FileSystemAccess hadoop = server.Get <FileSystemAccess>(); FileSystem fs1 = hadoop.CreateFileSystem("u", hadoop.GetFileSystemConfiguration() ); NUnit.Framework.Assert.IsNotNull(fs1); fs1.Mkdirs(new Path("/tmp/foo1")); hadoop.ReleaseFileSystem(fs1); //still around because of caching fs1.Mkdirs(new Path("/tmp/foo2")); FileSystem fs2 = hadoop.CreateFileSystem("u", hadoop.GetFileSystemConfiguration() ); //should be same instance because of caching NUnit.Framework.Assert.AreEqual(fs1, fs2); Sharpen.Thread.Sleep(4 * 1000); //still around because of lease count is 1 (fs2 is out) fs1.Mkdirs(new Path("/tmp/foo2")); Sharpen.Thread.Sleep(4 * 1000); //still around because of lease count is 1 (fs2 is out) fs2.Mkdirs(new Path("/tmp/foo")); hadoop.ReleaseFileSystem(fs2); Sharpen.Thread.Sleep(4 * 1000); //should not be around as lease count is 0 try { fs2.Mkdirs(new Path("/tmp/foo")); NUnit.Framework.Assert.Fail(); } catch (IOException) { } catch (Exception) { NUnit.Framework.Assert.Fail(); } } finally { server.Destroy(); } }
public virtual void FileSystemExecutorException() { string dir = TestDirHelper.GetTestDir().GetAbsolutePath(); string services = StringUtils.Join(",", Arrays.AsList(typeof(InstrumentationService ).FullName, typeof(SchedulerService).FullName, typeof(FileSystemAccessService).FullName )); Configuration hadoopConf = new Configuration(false); hadoopConf.Set(CommonConfigurationKeysPublic.FsDefaultNameKey, TestHdfsHelper.GetHdfsConf ().Get(CommonConfigurationKeysPublic.FsDefaultNameKey)); CreateHadoopConf(hadoopConf); Configuration conf = new Configuration(false); conf.Set("server.services", services); conf.Set("server.hadoop.filesystem.cache.purge.timeout", "0"); Org.Apache.Hadoop.Lib.Server.Server server = new Org.Apache.Hadoop.Lib.Server.Server ("server", dir, dir, dir, dir, conf); server.Init(); FileSystemAccess hadoop = server.Get <FileSystemAccess>(); FileSystem[] fsa = new FileSystem[1]; try { hadoop.Execute("u", hadoop.GetFileSystemConfiguration(), new _FileSystemExecutor_377 (fsa)); NUnit.Framework.Assert.Fail(); } catch (FileSystemAccessException ex) { NUnit.Framework.Assert.AreEqual(ex.GetError(), FileSystemAccessException.ERROR.H03 ); } catch (Exception) { NUnit.Framework.Assert.Fail(); } try { fsa[0].Mkdirs(new Path("/tmp/foo")); NUnit.Framework.Assert.Fail(); } catch (IOException) { } catch (Exception) { NUnit.Framework.Assert.Fail(); } server.Destroy(); }
/// <exception cref="System.Exception"/> private void CreateHttpFSServer() { FilePath homeDir = TestDirHelper.GetTestDir(); NUnit.Framework.Assert.IsTrue(new FilePath(homeDir, "conf").Mkdir()); NUnit.Framework.Assert.IsTrue(new FilePath(homeDir, "log").Mkdir()); NUnit.Framework.Assert.IsTrue(new FilePath(homeDir, "temp").Mkdir()); HttpFSServerWebApp.SetHomeDirForCurrentThread(homeDir.GetAbsolutePath()); FilePath secretFile = new FilePath(new FilePath(homeDir, "conf"), "secret"); TextWriter w = new FileWriter(secretFile); w.Write("secret"); w.Close(); //HDFS configuration FilePath hadoopConfDir = new FilePath(new FilePath(homeDir, "conf"), "hadoop-conf" ); hadoopConfDir.Mkdirs(); string fsDefaultName = TestHdfsHelper.GetHdfsConf().Get(CommonConfigurationKeysPublic .FsDefaultNameKey); Configuration conf = new Configuration(false); conf.Set(CommonConfigurationKeysPublic.FsDefaultNameKey, fsDefaultName); FilePath hdfsSite = new FilePath(hadoopConfDir, "hdfs-site.xml"); OutputStream os = new FileOutputStream(hdfsSite); conf.WriteXml(os); os.Close(); conf = new Configuration(false); conf.Set("httpfs.proxyuser.client.hosts", "*"); conf.Set("httpfs.proxyuser.client.groups", "*"); conf.Set("httpfs.authentication.type", "kerberos"); conf.Set("httpfs.authentication.signature.secret.file", secretFile.GetAbsolutePath ()); FilePath httpfsSite = new FilePath(new FilePath(homeDir, "conf"), "httpfs-site.xml" ); os = new FileOutputStream(httpfsSite); conf.WriteXml(os); os.Close(); ClassLoader cl = Sharpen.Thread.CurrentThread().GetContextClassLoader(); Uri url = cl.GetResource("webapp"); WebAppContext context = new WebAppContext(url.AbsolutePath, "/webhdfs"); Org.Mortbay.Jetty.Server server = TestJettyHelper.GetJettyServer(); server.AddHandler(context); server.Start(); HttpFSServerWebApp.Get().SetAuthority(TestJettyHelper.GetAuthority()); }
public virtual void TestGlobFilter() { CreateHttpFSServer(false); FileSystem fs = FileSystem.Get(TestHdfsHelper.GetHdfsConf()); fs.Mkdirs(new Path("/tmp")); fs.Create(new Path("/tmp/foo.txt")).Close(); string user = HadoopUsersConfTestHelper.GetHadoopUsers()[0]; Uri url = new Uri(TestJettyHelper.GetJettyURL(), MessageFormat.Format("/webhdfs/v1/tmp?user.name={0}&op=liststatus&filter=f*" , user)); HttpURLConnection conn = (HttpURLConnection)url.OpenConnection(); NUnit.Framework.Assert.AreEqual(conn.GetResponseCode(), HttpURLConnection.HttpOk); BufferedReader reader = new BufferedReader(new InputStreamReader(conn.GetInputStream ())); reader.ReadLine(); reader.Close(); }
public virtual void TestXAttrs() { string name1 = "user.a1"; byte[] value1 = new byte[] { unchecked ((int)(0x31)), unchecked ((int)(0x32)), unchecked ( (int)(0x33)) }; string name2 = "user.a2"; byte[] value2 = new byte[] { unchecked ((int)(0x41)), unchecked ((int)(0x42)), unchecked ( (int)(0x43)) }; string dir = "/xattrTest"; string path = dir + "/file"; CreateHttpFSServer(false); FileSystem fs = FileSystem.Get(TestHdfsHelper.GetHdfsConf()); fs.Mkdirs(new Path(dir)); CreateWithHttp(path, null); string statusJson = GetStatus(path, "GETXATTRS"); IDictionary <string, byte[]> xAttrs = GetXAttrs(statusJson); NUnit.Framework.Assert.AreEqual(0, xAttrs.Count); // Set two xattrs PutCmd(path, "SETXATTR", SetXAttrParam(name1, value1)); PutCmd(path, "SETXATTR", SetXAttrParam(name2, value2)); statusJson = GetStatus(path, "GETXATTRS"); xAttrs = GetXAttrs(statusJson); NUnit.Framework.Assert.AreEqual(2, xAttrs.Count); Assert.AssertArrayEquals(value1, xAttrs[name1]); Assert.AssertArrayEquals(value2, xAttrs[name2]); // Remove one xattr PutCmd(path, "REMOVEXATTR", "xattr.name=" + name1); statusJson = GetStatus(path, "GETXATTRS"); xAttrs = GetXAttrs(statusJson); NUnit.Framework.Assert.AreEqual(1, xAttrs.Count); Assert.AssertArrayEquals(value2, xAttrs[name2]); // Remove another xattr, then there is no xattr PutCmd(path, "REMOVEXATTR", "xattr.name=" + name2); statusJson = GetStatus(path, "GETXATTRS"); xAttrs = GetXAttrs(statusJson); NUnit.Framework.Assert.AreEqual(0, xAttrs.Count); }
public virtual void TestPerms() { CreateHttpFSServer(false); FileSystem fs = FileSystem.Get(TestHdfsHelper.GetHdfsConf()); fs.Mkdirs(new Path("/perm")); CreateWithHttp("/perm/none", null); string statusJson = GetStatus("/perm/none", "GETFILESTATUS"); NUnit.Framework.Assert.IsTrue("755".Equals(GetPerms(statusJson))); CreateWithHttp("/perm/p-777", "777"); statusJson = GetStatus("/perm/p-777", "GETFILESTATUS"); NUnit.Framework.Assert.IsTrue("777".Equals(GetPerms(statusJson))); CreateWithHttp("/perm/p-654", "654"); statusJson = GetStatus("/perm/p-654", "GETFILESTATUS"); NUnit.Framework.Assert.IsTrue("654".Equals(GetPerms(statusJson))); CreateWithHttp("/perm/p-321", "321"); statusJson = GetStatus("/perm/p-321", "GETFILESTATUS"); NUnit.Framework.Assert.IsTrue("321".Equals(GetPerms(statusJson))); }
public virtual void CreateFileSystem() { string dir = TestDirHelper.GetTestDir().GetAbsolutePath(); string services = StringUtils.Join(",", Arrays.AsList(typeof(InstrumentationService ).FullName, typeof(SchedulerService).FullName, typeof(FileSystemAccessService).FullName )); Configuration hadoopConf = new Configuration(false); hadoopConf.Set(CommonConfigurationKeysPublic.FsDefaultNameKey, TestHdfsHelper.GetHdfsConf ().Get(CommonConfigurationKeysPublic.FsDefaultNameKey)); CreateHadoopConf(hadoopConf); Configuration conf = new Configuration(false); conf.Set("server.services", services); conf.Set("server.hadoop.filesystem.cache.purge.timeout", "0"); Org.Apache.Hadoop.Lib.Server.Server server = new Org.Apache.Hadoop.Lib.Server.Server ("server", dir, dir, dir, dir, conf); server.Init(); FileSystemAccess hadoop = server.Get <FileSystemAccess>(); FileSystem fs = hadoop.CreateFileSystem("u", hadoop.GetFileSystemConfiguration()); NUnit.Framework.Assert.IsNotNull(fs); fs.Mkdirs(new Path("/tmp/foo")); hadoop.ReleaseFileSystem(fs); try { fs.Mkdirs(new Path("/tmp/foo")); NUnit.Framework.Assert.Fail(); } catch (IOException) { } catch (Exception) { NUnit.Framework.Assert.Fail(); } server.Destroy(); }
public virtual void TestOpenOffsetLength() { CreateHttpFSServer(false); byte[] array = new byte[] { 0, 1, 2, 3 }; FileSystem fs = FileSystem.Get(TestHdfsHelper.GetHdfsConf()); fs.Mkdirs(new Path("/tmp")); OutputStream os = fs.Create(new Path("/tmp/foo")); os.Write(array); os.Close(); string user = HadoopUsersConfTestHelper.GetHadoopUsers()[0]; Uri url = new Uri(TestJettyHelper.GetJettyURL(), MessageFormat.Format("/webhdfs/v1/tmp/foo?user.name={0}&op=open&offset=1&length=2" , user)); HttpURLConnection conn = (HttpURLConnection)url.OpenConnection(); NUnit.Framework.Assert.AreEqual(HttpURLConnection.HttpOk, conn.GetResponseCode()); InputStream @is = conn.GetInputStream(); NUnit.Framework.Assert.AreEqual(1, @is.Read()); NUnit.Framework.Assert.AreEqual(2, @is.Read()); NUnit.Framework.Assert.AreEqual(-1, @is.Read()); }
public virtual void FileSystemExecutorNoNameNode() { string dir = TestDirHelper.GetTestDir().GetAbsolutePath(); string services = StringUtils.Join(",", Arrays.AsList(typeof(InstrumentationService ).FullName, typeof(SchedulerService).FullName, typeof(FileSystemAccessService).FullName )); Configuration hadoopConf = new Configuration(false); hadoopConf.Set(CommonConfigurationKeysPublic.FsDefaultNameKey, TestHdfsHelper.GetHdfsConf ().Get(CommonConfigurationKeysPublic.FsDefaultNameKey)); CreateHadoopConf(hadoopConf); Configuration conf = new Configuration(false); conf.Set("server.services", services); Org.Apache.Hadoop.Lib.Server.Server server = new Org.Apache.Hadoop.Lib.Server.Server ("server", dir, dir, dir, dir, conf); server.Init(); FileSystemAccess fsAccess = server.Get <FileSystemAccess>(); Configuration hdfsConf = fsAccess.GetFileSystemConfiguration(); hdfsConf.Set(CommonConfigurationKeysPublic.FsDefaultNameKey, string.Empty); fsAccess.Execute("u", hdfsConf, new _FileSystemExecutor_346()); }
public virtual void TestDirAcls() { string defUser1 = "default:user:glarch:r-x"; string defSpec1 = "aclspec=" + defUser1; string dir = "/aclDirTest"; string statusJson; IList <string> aclEntries; CreateHttpFSServer(false); FileSystem fs = FileSystem.Get(TestHdfsHelper.GetHdfsConf()); fs.Mkdirs(new Path(dir)); /* getfilestatus and liststatus don't have 'aclBit' in their reply */ statusJson = GetStatus(dir, "GETFILESTATUS"); NUnit.Framework.Assert.AreEqual(-1, statusJson.IndexOf("aclBit")); /* No ACLs, either */ statusJson = GetStatus(dir, "GETACLSTATUS"); aclEntries = GetAclEntries(statusJson); NUnit.Framework.Assert.IsTrue(aclEntries.Count == 0); /* Give it a default ACL and verify */ PutCmd(dir, "SETACL", defSpec1); statusJson = GetStatus(dir, "GETFILESTATUS"); Assert.AssertNotEquals(-1, statusJson.IndexOf("aclBit")); statusJson = GetStatus(dir, "GETACLSTATUS"); aclEntries = GetAclEntries(statusJson); NUnit.Framework.Assert.IsTrue(aclEntries.Count == 5); /* 4 Entries are default:(user|group|mask|other):perm */ NUnit.Framework.Assert.IsTrue(aclEntries.Contains(defUser1)); /* Remove the default ACL and re-verify */ PutCmd(dir, "REMOVEDEFAULTACL", null); statusJson = GetStatus(dir, "GETFILESTATUS"); NUnit.Framework.Assert.AreEqual(-1, statusJson.IndexOf("aclBit")); statusJson = GetStatus(dir, "GETACLSTATUS"); aclEntries = GetAclEntries(statusJson); NUnit.Framework.Assert.IsTrue(aclEntries.Count == 0); }
/// <exception cref="System.Exception"/> private void CreateHttpFSServer(bool addDelegationTokenAuthHandler) { FilePath homeDir = TestDirHelper.GetTestDir(); NUnit.Framework.Assert.IsTrue(new FilePath(homeDir, "conf").Mkdir()); NUnit.Framework.Assert.IsTrue(new FilePath(homeDir, "log").Mkdir()); NUnit.Framework.Assert.IsTrue(new FilePath(homeDir, "temp").Mkdir()); HttpFSServerWebApp.SetHomeDirForCurrentThread(homeDir.GetAbsolutePath()); FilePath secretFile = new FilePath(new FilePath(homeDir, "conf"), "secret"); TextWriter w = new FileWriter(secretFile); w.Write("secret"); w.Close(); //HDFS configuration FilePath hadoopConfDir = new FilePath(new FilePath(homeDir, "conf"), "hadoop-conf" ); hadoopConfDir.Mkdirs(); string fsDefaultName = TestHdfsHelper.GetHdfsConf().Get(CommonConfigurationKeysPublic .FsDefaultNameKey); Configuration conf = new Configuration(false); conf.Set(CommonConfigurationKeysPublic.FsDefaultNameKey, fsDefaultName); conf.SetBoolean(DFSConfigKeys.DfsNamenodeAclsEnabledKey, true); conf.SetBoolean(DFSConfigKeys.DfsNamenodeXattrsEnabledKey, true); FilePath hdfsSite = new FilePath(hadoopConfDir, "hdfs-site.xml"); OutputStream os = new FileOutputStream(hdfsSite); conf.WriteXml(os); os.Close(); //HTTPFS configuration conf = new Configuration(false); if (addDelegationTokenAuthHandler) { conf.Set("httpfs.authentication.type", typeof(HttpFSKerberosAuthenticationHandlerForTesting ).FullName); } conf.Set("httpfs.services.ext", typeof(TestHttpFSServer.MockGroups).FullName); conf.Set("httpfs.admin.group", HadoopUsersConfTestHelper.GetHadoopUserGroups(HadoopUsersConfTestHelper .GetHadoopUsers()[0])[0]); conf.Set("httpfs.proxyuser." + HadoopUsersConfTestHelper.GetHadoopProxyUser() + ".groups" , HadoopUsersConfTestHelper.GetHadoopProxyUserGroups()); conf.Set("httpfs.proxyuser." + HadoopUsersConfTestHelper.GetHadoopProxyUser() + ".hosts" , HadoopUsersConfTestHelper.GetHadoopProxyUserHosts()); conf.Set("httpfs.authentication.signature.secret.file", secretFile.GetAbsolutePath ()); conf.Set("httpfs.hadoop.config.dir", hadoopConfDir.ToString()); FilePath httpfsSite = new FilePath(new FilePath(homeDir, "conf"), "httpfs-site.xml" ); os = new FileOutputStream(httpfsSite); conf.WriteXml(os); os.Close(); ClassLoader cl = Sharpen.Thread.CurrentThread().GetContextClassLoader(); Uri url = cl.GetResource("webapp"); WebAppContext context = new WebAppContext(url.AbsolutePath, "/webhdfs"); Org.Mortbay.Jetty.Server server = TestJettyHelper.GetJettyServer(); server.AddHandler(context); server.Start(); if (addDelegationTokenAuthHandler) { HttpFSServerWebApp.Get().SetAuthority(TestJettyHelper.GetAuthority()); } }
public virtual void TestFileAcls() { string aclUser1 = "user:foo:rw-"; string aclUser2 = "user:bar:r--"; string aclGroup1 = "group::r--"; string aclSpec = "aclspec=user::rwx," + aclUser1 + "," + aclGroup1 + ",other::---"; string modAclSpec = "aclspec=" + aclUser2; string remAclSpec = "aclspec=" + aclUser1; string dir = "/aclFileTest"; string path = dir + "/test"; string statusJson; IList <string> aclEntries; CreateHttpFSServer(false); FileSystem fs = FileSystem.Get(TestHdfsHelper.GetHdfsConf()); fs.Mkdirs(new Path(dir)); CreateWithHttp(path, null); /* getfilestatus and liststatus don't have 'aclBit' in their reply */ statusJson = GetStatus(path, "GETFILESTATUS"); NUnit.Framework.Assert.AreEqual(-1, statusJson.IndexOf("aclBit")); statusJson = GetStatus(dir, "LISTSTATUS"); NUnit.Framework.Assert.AreEqual(-1, statusJson.IndexOf("aclBit")); /* getaclstatus works and returns no entries */ statusJson = GetStatus(path, "GETACLSTATUS"); aclEntries = GetAclEntries(statusJson); NUnit.Framework.Assert.IsTrue(aclEntries.Count == 0); /* * Now set an ACL on the file. (getfile|list)status have aclBit, * and aclstatus has entries that looks familiar. */ PutCmd(path, "SETACL", aclSpec); statusJson = GetStatus(path, "GETFILESTATUS"); Assert.AssertNotEquals(-1, statusJson.IndexOf("aclBit")); statusJson = GetStatus(dir, "LISTSTATUS"); Assert.AssertNotEquals(-1, statusJson.IndexOf("aclBit")); statusJson = GetStatus(path, "GETACLSTATUS"); aclEntries = GetAclEntries(statusJson); NUnit.Framework.Assert.IsTrue(aclEntries.Count == 2); NUnit.Framework.Assert.IsTrue(aclEntries.Contains(aclUser1)); NUnit.Framework.Assert.IsTrue(aclEntries.Contains(aclGroup1)); /* Modify acl entries to add another user acl */ PutCmd(path, "MODIFYACLENTRIES", modAclSpec); statusJson = GetStatus(path, "GETACLSTATUS"); aclEntries = GetAclEntries(statusJson); NUnit.Framework.Assert.IsTrue(aclEntries.Count == 3); NUnit.Framework.Assert.IsTrue(aclEntries.Contains(aclUser1)); NUnit.Framework.Assert.IsTrue(aclEntries.Contains(aclUser2)); NUnit.Framework.Assert.IsTrue(aclEntries.Contains(aclGroup1)); /* Remove the first user acl entry and verify */ PutCmd(path, "REMOVEACLENTRIES", remAclSpec); statusJson = GetStatus(path, "GETACLSTATUS"); aclEntries = GetAclEntries(statusJson); NUnit.Framework.Assert.IsTrue(aclEntries.Count == 2); NUnit.Framework.Assert.IsTrue(aclEntries.Contains(aclUser2)); NUnit.Framework.Assert.IsTrue(aclEntries.Contains(aclGroup1)); /* Remove all acls and verify */ PutCmd(path, "REMOVEACL", null); statusJson = GetStatus(path, "GETACLSTATUS"); aclEntries = GetAclEntries(statusJson); NUnit.Framework.Assert.IsTrue(aclEntries.Count == 0); statusJson = GetStatus(path, "GETFILESTATUS"); NUnit.Framework.Assert.AreEqual(-1, statusJson.IndexOf("aclBit")); statusJson = GetStatus(dir, "LISTSTATUS"); NUnit.Framework.Assert.AreEqual(-1, statusJson.IndexOf("aclBit")); }
protected internal override Configuration GetProxiedFSConf() { return(TestHdfsHelper.GetHdfsConf()); }
protected internal override string GetProxiedFSURI() { return(TestHdfsHelper.GetHdfsConf().Get(CommonConfigurationKeysPublic.FsDefaultNameKey )); }
protected internal override Path GetProxiedFSTestDir() { return(TestHdfsHelper.GetHdfsTestDir()); }