예제 #1
0
            /// <summary>Executes the filesystem operation.</summary>
            /// <param name="fs">filesystem instance to use.</param>
            /// <returns>The inputstream of the file.</returns>
            /// <exception cref="System.IO.IOException">thrown if an IO error occured.</exception>
            public virtual InputStream Execute(FileSystem fs)
            {
                int bufferSize = HttpFSServerWebApp.Get().GetConfig().GetInt("httpfs.buffer.size"
                                                                             , 4096);

                return(fs.Open(path, bufferSize));
            }
        /// <summary>Create an HttpFS Server to talk to the MiniDFSCluster we created.</summary>
        /// <exception cref="System.Exception"/>
        private void CreateHttpFSServer()
        {
            FilePath homeDir = TestDirHelper.GetTestDir();

            NUnit.Framework.Assert.IsTrue(new FilePath(homeDir, "conf").Mkdir());
            NUnit.Framework.Assert.IsTrue(new FilePath(homeDir, "log").Mkdir());
            NUnit.Framework.Assert.IsTrue(new FilePath(homeDir, "temp").Mkdir());
            HttpFSServerWebApp.SetHomeDirForCurrentThread(homeDir.GetAbsolutePath());
            FilePath   secretFile = new FilePath(new FilePath(homeDir, "conf"), "secret");
            TextWriter w          = new FileWriter(secretFile);

            w.Write("secret");
            w.Close();
            // HDFS configuration
            FilePath hadoopConfDir = new FilePath(new FilePath(homeDir, "conf"), "hadoop-conf"
                                                  );

            if (!hadoopConfDir.Mkdirs())
            {
                throw new IOException();
            }
            string        fsDefaultName = nnConf.Get(CommonConfigurationKeysPublic.FsDefaultNameKey);
            Configuration conf          = new Configuration(false);

            conf.Set(CommonConfigurationKeysPublic.FsDefaultNameKey, fsDefaultName);
            // Explicitly turn off XAttr support
            conf.SetBoolean(DFSConfigKeys.DfsNamenodeXattrsEnabledKey, false);
            FilePath     hdfsSite = new FilePath(hadoopConfDir, "hdfs-site.xml");
            OutputStream os       = new FileOutputStream(hdfsSite);

            conf.WriteXml(os);
            os.Close();
            // HTTPFS configuration
            conf = new Configuration(false);
            conf.Set("httpfs.hadoop.config.dir", hadoopConfDir.ToString());
            conf.Set("httpfs.proxyuser." + HadoopUsersConfTestHelper.GetHadoopProxyUser() + ".groups"
                     , HadoopUsersConfTestHelper.GetHadoopProxyUserGroups());
            conf.Set("httpfs.proxyuser." + HadoopUsersConfTestHelper.GetHadoopProxyUser() + ".hosts"
                     , HadoopUsersConfTestHelper.GetHadoopProxyUserHosts());
            conf.Set("httpfs.authentication.signature.secret.file", secretFile.GetAbsolutePath
                         ());
            FilePath httpfsSite = new FilePath(new FilePath(homeDir, "conf"), "httpfs-site.xml"
                                               );

            os = new FileOutputStream(httpfsSite);
            conf.WriteXml(os);
            os.Close();
            ClassLoader cl  = Sharpen.Thread.CurrentThread().GetContextClassLoader();
            Uri         url = cl.GetResource("webapp");

            if (url == null)
            {
                throw new IOException();
            }
            WebAppContext context = new WebAppContext(url.AbsolutePath, "/webhdfs");

            Org.Mortbay.Jetty.Server server = TestJettyHelper.GetJettyServer();
            server.AddHandler(context);
            server.Start();
        }
예제 #3
0
        /// <summary>
        /// Executes a
        /// <see cref="Org.Apache.Hadoop.Lib.Service.FileSystemAccess.FileSystemExecutor{T}"/
        ///     >
        /// using a filesystem for the effective
        /// user.
        /// </summary>
        /// <param name="ugi">user making the request.</param>
        /// <param name="executor">FileSystemExecutor to execute.</param>
        /// <returns>FileSystemExecutor response</returns>
        /// <exception cref="System.IO.IOException">thrown if an IO error occurrs.</exception>
        /// <exception cref="Org.Apache.Hadoop.Lib.Service.FileSystemAccessException">
        /// thrown if a FileSystemAccess releated error occurred. Thrown
        /// exceptions are handled by
        /// <see cref="HttpFSExceptionProvider"/>
        /// .
        /// </exception>
        private T FsExecute <T>(UserGroupInformation ugi, FileSystemAccess.FileSystemExecutor
                                <T> executor)
        {
            FileSystemAccess fsAccess = HttpFSServerWebApp.Get().Get <FileSystemAccess>();
            Configuration    conf     = HttpFSServerWebApp.Get().Get <FileSystemAccess>().GetFileSystemConfiguration
                                            ();

            return(fsAccess.Execute(ugi.GetShortUserName(), conf, executor));
        }
예제 #4
0
        public virtual void Server()
        {
            string             dir        = TestDirHelper.GetTestDir().GetAbsolutePath();
            Configuration      httpfsConf = new Configuration(false);
            HttpFSServerWebApp server     = new HttpFSServerWebApp(dir, dir, dir, dir, httpfsConf
                                                                   );

            server.Init();
            server.Destroy();
        }
예제 #5
0
        /// <summary>Returns a filesystem instance.</summary>
        /// <remarks>
        /// Returns a filesystem instance. The fileystem instance is wired for release at the completion of
        /// the current Servlet request via the
        /// <see cref="Org.Apache.Hadoop.Lib.Servlet.FileSystemReleaseFilter"/>
        /// .
        /// <p>
        /// If a do-as user is specified, the current user must be a valid proxyuser, otherwise an
        /// <code>AccessControlException</code> will be thrown.
        /// </remarks>
        /// <param name="ugi">principal for whom the filesystem instance is.</param>
        /// <returns>a filesystem for the specified user or do-as user.</returns>
        /// <exception cref="System.IO.IOException">
        /// thrown if an IO error occurred. Thrown exceptions are
        /// handled by
        /// <see cref="HttpFSExceptionProvider"/>
        /// .
        /// </exception>
        /// <exception cref="Org.Apache.Hadoop.Lib.Service.FileSystemAccessException">
        /// thrown if a FileSystemAccess releated error occurred. Thrown
        /// exceptions are handled by
        /// <see cref="HttpFSExceptionProvider"/>
        /// .
        /// </exception>
        private FileSystem CreateFileSystem(UserGroupInformation ugi)
        {
            string           hadoopUser = ugi.GetShortUserName();
            FileSystemAccess fsAccess   = HttpFSServerWebApp.Get().Get <FileSystemAccess>();
            Configuration    conf       = HttpFSServerWebApp.Get().Get <FileSystemAccess>().GetFileSystemConfiguration
                                              ();
            FileSystem fs = fsAccess.CreateFileSystem(hadoopUser, conf);

            FileSystemReleaseFilter.SetFileSystem(fs);
            return(fs);
        }
        protected override Configuration GetProxyuserConfiguration(FilterConfig filterConfig
                                                                   )
        {
            IDictionary <string, string> proxyuserConf = HttpFSServerWebApp.Get().GetConfig().
                                                         GetValByRegex("httpfs\\.proxyuser\\.");
            Configuration conf = new Configuration(false);

            foreach (KeyValuePair <string, string> entry in proxyuserConf)
            {
                conf.Set(Sharpen.Runtime.Substring(entry.Key, "httpfs.".Length), entry.Value);
            }
            return(conf);
        }
예제 #7
0
        /// <exception cref="System.Exception"/>
        private void CreateHttpFSServer()
        {
            FilePath homeDir = TestDirHelper.GetTestDir();

            NUnit.Framework.Assert.IsTrue(new FilePath(homeDir, "conf").Mkdir());
            NUnit.Framework.Assert.IsTrue(new FilePath(homeDir, "log").Mkdir());
            NUnit.Framework.Assert.IsTrue(new FilePath(homeDir, "temp").Mkdir());
            HttpFSServerWebApp.SetHomeDirForCurrentThread(homeDir.GetAbsolutePath());
            FilePath   secretFile = new FilePath(new FilePath(homeDir, "conf"), "secret");
            TextWriter w          = new FileWriter(secretFile);

            w.Write("secret");
            w.Close();
            //HDFS configuration
            FilePath hadoopConfDir = new FilePath(new FilePath(homeDir, "conf"), "hadoop-conf"
                                                  );

            hadoopConfDir.Mkdirs();
            string fsDefaultName = TestHdfsHelper.GetHdfsConf().Get(CommonConfigurationKeysPublic
                                                                    .FsDefaultNameKey);
            Configuration conf = new Configuration(false);

            conf.Set(CommonConfigurationKeysPublic.FsDefaultNameKey, fsDefaultName);
            FilePath     hdfsSite = new FilePath(hadoopConfDir, "hdfs-site.xml");
            OutputStream os       = new FileOutputStream(hdfsSite);

            conf.WriteXml(os);
            os.Close();
            conf = new Configuration(false);
            conf.Set("httpfs.proxyuser.client.hosts", "*");
            conf.Set("httpfs.proxyuser.client.groups", "*");
            conf.Set("httpfs.authentication.type", "kerberos");
            conf.Set("httpfs.authentication.signature.secret.file", secretFile.GetAbsolutePath
                         ());
            FilePath httpfsSite = new FilePath(new FilePath(homeDir, "conf"), "httpfs-site.xml"
                                               );

            os = new FileOutputStream(httpfsSite);
            conf.WriteXml(os);
            os.Close();
            ClassLoader   cl      = Sharpen.Thread.CurrentThread().GetContextClassLoader();
            Uri           url     = cl.GetResource("webapp");
            WebAppContext context = new WebAppContext(url.AbsolutePath, "/webhdfs");

            Org.Mortbay.Jetty.Server server = TestJettyHelper.GetJettyServer();
            server.AddHandler(context);
            server.Start();
            HttpFSServerWebApp.Get().SetAuthority(TestJettyHelper.GetAuthority());
        }
        /// <summary>Returns the hadoop-auth configuration from HttpFSServer's configuration.
        ///     </summary>
        /// <remarks>
        /// Returns the hadoop-auth configuration from HttpFSServer's configuration.
        /// <p>
        /// It returns all HttpFSServer's configuration properties prefixed with
        /// <code>httpfs.authentication</code>. The <code>httpfs.authentication</code>
        /// prefix is removed from the returned property names.
        /// </remarks>
        /// <param name="configPrefix">parameter not used.</param>
        /// <param name="filterConfig">parameter not used.</param>
        /// <returns>hadoop-auth configuration read from HttpFSServer's configuration.</returns>
        /// <exception cref="Javax.Servlet.ServletException"/>
        protected override Properties GetConfiguration(string configPrefix, FilterConfig
                                                       filterConfig)
        {
            Properties    props = new Properties();
            Configuration conf  = HttpFSServerWebApp.Get().GetConfig();

            props.SetProperty(AuthenticationFilter.CookiePath, "/");
            foreach (KeyValuePair <string, string> entry in conf)
            {
                string name = entry.Key;
                if (name.StartsWith(ConfPrefix))
                {
                    string value = conf.Get(name);
                    name = Sharpen.Runtime.Substring(name, ConfPrefix.Length);
                    props.SetProperty(name, value);
                }
            }
            string signatureSecretFile = props.GetProperty(SignatureSecretFile, null);

            if (signatureSecretFile == null)
            {
                throw new RuntimeException("Undefined property: " + SignatureSecretFile);
            }
            try
            {
                StringBuilder secret = new StringBuilder();
                StreamReader  reader = new InputStreamReader(new FileInputStream(signatureSecretFile
                                                                                 ), Charsets.Utf8);
                int c = reader.Read();
                while (c > -1)
                {
                    secret.Append((char)c);
                    c = reader.Read();
                }
                reader.Close();
                props.SetProperty(AuthenticationFilter.SignatureSecret, secret.ToString());
            }
            catch (IOException)
            {
                throw new RuntimeException("Could not read HttpFS signature secret file: " + signatureSecretFile
                                           );
            }
            return(props);
        }
예제 #9
0
 /// <summary>
 /// Returns the
 /// <see cref="Org.Apache.Hadoop.Lib.Service.FileSystemAccess"/>
 /// service to return the FileSystemAccess filesystem
 /// instance to.
 /// </summary>
 /// <returns>the FileSystemAccess service.</returns>
 protected internal override FileSystemAccess GetFileSystemAccess()
 {
     return(HttpFSServerWebApp.Get().Get <FileSystemAccess>());
 }
예제 #10
0
        /// <exception cref="System.Exception"/>
        private void CreateHttpFSServer(bool addDelegationTokenAuthHandler)
        {
            FilePath homeDir = TestDirHelper.GetTestDir();

            NUnit.Framework.Assert.IsTrue(new FilePath(homeDir, "conf").Mkdir());
            NUnit.Framework.Assert.IsTrue(new FilePath(homeDir, "log").Mkdir());
            NUnit.Framework.Assert.IsTrue(new FilePath(homeDir, "temp").Mkdir());
            HttpFSServerWebApp.SetHomeDirForCurrentThread(homeDir.GetAbsolutePath());
            FilePath   secretFile = new FilePath(new FilePath(homeDir, "conf"), "secret");
            TextWriter w          = new FileWriter(secretFile);

            w.Write("secret");
            w.Close();
            //HDFS configuration
            FilePath hadoopConfDir = new FilePath(new FilePath(homeDir, "conf"), "hadoop-conf"
                                                  );

            hadoopConfDir.Mkdirs();
            string fsDefaultName = TestHdfsHelper.GetHdfsConf().Get(CommonConfigurationKeysPublic
                                                                    .FsDefaultNameKey);
            Configuration conf = new Configuration(false);

            conf.Set(CommonConfigurationKeysPublic.FsDefaultNameKey, fsDefaultName);
            conf.SetBoolean(DFSConfigKeys.DfsNamenodeAclsEnabledKey, true);
            conf.SetBoolean(DFSConfigKeys.DfsNamenodeXattrsEnabledKey, true);
            FilePath     hdfsSite = new FilePath(hadoopConfDir, "hdfs-site.xml");
            OutputStream os       = new FileOutputStream(hdfsSite);

            conf.WriteXml(os);
            os.Close();
            //HTTPFS configuration
            conf = new Configuration(false);
            if (addDelegationTokenAuthHandler)
            {
                conf.Set("httpfs.authentication.type", typeof(HttpFSKerberosAuthenticationHandlerForTesting
                                                              ).FullName);
            }
            conf.Set("httpfs.services.ext", typeof(TestHttpFSServer.MockGroups).FullName);
            conf.Set("httpfs.admin.group", HadoopUsersConfTestHelper.GetHadoopUserGroups(HadoopUsersConfTestHelper
                                                                                         .GetHadoopUsers()[0])[0]);
            conf.Set("httpfs.proxyuser." + HadoopUsersConfTestHelper.GetHadoopProxyUser() + ".groups"
                     , HadoopUsersConfTestHelper.GetHadoopProxyUserGroups());
            conf.Set("httpfs.proxyuser." + HadoopUsersConfTestHelper.GetHadoopProxyUser() + ".hosts"
                     , HadoopUsersConfTestHelper.GetHadoopProxyUserHosts());
            conf.Set("httpfs.authentication.signature.secret.file", secretFile.GetAbsolutePath
                         ());
            conf.Set("httpfs.hadoop.config.dir", hadoopConfDir.ToString());
            FilePath httpfsSite = new FilePath(new FilePath(homeDir, "conf"), "httpfs-site.xml"
                                               );

            os = new FileOutputStream(httpfsSite);
            conf.WriteXml(os);
            os.Close();
            ClassLoader   cl      = Sharpen.Thread.CurrentThread().GetContextClassLoader();
            Uri           url     = cl.GetResource("webapp");
            WebAppContext context = new WebAppContext(url.AbsolutePath, "/webhdfs");

            Org.Mortbay.Jetty.Server server = TestJettyHelper.GetJettyServer();
            server.AddHandler(context);
            server.Start();
            if (addDelegationTokenAuthHandler)
            {
                HttpFSServerWebApp.Get().SetAuthority(TestJettyHelper.GetAuthority());
            }
        }
예제 #11
0
        public virtual Response Get(string path, HttpFSParametersProvider.OperationParam
                                    op, Parameters @params, HttpServletRequest request)
        {
            UserGroupInformation user = HttpUserGroupInformation.Get();
            Response             response;

            path = MakeAbsolute(path);
            MDC.Put(HttpFSFileSystem.OpParam, op.Value().ToString());
            MDC.Put("hostname", request.GetRemoteAddr());
            switch (op.Value())
            {
            case HttpFSFileSystem.Operation.Open:
            {
                //Invoking the command directly using an unmanaged FileSystem that is
                // released by the FileSystemReleaseFilter
                FSOperations.FSOpen command = new FSOperations.FSOpen(path);
                FileSystem          fs      = CreateFileSystem(user);
                InputStream         @is     = command.Execute(fs);
                long offset = @params.Get <HttpFSParametersProvider.OffsetParam>(HttpFSParametersProvider.OffsetParam
                                                                                 .Name);
                long len = @params.Get <HttpFSParametersProvider.LenParam>(HttpFSParametersProvider.LenParam
                                                                           .Name);
                AuditLog.Info("[{}] offset [{}] len [{}]", new object[] { path, offset, len });
                InputStreamEntity entity = new InputStreamEntity(@is, offset, len);
                response = Response.Ok(entity).Type(MediaType.ApplicationOctetStream).Build();
                break;
            }

            case HttpFSFileSystem.Operation.Getfilestatus:
            {
                FSOperations.FSFileStatus command = new FSOperations.FSFileStatus(path);
                IDictionary json = FsExecute(user, command);
                AuditLog.Info("[{}]", path);
                response = Response.Ok(json).Type(MediaType.ApplicationJson).Build();
                break;
            }

            case HttpFSFileSystem.Operation.Liststatus:
            {
                string filter = @params.Get <HttpFSParametersProvider.FilterParam>(HttpFSParametersProvider.FilterParam
                                                                                   .Name);
                FSOperations.FSListStatus command = new FSOperations.FSListStatus(path, filter);
                IDictionary json = FsExecute(user, command);
                AuditLog.Info("[{}] filter [{}]", path, (filter != null) ? filter : "-");
                response = Response.Ok(json).Type(MediaType.ApplicationJson).Build();
                break;
            }

            case HttpFSFileSystem.Operation.Gethomedirectory:
            {
                EnforceRootPath(op.Value(), path);
                FSOperations.FSHomeDir command = new FSOperations.FSHomeDir();
                JSONObject             json    = FsExecute(user, command);
                AuditLog.Info(string.Empty);
                response = Response.Ok(json).Type(MediaType.ApplicationJson).Build();
                break;
            }

            case HttpFSFileSystem.Operation.Instrumentation:
            {
                EnforceRootPath(op.Value(), path);
                Groups         groups     = HttpFSServerWebApp.Get().Get <Groups>();
                IList <string> userGroups = groups.GetGroups(user.GetShortUserName());
                if (!userGroups.Contains(HttpFSServerWebApp.Get().GetAdminGroup()))
                {
                    throw new AccessControlException("User not in HttpFSServer admin group");
                }
                Instrumentation instrumentation = HttpFSServerWebApp.Get().Get <Instrumentation>();
                IDictionary     snapshot        = instrumentation.GetSnapshot();
                response = Response.Ok(snapshot).Build();
                break;
            }

            case HttpFSFileSystem.Operation.Getcontentsummary:
            {
                FSOperations.FSContentSummary command = new FSOperations.FSContentSummary(path);
                IDictionary json = FsExecute(user, command);
                AuditLog.Info("[{}]", path);
                response = Response.Ok(json).Type(MediaType.ApplicationJson).Build();
                break;
            }

            case HttpFSFileSystem.Operation.Getfilechecksum:
            {
                FSOperations.FSFileChecksum command = new FSOperations.FSFileChecksum(path);
                IDictionary json = FsExecute(user, command);
                AuditLog.Info("[{}]", path);
                response = Response.Ok(json).Type(MediaType.ApplicationJson).Build();
                break;
            }

            case HttpFSFileSystem.Operation.Getfileblocklocations:
            {
                response = Response.Status(Response.Status.BadRequest).Build();
                break;
            }

            case HttpFSFileSystem.Operation.Getaclstatus:
            {
                FSOperations.FSAclStatus command = new FSOperations.FSAclStatus(path);
                IDictionary json = FsExecute(user, command);
                AuditLog.Info("ACL status for [{}]", path);
                response = Response.Ok(json).Type(MediaType.ApplicationJson).Build();
                break;
            }

            case HttpFSFileSystem.Operation.Getxattrs:
            {
                IList <string> xattrNames = @params.GetValues <HttpFSParametersProvider.XAttrNameParam
                                                               >(HttpFSParametersProvider.XAttrNameParam.Name);
                XAttrCodec encoding = @params.Get <HttpFSParametersProvider.XAttrEncodingParam>(HttpFSParametersProvider.XAttrEncodingParam
                                                                                                .Name);
                FSOperations.FSGetXAttrs command = new FSOperations.FSGetXAttrs(path, xattrNames,
                                                                                encoding);
                IDictionary json = FsExecute(user, command);
                AuditLog.Info("XAttrs for [{}]", path);
                response = Response.Ok(json).Type(MediaType.ApplicationJson).Build();
                break;
            }

            case HttpFSFileSystem.Operation.Listxattrs:
            {
                FSOperations.FSListXAttrs command = new FSOperations.FSListXAttrs(path);
                IDictionary json = FsExecute(user, command);
                AuditLog.Info("XAttr names for [{}]", path);
                response = Response.Ok(json).Type(MediaType.ApplicationJson).Build();
                break;
            }

            default:
            {
                throw new IOException(MessageFormat.Format("Invalid HTTP GET operation [{0}]", op
                                                           .Value()));
            }
            }
            return(response);
        }