Ejemplo n.º 1
0
        public virtual void TestHttpPolicy()
        {
            conf.Set(DFSConfigKeys.DfsHttpPolicyKey, policy.ToString());
            conf.Set(DFSConfigKeys.DfsNamenodeHttpsAddressKey, "localhost:0");
            IPEndPoint         addr   = IPEndPoint.CreateUnresolved("localhost", 0);
            NameNodeHttpServer server = null;

            try
            {
                server = new NameNodeHttpServer(conf, null, addr);
                server.Start();
                NUnit.Framework.Assert.IsTrue(Implies(policy.IsHttpEnabled(), CanAccess("http", server
                                                                                        .GetHttpAddress())));
                NUnit.Framework.Assert.IsTrue(Implies(!policy.IsHttpEnabled(), server.GetHttpAddress
                                                          () == null));
                NUnit.Framework.Assert.IsTrue(Implies(policy.IsHttpsEnabled(), CanAccess("https",
                                                                                         server.GetHttpsAddress())));
                NUnit.Framework.Assert.IsTrue(Implies(!policy.IsHttpsEnabled(), server.GetHttpsAddress
                                                          () == null));
            }
            finally
            {
                if (server != null)
                {
                    server.Stop();
                }
            }
        }
Ejemplo n.º 2
0
        /// <seealso cref="Org.Apache.Hadoop.Hdfs.DFSUtil.GetHttpPolicy(Org.Apache.Hadoop.Conf.Configuration)
        ///     ">
        /// for information related to the different configuration options and
        /// Http Policy is decided.
        /// </seealso>
        /// <exception cref="System.IO.IOException"/>
        internal virtual void Start()
        {
            HttpConfig.Policy policy          = DFSUtil.GetHttpPolicy(conf);
            string            infoHost        = bindAddress.GetHostName();
            IPEndPoint        httpAddr        = bindAddress;
            string            httpsAddrString = conf.GetTrimmed(DFSConfigKeys.DfsNamenodeHttpsAddressKey
                                                                , DFSConfigKeys.DfsNamenodeHttpsAddressDefault);
            IPEndPoint httpsAddr = NetUtils.CreateSocketAddr(httpsAddrString);

            if (httpsAddr != null)
            {
                // If DFS_NAMENODE_HTTPS_BIND_HOST_KEY exists then it overrides the
                // host name portion of DFS_NAMENODE_HTTPS_ADDRESS_KEY.
                string bindHost = conf.GetTrimmed(DFSConfigKeys.DfsNamenodeHttpsBindHostKey);
                if (bindHost != null && !bindHost.IsEmpty())
                {
                    httpsAddr = new IPEndPoint(bindHost, httpsAddr.Port);
                }
            }
            HttpServer2.Builder builder = DFSUtil.HttpServerTemplateForNNAndJN(conf, httpAddr
                                                                               , httpsAddr, "hdfs", DFSConfigKeys.DfsNamenodeKerberosInternalSpnegoPrincipalKey
                                                                               , DFSConfigKeys.DfsNamenodeKeytabFileKey);
            httpServer = builder.Build();
            if (policy.IsHttpsEnabled())
            {
                // assume same ssl port for all datanodes
                IPEndPoint datanodeSslPort = NetUtils.CreateSocketAddr(conf.GetTrimmed(DFSConfigKeys
                                                                                       .DfsDatanodeHttpsAddressKey, infoHost + ":" + DFSConfigKeys.DfsDatanodeHttpsDefaultPort
                                                                                       ));
                httpServer.SetAttribute(DFSConfigKeys.DfsDatanodeHttpsPortKey, datanodeSslPort.Port
                                        );
            }
            InitWebHdfs(conf);
            httpServer.SetAttribute(NamenodeAttributeKey, nn);
            httpServer.SetAttribute(JspHelper.CurrentConf, conf);
            SetupServlets(httpServer, conf);
            httpServer.Start();
            int connIdx = 0;

            if (policy.IsHttpEnabled())
            {
                httpAddress = httpServer.GetConnectorAddress(connIdx++);
                conf.Set(DFSConfigKeys.DfsNamenodeHttpAddressKey, NetUtils.GetHostPortString(httpAddress
                                                                                             ));
            }
            if (policy.IsHttpsEnabled())
            {
                httpsAddress = httpServer.GetConnectorAddress(connIdx);
                conf.Set(DFSConfigKeys.DfsNamenodeHttpsAddressKey, NetUtils.GetHostPortString(httpsAddress
                                                                                              ));
            }
        }
Ejemplo n.º 3
0
        /// <exception cref="System.IO.IOException"/>
        internal virtual void Start()
        {
            IPEndPoint httpAddr        = GetHttpAddress(conf);
            string     httpsAddrString = conf.Get(NfsConfigKeys.NfsHttpsAddressKey, NfsConfigKeys
                                                  .NfsHttpsAddressDefault);
            IPEndPoint httpsAddr = NetUtils.CreateSocketAddr(httpsAddrString);

            HttpServer2.Builder builder = DFSUtil.HttpServerTemplateForNNAndJN(conf, httpAddr
                                                                               , httpsAddr, "nfs3", NfsConfigKeys.DfsNfsKerberosPrincipalKey, NfsConfigKeys.DfsNfsKeytabFileKey
                                                                               );
            this.httpServer = builder.Build();
            this.httpServer.Start();
            HttpConfig.Policy policy = DFSUtil.GetHttpPolicy(conf);
            int connIdx = 0;

            if (policy.IsHttpEnabled())
            {
                infoPort = httpServer.GetConnectorAddress(connIdx++).Port;
            }
            if (policy.IsHttpsEnabled())
            {
                infoSecurePort = httpServer.GetConnectorAddress(connIdx).Port;
            }
        }
Ejemplo n.º 4
0
        /// <exception cref="System.IO.IOException"/>
        public DatanodeHttpServer(Configuration conf, DataNode datanode, ServerSocketChannel
                                  externalHttpChannel)
        {
            this.conf = conf;
            Configuration confForInfoServer = new Configuration(conf);

            confForInfoServer.SetInt(HttpServer2.HttpMaxThreads, 10);
            HttpServer2.Builder builder = new HttpServer2.Builder().SetName("datanode").SetConf
                                              (confForInfoServer).SetACL(new AccessControlList(conf.Get(DFSConfigKeys.DfsAdmin
                                                                                                        , " "))).HostName(GetHostnameForSpnegoPrincipal(confForInfoServer)).AddEndpoint(
                URI.Create("http://localhost:0")).SetFindPort(true);
            this.infoServer = builder.Build();
            this.infoServer.AddInternalServlet(null, "/streamFile/*", typeof(StreamFile));
            this.infoServer.AddInternalServlet(null, "/getFileChecksum/*", typeof(FileChecksumServlets.GetServlet
                                                                                  ));
            this.infoServer.SetAttribute("datanode", datanode);
            this.infoServer.SetAttribute(JspHelper.CurrentConf, conf);
            this.infoServer.AddServlet(null, "/blockScannerReport", typeof(BlockScanner.Servlet
                                                                           ));
            this.infoServer.Start();
            IPEndPoint jettyAddr = infoServer.GetConnectorAddress(0);

            this.confForCreate = new Configuration(conf);
            confForCreate.Set(FsPermission.UmaskLabel, "000");
            this.bossGroup           = new NioEventLoopGroup();
            this.workerGroup         = new NioEventLoopGroup();
            this.externalHttpChannel = externalHttpChannel;
            HttpConfig.Policy policy = DFSUtil.GetHttpPolicy(conf);
            if (policy.IsHttpEnabled())
            {
                this.httpServer = new ServerBootstrap().Group(bossGroup, workerGroup).ChildHandler
                                      (new _ChannelInitializer_117(this, jettyAddr, conf));
                if (externalHttpChannel == null)
                {
                    httpServer.Channel(typeof(NioServerSocketChannel));
                }
                else
                {
                    httpServer.ChannelFactory(new _ChannelFactory_130(externalHttpChannel));
                }
            }
            else
            {
                // The channel has been bounded externally via JSVC,
                // thus bind() becomes a no-op.
                this.httpServer = null;
            }
            if (policy.IsHttpsEnabled())
            {
                this.sslFactory = new SSLFactory(SSLFactory.Mode.Server, conf);
                try
                {
                    sslFactory.Init();
                }
                catch (GeneralSecurityException e)
                {
                    throw new IOException(e);
                }
                this.httpsServer = new ServerBootstrap().Group(bossGroup, workerGroup).Channel(typeof(
                                                                                                   NioServerSocketChannel)).ChildHandler(new _ChannelInitializer_155(this, jettyAddr
                                                                                                                                                                     , conf));
            }
            else
            {
                this.httpsServer = null;
                this.sslFactory  = null;
            }
        }
Ejemplo n.º 5
0
        /// <summary>Initialize SecondaryNameNode.</summary>
        /// <exception cref="System.IO.IOException"/>
        private void Initialize(Configuration conf, SecondaryNameNode.CommandLineOpts commandLineOpts
                                )
        {
            IPEndPoint infoSocAddr     = GetHttpAddress(conf);
            string     infoBindAddress = infoSocAddr.GetHostName();

            UserGroupInformation.SetConfiguration(conf);
            if (UserGroupInformation.IsSecurityEnabled())
            {
                SecurityUtil.Login(conf, DFSConfigKeys.DfsSecondaryNamenodeKeytabFileKey, DFSConfigKeys
                                   .DfsSecondaryNamenodeKerberosPrincipalKey, infoBindAddress);
            }
            // initiate Java VM metrics
            DefaultMetricsSystem.Initialize("SecondaryNameNode");
            JvmMetrics.Create("SecondaryNameNode", conf.Get(DFSConfigKeys.DfsMetricsSessionIdKey
                                                            ), DefaultMetricsSystem.Instance());
            // Create connection to the namenode.
            shouldRun     = true;
            nameNodeAddr  = NameNode.GetServiceAddress(conf, true);
            this.conf     = conf;
            this.namenode = NameNodeProxies.CreateNonHAProxy <NamenodeProtocol>(conf, nameNodeAddr
                                                                                , UserGroupInformation.GetCurrentUser(), true).GetProxy();
            // initialize checkpoint directories
            fsName              = GetInfoServer();
            checkpointDirs      = FSImage.GetCheckpointDirs(conf, "/tmp/hadoop/dfs/namesecondary");
            checkpointEditsDirs = FSImage.GetCheckpointEditsDirs(conf, "/tmp/hadoop/dfs/namesecondary"
                                                                 );
            checkpointImage = new SecondaryNameNode.CheckpointStorage(conf, checkpointDirs, checkpointEditsDirs
                                                                      );
            checkpointImage.RecoverCreate(commandLineOpts.ShouldFormat());
            checkpointImage.DeleteTempEdits();
            namesystem = new FSNamesystem(conf, checkpointImage, true);
            // Disable quota checks
            namesystem.dir.DisableQuotaChecks();
            // Initialize other scheduling parameters from the configuration
            checkpointConf = new CheckpointConf(conf);
            IPEndPoint httpAddr        = infoSocAddr;
            string     httpsAddrString = conf.GetTrimmed(DFSConfigKeys.DfsNamenodeSecondaryHttpsAddressKey
                                                         , DFSConfigKeys.DfsNamenodeSecondaryHttpsAddressDefault);
            IPEndPoint httpsAddr = NetUtils.CreateSocketAddr(httpsAddrString);

            HttpServer2.Builder builder = DFSUtil.HttpServerTemplateForNNAndJN(conf, httpAddr
                                                                               , httpsAddr, "secondary", DFSConfigKeys.DfsSecondaryNamenodeKerberosInternalSpnegoPrincipalKey
                                                                               , DFSConfigKeys.DfsSecondaryNamenodeKeytabFileKey);
            nameNodeStatusBeanName = MBeans.Register("SecondaryNameNode", "SecondaryNameNodeInfo"
                                                     , this);
            infoServer = builder.Build();
            infoServer.SetAttribute("secondary.name.node", this);
            infoServer.SetAttribute("name.system.image", checkpointImage);
            infoServer.SetAttribute(JspHelper.CurrentConf, conf);
            infoServer.AddInternalServlet("imagetransfer", ImageServlet.PathSpec, typeof(ImageServlet
                                                                                         ), true);
            infoServer.Start();
            Log.Info("Web server init done");
            HttpConfig.Policy policy = DFSUtil.GetHttpPolicy(conf);
            int connIdx = 0;

            if (policy.IsHttpEnabled())
            {
                IPEndPoint httpAddress = infoServer.GetConnectorAddress(connIdx++);
                conf.Set(DFSConfigKeys.DfsNamenodeSecondaryHttpAddressKey, NetUtils.GetHostPortString
                             (httpAddress));
            }
            if (policy.IsHttpsEnabled())
            {
                IPEndPoint httpsAddress = infoServer.GetConnectorAddress(connIdx);
                conf.Set(DFSConfigKeys.DfsNamenodeSecondaryHttpsAddressKey, NetUtils.GetHostPortString
                             (httpsAddress));
            }
            legacyOivImageDir = conf.Get(DFSConfigKeys.DfsNamenodeLegacyOivImageDirKey);
            Log.Info("Checkpoint Period   :" + checkpointConf.GetPeriod() + " secs " + "(" +
                     checkpointConf.GetPeriod() / 60 + " min)");
            Log.Info("Log Size Trigger    :" + checkpointConf.GetTxnCount() + " txns");
        }