public virtual void TestDisabledAuthorizationOfDefaultServlets()
{
Configuration conf = new Configuration();
// Authorization is disabled by default
conf.Set(HttpServer2.FilterInitializerProperty, typeof(TestHttpServer.DummyFilterInitializer
).FullName);
conf.Set(CommonConfigurationKeys.HadoopSecurityGroupMapping, typeof(TestHttpServer.MyGroupsProvider
).FullName);
Groups.GetUserToGroupsMappingService(conf);
TestHttpServer.MyGroupsProvider.ClearMapping();
TestHttpServer.MyGroupsProvider.mapping["userA"] = Arrays.AsList("groupA");
TestHttpServer.MyGroupsProvider.mapping["userB"] = Arrays.AsList("groupB");
HttpServer2 myServer = new HttpServer2.Builder().SetName("test").AddEndpoint(new
URI("http://localhost:0")).SetFindPort(true).Build();
myServer.SetAttribute(HttpServer2.ConfContextAttribute, conf);
myServer.Start();
string serverURL = "http://" + NetUtils.GetHostPortString(myServer.GetConnectorAddress
(0)) + "/";
foreach (string servlet in new string[] { "conf", "logs", "stacks", "logLevel", "metrics" })
{
foreach (string user in new string[] { "userA", "userB" })
{
Assert.Equal(HttpURLConnection.HttpOk, GetHttpStatusCode(serverURL
+ servlet, user));
}
}
myServer.Stop();
}
/// <exception cref="System.Exception"/>
protected override void ServiceStart()
{
try
{
Configuration conf = GetConfig();
HttpServer2.Builder b = new HttpServer2.Builder().SetName("proxy").AddEndpoint(URI
.Create(WebAppUtils.GetHttpSchemePrefix(conf) + bindAddress + ":" + port)).SetFindPort
(port == 0).SetConf(GetConfig()).SetACL(acl);
if (YarnConfiguration.UseHttps(conf))
{
WebAppUtils.LoadSslConfiguration(b);
}
proxyServer = b.Build();
proxyServer.AddServlet(ProxyUriUtils.ProxyServletName, ProxyUriUtils.ProxyPathSpec
, typeof(WebAppProxyServlet));
proxyServer.SetAttribute(FetcherAttribute, fetcher);
proxyServer.SetAttribute(IsSecurityEnabledAttribute, isSecurityEnabled);
proxyServer.SetAttribute(ProxyHostAttribute, proxyHost);
proxyServer.Start();
}
catch (IOException e)
{
Log.Error("Could not start proxy web server", e);
throw;
}
base.ServiceStart();
}
/// <seealso cref="Org.Apache.Hadoop.Hdfs.DFSUtil.GetHttpPolicy(Org.Apache.Hadoop.Conf.Configuration)
/// ">
/// for information related to the different configuration options and
/// Http Policy is decided.
/// </seealso>
/// <exception cref="System.IO.IOException"/>
internal virtual void Start()
{
HttpConfig.Policy policy = DFSUtil.GetHttpPolicy(conf);
string infoHost = bindAddress.GetHostName();
IPEndPoint httpAddr = bindAddress;
string httpsAddrString = conf.GetTrimmed(DFSConfigKeys.DfsNamenodeHttpsAddressKey
, DFSConfigKeys.DfsNamenodeHttpsAddressDefault);
IPEndPoint httpsAddr = NetUtils.CreateSocketAddr(httpsAddrString);
if (httpsAddr != null)
{
// If DFS_NAMENODE_HTTPS_BIND_HOST_KEY exists then it overrides the
// host name portion of DFS_NAMENODE_HTTPS_ADDRESS_KEY.
string bindHost = conf.GetTrimmed(DFSConfigKeys.DfsNamenodeHttpsBindHostKey);
if (bindHost != null && !bindHost.IsEmpty())
{
httpsAddr = new IPEndPoint(bindHost, httpsAddr.Port);
}
}
HttpServer2.Builder builder = DFSUtil.HttpServerTemplateForNNAndJN(conf, httpAddr
, httpsAddr, "hdfs", DFSConfigKeys.DfsNamenodeKerberosInternalSpnegoPrincipalKey
, DFSConfigKeys.DfsNamenodeKeytabFileKey);
httpServer = builder.Build();
if (policy.IsHttpsEnabled())
{
// assume same ssl port for all datanodes
IPEndPoint datanodeSslPort = NetUtils.CreateSocketAddr(conf.GetTrimmed(DFSConfigKeys
.DfsDatanodeHttpsAddressKey, infoHost + ":" + DFSConfigKeys.DfsDatanodeHttpsDefaultPort
));
httpServer.SetAttribute(DFSConfigKeys.DfsDatanodeHttpsPortKey, datanodeSslPort.Port
);
}
InitWebHdfs(conf);
httpServer.SetAttribute(NamenodeAttributeKey, nn);
httpServer.SetAttribute(JspHelper.CurrentConf, conf);
SetupServlets(httpServer, conf);
httpServer.Start();
int connIdx = 0;
if (policy.IsHttpEnabled())
{
httpAddress = httpServer.GetConnectorAddress(connIdx++);
conf.Set(DFSConfigKeys.DfsNamenodeHttpAddressKey, NetUtils.GetHostPortString(httpAddress
));
}
if (policy.IsHttpsEnabled())
{
httpsAddress = httpServer.GetConnectorAddress(connIdx);
conf.Set(DFSConfigKeys.DfsNamenodeHttpsAddressKey, NetUtils.GetHostPortString(httpsAddress
));
}
}
/// <exception cref="System.Exception"/>
private static HttpServer2 StartHttpServer()
{
new FilePath(Runtime.GetProperty("build.webapps", "build/webapps") + "/test").Mkdirs
();
HttpServer2 server = new HttpServer2.Builder().SetName("test").AddEndpoint(URI.Create
("http://localhost:0")).SetFindPort(true).Build();
server.AddServlet("jobend", "/jobend", typeof(TestJobEndNotifier.JobEndServlet));
server.Start();
TestJobEndNotifier.JobEndServlet.calledTimes = 0;
TestJobEndNotifier.JobEndServlet.requestUri = null;
TestJobEndNotifier.JobEndServlet.baseUrl = "http://localhost:" + server.GetConnectorAddress
(0).Port + "/";
TestJobEndNotifier.JobEndServlet.foundJobState = null;
return(server);
}
/// <exception cref="System.Exception"/>
public virtual void TestDynamicLogLevel()
{
string logName = typeof(TestLogLevel).FullName;
Org.Apache.Commons.Logging.Log testlog = LogFactory.GetLog(logName);
//only test Log4JLogger
if (testlog is Log4JLogger)
{
Logger log = ((Log4JLogger)testlog).GetLogger();
log.Debug("log.debug1");
log.Info("log.info1");
log.Error("log.error1");
Assert.True(!Level.Error.Equals(log.GetEffectiveLevel()));
HttpServer2 server = new HttpServer2.Builder().SetName("..").AddEndpoint(new URI(
"http://*****:*****@out.WriteLine("*** Connecting to " + url);
URLConnection connection = url.OpenConnection();
connection.Connect();
BufferedReader @in = new BufferedReader(new InputStreamReader(connection.GetInputStream
()));
for (string line; (line = @in.ReadLine()) != null; @out.WriteLine(line))
{
}
@in.Close();
log.Debug("log.debug2");
log.Info("log.info2");
log.Error("log.error2");
Assert.True(Level.Error.Equals(log.GetEffectiveLevel()));
//command line
string[] args = new string[] { "-setlevel", authority, logName, Level.Debug.ToString
() };
LogLevel.Main(args);
log.Debug("log.debug3");
log.Info("log.info3");
log.Error("log.error3");
Assert.True(Level.Debug.Equals(log.GetEffectiveLevel()));
}
else
{
@out.WriteLine(testlog.GetType() + " not tested.");
}
}
/// <summary>Load the SSL keystore / truststore into the HttpServer builder.</summary>
/// <param name="builder">the HttpServer2.Builder to populate with ssl config</param>
/// <param name="sslConf">the Configuration instance to use during loading of SSL conf
/// </param>
public static HttpServer2.Builder LoadSslConfiguration(HttpServer2.Builder builder
, Configuration sslConf)
{
if (sslConf == null)
{
sslConf = new Configuration(false);
}
bool needsClientAuth = YarnConfiguration.YarnSslClientHttpsNeedAuthDefault;
sslConf.AddResource(YarnConfiguration.YarnSslServerResourceDefault);
return(builder.NeedsClientAuth(needsClientAuth).KeyPassword(GetPassword(sslConf,
WebAppKeyPasswordKey)).KeyStore(sslConf.Get("ssl.server.keystore.location"), GetPassword
(sslConf, WebAppKeystorePasswordKey), sslConf.Get("ssl.server.keystore.type", "jks"
)).TrustStore(sslConf.Get("ssl.server.truststore.location"), GetPassword(sslConf
, WebAppTruststorePasswordKey), sslConf.Get("ssl.server.truststore.type", "jks")
));
}
/// <exception cref="System.IO.IOException"/>
internal virtual void Start()
{
IPEndPoint httpAddr = GetAddress(conf);
string httpsAddrString = conf.Get(DFSConfigKeys.DfsJournalnodeHttpsAddressKey, DFSConfigKeys
.DfsJournalnodeHttpsAddressDefault);
IPEndPoint httpsAddr = NetUtils.CreateSocketAddr(httpsAddrString);
HttpServer2.Builder builder = DFSUtil.HttpServerTemplateForNNAndJN(conf, httpAddr
, httpsAddr, "journal", DFSConfigKeys.DfsJournalnodeKerberosInternalSpnegoPrincipalKey
, DFSConfigKeys.DfsJournalnodeKeytabFileKey);
httpServer = builder.Build();
httpServer.SetAttribute(JnAttributeKey, localJournalNode);
httpServer.SetAttribute(JspHelper.CurrentConf, conf);
httpServer.AddInternalServlet("getJournal", "/getJournal", typeof(GetJournalEditServlet
), true);
httpServer.Start();
}
/// <exception cref="System.IO.IOException"/>
internal virtual void Start()
{
IPEndPoint httpAddr = GetHttpAddress(conf);
string httpsAddrString = conf.Get(NfsConfigKeys.NfsHttpsAddressKey, NfsConfigKeys
.NfsHttpsAddressDefault);
IPEndPoint httpsAddr = NetUtils.CreateSocketAddr(httpsAddrString);
HttpServer2.Builder builder = DFSUtil.HttpServerTemplateForNNAndJN(conf, httpAddr
, httpsAddr, "nfs3", NfsConfigKeys.DfsNfsKerberosPrincipalKey, NfsConfigKeys.DfsNfsKeytabFileKey
);
this.httpServer = builder.Build();
this.httpServer.Start();
HttpConfig.Policy policy = DFSUtil.GetHttpPolicy(conf);
int connIdx = 0;
if (policy.IsHttpEnabled())
{
infoPort = httpServer.GetConnectorAddress(connIdx++).Port;
}
if (policy.IsHttpsEnabled())
{
infoSecurePort = httpServer.GetConnectorAddress(connIdx).Port;
}
}
/// <exception cref="System.IO.IOException"/>
public DatanodeHttpServer(Configuration conf, DataNode datanode, ServerSocketChannel
externalHttpChannel)
{
this.conf = conf;
Configuration confForInfoServer = new Configuration(conf);
confForInfoServer.SetInt(HttpServer2.HttpMaxThreads, 10);
HttpServer2.Builder builder = new HttpServer2.Builder().SetName("datanode").SetConf
(confForInfoServer).SetACL(new AccessControlList(conf.Get(DFSConfigKeys.DfsAdmin
, " "))).HostName(GetHostnameForSpnegoPrincipal(confForInfoServer)).AddEndpoint(
URI.Create("http://localhost:0")).SetFindPort(true);
this.infoServer = builder.Build();
this.infoServer.AddInternalServlet(null, "/streamFile/*", typeof(StreamFile));
this.infoServer.AddInternalServlet(null, "/getFileChecksum/*", typeof(FileChecksumServlets.GetServlet
));
this.infoServer.SetAttribute("datanode", datanode);
this.infoServer.SetAttribute(JspHelper.CurrentConf, conf);
this.infoServer.AddServlet(null, "/blockScannerReport", typeof(BlockScanner.Servlet
));
this.infoServer.Start();
IPEndPoint jettyAddr = infoServer.GetConnectorAddress(0);
this.confForCreate = new Configuration(conf);
confForCreate.Set(FsPermission.UmaskLabel, "000");
this.bossGroup = new NioEventLoopGroup();
this.workerGroup = new NioEventLoopGroup();
this.externalHttpChannel = externalHttpChannel;
HttpConfig.Policy policy = DFSUtil.GetHttpPolicy(conf);
if (policy.IsHttpEnabled())
{
this.httpServer = new ServerBootstrap().Group(bossGroup, workerGroup).ChildHandler
(new _ChannelInitializer_117(this, jettyAddr, conf));
if (externalHttpChannel == null)
{
httpServer.Channel(typeof(NioServerSocketChannel));
}
else
{
httpServer.ChannelFactory(new _ChannelFactory_130(externalHttpChannel));
}
}
else
{
// The channel has been bounded externally via JSVC,
// thus bind() becomes a no-op.
this.httpServer = null;
}
if (policy.IsHttpsEnabled())
{
this.sslFactory = new SSLFactory(SSLFactory.Mode.Server, conf);
try
{
sslFactory.Init();
}
catch (GeneralSecurityException e)
{
throw new IOException(e);
}
this.httpsServer = new ServerBootstrap().Group(bossGroup, workerGroup).Channel(typeof(
NioServerSocketChannel)).ChildHandler(new _ChannelInitializer_155(this, jettyAddr
, conf));
}
else
{
this.httpsServer = null;
this.sslFactory = null;
}
}
/// <summary>Load the SSL keystore / truststore into the HttpServer builder.</summary>
/// <param name="builder">the HttpServer2.Builder to populate with ssl config</param>
public static HttpServer2.Builder LoadSslConfiguration(HttpServer2.Builder builder
)
{
return(LoadSslConfiguration(builder, null));
}
/// <summary>Initialize SecondaryNameNode.</summary>
/// <exception cref="System.IO.IOException"/>
private void Initialize(Configuration conf, SecondaryNameNode.CommandLineOpts commandLineOpts
)
{
IPEndPoint infoSocAddr = GetHttpAddress(conf);
string infoBindAddress = infoSocAddr.GetHostName();
UserGroupInformation.SetConfiguration(conf);
if (UserGroupInformation.IsSecurityEnabled())
{
SecurityUtil.Login(conf, DFSConfigKeys.DfsSecondaryNamenodeKeytabFileKey, DFSConfigKeys
.DfsSecondaryNamenodeKerberosPrincipalKey, infoBindAddress);
}
// initiate Java VM metrics
DefaultMetricsSystem.Initialize("SecondaryNameNode");
JvmMetrics.Create("SecondaryNameNode", conf.Get(DFSConfigKeys.DfsMetricsSessionIdKey
), DefaultMetricsSystem.Instance());
// Create connection to the namenode.
shouldRun = true;
nameNodeAddr = NameNode.GetServiceAddress(conf, true);
this.conf = conf;
this.namenode = NameNodeProxies.CreateNonHAProxy <NamenodeProtocol>(conf, nameNodeAddr
, UserGroupInformation.GetCurrentUser(), true).GetProxy();
// initialize checkpoint directories
fsName = GetInfoServer();
checkpointDirs = FSImage.GetCheckpointDirs(conf, "/tmp/hadoop/dfs/namesecondary");
checkpointEditsDirs = FSImage.GetCheckpointEditsDirs(conf, "/tmp/hadoop/dfs/namesecondary"
);
checkpointImage = new SecondaryNameNode.CheckpointStorage(conf, checkpointDirs, checkpointEditsDirs
);
checkpointImage.RecoverCreate(commandLineOpts.ShouldFormat());
checkpointImage.DeleteTempEdits();
namesystem = new FSNamesystem(conf, checkpointImage, true);
// Disable quota checks
namesystem.dir.DisableQuotaChecks();
// Initialize other scheduling parameters from the configuration
checkpointConf = new CheckpointConf(conf);
IPEndPoint httpAddr = infoSocAddr;
string httpsAddrString = conf.GetTrimmed(DFSConfigKeys.DfsNamenodeSecondaryHttpsAddressKey
, DFSConfigKeys.DfsNamenodeSecondaryHttpsAddressDefault);
IPEndPoint httpsAddr = NetUtils.CreateSocketAddr(httpsAddrString);
HttpServer2.Builder builder = DFSUtil.HttpServerTemplateForNNAndJN(conf, httpAddr
, httpsAddr, "secondary", DFSConfigKeys.DfsSecondaryNamenodeKerberosInternalSpnegoPrincipalKey
, DFSConfigKeys.DfsSecondaryNamenodeKeytabFileKey);
nameNodeStatusBeanName = MBeans.Register("SecondaryNameNode", "SecondaryNameNodeInfo"
, this);
infoServer = builder.Build();
infoServer.SetAttribute("secondary.name.node", this);
infoServer.SetAttribute("name.system.image", checkpointImage);
infoServer.SetAttribute(JspHelper.CurrentConf, conf);
infoServer.AddInternalServlet("imagetransfer", ImageServlet.PathSpec, typeof(ImageServlet
), true);
infoServer.Start();
Log.Info("Web server init done");
HttpConfig.Policy policy = DFSUtil.GetHttpPolicy(conf);
int connIdx = 0;
if (policy.IsHttpEnabled())
{
IPEndPoint httpAddress = infoServer.GetConnectorAddress(connIdx++);
conf.Set(DFSConfigKeys.DfsNamenodeSecondaryHttpAddressKey, NetUtils.GetHostPortString
(httpAddress));
}
if (policy.IsHttpsEnabled())
{
IPEndPoint httpsAddress = infoServer.GetConnectorAddress(connIdx);
conf.Set(DFSConfigKeys.DfsNamenodeSecondaryHttpsAddressKey, NetUtils.GetHostPortString
(httpsAddress));
}
legacyOivImageDir = conf.Get(DFSConfigKeys.DfsNamenodeLegacyOivImageDirKey);
Log.Info("Checkpoint Period :" + checkpointConf.GetPeriod() + " secs " + "(" +
checkpointConf.GetPeriod() / 60 + " min)");
Log.Info("Log Size Trigger :" + checkpointConf.GetTxnCount() + " txns");
}
public virtual WebApp Build(WebApp webapp)
{
if (webapp == null)
{
webapp = new _WebApp_171();
}
// Defaults should be fine in usual cases
webapp.SetName(name);
webapp.SetWebServices(wsName);
string basePath = "/" + name;
webapp.SetRedirectPath(basePath);
IList <string> pathList = new AList <string>();
if (basePath.Equals("/"))
{
webapp.AddServePathSpec("/*");
pathList.AddItem("/*");
}
else
{
webapp.AddServePathSpec(basePath);
webapp.AddServePathSpec(basePath + "/*");
pathList.AddItem(basePath + "/*");
}
if (wsName != null && !wsName.Equals(basePath))
{
if (wsName.Equals("/"))
{
webapp.AddServePathSpec("/*");
pathList.AddItem("/*");
}
else
{
webapp.AddServePathSpec("/" + wsName);
webapp.AddServePathSpec("/" + wsName + "/*");
pathList.AddItem("/" + wsName + "/*");
}
}
if (conf == null)
{
conf = new Configuration();
}
try
{
if (application != null)
{
webapp.SetHostClass(application.GetType());
}
else
{
string cls = InferHostClass();
Log.Debug("setting webapp host class to {}", cls);
webapp.SetHostClass(Sharpen.Runtime.GetType(cls));
}
if (devMode)
{
if (port > 0)
{
try
{
new Uri("http://localhost:" + port + "/__stop").GetContent();
Log.Info("stopping existing webapp instance");
Sharpen.Thread.Sleep(100);
}
catch (ConnectException e)
{
Log.Info("no existing webapp instance found: {}", e.ToString());
}
catch (Exception e)
{
// should not be fatal
Log.Warn("error stopping existing instance: {}", e.ToString());
}
}
else
{
Log.Error("dev mode does NOT work with ephemeral port!");
System.Environment.Exit(1);
}
}
string httpScheme;
if (this.httpPolicy == null)
{
httpScheme = WebAppUtils.GetHttpSchemePrefix(conf);
}
else
{
httpScheme = (httpPolicy == HttpConfig.Policy.HttpsOnly) ? WebAppUtils.HttpsPrefix
: WebAppUtils.HttpPrefix;
}
HttpServer2.Builder builder = new HttpServer2.Builder().SetName(name).AddEndpoint
(URI.Create(httpScheme + bindAddress + ":" + port)).SetConf(conf).SetFindPort(findPort
).SetACL(new AccessControlList(conf.Get(YarnConfiguration.YarnAdminAcl, YarnConfiguration
.DefaultYarnAdminAcl))).SetPathSpec(Sharpen.Collections.ToArray(pathList, new string
[0]));
bool hasSpnegoConf = spnegoPrincipalKey != null && conf.Get(spnegoPrincipalKey) !=
null && spnegoKeytabKey != null && conf.Get(spnegoKeytabKey) != null;
if (hasSpnegoConf)
{
builder.SetUsernameConfKey(spnegoPrincipalKey).SetKeytabConfKey(spnegoKeytabKey).
SetSecurityEnabled(UserGroupInformation.IsSecurityEnabled());
}
if (httpScheme.Equals(WebAppUtils.HttpsPrefix))
{
WebAppUtils.LoadSslConfiguration(builder);
}
HttpServer2 server = builder.Build();
foreach (WebApps.Builder.ServletStruct @struct in servlets)
{
server.AddServlet(@struct.name, @struct.spec, @struct.clazz);
}
foreach (KeyValuePair <string, object> entry in attributes)
{
server.SetAttribute(entry.Key, entry.Value);
}
HttpServer2.DefineFilter(server.GetWebAppContext(), "guice", typeof(GuiceFilter).
FullName, null, new string[] { "/*" });
webapp.SetConf(conf);
webapp.SetHttpServer(server);
}
catch (TypeLoadException e)
{
throw new WebAppException("Error starting http server", e);
}
catch (IOException e)
{
throw new WebAppException("Error starting http server", e);
}
Injector injector = Guice.CreateInjector(webapp, new _AbstractModule_280(this));
Log.Info("Registered webapp guice modules");
// save a guice filter instance for webapp stop (mostly for unit tests)
webapp.SetGuiceFilter(injector.GetInstance <GuiceFilter>());
if (devMode)
{
injector.GetInstance <Dispatcher>().SetDevMode(devMode);
Log.Info("in dev mode!");
}
return(webapp);
}
