public virtual IPEndPoint GetAddress()
{
IPEndPoint addr = httpServer.GetConnectorAddress(0);
System.Diagnostics.Debug.Assert(addr.Port != 0);
return(addr);
}
public virtual void TestBindAddress()
{
CheckBindAddress("localhost", 0, false).Stop();
// hang onto this one for a bit more testing
HttpServer2 myServer = CheckBindAddress("localhost", 0, false);
HttpServer2 myServer2 = null;
try
{
int port = myServer.GetConnectorAddress(0).Port;
// it's already in use, true = expect a higher port
myServer2 = CheckBindAddress("localhost", port, true);
// try to reuse the port
port = myServer2.GetConnectorAddress(0).Port;
myServer2.Stop();
NUnit.Framework.Assert.IsNull(myServer2.GetConnectorAddress(0));
// not bound
myServer2.OpenListeners();
Assert.Equal(port, myServer2.GetConnectorAddress(0).Port);
}
finally
{
// expect same port
myServer.Stop();
if (myServer2 != null)
{
myServer2.Stop();
}
}
}
/// <seealso cref="Org.Apache.Hadoop.Hdfs.DFSUtil.GetHttpPolicy(Org.Apache.Hadoop.Conf.Configuration)
/// ">
/// for information related to the different configuration options and
/// Http Policy is decided.
/// </seealso>
/// <exception cref="System.IO.IOException"/>
internal virtual void Start()
{
HttpConfig.Policy policy = DFSUtil.GetHttpPolicy(conf);
string infoHost = bindAddress.GetHostName();
IPEndPoint httpAddr = bindAddress;
string httpsAddrString = conf.GetTrimmed(DFSConfigKeys.DfsNamenodeHttpsAddressKey
, DFSConfigKeys.DfsNamenodeHttpsAddressDefault);
IPEndPoint httpsAddr = NetUtils.CreateSocketAddr(httpsAddrString);
if (httpsAddr != null)
{
// If DFS_NAMENODE_HTTPS_BIND_HOST_KEY exists then it overrides the
// host name portion of DFS_NAMENODE_HTTPS_ADDRESS_KEY.
string bindHost = conf.GetTrimmed(DFSConfigKeys.DfsNamenodeHttpsBindHostKey);
if (bindHost != null && !bindHost.IsEmpty())
{
httpsAddr = new IPEndPoint(bindHost, httpsAddr.Port);
}
}
HttpServer2.Builder builder = DFSUtil.HttpServerTemplateForNNAndJN(conf, httpAddr
, httpsAddr, "hdfs", DFSConfigKeys.DfsNamenodeKerberosInternalSpnegoPrincipalKey
, DFSConfigKeys.DfsNamenodeKeytabFileKey);
httpServer = builder.Build();
if (policy.IsHttpsEnabled())
{
// assume same ssl port for all datanodes
IPEndPoint datanodeSslPort = NetUtils.CreateSocketAddr(conf.GetTrimmed(DFSConfigKeys
.DfsDatanodeHttpsAddressKey, infoHost + ":" + DFSConfigKeys.DfsDatanodeHttpsDefaultPort
));
httpServer.SetAttribute(DFSConfigKeys.DfsDatanodeHttpsPortKey, datanodeSslPort.Port
);
}
InitWebHdfs(conf);
httpServer.SetAttribute(NamenodeAttributeKey, nn);
httpServer.SetAttribute(JspHelper.CurrentConf, conf);
SetupServlets(httpServer, conf);
httpServer.Start();
int connIdx = 0;
if (policy.IsHttpEnabled())
{
httpAddress = httpServer.GetConnectorAddress(connIdx++);
conf.Set(DFSConfigKeys.DfsNamenodeHttpAddressKey, NetUtils.GetHostPortString(httpAddress
));
}
if (policy.IsHttpsEnabled())
{
httpsAddress = httpServer.GetConnectorAddress(connIdx);
conf.Set(DFSConfigKeys.DfsNamenodeHttpsAddressKey, NetUtils.GetHostPortString(httpsAddress
));
}
}
public virtual void TestHttpCookie()
{
Uri @base = new Uri("http://" + NetUtils.GetHostPortString(server.GetConnectorAddress
(0)));
HttpURLConnection conn = (HttpURLConnection) new Uri(@base, "/echo").OpenConnection
();
string header = conn.GetHeaderField("Set-Cookie");
IList <HttpCookie> cookies = HttpCookie.Parse(header);
Assert.True(!cookies.IsEmpty());
Assert.True(header.Contains("; HttpOnly"));
Assert.True("token".Equals(cookies[0].GetValue()));
}
public static void Setup()
{
conf = new Configuration();
conf.SetInt(HttpServer2.HttpMaxThreads, 10);
FilePath @base = new FilePath(Basedir);
FileUtil.FullyDelete(@base);
@base.Mkdirs();
keystoresDir = new FilePath(Basedir).GetAbsolutePath();
sslConfDir = KeyStoreTestUtil.GetClasspathDir(typeof(TestSSLHttpServer));
KeyStoreTestUtil.SetupSSLConfig(keystoresDir, sslConfDir, conf, false);
Configuration sslConf = new Configuration(false);
sslConf.AddResource("ssl-server.xml");
sslConf.AddResource("ssl-client.xml");
clientSslFactory = new SSLFactory(SSLFactory.Mode.Client, sslConf);
clientSslFactory.Init();
server = new HttpServer2.Builder().SetName("test").AddEndpoint(new URI("https://localhost"
)).SetConf(conf).KeyPassword(sslConf.Get("ssl.server.keystore.keypassword")).KeyStore
(sslConf.Get("ssl.server.keystore.location"), sslConf.Get("ssl.server.keystore.password"
), sslConf.Get("ssl.server.keystore.type", "jks")).TrustStore(sslConf.Get("ssl.server.truststore.location"
), sslConf.Get("ssl.server.truststore.password"), sslConf.Get("ssl.server.truststore.type"
, "jks")).Build();
server.AddServlet("echo", "/echo", typeof(TestHttpServer.EchoServlet));
server.AddServlet("longheader", "/longheader", typeof(HttpServerFunctionalTest.LongHeaderServlet
));
server.Start();
baseUrl = new Uri("https://" + NetUtils.GetHostPortString(server.GetConnectorAddress
(0)));
Log.Info("HTTP server started: " + baseUrl);
}
/// <summary>Return the URI that locates the HTTP server.</summary>
public virtual URI GetServerURI()
{
// getHttpClientScheme() only returns https for HTTPS_ONLY policy. This
// matches the behavior that the first connector is a HTTPS connector only
// for HTTPS_ONLY policy.
IPEndPoint addr = httpServer.GetConnectorAddress(0);
return(URI.Create(DFSUtil.GetHttpClientScheme(conf) + "://" + NetUtils.GetHostPortString
(addr)));
}
/// <exception cref="System.Exception"/>
private HttpServer2 CheckBindAddress(string host, int port, bool findPort)
{
HttpServer2 server = CreateServer(host, port);
try
{
// not bound, ephemeral should return requested port (0 for ephemeral)
IList <object> listeners = (IList <object>)Whitebox.GetInternalState(server, "listeners"
);
Connector listener = (Connector)listeners[0];
Assert.Equal(port, listener.GetPort());
// verify hostname is what was given
server.OpenListeners();
Assert.Equal(host, server.GetConnectorAddress(0).GetHostName()
);
int boundPort = server.GetConnectorAddress(0).Port;
if (port == 0)
{
Assert.True(boundPort != 0);
}
else
{
// ephemeral should now return bound port
if (findPort)
{
Assert.True(boundPort > port);
// allow a little wiggle room to prevent random test failures if
// some consecutive ports are already in use
Assert.True(boundPort - port < 8);
}
}
}
catch (Exception e)
{
server.Stop();
throw;
}
return(server);
}
public virtual void TestServletFilter()
{
Configuration conf = new Configuration();
//start a http server with CountingFilter
conf.Set(HttpServer2.FilterInitializerProperty, typeof(TestServletFilter.SimpleFilter.Initializer
).FullName);
HttpServer2 http = CreateTestServer(conf);
http.Start();
string fsckURL = "/fsck";
string stacksURL = "/stacks";
string ajspURL = "/a.jsp";
string logURL = "/logs/a.log";
string hadooplogoURL = "/static/hadoop-logo.jpg";
string[] urls = new string[] { fsckURL, stacksURL, ajspURL, logURL, hadooplogoURL };
Random ran = new Random();
int[] sequence = new int[50];
//generate a random sequence and update counts
for (int i = 0; i < sequence.Length; i++)
{
sequence[i] = ran.Next(urls.Length);
}
//access the urls as the sequence
string prefix = "http://" + NetUtils.GetHostPortString(http.GetConnectorAddress(0
));
try
{
for (int i_1 = 0; i_1 < sequence.Length; i_1++)
{
Access(prefix + urls[sequence[i_1]]);
//make sure everything except fsck get filtered
if (sequence[i_1] == 0)
{
Assert.Equal(null, uri);
}
else
{
Assert.Equal(urls[sequence[i_1]], uri);
uri = null;
}
}
}
finally
{
http.Stop();
}
}
public virtual void TestPathSpecFilters()
{
Configuration conf = new Configuration();
//start a http server with CountingFilter
conf.Set(HttpServer2.FilterInitializerProperty, typeof(TestPathFilter.RecordingFilter.Initializer
).FullName);
string[] pathSpecs = new string[] { "/path", "/path/*" };
HttpServer2 http = CreateTestServer(conf, pathSpecs);
http.Start();
string baseURL = "/path";
string baseSlashURL = "/path/";
string addedURL = "/path/nodes";
string addedSlashURL = "/path/nodes/";
string longURL = "/path/nodes/foo/job";
string rootURL = "/";
string allURL = "/*";
string[] filteredUrls = new string[] { baseURL, baseSlashURL, addedURL, addedSlashURL
, longURL };
string[] notFilteredUrls = new string[] { rootURL, allURL };
// access the urls and verify our paths specs got added to the
// filters
string prefix = "http://" + NetUtils.GetHostPortString(http.GetConnectorAddress(0
));
try
{
for (int i = 0; i < filteredUrls.Length; i++)
{
Access(prefix + filteredUrls[i]);
}
for (int i_1 = 0; i_1 < notFilteredUrls.Length; i_1++)
{
Access(prefix + notFilteredUrls[i_1]);
}
}
finally
{
http.Stop();
}
Log.Info("RECORDS = " + Records);
//verify records
for (int i_2 = 0; i_2 < filteredUrls.Length; i_2++)
{
Assert.True(Records.Remove(filteredUrls[i_2]));
}
Assert.True(Records.IsEmpty());
}
/// <exception cref="System.IO.IOException"/>
internal virtual void Start()
{
IPEndPoint httpAddr = GetHttpAddress(conf);
string httpsAddrString = conf.Get(NfsConfigKeys.NfsHttpsAddressKey, NfsConfigKeys
.NfsHttpsAddressDefault);
IPEndPoint httpsAddr = NetUtils.CreateSocketAddr(httpsAddrString);
HttpServer2.Builder builder = DFSUtil.HttpServerTemplateForNNAndJN(conf, httpAddr
, httpsAddr, "nfs3", NfsConfigKeys.DfsNfsKerberosPrincipalKey, NfsConfigKeys.DfsNfsKeytabFileKey
);
this.httpServer = builder.Build();
this.httpServer.Start();
HttpConfig.Policy policy = DFSUtil.GetHttpPolicy(conf);
int connIdx = 0;
if (policy.IsHttpEnabled())
{
infoPort = httpServer.GetConnectorAddress(connIdx++).Port;
}
if (policy.IsHttpsEnabled())
{
infoSecurePort = httpServer.GetConnectorAddress(connIdx).Port;
}
}
public virtual void TestServletFilter()
{
Configuration conf = new Configuration();
//start a http server with CountingFilter
conf.Set(HttpServer2.FilterInitializerProperty, typeof(TestGlobalFilter.RecordingFilter.Initializer
).FullName);
HttpServer2 http = CreateTestServer(conf);
http.Start();
string fsckURL = "/fsck";
string stacksURL = "/stacks";
string ajspURL = "/a.jsp";
string listPathsURL = "/listPaths";
string dataURL = "/data";
string streamFile = "/streamFile";
string rootURL = "/";
string allURL = "/*";
string outURL = "/static/a.out";
string logURL = "/logs/a.log";
string[] urls = new string[] { fsckURL, stacksURL, ajspURL, listPathsURL, dataURL
, streamFile, rootURL, allURL, outURL, logURL };
//access the urls
string prefix = "http://" + NetUtils.GetHostPortString(http.GetConnectorAddress(0
));
try
{
for (int i = 0; i < urls.Length; i++)
{
Access(prefix + urls[i]);
}
}
finally
{
http.Stop();
}
Log.Info("RECORDS = " + Records);
//verify records
for (int i_1 = 0; i_1 < urls.Length; i_1++)
{
Assert.True(Records.Remove(urls[i_1]));
}
Assert.True(Records.IsEmpty());
}
public virtual WebApp Start(WebApp webapp)
{
WebApp webApp = Build(webapp);
HttpServer2 httpServer = webApp.HttpServer();
try
{
httpServer.Start();
Log.Info("Web app " + name + " started at " + httpServer.GetConnectorAddress(0).Port
);
}
catch (IOException e)
{
throw new WebAppException("Error starting http server", e);
}
return(webApp);
}
/// <exception cref="System.Exception"/>
protected override void SetUp()
{
new FilePath(Runtime.GetProperty("build.webapps", "build/webapps") + "/test").Mkdirs
();
server = new HttpServer2.Builder().SetName("test").AddEndpoint(URI.Create("http://localhost:0"
)).SetFindPort(true).Build();
server.AddServlet("delay", "/delay", typeof(TestJobEndNotifier.DelayServlet));
server.AddServlet("jobend", "/jobend", typeof(TestJobEndNotifier.JobEndServlet));
server.AddServlet("fail", "/fail", typeof(TestJobEndNotifier.FailServlet));
server.Start();
int port = server.GetConnectorAddress(0).Port;
baseUrl = new Uri("http://localhost:" + port + "/");
TestJobEndNotifier.JobEndServlet.calledTimes = 0;
TestJobEndNotifier.JobEndServlet.requestUri = null;
TestJobEndNotifier.DelayServlet.calledTimes = 0;
TestJobEndNotifier.FailServlet.calledTimes = 0;
}
/// <exception cref="System.IO.IOException"/>
public DatanodeHttpServer(Configuration conf, DataNode datanode, ServerSocketChannel
externalHttpChannel)
{
this.conf = conf;
Configuration confForInfoServer = new Configuration(conf);
confForInfoServer.SetInt(HttpServer2.HttpMaxThreads, 10);
HttpServer2.Builder builder = new HttpServer2.Builder().SetName("datanode").SetConf
(confForInfoServer).SetACL(new AccessControlList(conf.Get(DFSConfigKeys.DfsAdmin
, " "))).HostName(GetHostnameForSpnegoPrincipal(confForInfoServer)).AddEndpoint(
URI.Create("http://localhost:0")).SetFindPort(true);
this.infoServer = builder.Build();
this.infoServer.AddInternalServlet(null, "/streamFile/*", typeof(StreamFile));
this.infoServer.AddInternalServlet(null, "/getFileChecksum/*", typeof(FileChecksumServlets.GetServlet
));
this.infoServer.SetAttribute("datanode", datanode);
this.infoServer.SetAttribute(JspHelper.CurrentConf, conf);
this.infoServer.AddServlet(null, "/blockScannerReport", typeof(BlockScanner.Servlet
));
this.infoServer.Start();
IPEndPoint jettyAddr = infoServer.GetConnectorAddress(0);
this.confForCreate = new Configuration(conf);
confForCreate.Set(FsPermission.UmaskLabel, "000");
this.bossGroup = new NioEventLoopGroup();
this.workerGroup = new NioEventLoopGroup();
this.externalHttpChannel = externalHttpChannel;
HttpConfig.Policy policy = DFSUtil.GetHttpPolicy(conf);
if (policy.IsHttpEnabled())
{
this.httpServer = new ServerBootstrap().Group(bossGroup, workerGroup).ChildHandler
(new _ChannelInitializer_117(this, jettyAddr, conf));
if (externalHttpChannel == null)
{
httpServer.Channel(typeof(NioServerSocketChannel));
}
else
{
httpServer.ChannelFactory(new _ChannelFactory_130(externalHttpChannel));
}
}
else
{
// The channel has been bounded externally via JSVC,
// thus bind() becomes a no-op.
this.httpServer = null;
}
if (policy.IsHttpsEnabled())
{
this.sslFactory = new SSLFactory(SSLFactory.Mode.Server, conf);
try
{
sslFactory.Init();
}
catch (GeneralSecurityException e)
{
throw new IOException(e);
}
this.httpsServer = new ServerBootstrap().Group(bossGroup, workerGroup).Channel(typeof(
NioServerSocketChannel)).ChildHandler(new _ChannelInitializer_155(this, jettyAddr
, conf));
}
else
{
this.httpsServer = null;
this.sslFactory = null;
}
}
/// <summary>Pass in a server, return a URL bound to localhost and its port</summary>
/// <param name="server">server</param>
/// <returns>a URL bonded to the base of the server</returns>
/// <exception cref="System.UriFormatException">if the URL cannot be created.</exception>
public static Uri GetServerURL(HttpServer2 server)
{
NUnit.Framework.Assert.IsNotNull("No server", server);
return(new Uri("http://" + NetUtils.GetHostPortString(server.GetConnectorAddress(
0))));
}
/// <summary>Initialize SecondaryNameNode.</summary>
/// <exception cref="System.IO.IOException"/>
private void Initialize(Configuration conf, SecondaryNameNode.CommandLineOpts commandLineOpts
)
{
IPEndPoint infoSocAddr = GetHttpAddress(conf);
string infoBindAddress = infoSocAddr.GetHostName();
UserGroupInformation.SetConfiguration(conf);
if (UserGroupInformation.IsSecurityEnabled())
{
SecurityUtil.Login(conf, DFSConfigKeys.DfsSecondaryNamenodeKeytabFileKey, DFSConfigKeys
.DfsSecondaryNamenodeKerberosPrincipalKey, infoBindAddress);
}
// initiate Java VM metrics
DefaultMetricsSystem.Initialize("SecondaryNameNode");
JvmMetrics.Create("SecondaryNameNode", conf.Get(DFSConfigKeys.DfsMetricsSessionIdKey
), DefaultMetricsSystem.Instance());
// Create connection to the namenode.
shouldRun = true;
nameNodeAddr = NameNode.GetServiceAddress(conf, true);
this.conf = conf;
this.namenode = NameNodeProxies.CreateNonHAProxy <NamenodeProtocol>(conf, nameNodeAddr
, UserGroupInformation.GetCurrentUser(), true).GetProxy();
// initialize checkpoint directories
fsName = GetInfoServer();
checkpointDirs = FSImage.GetCheckpointDirs(conf, "/tmp/hadoop/dfs/namesecondary");
checkpointEditsDirs = FSImage.GetCheckpointEditsDirs(conf, "/tmp/hadoop/dfs/namesecondary"
);
checkpointImage = new SecondaryNameNode.CheckpointStorage(conf, checkpointDirs, checkpointEditsDirs
);
checkpointImage.RecoverCreate(commandLineOpts.ShouldFormat());
checkpointImage.DeleteTempEdits();
namesystem = new FSNamesystem(conf, checkpointImage, true);
// Disable quota checks
namesystem.dir.DisableQuotaChecks();
// Initialize other scheduling parameters from the configuration
checkpointConf = new CheckpointConf(conf);
IPEndPoint httpAddr = infoSocAddr;
string httpsAddrString = conf.GetTrimmed(DFSConfigKeys.DfsNamenodeSecondaryHttpsAddressKey
, DFSConfigKeys.DfsNamenodeSecondaryHttpsAddressDefault);
IPEndPoint httpsAddr = NetUtils.CreateSocketAddr(httpsAddrString);
HttpServer2.Builder builder = DFSUtil.HttpServerTemplateForNNAndJN(conf, httpAddr
, httpsAddr, "secondary", DFSConfigKeys.DfsSecondaryNamenodeKerberosInternalSpnegoPrincipalKey
, DFSConfigKeys.DfsSecondaryNamenodeKeytabFileKey);
nameNodeStatusBeanName = MBeans.Register("SecondaryNameNode", "SecondaryNameNodeInfo"
, this);
infoServer = builder.Build();
infoServer.SetAttribute("secondary.name.node", this);
infoServer.SetAttribute("name.system.image", checkpointImage);
infoServer.SetAttribute(JspHelper.CurrentConf, conf);
infoServer.AddInternalServlet("imagetransfer", ImageServlet.PathSpec, typeof(ImageServlet
), true);
infoServer.Start();
Log.Info("Web server init done");
HttpConfig.Policy policy = DFSUtil.GetHttpPolicy(conf);
int connIdx = 0;
if (policy.IsHttpEnabled())
{
IPEndPoint httpAddress = infoServer.GetConnectorAddress(connIdx++);
conf.Set(DFSConfigKeys.DfsNamenodeSecondaryHttpAddressKey, NetUtils.GetHostPortString
(httpAddress));
}
if (policy.IsHttpsEnabled())
{
IPEndPoint httpsAddress = infoServer.GetConnectorAddress(connIdx);
conf.Set(DFSConfigKeys.DfsNamenodeSecondaryHttpsAddressKey, NetUtils.GetHostPortString
(httpsAddress));
}
legacyOivImageDir = conf.Get(DFSConfigKeys.DfsNamenodeLegacyOivImageDirKey);
Log.Info("Checkpoint Period :" + checkpointConf.GetPeriod() + " secs " + "(" +
checkpointConf.GetPeriod() / 60 + " min)");
Log.Info("Log Size Trigger :" + checkpointConf.GetTxnCount() + " txns");
}