public TestHttpFSFWithSWebhdfsFileSystem(BaseTestHttpFSWith.Operation operation)
: base(operation)
{
{
Uri url = Sharpen.Thread.CurrentThread().GetContextClassLoader().GetResource("classutils.txt"
);
classpathDir = url.ToExternalForm();
if (classpathDir.StartsWith("file:"))
{
classpathDir = Sharpen.Runtime.Substring(classpathDir, "file:".Length);
classpathDir = Sharpen.Runtime.Substring(classpathDir, 0, classpathDir.Length - "/classutils.txt"
.Length);
}
else
{
throw new RuntimeException("Cannot find test classes dir");
}
FilePath @base = new FilePath(Basedir);
FileUtil.FullyDelete(@base);
@base.Mkdirs();
string keyStoreDir = new FilePath(Basedir).GetAbsolutePath();
try
{
sslConf = new Configuration();
KeyStoreTestUtil.SetupSSLConfig(keyStoreDir, classpathDir, sslConf, false);
}
catch (Exception ex)
{
throw new RuntimeException(ex);
}
jettyTestHelper = new TestJettyHelper("jks", keyStoreDir + "/serverKS.jks", "serverP"
);
}
}
public static void SetUp()
{
conf = new Configuration();
conf.SetBoolean(DFSConfigKeys.DfsWebhdfsEnabledKey, true);
conf.Set(DFSConfigKeys.DfsHttpPolicyKey, HttpConfig.Policy.HttpsOnly.ToString());
conf.Set(DFSConfigKeys.DfsNamenodeHttpsAddressKey, "localhost:0");
conf.Set(DFSConfigKeys.DfsDatanodeHttpsAddressKey, "localhost:0");
FilePath @base = new FilePath(Basedir);
FileUtil.FullyDelete(@base);
@base.Mkdirs();
keystoresDir = new FilePath(Basedir).GetAbsolutePath();
sslConfDir = KeyStoreTestUtil.GetClasspathDir(typeof(TestHttpsFileSystem));
KeyStoreTestUtil.SetupSSLConfig(keystoresDir, sslConfDir, conf, false);
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();
cluster.WaitActive();
OutputStream os = cluster.GetFileSystem().Create(new Path("/test"));
os.Write(23);
os.Close();
IPEndPoint addr = cluster.GetNameNode().GetHttpsAddress();
nnAddr = NetUtils.GetHostPortString(addr);
conf.Set(DFSConfigKeys.DfsNamenodeHttpsAddressKey, nnAddr);
}
public virtual void CleanUpMiniClusterSpecialConfig()
{
new FilePath(classpathDir, "core-site.xml").Delete();
string keystoresDir = new FilePath(Basedir).GetAbsolutePath();
KeyStoreTestUtil.CleanupSSLConfig(keystoresDir, classpathDir);
}
public static void Cleanup()
{
server.Stop();
FileUtil.FullyDelete(new FilePath(Basedir));
KeyStoreTestUtil.CleanupSSLConfig(keystoresDir, sslConfDir);
clientSslFactory.Destroy();
}
public static void Setup()
{
conf = new Configuration();
conf.SetInt(HttpServer2.HttpMaxThreads, 10);
FilePath @base = new FilePath(Basedir);
FileUtil.FullyDelete(@base);
@base.Mkdirs();
keystoresDir = new FilePath(Basedir).GetAbsolutePath();
sslConfDir = KeyStoreTestUtil.GetClasspathDir(typeof(TestSSLHttpServer));
KeyStoreTestUtil.SetupSSLConfig(keystoresDir, sslConfDir, conf, false);
Configuration sslConf = new Configuration(false);
sslConf.AddResource("ssl-server.xml");
sslConf.AddResource("ssl-client.xml");
clientSslFactory = new SSLFactory(SSLFactory.Mode.Client, sslConf);
clientSslFactory.Init();
server = new HttpServer2.Builder().SetName("test").AddEndpoint(new URI("https://localhost"
)).SetConf(conf).KeyPassword(sslConf.Get("ssl.server.keystore.keypassword")).KeyStore
(sslConf.Get("ssl.server.keystore.location"), sslConf.Get("ssl.server.keystore.password"
), sslConf.Get("ssl.server.keystore.type", "jks")).TrustStore(sslConf.Get("ssl.server.truststore.location"
), sslConf.Get("ssl.server.truststore.password"), sslConf.Get("ssl.server.truststore.type"
, "jks")).Build();
server.AddServlet("echo", "/echo", typeof(TestHttpServer.EchoServlet));
server.AddServlet("longheader", "/longheader", typeof(HttpServerFunctionalTest.LongHeaderServlet
));
server.Start();
baseUrl = new Uri("https://" + NetUtils.GetHostPortString(server.GetConnectorAddress
(0)));
Log.Info("HTTP server started: " + baseUrl);
}
public static void SetUp()
{
Configuration conf = new Configuration();
conf.Set(HttpServer2.FilterInitializerProperty, typeof(TestHttpCookieFlag.DummyFilterInitializer
).FullName);
FilePath @base = new FilePath(Basedir);
FileUtil.FullyDelete(@base);
@base.Mkdirs();
keystoresDir = new FilePath(Basedir).GetAbsolutePath();
sslConfDir = KeyStoreTestUtil.GetClasspathDir(typeof(TestSSLHttpServer));
KeyStoreTestUtil.SetupSSLConfig(keystoresDir, sslConfDir, conf, false);
Configuration sslConf = new Configuration(false);
sslConf.AddResource("ssl-server.xml");
sslConf.AddResource("ssl-client.xml");
clientSslFactory = new SSLFactory(SSLFactory.Mode.Client, sslConf);
clientSslFactory.Init();
server = new HttpServer2.Builder().SetName("test").AddEndpoint(new URI("http://localhost"
)).AddEndpoint(new URI("https://localhost")).SetConf(conf).KeyPassword(sslConf.Get
("ssl.server.keystore.keypassword")).KeyStore(sslConf.Get("ssl.server.keystore.location"
), sslConf.Get("ssl.server.keystore.password"), sslConf.Get("ssl.server.keystore.type"
, "jks")).TrustStore(sslConf.Get("ssl.server.truststore.location"), sslConf.Get(
"ssl.server.truststore.password"), sslConf.Get("ssl.server.truststore.type", "jks"
)).Build();
server.AddServlet("echo", "/echo", typeof(TestHttpServer.EchoServlet));
server.Start();
}
public static void TearDown()
{
if (cluster != null)
{
cluster.Shutdown();
}
FileUtil.FullyDelete(new FilePath(Basedir));
KeyStoreTestUtil.CleanupSSLConfig(keystoresDir, sslConfDir);
}
public static void Setup()
{
try
{
testMiniKDC = new MiniKdc(MiniKdc.CreateConf(), testRootDir);
testMiniKDC.Start();
testMiniKDC.CreatePrincipal(httpSpnegoKeytabFile, HttpUser + "/localhost");
}
catch (Exception)
{
NUnit.Framework.Assert.IsTrue("Couldn't setup MiniKDC", false);
}
try
{
testTimelineServer = new ApplicationHistoryServer();
conf = new Configuration(false);
conf.SetStrings(TimelineAuthenticationFilterInitializer.Prefix + "type", "kerberos"
);
conf.Set(TimelineAuthenticationFilterInitializer.Prefix + KerberosAuthenticationHandler
.Principal, httpSpnegoPrincipal);
conf.Set(TimelineAuthenticationFilterInitializer.Prefix + KerberosAuthenticationHandler
.Keytab, httpSpnegoKeytabFile.GetAbsolutePath());
conf.Set(CommonConfigurationKeysPublic.HadoopSecurityAuthentication, "kerberos");
conf.Set(YarnConfiguration.TimelineServicePrincipal, httpSpnegoPrincipal);
conf.Set(YarnConfiguration.TimelineServiceKeytab, httpSpnegoKeytabFile.GetAbsolutePath
());
conf.SetBoolean(YarnConfiguration.TimelineServiceEnabled, true);
conf.SetClass(YarnConfiguration.TimelineServiceStore, typeof(MemoryTimelineStore)
, typeof(TimelineStore));
conf.Set(YarnConfiguration.TimelineServiceAddress, "localhost:10200");
conf.Set(YarnConfiguration.TimelineServiceWebappAddress, "localhost:8188");
conf.Set(YarnConfiguration.TimelineServiceWebappHttpsAddress, "localhost:8190");
conf.Set("hadoop.proxyuser.HTTP.hosts", "*");
conf.Set("hadoop.proxyuser.HTTP.users", FooUser);
conf.SetInt(YarnConfiguration.TimelineServiceClientMaxRetries, 1);
if (withSsl)
{
conf.Set(YarnConfiguration.YarnHttpPolicyKey, HttpConfig.Policy.HttpsOnly.ToString
());
FilePath @base = new FilePath(Basedir);
FileUtil.FullyDelete(@base);
@base.Mkdirs();
keystoresDir = new FilePath(Basedir).GetAbsolutePath();
sslConfDir = KeyStoreTestUtil.GetClasspathDir(typeof(Org.Apache.Hadoop.Yarn.Server.Timeline.Security.TestTimelineAuthenticationFilter
));
KeyStoreTestUtil.SetupSSLConfig(keystoresDir, sslConfDir, conf, false);
}
UserGroupInformation.SetConfiguration(conf);
testTimelineServer.Init(conf);
testTimelineServer.Start();
}
catch (Exception)
{
NUnit.Framework.Assert.IsTrue("Couldn't setup TimelineServer", false);
}
}
public static void SetUp()
{
FilePath @base = new FilePath(Basedir);
FileUtil.FullyDelete(@base);
@base.Mkdirs();
conf = new Configuration();
keystoresDir = new FilePath(Basedir).GetAbsolutePath();
sslConfDir = KeyStoreTestUtil.GetClasspathDir(typeof(Org.Apache.Hadoop.Hdfs.Server.Namenode.TestNameNodeHttpServer
));
KeyStoreTestUtil.SetupSSLConfig(keystoresDir, sslConfDir, conf, false);
connectionFactory = URLConnectionFactory.NewDefaultURLConnectionFactory(conf);
}
public virtual void TestLazyTokenFetchForSWebhdfs()
{
// for any(Token.class)
MiniDFSCluster cluster = null;
SWebHdfsFileSystem fs = null;
try
{
Configuration clusterConf = new HdfsConfiguration(conf);
SecurityUtil.SetAuthenticationMethod(UserGroupInformation.AuthenticationMethod.Simple
, clusterConf);
clusterConf.SetBoolean(DFSConfigKeys.DfsNamenodeDelegationTokenAlwaysUseKey, true
);
string Basedir = Runtime.GetProperty("test.build.dir", "target/test-dir") + "/" +
typeof(TestWebHdfsTokens).Name;
string keystoresDir;
string sslConfDir;
clusterConf.SetBoolean(DFSConfigKeys.DfsWebhdfsEnabledKey, true);
clusterConf.Set(DFSConfigKeys.DfsHttpPolicyKey, HttpConfig.Policy.HttpsOnly.ToString
());
clusterConf.Set(DFSConfigKeys.DfsNamenodeHttpsAddressKey, "localhost:0");
clusterConf.Set(DFSConfigKeys.DfsDatanodeHttpsAddressKey, "localhost:0");
FilePath @base = new FilePath(Basedir);
FileUtil.FullyDelete(@base);
@base.Mkdirs();
keystoresDir = new FilePath(Basedir).GetAbsolutePath();
sslConfDir = KeyStoreTestUtil.GetClasspathDir(typeof(TestWebHdfsTokens));
KeyStoreTestUtil.SetupSSLConfig(keystoresDir, sslConfDir, clusterConf, false);
// trick the NN into thinking security is enabled w/o it trying
// to login from a keytab
UserGroupInformation.SetConfiguration(clusterConf);
cluster = new MiniDFSCluster.Builder(clusterConf).NumDataNodes(1).Build();
cluster.WaitActive();
IPEndPoint addr = cluster.GetNameNode().GetHttpsAddress();
string nnAddr = NetUtils.GetHostPortString(addr);
clusterConf.Set(DFSConfigKeys.DfsNamenodeHttpsAddressKey, nnAddr);
SecurityUtil.SetAuthenticationMethod(UserGroupInformation.AuthenticationMethod.Kerberos
, clusterConf);
UserGroupInformation.SetConfiguration(clusterConf);
uri = DFSUtil.CreateUri("swebhdfs", cluster.GetNameNode().GetHttpsAddress());
ValidateLazyTokenFetch(clusterConf);
}
finally
{
IOUtils.Cleanup(null, fs);
if (cluster != null)
{
cluster.Shutdown();
}
}
}
public static void SetUp()
{
config = new Configuration();
cluster = new MiniDFSCluster.Builder(config).NumDataNodes(2).Build();
blockPoolId = cluster.GetNamesystem().GetBlockPoolId();
hftpUri = "hftp://" + config.Get(DFSConfigKeys.DfsNamenodeHttpAddressKey);
FilePath @base = new FilePath(Basedir);
FileUtil.FullyDelete(@base);
@base.Mkdirs();
keystoresDir = new FilePath(Basedir).GetAbsolutePath();
sslConfDir = KeyStoreTestUtil.GetClasspathDir(typeof(TestHftpFileSystem));
KeyStoreTestUtil.SetupSSLConfig(keystoresDir, sslConfDir, config, false);
}
public static void Init()
{
baseDir = new FilePath(Runtime.GetProperty("test.build.dir", "target/test-dir"),
typeof(TestSecureNNWithQJM).Name);
FileUtil.FullyDelete(baseDir);
NUnit.Framework.Assert.IsTrue(baseDir.Mkdirs());
Properties kdcConf = MiniKdc.CreateConf();
kdc = new MiniKdc(kdcConf, baseDir);
kdc.Start();
baseConf = new HdfsConfiguration();
SecurityUtil.SetAuthenticationMethod(UserGroupInformation.AuthenticationMethod.Kerberos
, baseConf);
UserGroupInformation.SetConfiguration(baseConf);
NUnit.Framework.Assert.IsTrue("Expected configuration to enable security", UserGroupInformation
.IsSecurityEnabled());
string userName = UserGroupInformation.GetLoginUser().GetShortUserName();
FilePath keytabFile = new FilePath(baseDir, userName + ".keytab");
string keytab = keytabFile.GetAbsolutePath();
// Windows will not reverse name lookup "127.0.0.1" to "localhost".
string krbInstance = Path.Windows ? "127.0.0.1" : "localhost";
kdc.CreatePrincipal(keytabFile, userName + "/" + krbInstance, "HTTP/" + krbInstance
);
string hdfsPrincipal = userName + "/" + krbInstance + "@" + kdc.GetRealm();
string spnegoPrincipal = "HTTP/" + krbInstance + "@" + kdc.GetRealm();
baseConf.Set(DFSConfigKeys.DfsNamenodeKerberosPrincipalKey, hdfsPrincipal);
baseConf.Set(DFSConfigKeys.DfsNamenodeKeytabFileKey, keytab);
baseConf.Set(DFSConfigKeys.DfsDatanodeKerberosPrincipalKey, hdfsPrincipal);
baseConf.Set(DFSConfigKeys.DfsDatanodeKeytabFileKey, keytab);
baseConf.Set(DFSConfigKeys.DfsWebAuthenticationKerberosPrincipalKey, spnegoPrincipal
);
baseConf.Set(DFSConfigKeys.DfsJournalnodeKeytabFileKey, keytab);
baseConf.Set(DFSConfigKeys.DfsJournalnodeKerberosPrincipalKey, hdfsPrincipal);
baseConf.Set(DFSConfigKeys.DfsJournalnodeKerberosInternalSpnegoPrincipalKey, spnegoPrincipal
);
baseConf.SetBoolean(DFSConfigKeys.DfsBlockAccessTokenEnableKey, true);
baseConf.Set(DFSConfigKeys.DfsDataTransferProtectionKey, "authentication");
baseConf.Set(DFSConfigKeys.DfsHttpPolicyKey, HttpConfig.Policy.HttpsOnly.ToString
());
baseConf.Set(DFSConfigKeys.DfsNamenodeHttpsAddressKey, "localhost:0");
baseConf.Set(DFSConfigKeys.DfsDatanodeHttpsAddressKey, "localhost:0");
baseConf.Set(DFSConfigKeys.DfsJournalnodeHttpsAddressKey, "localhost:0");
baseConf.SetInt(CommonConfigurationKeys.IpcClientConnectMaxRetriesOnSaslKey, 10);
string keystoresDir = baseDir.GetAbsolutePath();
string sslConfDir = KeyStoreTestUtil.GetClasspathDir(typeof(TestSecureNNWithQJM));
KeyStoreTestUtil.SetupSSLConfig(keystoresDir, sslConfDir, baseConf, false);
}
public static void TearDown()
{
if (testMiniKDC != null)
{
testMiniKDC.Stop();
}
if (testTimelineServer != null)
{
testTimelineServer.Stop();
}
if (withSsl)
{
KeyStoreTestUtil.CleanupSSLConfig(keystoresDir, sslConfDir);
FilePath @base = new FilePath(Basedir);
FileUtil.FullyDelete(@base);
}
}
/// <exception cref="System.Exception"/>
private static void SetupSsl()
{
Configuration conf = new Configuration();
conf.SetBoolean(DFSConfigKeys.DfsWebhdfsEnabledKey, true);
conf.Set(DFSConfigKeys.DfsHttpPolicyKey, HttpConfig.Policy.HttpsOnly.ToString());
conf.Set(DFSConfigKeys.DfsNamenodeHttpsAddressKey, "localhost:0");
conf.Set(DFSConfigKeys.DfsDatanodeHttpsAddressKey, "localhost:0");
FilePath @base = new FilePath(Basedir);
FileUtil.FullyDelete(@base);
NUnit.Framework.Assert.IsTrue(@base.Mkdirs());
string keystoresDir = new FilePath(Basedir).GetAbsolutePath();
string sslConfDir = KeyStoreTestUtil.GetClasspathDir(typeof(TestNameNodeRespectsBindHostKeys
));
KeyStoreTestUtil.SetupSSLConfig(keystoresDir, sslConfDir, conf, false);
}
public static void SetUp()
{
conf.Set(DFSConfigKeys.DfsHttpPolicyKey, HttpConfig.Policy.HttpAndHttps.ToString(
));
conf.Set(NfsConfigKeys.NfsHttpAddressKey, "localhost:0");
conf.Set(NfsConfigKeys.NfsHttpsAddressKey, "localhost:0");
// Use emphral port in case tests are running in parallel
conf.SetInt(NfsConfigKeys.DfsNfsServerPortKey, 0);
conf.SetInt(NfsConfigKeys.DfsNfsMountdPortKey, 0);
FilePath @base = new FilePath(Basedir);
FileUtil.FullyDelete(@base);
@base.Mkdirs();
keystoresDir = new FilePath(Basedir).GetAbsolutePath();
sslConfDir = KeyStoreTestUtil.GetClasspathDir(typeof(TestNfs3HttpServer));
KeyStoreTestUtil.SetupSSLConfig(keystoresDir, sslConfDir, conf, false);
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();
cluster.WaitActive();
}
public static void SetupServer()
{
conf = new YarnConfiguration();
conf.SetBoolean(YarnConfiguration.TimelineServiceEnabled, true);
conf.SetClass(YarnConfiguration.TimelineServiceStore, typeof(MemoryTimelineStore)
, typeof(TimelineStore));
conf.Set(YarnConfiguration.YarnHttpPolicyKey, "HTTPS_ONLY");
FilePath @base = new FilePath(Basedir);
FileUtil.FullyDelete(@base);
@base.Mkdirs();
keystoresDir = new FilePath(Basedir).GetAbsolutePath();
sslConfDir = KeyStoreTestUtil.GetClasspathDir(typeof(TestTimelineWebServicesWithSSL
));
KeyStoreTestUtil.SetupSSLConfig(keystoresDir, sslConfDir, conf, false);
conf.AddResource("ssl-server.xml");
conf.AddResource("ssl-client.xml");
timelineServer = new ApplicationHistoryServer();
timelineServer.Init(conf);
timelineServer.Start();
store = timelineServer.GetTimelineStore();
}
/// <summary>Creates configuration for starting a secure cluster.</summary>
/// <param name="dataTransferProtection">supported QOPs</param>
/// <returns>configuration for starting a secure cluster</returns>
/// <exception cref="System.Exception">if there is any failure</exception>
protected internal virtual HdfsConfiguration CreateSecureConfig(string dataTransferProtection
)
{
HdfsConfiguration conf = new HdfsConfiguration();
SecurityUtil.SetAuthenticationMethod(UserGroupInformation.AuthenticationMethod.Kerberos
, conf);
conf.Set(DFSConfigKeys.DfsNamenodeKerberosPrincipalKey, hdfsPrincipal);
conf.Set(DFSConfigKeys.DfsNamenodeKeytabFileKey, keytab);
conf.Set(DFSConfigKeys.DfsDatanodeKerberosPrincipalKey, hdfsPrincipal);
conf.Set(DFSConfigKeys.DfsDatanodeKeytabFileKey, keytab);
conf.Set(DFSConfigKeys.DfsWebAuthenticationKerberosPrincipalKey, spnegoPrincipal);
conf.SetBoolean(DFSConfigKeys.DfsBlockAccessTokenEnableKey, true);
conf.Set(DFSConfigKeys.DfsDataTransferProtectionKey, dataTransferProtection);
conf.Set(DFSConfigKeys.DfsHttpPolicyKey, HttpConfig.Policy.HttpsOnly.ToString());
conf.Set(DFSConfigKeys.DfsNamenodeHttpsAddressKey, "localhost:0");
conf.Set(DFSConfigKeys.DfsDatanodeHttpsAddressKey, "localhost:0");
conf.SetInt(CommonConfigurationKeys.IpcClientConnectMaxRetriesOnSaslKey, 10);
string keystoresDir = baseDir.GetAbsolutePath();
string sslConfDir = KeyStoreTestUtil.GetClasspathDir(this.GetType());
KeyStoreTestUtil.SetupSSLConfig(keystoresDir, sslConfDir, conf, false);
return(conf);
}
/// <exception cref="System.Exception"/>
private void EncryptedShuffleWithCerts(bool useClientCerts)
{
try
{
Configuration conf = new Configuration();
string keystoresDir = new FilePath(Basedir).GetAbsolutePath();
string sslConfsDir = KeyStoreTestUtil.GetClasspathDir(typeof(TestEncryptedShuffle
));
KeyStoreTestUtil.SetupSSLConfig(keystoresDir, sslConfsDir, conf, useClientCerts);
conf.SetBoolean(MRConfig.ShuffleSslEnabledKey, true);
StartCluster(conf);
FileSystem fs = FileSystem.Get(GetJobConf());
Path inputDir = new Path("input");
fs.Mkdirs(inputDir);
TextWriter writer = new OutputStreamWriter(fs.Create(new Path(inputDir, "data.txt"
)));
writer.Write("hello");
writer.Close();
Path outputDir = new Path("output", "output");
JobConf jobConf = new JobConf(GetJobConf());
jobConf.SetInt("mapred.map.tasks", 1);
jobConf.SetInt("mapred.map.max.attempts", 1);
jobConf.SetInt("mapred.reduce.max.attempts", 1);
jobConf.Set("mapred.input.dir", inputDir.ToString());
jobConf.Set("mapred.output.dir", outputDir.ToString());
JobClient jobClient = new JobClient(jobConf);
RunningJob runJob = jobClient.SubmitJob(jobConf);
runJob.WaitForCompletion();
NUnit.Framework.Assert.IsTrue(runJob.IsComplete());
NUnit.Framework.Assert.IsTrue(runJob.IsSuccessful());
}
finally
{
StopCluster();
}
}
public virtual void CreateCustomYarnClasspath()
{
classpathDir = KeyStoreTestUtil.GetClasspathDir(typeof(TestEncryptedShuffle));
new FilePath(classpathDir, "core-site.xml").Delete();
}
public static void TearDown()
{
FileUtil.FullyDelete(new FilePath(Basedir));
KeyStoreTestUtil.CleanupSSLConfig(keystoresDir, sslConfDir);
}
