public virtual void TestBindAddress()
{
CheckBindAddress("localhost", 0, false).Stop();
// hang onto this one for a bit more testing
HttpServer2 myServer = CheckBindAddress("localhost", 0, false);
HttpServer2 myServer2 = null;
try
{
int port = myServer.GetConnectorAddress(0).Port;
// it's already in use, true = expect a higher port
myServer2 = CheckBindAddress("localhost", port, true);
// try to reuse the port
port = myServer2.GetConnectorAddress(0).Port;
myServer2.Stop();
NUnit.Framework.Assert.IsNull(myServer2.GetConnectorAddress(0));
// not bound
myServer2.OpenListeners();
Assert.Equal(port, myServer2.GetConnectorAddress(0).Port);
}
finally
{
// expect same port
myServer.Stop();
if (myServer2 != null)
{
myServer2.Stop();
}
}
}
/// <summary>Test to verify the read timeout</summary>
/// <exception cref="System.Exception"/>
public virtual void TestGetImageTimeout()
{
HttpServer2 testServer = HttpServerFunctionalTest.CreateServer("hdfs");
try
{
testServer.AddServlet("ImageTransfer", ImageServlet.PathSpec, typeof(TestTransferFsImage.TestImageTransferServlet
));
testServer.Start();
Uri serverURL = HttpServerFunctionalTest.GetServerURL(testServer);
TransferFsImage.timeout = 2000;
try
{
TransferFsImage.GetFileClient(serverURL, "txid=1", null, null, false);
NUnit.Framework.Assert.Fail("TransferImage Should fail with timeout");
}
catch (SocketTimeoutException e)
{
NUnit.Framework.Assert.AreEqual("Read should timeout", "Read timed out", e.Message
);
}
}
finally
{
if (testServer != null)
{
testServer.Stop();
}
}
}
public virtual void TestNotificationOnLastRetryUnregistrationFailure()
{
HttpServer2 server = StartHttpServer();
MRApp app = Org.Mockito.Mockito.Spy(new TestJobEndNotifier.MRAppWithCustomContainerAllocator
(this, 2, 2, false, this.GetType().FullName, true, 2, false));
// Currently, we will have isLastRetry always equals to false at beginning
// of MRAppMaster, except staging area exists or commit already started at
// the beginning.
// Now manually set isLastRetry to true and this should reset to false when
// unregister failed.
app.isLastAMRetry = true;
Org.Mockito.Mockito.DoNothing().When(app).Sysexit();
JobConf conf = new JobConf();
conf.Set(JobContext.MrJobEndNotificationUrl, TestJobEndNotifier.JobEndServlet.baseUrl
+ "jobend?jobid=$jobId&status=$jobStatus");
JobImpl job = (JobImpl)app.Submit(conf);
app.WaitForState(job, JobState.Running);
app.GetContext().GetEventHandler().Handle(new JobEvent(app.GetJobId(), JobEventType
.JobAmReboot));
app.WaitForInternalState(job, JobStateInternal.Reboot);
// Now shutdown. User should see FAILED state.
// Unregistration fails: isLastAMRetry is recalculated, this is
///reboot will stop service internally, we don't need to shutdown twice
app.WaitForServiceToStop(10000);
NUnit.Framework.Assert.IsFalse(app.IsLastAMRetry());
// Since it's not last retry, JobEndServlet didn't called
NUnit.Framework.Assert.AreEqual(0, TestJobEndNotifier.JobEndServlet.calledTimes);
NUnit.Framework.Assert.IsNull(TestJobEndNotifier.JobEndServlet.requestUri);
NUnit.Framework.Assert.IsNull(TestJobEndNotifier.JobEndServlet.foundJobState);
server.Stop();
}
public virtual void TestAbsentNotificationOnNotLastRetryUnregistrationFailure()
{
HttpServer2 server = StartHttpServer();
MRApp app = Org.Mockito.Mockito.Spy(new TestJobEndNotifier.MRAppWithCustomContainerAllocator
(this, 2, 2, false, this.GetType().FullName, true, 1, false));
Org.Mockito.Mockito.DoNothing().When(app).Sysexit();
JobConf conf = new JobConf();
conf.Set(JobContext.MrJobEndNotificationUrl, TestJobEndNotifier.JobEndServlet.baseUrl
+ "jobend?jobid=$jobId&status=$jobStatus");
JobImpl job = (JobImpl)app.Submit(conf);
app.WaitForState(job, JobState.Running);
app.GetContext().GetEventHandler().Handle(new JobEvent(app.GetJobId(), JobEventType
.JobAmReboot));
app.WaitForInternalState(job, JobStateInternal.Reboot);
// Now shutdown.
// Unregistration fails: isLastAMRetry is recalculated, this is not
app.ShutDownJob();
// Not the last AM attempt. So user should that the job is still running.
app.WaitForState(job, JobState.Running);
NUnit.Framework.Assert.IsFalse(app.IsLastAMRetry());
NUnit.Framework.Assert.AreEqual(0, TestJobEndNotifier.JobEndServlet.calledTimes);
NUnit.Framework.Assert.IsNull(TestJobEndNotifier.JobEndServlet.requestUri);
NUnit.Framework.Assert.IsNull(TestJobEndNotifier.JobEndServlet.foundJobState);
server.Stop();
}
public virtual void TestNotificationOnLastRetryNormalShutdown()
{
HttpServer2 server = StartHttpServer();
// Act like it is the second attempt. Default max attempts is 2
MRApp app = Org.Mockito.Mockito.Spy(new TestJobEndNotifier.MRAppWithCustomContainerAllocator
(this, 2, 2, true, this.GetType().FullName, true, 2, true));
Org.Mockito.Mockito.DoNothing().When(app).Sysexit();
JobConf conf = new JobConf();
conf.Set(JobContext.MrJobEndNotificationUrl, TestJobEndNotifier.JobEndServlet.baseUrl
+ "jobend?jobid=$jobId&status=$jobStatus");
JobImpl job = (JobImpl)app.Submit(conf);
app.WaitForInternalState(job, JobStateInternal.Succeeded);
// Unregistration succeeds: successfullyUnregistered is set
app.ShutDownJob();
NUnit.Framework.Assert.IsTrue(app.IsLastAMRetry());
NUnit.Framework.Assert.AreEqual(1, TestJobEndNotifier.JobEndServlet.calledTimes);
NUnit.Framework.Assert.AreEqual("jobid=" + job.GetID() + "&status=SUCCEEDED", TestJobEndNotifier.JobEndServlet
.requestUri.GetQuery());
NUnit.Framework.Assert.AreEqual(JobState.Succeeded.ToString(), TestJobEndNotifier.JobEndServlet
.foundJobState);
server.Stop();
}
/// <summary>If the server is non null, stop it</summary>
/// <param name="server">to stop</param>
/// <exception cref="System.Exception">on any failure</exception>
public static void Stop(HttpServer2 server)
{
if (server != null)
{
server.Stop();
}
}
/// <exception cref="System.Exception"/>
internal virtual void Stop()
{
if (httpServer != null)
{
httpServer.Stop();
}
}
public static void Cleanup()
{
server.Stop();
FileUtil.FullyDelete(new FilePath(Basedir));
KeyStoreTestUtil.CleanupSSLConfig(keystoresDir, sslConfDir);
clientSslFactory.Destroy();
}
/// <summary>Shut down this instance of the datanode.</summary>
/// <remarks>
/// Shut down this instance of the datanode.
/// Returns only after shutdown is complete.
/// </remarks>
public virtual void Shutdown()
{
shouldRun = false;
if (checkpointThread != null)
{
checkpointThread.Interrupt();
try
{
checkpointThread.Join(10000);
}
catch (Exception)
{
Log.Info("Interrupted waiting to join on checkpointer thread");
Sharpen.Thread.CurrentThread().Interrupt();
}
}
// maintain status
try
{
if (infoServer != null)
{
infoServer.Stop();
infoServer = null;
}
}
catch (Exception e)
{
Log.Warn("Exception shutting down SecondaryNameNode", e);
}
if (nameNodeStatusBeanName != null)
{
MBeans.Unregister(nameNodeStatusBeanName);
nameNodeStatusBeanName = null;
}
try
{
if (checkpointImage != null)
{
checkpointImage.Close();
checkpointImage = null;
}
}
catch (IOException e)
{
Log.Warn("Exception while closing CheckpointStorage", e);
}
if (namesystem != null)
{
namesystem.Shutdown();
namesystem = null;
}
}
public virtual void TestServletFilter()
{
Configuration conf = new Configuration();
//start a http server with CountingFilter
conf.Set(HttpServer2.FilterInitializerProperty, typeof(TestServletFilter.SimpleFilter.Initializer
).FullName);
HttpServer2 http = CreateTestServer(conf);
http.Start();
string fsckURL = "/fsck";
string stacksURL = "/stacks";
string ajspURL = "/a.jsp";
string logURL = "/logs/a.log";
string hadooplogoURL = "/static/hadoop-logo.jpg";
string[] urls = new string[] { fsckURL, stacksURL, ajspURL, logURL, hadooplogoURL };
Random ran = new Random();
int[] sequence = new int[50];
//generate a random sequence and update counts
for (int i = 0; i < sequence.Length; i++)
{
sequence[i] = ran.Next(urls.Length);
}
//access the urls as the sequence
string prefix = "http://" + NetUtils.GetHostPortString(http.GetConnectorAddress(0
));
try
{
for (int i_1 = 0; i_1 < sequence.Length; i_1++)
{
Access(prefix + urls[sequence[i_1]]);
//make sure everything except fsck get filtered
if (sequence[i_1] == 0)
{
Assert.Equal(null, uri);
}
else
{
Assert.Equal(urls[sequence[i_1]], uri);
uri = null;
}
}
}
finally
{
http.Stop();
}
}
/// <exception cref="System.IO.IOException"/>
internal virtual void Stop()
{
if (httpServer != null)
{
try
{
httpServer.Stop();
}
catch (Exception e)
{
throw new IOException(e);
}
}
}
public virtual void TestPathSpecFilters()
{
Configuration conf = new Configuration();
//start a http server with CountingFilter
conf.Set(HttpServer2.FilterInitializerProperty, typeof(TestPathFilter.RecordingFilter.Initializer
).FullName);
string[] pathSpecs = new string[] { "/path", "/path/*" };
HttpServer2 http = CreateTestServer(conf, pathSpecs);
http.Start();
string baseURL = "/path";
string baseSlashURL = "/path/";
string addedURL = "/path/nodes";
string addedSlashURL = "/path/nodes/";
string longURL = "/path/nodes/foo/job";
string rootURL = "/";
string allURL = "/*";
string[] filteredUrls = new string[] { baseURL, baseSlashURL, addedURL, addedSlashURL
, longURL };
string[] notFilteredUrls = new string[] { rootURL, allURL };
// access the urls and verify our paths specs got added to the
// filters
string prefix = "http://" + NetUtils.GetHostPortString(http.GetConnectorAddress(0
));
try
{
for (int i = 0; i < filteredUrls.Length; i++)
{
Access(prefix + filteredUrls[i]);
}
for (int i_1 = 0; i_1 < notFilteredUrls.Length; i_1++)
{
Access(prefix + notFilteredUrls[i_1]);
}
}
finally
{
http.Stop();
}
Log.Info("RECORDS = " + Records);
//verify records
for (int i_2 = 0; i_2 < filteredUrls.Length; i_2++)
{
Assert.True(Records.Remove(filteredUrls[i_2]));
}
Assert.True(Records.IsEmpty());
}
public virtual void TestServletFilter()
{
Configuration conf = new Configuration();
//start a http server with CountingFilter
conf.Set(HttpServer2.FilterInitializerProperty, typeof(TestGlobalFilter.RecordingFilter.Initializer
).FullName);
HttpServer2 http = CreateTestServer(conf);
http.Start();
string fsckURL = "/fsck";
string stacksURL = "/stacks";
string ajspURL = "/a.jsp";
string listPathsURL = "/listPaths";
string dataURL = "/data";
string streamFile = "/streamFile";
string rootURL = "/";
string allURL = "/*";
string outURL = "/static/a.out";
string logURL = "/logs/a.log";
string[] urls = new string[] { fsckURL, stacksURL, ajspURL, listPathsURL, dataURL
, streamFile, rootURL, allURL, outURL, logURL };
//access the urls
string prefix = "http://" + NetUtils.GetHostPortString(http.GetConnectorAddress(0
));
try
{
for (int i = 0; i < urls.Length; i++)
{
Access(prefix + urls[i]);
}
}
finally
{
http.Stop();
}
Log.Info("RECORDS = " + Records);
//verify records
for (int i_1 = 0; i_1 < urls.Length; i_1++)
{
Assert.True(Records.Remove(urls[i_1]));
}
Assert.True(Records.IsEmpty());
}
/// <exception cref="System.Exception"/>
protected override void ServiceStop()
{
if (proxyServer != null)
{
try
{
proxyServer.Stop();
}
catch (Exception e)
{
Log.Error("Error stopping proxy web server", e);
throw new YarnRuntimeException("Error stopping proxy web server", e);
}
}
if (this.fetcher != null)
{
this.fetcher.Stop();
}
base.ServiceStop();
}
/// <summary>Test to verify the timeout of Image upload</summary>
/// <exception cref="System.Exception"/>
public virtual void TestImageUploadTimeout()
{
Configuration conf = new HdfsConfiguration();
NNStorage mockStorage = Org.Mockito.Mockito.Mock <NNStorage>();
HttpServer2 testServer = HttpServerFunctionalTest.CreateServer("hdfs");
try
{
testServer.AddServlet("ImageTransfer", ImageServlet.PathSpec, typeof(TestTransferFsImage.TestImageTransferServlet
));
testServer.Start();
Uri serverURL = HttpServerFunctionalTest.GetServerURL(testServer);
// set the timeout here, otherwise it will take default.
TransferFsImage.timeout = 2000;
FilePath tmpDir = new FilePath(new FileSystemTestHelper().GetTestRootDir());
tmpDir.Mkdirs();
FilePath mockImageFile = FilePath.CreateTempFile("image", string.Empty, tmpDir);
FileOutputStream imageFile = new FileOutputStream(mockImageFile);
imageFile.Write(Sharpen.Runtime.GetBytesForString("data"));
imageFile.Close();
Org.Mockito.Mockito.When(mockStorage.FindImageFile(Org.Mockito.Mockito.Any <NNStorage.NameNodeFile
>(), Org.Mockito.Mockito.AnyLong())).ThenReturn(mockImageFile);
Org.Mockito.Mockito.When(mockStorage.ToColonSeparatedString()).ThenReturn("storage:info:string"
);
try
{
TransferFsImage.UploadImageFromStorage(serverURL, conf, mockStorage, NNStorage.NameNodeFile
.Image, 1L);
NUnit.Framework.Assert.Fail("TransferImage Should fail with timeout");
}
catch (SocketTimeoutException e)
{
NUnit.Framework.Assert.AreEqual("Upload should timeout", "Read timed out", e.Message
);
}
}
finally
{
testServer.Stop();
}
}
/// <exception cref="System.IO.IOException"/>
public virtual void Close()
{
bossGroup.ShutdownGracefully();
workerGroup.ShutdownGracefully();
if (sslFactory != null)
{
sslFactory.Destroy();
}
if (externalHttpChannel != null)
{
externalHttpChannel.Close();
}
try
{
infoServer.Stop();
}
catch (Exception e)
{
throw new IOException(e);
}
}
/// <exception cref="System.Exception"/>
private HttpServer2 CheckBindAddress(string host, int port, bool findPort)
{
HttpServer2 server = CreateServer(host, port);
try
{
// not bound, ephemeral should return requested port (0 for ephemeral)
IList <object> listeners = (IList <object>)Whitebox.GetInternalState(server, "listeners"
);
Connector listener = (Connector)listeners[0];
Assert.Equal(port, listener.GetPort());
// verify hostname is what was given
server.OpenListeners();
Assert.Equal(host, server.GetConnectorAddress(0).GetHostName()
);
int boundPort = server.GetConnectorAddress(0).Port;
if (port == 0)
{
Assert.True(boundPort != 0);
}
else
{
// ephemeral should now return bound port
if (findPort)
{
Assert.True(boundPort > port);
// allow a little wiggle room to prevent random test failures if
// some consecutive ports are already in use
Assert.True(boundPort - port < 8);
}
}
}
catch (Exception e)
{
server.Stop();
throw;
}
return(server);
}
/// <exception cref="System.Exception"/>
protected override void TearDown()
{
server.Stop();
}
public static void Cleanup()
{
server.Stop();
}