/// <exception cref="System.Exception"/>
private void WaitForEvents(TestContainerLauncher.CustomContainerLauncher containerLauncher
, int expectedNumEvents)
{
int timeOut = 0;
while (containerLauncher.numEventsProcessing.Get() < expectedNumEvents && timeOut
++ < 20)
{
Log.Info("Waiting for number of events to become " + expectedNumEvents + ". It is now "
+ containerLauncher.numEventsProcessing.Get());
Sharpen.Thread.Sleep(1000);
}
NUnit.Framework.Assert.AreEqual(expectedNumEvents, containerLauncher.numEventsProcessing
.Get());
}
/// <exception cref="System.Exception"/>
public virtual void TestPoolLimits()
{
ApplicationId appId = ApplicationId.NewInstance(12345, 67);
ApplicationAttemptId appAttemptId = ApplicationAttemptId.NewInstance(appId, 3);
JobId jobId = MRBuilderUtils.NewJobId(appId, 8);
TaskId taskId = MRBuilderUtils.NewTaskId(jobId, 9, TaskType.Map);
TaskAttemptId taskAttemptId = MRBuilderUtils.NewTaskAttemptId(taskId, 0);
ContainerId containerId = ContainerId.NewContainerId(appAttemptId, 10);
AppContext context = Org.Mockito.Mockito.Mock<AppContext>();
TestContainerLauncher.CustomContainerLauncher containerLauncher = new TestContainerLauncher.CustomContainerLauncher
(this, context);
Configuration conf = new Configuration();
conf.SetInt(MRJobConfig.MrAmContainerlauncherThreadCountLimit, 12);
containerLauncher.Init(conf);
containerLauncher.Start();
ThreadPoolExecutor threadPool = containerLauncher.GetThreadPool();
// 10 different hosts
containerLauncher.expectedCorePoolSize = containerLauncher.initialPoolSize;
for (int i = 0; i < 10; i++)
{
containerLauncher.Handle(new ContainerLauncherEvent(taskAttemptId, containerId, "host"
+ i + ":1234", null, ContainerLauncher.EventType.ContainerRemoteLaunch));
}
WaitForEvents(containerLauncher, 10);
NUnit.Framework.Assert.AreEqual(10, threadPool.GetPoolSize());
NUnit.Framework.Assert.IsNull(containerLauncher.foundErrors);
// 4 more different hosts, but thread pool size should be capped at 12
containerLauncher.expectedCorePoolSize = 12;
for (int i_1 = 1; i_1 <= 4; i_1++)
{
containerLauncher.Handle(new ContainerLauncherEvent(taskAttemptId, containerId, "host1"
+ i_1 + ":1234", null, ContainerLauncher.EventType.ContainerRemoteLaunch));
}
WaitForEvents(containerLauncher, 12);
NUnit.Framework.Assert.AreEqual(12, threadPool.GetPoolSize());
NUnit.Framework.Assert.IsNull(containerLauncher.foundErrors);
// Make some threads ideal so that remaining events are also done.
containerLauncher.finishEventHandling = true;
WaitForEvents(containerLauncher, 14);
NUnit.Framework.Assert.AreEqual(12, threadPool.GetPoolSize());
NUnit.Framework.Assert.IsNull(containerLauncher.foundErrors);
containerLauncher.Stop();
}
/// <exception cref="System.Exception"/>
public virtual void TestPoolSize()
{
ApplicationId appId = ApplicationId.NewInstance(12345, 67);
ApplicationAttemptId appAttemptId = ApplicationAttemptId.NewInstance(appId, 3);
JobId jobId = MRBuilderUtils.NewJobId(appId, 8);
TaskId taskId = MRBuilderUtils.NewTaskId(jobId, 9, TaskType.Map);
AppContext context = Org.Mockito.Mockito.Mock<AppContext>();
TestContainerLauncher.CustomContainerLauncher containerLauncher = new TestContainerLauncher.CustomContainerLauncher
(this, context);
containerLauncher.Init(new Configuration());
containerLauncher.Start();
ThreadPoolExecutor threadPool = containerLauncher.GetThreadPool();
// No events yet
NUnit.Framework.Assert.AreEqual(containerLauncher.initialPoolSize, MRJobConfig.DefaultMrAmContainerlauncherThreadpoolInitialSize
);
NUnit.Framework.Assert.AreEqual(0, threadPool.GetPoolSize());
NUnit.Framework.Assert.AreEqual(containerLauncher.initialPoolSize, threadPool.GetCorePoolSize
());
NUnit.Framework.Assert.IsNull(containerLauncher.foundErrors);
containerLauncher.expectedCorePoolSize = containerLauncher.initialPoolSize;
for (int i = 0; i < 10; i++)
{
ContainerId containerId = ContainerId.NewContainerId(appAttemptId, i);
TaskAttemptId taskAttemptId = MRBuilderUtils.NewTaskAttemptId(taskId, i);
containerLauncher.Handle(new ContainerLauncherEvent(taskAttemptId, containerId, "host"
+ i + ":1234", null, ContainerLauncher.EventType.ContainerRemoteLaunch));
}
WaitForEvents(containerLauncher, 10);
NUnit.Framework.Assert.AreEqual(10, threadPool.GetPoolSize());
NUnit.Framework.Assert.IsNull(containerLauncher.foundErrors);
// Same set of hosts, so no change
containerLauncher.finishEventHandling = true;
int timeOut = 0;
while (containerLauncher.numEventsProcessed.Get() < 10 && timeOut++ < 200)
{
Log.Info("Waiting for number of events processed to become " + 10 + ". It is now "
+ containerLauncher.numEventsProcessed.Get() + ". Timeout is " + timeOut);
Sharpen.Thread.Sleep(1000);
}
NUnit.Framework.Assert.AreEqual(10, containerLauncher.numEventsProcessed.Get());
containerLauncher.finishEventHandling = false;
for (int i_1 = 0; i_1 < 10; i_1++)
{
ContainerId containerId = ContainerId.NewContainerId(appAttemptId, i_1 + 10);
TaskAttemptId taskAttemptId = MRBuilderUtils.NewTaskAttemptId(taskId, i_1 + 10);
containerLauncher.Handle(new ContainerLauncherEvent(taskAttemptId, containerId, "host"
+ i_1 + ":1234", null, ContainerLauncher.EventType.ContainerRemoteLaunch));
}
WaitForEvents(containerLauncher, 20);
NUnit.Framework.Assert.AreEqual(10, threadPool.GetPoolSize());
NUnit.Framework.Assert.IsNull(containerLauncher.foundErrors);
// Different hosts, there should be an increase in core-thread-pool size to
// 21(11hosts+10buffer)
// Core pool size should be 21 but the live pool size should be only 11.
containerLauncher.expectedCorePoolSize = 11 + containerLauncher.initialPoolSize;
containerLauncher.finishEventHandling = false;
ContainerId containerId_1 = ContainerId.NewContainerId(appAttemptId, 21);
TaskAttemptId taskAttemptId_1 = MRBuilderUtils.NewTaskAttemptId(taskId, 21);
containerLauncher.Handle(new ContainerLauncherEvent(taskAttemptId_1, containerId_1
, "host11:1234", null, ContainerLauncher.EventType.ContainerRemoteLaunch));
WaitForEvents(containerLauncher, 21);
NUnit.Framework.Assert.AreEqual(11, threadPool.GetPoolSize());
NUnit.Framework.Assert.IsNull(containerLauncher.foundErrors);
containerLauncher.Stop();
// change configuration MR_AM_CONTAINERLAUNCHER_THREADPOOL_INITIAL_SIZE
// and verify initialPoolSize value.
Configuration conf = new Configuration();
conf.SetInt(MRJobConfig.MrAmContainerlauncherThreadpoolInitialSize, 20);
containerLauncher = new TestContainerLauncher.CustomContainerLauncher(this, context
);
containerLauncher.Init(conf);
NUnit.Framework.Assert.AreEqual(containerLauncher.initialPoolSize, 20);
}