/// <exception cref="System.Exception"/> public virtual void TestPoolLimits() { ApplicationId appId = ApplicationId.NewInstance(12345, 67); ApplicationAttemptId appAttemptId = ApplicationAttemptId.NewInstance(appId, 3); JobId jobId = MRBuilderUtils.NewJobId(appId, 8); TaskId taskId = MRBuilderUtils.NewTaskId(jobId, 9, TaskType.Map); TaskAttemptId taskAttemptId = MRBuilderUtils.NewTaskAttemptId(taskId, 0); ContainerId containerId = ContainerId.NewContainerId(appAttemptId, 10); AppContext context = Org.Mockito.Mockito.Mock<AppContext>(); TestContainerLauncher.CustomContainerLauncher containerLauncher = new TestContainerLauncher.CustomContainerLauncher (this, context); Configuration conf = new Configuration(); conf.SetInt(MRJobConfig.MrAmContainerlauncherThreadCountLimit, 12); containerLauncher.Init(conf); containerLauncher.Start(); ThreadPoolExecutor threadPool = containerLauncher.GetThreadPool(); // 10 different hosts containerLauncher.expectedCorePoolSize = containerLauncher.initialPoolSize; for (int i = 0; i < 10; i++) { containerLauncher.Handle(new ContainerLauncherEvent(taskAttemptId, containerId, "host" + i + ":1234", null, ContainerLauncher.EventType.ContainerRemoteLaunch)); } WaitForEvents(containerLauncher, 10); NUnit.Framework.Assert.AreEqual(10, threadPool.GetPoolSize()); NUnit.Framework.Assert.IsNull(containerLauncher.foundErrors); // 4 more different hosts, but thread pool size should be capped at 12 containerLauncher.expectedCorePoolSize = 12; for (int i_1 = 1; i_1 <= 4; i_1++) { containerLauncher.Handle(new ContainerLauncherEvent(taskAttemptId, containerId, "host1" + i_1 + ":1234", null, ContainerLauncher.EventType.ContainerRemoteLaunch)); } WaitForEvents(containerLauncher, 12); NUnit.Framework.Assert.AreEqual(12, threadPool.GetPoolSize()); NUnit.Framework.Assert.IsNull(containerLauncher.foundErrors); // Make some threads ideal so that remaining events are also done. containerLauncher.finishEventHandling = true; WaitForEvents(containerLauncher, 14); NUnit.Framework.Assert.AreEqual(12, threadPool.GetPoolSize()); NUnit.Framework.Assert.IsNull(containerLauncher.foundErrors); containerLauncher.Stop(); }
/// <exception cref="System.Exception"/> public virtual void TestPoolSize() { ApplicationId appId = ApplicationId.NewInstance(12345, 67); ApplicationAttemptId appAttemptId = ApplicationAttemptId.NewInstance(appId, 3); JobId jobId = MRBuilderUtils.NewJobId(appId, 8); TaskId taskId = MRBuilderUtils.NewTaskId(jobId, 9, TaskType.Map); AppContext context = Org.Mockito.Mockito.Mock<AppContext>(); TestContainerLauncher.CustomContainerLauncher containerLauncher = new TestContainerLauncher.CustomContainerLauncher (this, context); containerLauncher.Init(new Configuration()); containerLauncher.Start(); ThreadPoolExecutor threadPool = containerLauncher.GetThreadPool(); // No events yet NUnit.Framework.Assert.AreEqual(containerLauncher.initialPoolSize, MRJobConfig.DefaultMrAmContainerlauncherThreadpoolInitialSize ); NUnit.Framework.Assert.AreEqual(0, threadPool.GetPoolSize()); NUnit.Framework.Assert.AreEqual(containerLauncher.initialPoolSize, threadPool.GetCorePoolSize ()); NUnit.Framework.Assert.IsNull(containerLauncher.foundErrors); containerLauncher.expectedCorePoolSize = containerLauncher.initialPoolSize; for (int i = 0; i < 10; i++) { ContainerId containerId = ContainerId.NewContainerId(appAttemptId, i); TaskAttemptId taskAttemptId = MRBuilderUtils.NewTaskAttemptId(taskId, i); containerLauncher.Handle(new ContainerLauncherEvent(taskAttemptId, containerId, "host" + i + ":1234", null, ContainerLauncher.EventType.ContainerRemoteLaunch)); } WaitForEvents(containerLauncher, 10); NUnit.Framework.Assert.AreEqual(10, threadPool.GetPoolSize()); NUnit.Framework.Assert.IsNull(containerLauncher.foundErrors); // Same set of hosts, so no change containerLauncher.finishEventHandling = true; int timeOut = 0; while (containerLauncher.numEventsProcessed.Get() < 10 && timeOut++ < 200) { Log.Info("Waiting for number of events processed to become " + 10 + ". It is now " + containerLauncher.numEventsProcessed.Get() + ". Timeout is " + timeOut); Sharpen.Thread.Sleep(1000); } NUnit.Framework.Assert.AreEqual(10, containerLauncher.numEventsProcessed.Get()); containerLauncher.finishEventHandling = false; for (int i_1 = 0; i_1 < 10; i_1++) { ContainerId containerId = ContainerId.NewContainerId(appAttemptId, i_1 + 10); TaskAttemptId taskAttemptId = MRBuilderUtils.NewTaskAttemptId(taskId, i_1 + 10); containerLauncher.Handle(new ContainerLauncherEvent(taskAttemptId, containerId, "host" + i_1 + ":1234", null, ContainerLauncher.EventType.ContainerRemoteLaunch)); } WaitForEvents(containerLauncher, 20); NUnit.Framework.Assert.AreEqual(10, threadPool.GetPoolSize()); NUnit.Framework.Assert.IsNull(containerLauncher.foundErrors); // Different hosts, there should be an increase in core-thread-pool size to // 21(11hosts+10buffer) // Core pool size should be 21 but the live pool size should be only 11. containerLauncher.expectedCorePoolSize = 11 + containerLauncher.initialPoolSize; containerLauncher.finishEventHandling = false; ContainerId containerId_1 = ContainerId.NewContainerId(appAttemptId, 21); TaskAttemptId taskAttemptId_1 = MRBuilderUtils.NewTaskAttemptId(taskId, 21); containerLauncher.Handle(new ContainerLauncherEvent(taskAttemptId_1, containerId_1 , "host11:1234", null, ContainerLauncher.EventType.ContainerRemoteLaunch)); WaitForEvents(containerLauncher, 21); NUnit.Framework.Assert.AreEqual(11, threadPool.GetPoolSize()); NUnit.Framework.Assert.IsNull(containerLauncher.foundErrors); containerLauncher.Stop(); // change configuration MR_AM_CONTAINERLAUNCHER_THREADPOOL_INITIAL_SIZE // and verify initialPoolSize value. Configuration conf = new Configuration(); conf.SetInt(MRJobConfig.MrAmContainerlauncherThreadpoolInitialSize, 20); containerLauncher = new TestContainerLauncher.CustomContainerLauncher(this, context ); containerLauncher.Init(conf); NUnit.Framework.Assert.AreEqual(containerLauncher.initialPoolSize, 20); }