예제 #1
0
		/// <exception cref="System.Exception"/>
		public virtual void TestPoolLimits()
		{
			ApplicationId appId = ApplicationId.NewInstance(12345, 67);
			ApplicationAttemptId appAttemptId = ApplicationAttemptId.NewInstance(appId, 3);
			JobId jobId = MRBuilderUtils.NewJobId(appId, 8);
			TaskId taskId = MRBuilderUtils.NewTaskId(jobId, 9, TaskType.Map);
			TaskAttemptId taskAttemptId = MRBuilderUtils.NewTaskAttemptId(taskId, 0);
			ContainerId containerId = ContainerId.NewContainerId(appAttemptId, 10);
			AppContext context = Org.Mockito.Mockito.Mock<AppContext>();
			TestContainerLauncher.CustomContainerLauncher containerLauncher = new TestContainerLauncher.CustomContainerLauncher
				(this, context);
			Configuration conf = new Configuration();
			conf.SetInt(MRJobConfig.MrAmContainerlauncherThreadCountLimit, 12);
			containerLauncher.Init(conf);
			containerLauncher.Start();
			ThreadPoolExecutor threadPool = containerLauncher.GetThreadPool();
			// 10 different hosts
			containerLauncher.expectedCorePoolSize = containerLauncher.initialPoolSize;
			for (int i = 0; i < 10; i++)
			{
				containerLauncher.Handle(new ContainerLauncherEvent(taskAttemptId, containerId, "host"
					 + i + ":1234", null, ContainerLauncher.EventType.ContainerRemoteLaunch));
			}
			WaitForEvents(containerLauncher, 10);
			NUnit.Framework.Assert.AreEqual(10, threadPool.GetPoolSize());
			NUnit.Framework.Assert.IsNull(containerLauncher.foundErrors);
			// 4 more different hosts, but thread pool size should be capped at 12
			containerLauncher.expectedCorePoolSize = 12;
			for (int i_1 = 1; i_1 <= 4; i_1++)
			{
				containerLauncher.Handle(new ContainerLauncherEvent(taskAttemptId, containerId, "host1"
					 + i_1 + ":1234", null, ContainerLauncher.EventType.ContainerRemoteLaunch));
			}
			WaitForEvents(containerLauncher, 12);
			NUnit.Framework.Assert.AreEqual(12, threadPool.GetPoolSize());
			NUnit.Framework.Assert.IsNull(containerLauncher.foundErrors);
			// Make some threads ideal so that remaining events are also done.
			containerLauncher.finishEventHandling = true;
			WaitForEvents(containerLauncher, 14);
			NUnit.Framework.Assert.AreEqual(12, threadPool.GetPoolSize());
			NUnit.Framework.Assert.IsNull(containerLauncher.foundErrors);
			containerLauncher.Stop();
		}
예제 #2
0
		/// <exception cref="System.Exception"/>
		public virtual void TestPoolSize()
		{
			ApplicationId appId = ApplicationId.NewInstance(12345, 67);
			ApplicationAttemptId appAttemptId = ApplicationAttemptId.NewInstance(appId, 3);
			JobId jobId = MRBuilderUtils.NewJobId(appId, 8);
			TaskId taskId = MRBuilderUtils.NewTaskId(jobId, 9, TaskType.Map);
			AppContext context = Org.Mockito.Mockito.Mock<AppContext>();
			TestContainerLauncher.CustomContainerLauncher containerLauncher = new TestContainerLauncher.CustomContainerLauncher
				(this, context);
			containerLauncher.Init(new Configuration());
			containerLauncher.Start();
			ThreadPoolExecutor threadPool = containerLauncher.GetThreadPool();
			// No events yet
			NUnit.Framework.Assert.AreEqual(containerLauncher.initialPoolSize, MRJobConfig.DefaultMrAmContainerlauncherThreadpoolInitialSize
				);
			NUnit.Framework.Assert.AreEqual(0, threadPool.GetPoolSize());
			NUnit.Framework.Assert.AreEqual(containerLauncher.initialPoolSize, threadPool.GetCorePoolSize
				());
			NUnit.Framework.Assert.IsNull(containerLauncher.foundErrors);
			containerLauncher.expectedCorePoolSize = containerLauncher.initialPoolSize;
			for (int i = 0; i < 10; i++)
			{
				ContainerId containerId = ContainerId.NewContainerId(appAttemptId, i);
				TaskAttemptId taskAttemptId = MRBuilderUtils.NewTaskAttemptId(taskId, i);
				containerLauncher.Handle(new ContainerLauncherEvent(taskAttemptId, containerId, "host"
					 + i + ":1234", null, ContainerLauncher.EventType.ContainerRemoteLaunch));
			}
			WaitForEvents(containerLauncher, 10);
			NUnit.Framework.Assert.AreEqual(10, threadPool.GetPoolSize());
			NUnit.Framework.Assert.IsNull(containerLauncher.foundErrors);
			// Same set of hosts, so no change
			containerLauncher.finishEventHandling = true;
			int timeOut = 0;
			while (containerLauncher.numEventsProcessed.Get() < 10 && timeOut++ < 200)
			{
				Log.Info("Waiting for number of events processed to become " + 10 + ". It is now "
					 + containerLauncher.numEventsProcessed.Get() + ". Timeout is " + timeOut);
				Sharpen.Thread.Sleep(1000);
			}
			NUnit.Framework.Assert.AreEqual(10, containerLauncher.numEventsProcessed.Get());
			containerLauncher.finishEventHandling = false;
			for (int i_1 = 0; i_1 < 10; i_1++)
			{
				ContainerId containerId = ContainerId.NewContainerId(appAttemptId, i_1 + 10);
				TaskAttemptId taskAttemptId = MRBuilderUtils.NewTaskAttemptId(taskId, i_1 + 10);
				containerLauncher.Handle(new ContainerLauncherEvent(taskAttemptId, containerId, "host"
					 + i_1 + ":1234", null, ContainerLauncher.EventType.ContainerRemoteLaunch));
			}
			WaitForEvents(containerLauncher, 20);
			NUnit.Framework.Assert.AreEqual(10, threadPool.GetPoolSize());
			NUnit.Framework.Assert.IsNull(containerLauncher.foundErrors);
			// Different hosts, there should be an increase in core-thread-pool size to
			// 21(11hosts+10buffer)
			// Core pool size should be 21 but the live pool size should be only 11.
			containerLauncher.expectedCorePoolSize = 11 + containerLauncher.initialPoolSize;
			containerLauncher.finishEventHandling = false;
			ContainerId containerId_1 = ContainerId.NewContainerId(appAttemptId, 21);
			TaskAttemptId taskAttemptId_1 = MRBuilderUtils.NewTaskAttemptId(taskId, 21);
			containerLauncher.Handle(new ContainerLauncherEvent(taskAttemptId_1, containerId_1
				, "host11:1234", null, ContainerLauncher.EventType.ContainerRemoteLaunch));
			WaitForEvents(containerLauncher, 21);
			NUnit.Framework.Assert.AreEqual(11, threadPool.GetPoolSize());
			NUnit.Framework.Assert.IsNull(containerLauncher.foundErrors);
			containerLauncher.Stop();
			// change configuration MR_AM_CONTAINERLAUNCHER_THREADPOOL_INITIAL_SIZE
			// and verify initialPoolSize value.
			Configuration conf = new Configuration();
			conf.SetInt(MRJobConfig.MrAmContainerlauncherThreadpoolInitialSize, 20);
			containerLauncher = new TestContainerLauncher.CustomContainerLauncher(this, context
				);
			containerLauncher.Init(conf);
			NUnit.Framework.Assert.AreEqual(containerLauncher.initialPoolSize, 20);
		}