public virtual void TestRenameMapOutputForReduce()
{
JobConf conf = new JobConf();
MROutputFiles mrOutputFiles = new MROutputFiles();
mrOutputFiles.SetConf(conf);
// make sure both dirs are distinct
//
conf.Set(MRConfig.LocalDir, localDirs[0].ToString());
Path mapOut = mrOutputFiles.GetOutputFileForWrite(1);
conf.Set(MRConfig.LocalDir, localDirs[1].ToString());
Path mapOutIdx = mrOutputFiles.GetOutputIndexFileForWrite(1);
Assert.AssertNotEquals("Paths must be different!", mapOut.GetParent(), mapOutIdx.
GetParent());
// make both dirs part of LOCAL_DIR
conf.SetStrings(MRConfig.LocalDir, localDirs);
FileContext lfc = FileContext.GetLocalFSFileContext(conf);
lfc.Create(mapOut, EnumSet.Of(CreateFlag.Create)).Close();
lfc.Create(mapOutIdx, EnumSet.Of(CreateFlag.Create)).Close();
JobId jobId = MRBuilderUtils.NewJobId(12345L, 1, 2);
TaskId tid = MRBuilderUtils.NewTaskId(jobId, 0, TaskType.Map);
TaskAttemptId taid = MRBuilderUtils.NewTaskAttemptId(tid, 0);
LocalContainerLauncher.RenameMapOutputForReduce(conf, taid, mrOutputFiles);
}
/// <exception cref="System.Exception"/>
public virtual void TestCompletedTask()
{
HistoryFileManager.HistoryFileInfo info = Org.Mockito.Mockito.Mock <HistoryFileManager.HistoryFileInfo
>();
Org.Mockito.Mockito.When(info.GetConfFile()).ThenReturn(fullConfPath);
completedJob = new CompletedJob(conf, jobId, fullHistoryPath, loadTasks, "user",
info, jobAclsManager);
TaskId mt1Id = MRBuilderUtils.NewTaskId(jobId, 0, TaskType.Map);
TaskId rt1Id = MRBuilderUtils.NewTaskId(jobId, 0, TaskType.Reduce);
IDictionary <TaskId, Task> mapTasks = completedJob.GetTasks(TaskType.Map);
IDictionary <TaskId, Task> reduceTasks = completedJob.GetTasks(TaskType.Reduce);
NUnit.Framework.Assert.AreEqual(10, mapTasks.Count);
NUnit.Framework.Assert.AreEqual(2, reduceTasks.Count);
Task mt1 = mapTasks[mt1Id];
NUnit.Framework.Assert.AreEqual(1, mt1.GetAttempts().Count);
NUnit.Framework.Assert.AreEqual(TaskState.Succeeded, mt1.GetState());
TaskReport mt1Report = mt1.GetReport();
NUnit.Framework.Assert.AreEqual(TaskState.Succeeded, mt1Report.GetTaskState());
NUnit.Framework.Assert.AreEqual(mt1Id, mt1Report.GetTaskId());
Task rt1 = reduceTasks[rt1Id];
NUnit.Framework.Assert.AreEqual(1, rt1.GetAttempts().Count);
NUnit.Framework.Assert.AreEqual(TaskState.Succeeded, rt1.GetState());
TaskReport rt1Report = rt1.GetReport();
NUnit.Framework.Assert.AreEqual(TaskState.Succeeded, rt1Report.GetTaskState());
NUnit.Framework.Assert.AreEqual(rt1Id, rt1Report.GetTaskId());
}
public virtual void TestTimeout()
{
EventHandler mockHandler = Org.Mockito.Mockito.Mock <EventHandler>();
Clock clock = new SystemClock();
TaskHeartbeatHandler hb = new TaskHeartbeatHandler(mockHandler, clock, 1);
Configuration conf = new Configuration();
conf.SetInt(MRJobConfig.TaskTimeout, 10);
//10 ms
conf.SetInt(MRJobConfig.TaskTimeoutCheckIntervalMs, 10);
//10 ms
hb.Init(conf);
hb.Start();
try
{
ApplicationId appId = ApplicationId.NewInstance(0l, 5);
JobId jobId = MRBuilderUtils.NewJobId(appId, 4);
TaskId tid = MRBuilderUtils.NewTaskId(jobId, 3, TaskType.Map);
TaskAttemptId taid = MRBuilderUtils.NewTaskAttemptId(tid, 2);
hb.Register(taid);
Sharpen.Thread.Sleep(100);
//Events only happen when the task is canceled
Org.Mockito.Mockito.Verify(mockHandler, Org.Mockito.Mockito.Times(2)).Handle(Matchers.Any
<Org.Apache.Hadoop.Yarn.Event.Event>());
}
finally
{
hb.Stop();
}
}
public static TaskAttemptId MakeTaskAttemptId(long ts, int appId, int taskId, TaskType
taskType, int id)
{
ApplicationId aID = ApplicationId.NewInstance(ts, appId);
JobId jID = MRBuilderUtils.NewJobId(aID, id);
TaskId tID = MRBuilderUtils.NewTaskId(jID, taskId, taskType);
return(MRBuilderUtils.NewTaskAttemptId(tID, id));
}
public virtual void TestShuffleProviders()
{
ApplicationId appId = ApplicationId.NewInstance(1, 1);
JobId jobId = MRBuilderUtils.NewJobId(appId, 1);
TaskId taskId = MRBuilderUtils.NewTaskId(jobId, 1, TaskType.Map);
Path jobFile = Org.Mockito.Mockito.Mock <Path>();
EventHandler eventHandler = Org.Mockito.Mockito.Mock <EventHandler>();
TaskAttemptListener taListener = Org.Mockito.Mockito.Mock <TaskAttemptListener>();
Org.Mockito.Mockito.When(taListener.GetAddress()).ThenReturn(new IPEndPoint("localhost"
, 0));
JobConf jobConf = new JobConf();
jobConf.SetClass("fs.file.impl", typeof(TestShuffleProvider.StubbedFS), typeof(FileSystem
));
jobConf.SetBoolean("fs.file.impl.disable.cache", true);
jobConf.Set(JobConf.MapredMapTaskEnv, string.Empty);
jobConf.Set(YarnConfiguration.NmAuxServices, TestShuffleProvider.TestShuffleHandler1
.MapreduceTestShuffleServiceid + "," + TestShuffleProvider.TestShuffleHandler2.MapreduceTestShuffleServiceid
);
string serviceName = TestShuffleProvider.TestShuffleHandler1.MapreduceTestShuffleServiceid;
string serviceStr = string.Format(YarnConfiguration.NmAuxServiceFmt, serviceName);
jobConf.Set(serviceStr, typeof(TestShuffleProvider.TestShuffleHandler1).FullName);
serviceName = TestShuffleProvider.TestShuffleHandler2.MapreduceTestShuffleServiceid;
serviceStr = string.Format(YarnConfiguration.NmAuxServiceFmt, serviceName);
jobConf.Set(serviceStr, typeof(TestShuffleProvider.TestShuffleHandler2).FullName);
jobConf.Set(MRJobConfig.MapreduceJobShuffleProviderServices, TestShuffleProvider.TestShuffleHandler1
.MapreduceTestShuffleServiceid + "," + TestShuffleProvider.TestShuffleHandler2.MapreduceTestShuffleServiceid
);
Credentials credentials = new Credentials();
Org.Apache.Hadoop.Security.Token.Token <JobTokenIdentifier> jobToken = new Org.Apache.Hadoop.Security.Token.Token
<JobTokenIdentifier>(Sharpen.Runtime.GetBytesForString(("tokenid")), Sharpen.Runtime.GetBytesForString
(("tokenpw")), new Text("tokenkind"), new Text("tokenservice"));
TaskAttemptImpl taImpl = new MapTaskAttemptImpl(taskId, 1, eventHandler, jobFile,
1, Org.Mockito.Mockito.Mock <JobSplit.TaskSplitMetaInfo>(), jobConf, taListener,
jobToken, credentials, new SystemClock(), null);
jobConf.Set(MRJobConfig.ApplicationAttemptId, taImpl.GetID().ToString());
ContainerLaunchContext launchCtx = TaskAttemptImpl.CreateContainerLaunchContext(null
, jobConf, jobToken, taImpl.CreateRemoteTask(), TypeConverter.FromYarn(jobId), Org.Mockito.Mockito.Mock
<WrappedJvmID>(), taListener, credentials);
IDictionary <string, ByteBuffer> serviceDataMap = launchCtx.GetServiceData();
NUnit.Framework.Assert.IsNotNull("TestShuffleHandler1 is missing", serviceDataMap
[TestShuffleProvider.TestShuffleHandler1.MapreduceTestShuffleServiceid]);
NUnit.Framework.Assert.IsNotNull("TestShuffleHandler2 is missing", serviceDataMap
[TestShuffleProvider.TestShuffleHandler2.MapreduceTestShuffleServiceid]);
NUnit.Framework.Assert.IsTrue("mismatch number of services in map", serviceDataMap
.Count == 3);
}
private static TaskAttemptCompletionEvent CreateTce(int eventId, bool isMap, TaskAttemptCompletionEventStatus
status)
{
JobId jid = MRBuilderUtils.NewJobId(12345, 1, 1);
TaskId tid = MRBuilderUtils.NewTaskId(jid, 0, isMap ? TaskType.Map : TaskType.Reduce
);
TaskAttemptId attemptId = MRBuilderUtils.NewTaskAttemptId(tid, 0);
RecordFactory recordFactory = RecordFactoryProvider.GetRecordFactory(null);
TaskAttemptCompletionEvent tce = recordFactory.NewRecordInstance <TaskAttemptCompletionEvent
>();
tce.SetEventId(eventId);
tce.SetAttemptId(attemptId);
tce.SetStatus(status);
return(tce);
}
/// <exception cref="System.Exception"/>
public virtual void TestPoolLimits()
{
ApplicationId appId = ApplicationId.NewInstance(12345, 67);
ApplicationAttemptId appAttemptId = ApplicationAttemptId.NewInstance(appId, 3);
JobId jobId = MRBuilderUtils.NewJobId(appId, 8);
TaskId taskId = MRBuilderUtils.NewTaskId(jobId, 9, TaskType.Map);
TaskAttemptId taskAttemptId = MRBuilderUtils.NewTaskAttemptId(taskId, 0);
ContainerId containerId = ContainerId.NewContainerId(appAttemptId, 10);
AppContext context = Org.Mockito.Mockito.Mock<AppContext>();
TestContainerLauncher.CustomContainerLauncher containerLauncher = new TestContainerLauncher.CustomContainerLauncher
(this, context);
Configuration conf = new Configuration();
conf.SetInt(MRJobConfig.MrAmContainerlauncherThreadCountLimit, 12);
containerLauncher.Init(conf);
containerLauncher.Start();
ThreadPoolExecutor threadPool = containerLauncher.GetThreadPool();
// 10 different hosts
containerLauncher.expectedCorePoolSize = containerLauncher.initialPoolSize;
for (int i = 0; i < 10; i++)
{
containerLauncher.Handle(new ContainerLauncherEvent(taskAttemptId, containerId, "host"
+ i + ":1234", null, ContainerLauncher.EventType.ContainerRemoteLaunch));
}
WaitForEvents(containerLauncher, 10);
NUnit.Framework.Assert.AreEqual(10, threadPool.GetPoolSize());
NUnit.Framework.Assert.IsNull(containerLauncher.foundErrors);
// 4 more different hosts, but thread pool size should be capped at 12
containerLauncher.expectedCorePoolSize = 12;
for (int i_1 = 1; i_1 <= 4; i_1++)
{
containerLauncher.Handle(new ContainerLauncherEvent(taskAttemptId, containerId, "host1"
+ i_1 + ":1234", null, ContainerLauncher.EventType.ContainerRemoteLaunch));
}
WaitForEvents(containerLauncher, 12);
NUnit.Framework.Assert.AreEqual(12, threadPool.GetPoolSize());
NUnit.Framework.Assert.IsNull(containerLauncher.foundErrors);
// Make some threads ideal so that remaining events are also done.
containerLauncher.finishEventHandling = true;
WaitForEvents(containerLauncher, 14);
NUnit.Framework.Assert.AreEqual(12, threadPool.GetPoolSize());
NUnit.Framework.Assert.IsNull(containerLauncher.foundErrors);
containerLauncher.Stop();
}
public virtual void TestAverageReduceTime()
{
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = Org.Mockito.Mockito.Mock <CompletedJob
>();
Task task1 = Org.Mockito.Mockito.Mock <Task>();
Task task2 = Org.Mockito.Mockito.Mock <Task>();
JobId jobId = MRBuilderUtils.NewJobId(1L, 1, 1);
TaskId taskId1 = MRBuilderUtils.NewTaskId(jobId, 1, TaskType.Reduce);
TaskId taskId2 = MRBuilderUtils.NewTaskId(jobId, 2, TaskType.Reduce);
TaskAttemptId taskAttemptId1 = MRBuilderUtils.NewTaskAttemptId(taskId1, 1);
TaskAttemptId taskAttemptId2 = MRBuilderUtils.NewTaskAttemptId(taskId2, 2);
TaskAttempt taskAttempt1 = Org.Mockito.Mockito.Mock <TaskAttempt>();
TaskAttempt taskAttempt2 = Org.Mockito.Mockito.Mock <TaskAttempt>();
JobReport jobReport = Org.Mockito.Mockito.Mock <JobReport>();
Org.Mockito.Mockito.When(taskAttempt1.GetState()).ThenReturn(TaskAttemptState.Succeeded
);
Org.Mockito.Mockito.When(taskAttempt1.GetLaunchTime()).ThenReturn(0L);
Org.Mockito.Mockito.When(taskAttempt1.GetShuffleFinishTime()).ThenReturn(4L);
Org.Mockito.Mockito.When(taskAttempt1.GetSortFinishTime()).ThenReturn(6L);
Org.Mockito.Mockito.When(taskAttempt1.GetFinishTime()).ThenReturn(8L);
Org.Mockito.Mockito.When(taskAttempt2.GetState()).ThenReturn(TaskAttemptState.Succeeded
);
Org.Mockito.Mockito.When(taskAttempt2.GetLaunchTime()).ThenReturn(5L);
Org.Mockito.Mockito.When(taskAttempt2.GetShuffleFinishTime()).ThenReturn(10L);
Org.Mockito.Mockito.When(taskAttempt2.GetSortFinishTime()).ThenReturn(22L);
Org.Mockito.Mockito.When(taskAttempt2.GetFinishTime()).ThenReturn(42L);
Org.Mockito.Mockito.When(task1.GetType()).ThenReturn(TaskType.Reduce);
Org.Mockito.Mockito.When(task2.GetType()).ThenReturn(TaskType.Reduce);
Org.Mockito.Mockito.When(task1.GetAttempts()).ThenReturn(new _Dictionary_120(taskAttemptId1
, taskAttempt1));
Org.Mockito.Mockito.When(task2.GetAttempts()).ThenReturn(new _Dictionary_123(taskAttemptId2
, taskAttempt2));
Org.Mockito.Mockito.When(job.GetTasks()).ThenReturn(new _Dictionary_127(taskId1,
task1, taskId2, task2));
Org.Mockito.Mockito.When(job.GetID()).ThenReturn(jobId);
Org.Mockito.Mockito.When(job.GetReport()).ThenReturn(jobReport);
Org.Mockito.Mockito.When(job.GetName()).ThenReturn("TestJobInfo");
Org.Mockito.Mockito.When(job.GetState()).ThenReturn(JobState.Succeeded);
JobInfo jobInfo = new JobInfo(job);
NUnit.Framework.Assert.AreEqual(11L, jobInfo.GetAvgReduceTime());
}
/// <exception cref="System.Exception"/>
public virtual void TestCompletedTaskAttempt()
{
HistoryFileManager.HistoryFileInfo info = Org.Mockito.Mockito.Mock <HistoryFileManager.HistoryFileInfo
>();
Org.Mockito.Mockito.When(info.GetConfFile()).ThenReturn(fullConfPath);
completedJob = new CompletedJob(conf, jobId, fullHistoryPath, loadTasks, "user",
info, jobAclsManager);
TaskId mt1Id = MRBuilderUtils.NewTaskId(jobId, 0, TaskType.Map);
TaskId rt1Id = MRBuilderUtils.NewTaskId(jobId, 0, TaskType.Reduce);
TaskAttemptId mta1Id = MRBuilderUtils.NewTaskAttemptId(mt1Id, 0);
TaskAttemptId rta1Id = MRBuilderUtils.NewTaskAttemptId(rt1Id, 0);
Task mt1 = completedJob.GetTask(mt1Id);
Task rt1 = completedJob.GetTask(rt1Id);
TaskAttempt mta1 = mt1.GetAttempt(mta1Id);
NUnit.Framework.Assert.AreEqual(TaskAttemptState.Succeeded, mta1.GetState());
NUnit.Framework.Assert.AreEqual("localhost:45454", mta1.GetAssignedContainerMgrAddress
());
NUnit.Framework.Assert.AreEqual("localhost:9999", mta1.GetNodeHttpAddress());
TaskAttemptReport mta1Report = mta1.GetReport();
NUnit.Framework.Assert.AreEqual(TaskAttemptState.Succeeded, mta1Report.GetTaskAttemptState
());
NUnit.Framework.Assert.AreEqual("localhost", mta1Report.GetNodeManagerHost());
NUnit.Framework.Assert.AreEqual(45454, mta1Report.GetNodeManagerPort());
NUnit.Framework.Assert.AreEqual(9999, mta1Report.GetNodeManagerHttpPort());
TaskAttempt rta1 = rt1.GetAttempt(rta1Id);
NUnit.Framework.Assert.AreEqual(TaskAttemptState.Succeeded, rta1.GetState());
NUnit.Framework.Assert.AreEqual("localhost:45454", rta1.GetAssignedContainerMgrAddress
());
NUnit.Framework.Assert.AreEqual("localhost:9999", rta1.GetNodeHttpAddress());
TaskAttemptReport rta1Report = rta1.GetReport();
NUnit.Framework.Assert.AreEqual(TaskAttemptState.Succeeded, rta1Report.GetTaskAttemptState
());
NUnit.Framework.Assert.AreEqual("localhost", rta1Report.GetNodeManagerHost());
NUnit.Framework.Assert.AreEqual(45454, rta1Report.GetNodeManagerPort());
NUnit.Framework.Assert.AreEqual(9999, rta1Report.GetNodeManagerHttpPort());
}
/// <exception cref="System.Exception"/>
public virtual void TestKillJob()
{
JobConf conf = new JobConf();
AppContext context = Org.Mockito.Mockito.Mock <AppContext>();
// a simple event handler solely to detect the container cleaned event
CountDownLatch isDone = new CountDownLatch(1);
EventHandler handler = new _EventHandler_106(isDone);
Org.Mockito.Mockito.When(context.GetEventHandler()).ThenReturn(handler);
// create and start the launcher
LocalContainerLauncher launcher = new LocalContainerLauncher(context, Org.Mockito.Mockito.Mock
<TaskUmbilicalProtocol>());
launcher.Init(conf);
launcher.Start();
// create mocked job, task, and task attempt
// a single-mapper job
JobId jobId = MRBuilderUtils.NewJobId(Runtime.CurrentTimeMillis(), 1, 1);
TaskId taskId = MRBuilderUtils.NewTaskId(jobId, 1, TaskType.Map);
TaskAttemptId taId = MRBuilderUtils.NewTaskAttemptId(taskId, 0);
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job
>();
Org.Mockito.Mockito.When(job.GetTotalMaps()).ThenReturn(1);
Org.Mockito.Mockito.When(job.GetTotalReduces()).ThenReturn(0);
IDictionary <JobId, Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job> jobs = new Dictionary
<JobId, Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job>();
jobs[jobId] = job;
// app context returns the one and only job
Org.Mockito.Mockito.When(context.GetAllJobs()).ThenReturn(jobs);
Task ytask = Org.Mockito.Mockito.Mock <Task>();
Org.Mockito.Mockito.When(ytask.GetType()).ThenReturn(TaskType.Map);
Org.Mockito.Mockito.When(job.GetTask(taskId)).ThenReturn(ytask);
// create a sleeping mapper that runs beyond the test timeout
MapTask mapTask = Org.Mockito.Mockito.Mock <MapTask>();
Org.Mockito.Mockito.When(mapTask.IsMapOrReduce()).ThenReturn(true);
Org.Mockito.Mockito.When(mapTask.IsMapTask()).ThenReturn(true);
TaskAttemptID taskID = TypeConverter.FromYarn(taId);
Org.Mockito.Mockito.When(mapTask.GetTaskID()).ThenReturn(taskID);
Org.Mockito.Mockito.When(mapTask.GetJobID()).ThenReturn(((JobID)taskID.GetJobID()
));
Org.Mockito.Mockito.DoAnswer(new _Answer_152()).When(mapTask).Run(Matchers.IsA <JobConf
>(), Matchers.IsA <TaskUmbilicalProtocol>());
// sleep for a long time
// pump in a task attempt launch event
ContainerLauncherEvent launchEvent = new ContainerRemoteLaunchEvent(taId, null, CreateMockContainer
(), mapTask);
launcher.Handle(launchEvent);
Sharpen.Thread.Sleep(200);
// now pump in a container clean-up event
ContainerLauncherEvent cleanupEvent = new ContainerLauncherEvent(taId, null, null
, null, ContainerLauncher.EventType.ContainerRemoteCleanup);
launcher.Handle(cleanupEvent);
// wait for the event to fire: this should be received promptly
isDone.Await();
launcher.Close();
}
/// <exception cref="System.Exception"/>
public virtual void TestPoolSize()
{
ApplicationId appId = ApplicationId.NewInstance(12345, 67);
ApplicationAttemptId appAttemptId = ApplicationAttemptId.NewInstance(appId, 3);
JobId jobId = MRBuilderUtils.NewJobId(appId, 8);
TaskId taskId = MRBuilderUtils.NewTaskId(jobId, 9, TaskType.Map);
AppContext context = Org.Mockito.Mockito.Mock<AppContext>();
TestContainerLauncher.CustomContainerLauncher containerLauncher = new TestContainerLauncher.CustomContainerLauncher
(this, context);
containerLauncher.Init(new Configuration());
containerLauncher.Start();
ThreadPoolExecutor threadPool = containerLauncher.GetThreadPool();
// No events yet
NUnit.Framework.Assert.AreEqual(containerLauncher.initialPoolSize, MRJobConfig.DefaultMrAmContainerlauncherThreadpoolInitialSize
);
NUnit.Framework.Assert.AreEqual(0, threadPool.GetPoolSize());
NUnit.Framework.Assert.AreEqual(containerLauncher.initialPoolSize, threadPool.GetCorePoolSize
());
NUnit.Framework.Assert.IsNull(containerLauncher.foundErrors);
containerLauncher.expectedCorePoolSize = containerLauncher.initialPoolSize;
for (int i = 0; i < 10; i++)
{
ContainerId containerId = ContainerId.NewContainerId(appAttemptId, i);
TaskAttemptId taskAttemptId = MRBuilderUtils.NewTaskAttemptId(taskId, i);
containerLauncher.Handle(new ContainerLauncherEvent(taskAttemptId, containerId, "host"
+ i + ":1234", null, ContainerLauncher.EventType.ContainerRemoteLaunch));
}
WaitForEvents(containerLauncher, 10);
NUnit.Framework.Assert.AreEqual(10, threadPool.GetPoolSize());
NUnit.Framework.Assert.IsNull(containerLauncher.foundErrors);
// Same set of hosts, so no change
containerLauncher.finishEventHandling = true;
int timeOut = 0;
while (containerLauncher.numEventsProcessed.Get() < 10 && timeOut++ < 200)
{
Log.Info("Waiting for number of events processed to become " + 10 + ". It is now "
+ containerLauncher.numEventsProcessed.Get() + ". Timeout is " + timeOut);
Sharpen.Thread.Sleep(1000);
}
NUnit.Framework.Assert.AreEqual(10, containerLauncher.numEventsProcessed.Get());
containerLauncher.finishEventHandling = false;
for (int i_1 = 0; i_1 < 10; i_1++)
{
ContainerId containerId = ContainerId.NewContainerId(appAttemptId, i_1 + 10);
TaskAttemptId taskAttemptId = MRBuilderUtils.NewTaskAttemptId(taskId, i_1 + 10);
containerLauncher.Handle(new ContainerLauncherEvent(taskAttemptId, containerId, "host"
+ i_1 + ":1234", null, ContainerLauncher.EventType.ContainerRemoteLaunch));
}
WaitForEvents(containerLauncher, 20);
NUnit.Framework.Assert.AreEqual(10, threadPool.GetPoolSize());
NUnit.Framework.Assert.IsNull(containerLauncher.foundErrors);
// Different hosts, there should be an increase in core-thread-pool size to
// 21(11hosts+10buffer)
// Core pool size should be 21 but the live pool size should be only 11.
containerLauncher.expectedCorePoolSize = 11 + containerLauncher.initialPoolSize;
containerLauncher.finishEventHandling = false;
ContainerId containerId_1 = ContainerId.NewContainerId(appAttemptId, 21);
TaskAttemptId taskAttemptId_1 = MRBuilderUtils.NewTaskAttemptId(taskId, 21);
containerLauncher.Handle(new ContainerLauncherEvent(taskAttemptId_1, containerId_1
, "host11:1234", null, ContainerLauncher.EventType.ContainerRemoteLaunch));
WaitForEvents(containerLauncher, 21);
NUnit.Framework.Assert.AreEqual(11, threadPool.GetPoolSize());
NUnit.Framework.Assert.IsNull(containerLauncher.foundErrors);
containerLauncher.Stop();
// change configuration MR_AM_CONTAINERLAUNCHER_THREADPOOL_INITIAL_SIZE
// and verify initialPoolSize value.
Configuration conf = new Configuration();
conf.SetInt(MRJobConfig.MrAmContainerlauncherThreadpoolInitialSize, 20);
containerLauncher = new TestContainerLauncher.CustomContainerLauncher(this, context
);
containerLauncher.Init(conf);
NUnit.Framework.Assert.AreEqual(containerLauncher.initialPoolSize, 20);
}
private TaskId CreateTaskId(long clusterTimestamp, int jobIdInt, int taskIdInt, TaskType
taskType)
{
return(MRBuilderUtils.NewTaskId(CreateJobId(clusterTimestamp, jobIdInt), taskIdInt
, taskType));
}
public virtual void TestAttemptContainerRequest()
{
Text SecretKeyAlias = new Text("secretkeyalias");
byte[] SecretKey = Sharpen.Runtime.GetBytesForString(("secretkey"));
IDictionary <ApplicationAccessType, string> acls = new Dictionary <ApplicationAccessType
, string>(1);
acls[ApplicationAccessType.ViewApp] = "otheruser";
ApplicationId appId = ApplicationId.NewInstance(1, 1);
JobId jobId = MRBuilderUtils.NewJobId(appId, 1);
TaskId taskId = MRBuilderUtils.NewTaskId(jobId, 1, TaskType.Map);
Path jobFile = Org.Mockito.Mockito.Mock <Path>();
EventHandler eventHandler = Org.Mockito.Mockito.Mock <EventHandler>();
TaskAttemptListener taListener = Org.Mockito.Mockito.Mock <TaskAttemptListener>();
Org.Mockito.Mockito.When(taListener.GetAddress()).ThenReturn(new IPEndPoint("localhost"
, 0));
JobConf jobConf = new JobConf();
jobConf.SetClass("fs.file.impl", typeof(TestTaskAttemptContainerRequest.StubbedFS
), typeof(FileSystem));
jobConf.SetBoolean("fs.file.impl.disable.cache", true);
jobConf.Set(JobConf.MapredMapTaskEnv, string.Empty);
// setup UGI for security so tokens and keys are preserved
jobConf.Set(CommonConfigurationKeysPublic.HadoopSecurityAuthentication, "kerberos"
);
UserGroupInformation.SetConfiguration(jobConf);
Credentials credentials = new Credentials();
credentials.AddSecretKey(SecretKeyAlias, SecretKey);
Org.Apache.Hadoop.Security.Token.Token <JobTokenIdentifier> jobToken = new Org.Apache.Hadoop.Security.Token.Token
<JobTokenIdentifier>(Sharpen.Runtime.GetBytesForString(("tokenid")), Sharpen.Runtime.GetBytesForString
(("tokenpw")), new Text("tokenkind"), new Text("tokenservice"));
TaskAttemptImpl taImpl = new MapTaskAttemptImpl(taskId, 1, eventHandler, jobFile,
1, Org.Mockito.Mockito.Mock <JobSplit.TaskSplitMetaInfo>(), jobConf, taListener,
jobToken, credentials, new SystemClock(), null);
jobConf.Set(MRJobConfig.ApplicationAttemptId, taImpl.GetID().ToString());
ContainerLaunchContext launchCtx = TaskAttemptImpl.CreateContainerLaunchContext(acls
, jobConf, jobToken, taImpl.CreateRemoteTask(), TypeConverter.FromYarn(jobId), Org.Mockito.Mockito.Mock
<WrappedJvmID>(), taListener, credentials);
NUnit.Framework.Assert.AreEqual("ACLs mismatch", acls, launchCtx.GetApplicationACLs
());
Credentials launchCredentials = new Credentials();
DataInputByteBuffer dibb = new DataInputByteBuffer();
dibb.Reset(launchCtx.GetTokens());
launchCredentials.ReadTokenStorageStream(dibb);
// verify all tokens specified for the task attempt are in the launch context
foreach (Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> token in credentials
.GetAllTokens())
{
Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> launchToken = launchCredentials
.GetToken(token.GetService());
NUnit.Framework.Assert.IsNotNull("Token " + token.GetService() + " is missing", launchToken
);
NUnit.Framework.Assert.AreEqual("Token " + token.GetService() + " mismatch", token
, launchToken);
}
// verify the secret key is in the launch context
NUnit.Framework.Assert.IsNotNull("Secret key missing", launchCredentials.GetSecretKey
(SecretKeyAlias));
NUnit.Framework.Assert.IsTrue("Secret key mismatch", Arrays.Equals(SecretKey, launchCredentials
.GetSecretKey(SecretKeyAlias)));
}