/// <summary>
/// Trivial test case that verifies basic functionality of
/// <see cref="JobIdHistoryFileInfoMap"/>
/// </summary>
/// <exception cref="System.Exception"/>
public virtual void TestWithSingleElement()
{
HistoryFileManager.JobIdHistoryFileInfoMap mapWithSize = new HistoryFileManager.JobIdHistoryFileInfoMap
();
JobId jobId = MRBuilderUtils.NewJobId(1, 1, 1);
HistoryFileManager.HistoryFileInfo fileInfo1 = Org.Mockito.Mockito.Mock <HistoryFileManager.HistoryFileInfo
>();
Org.Mockito.Mockito.When(fileInfo1.GetJobId()).ThenReturn(jobId);
// add it twice
NUnit.Framework.Assert.AreEqual("Incorrect return on putIfAbsent()", null, mapWithSize
.PutIfAbsent(jobId, fileInfo1));
NUnit.Framework.Assert.AreEqual("Incorrect return on putIfAbsent()", fileInfo1, mapWithSize
.PutIfAbsent(jobId, fileInfo1));
// check get()
NUnit.Framework.Assert.AreEqual("Incorrect get()", fileInfo1, mapWithSize.Get(jobId
));
NUnit.Framework.Assert.IsTrue("Incorrect size()", CheckSize(mapWithSize, 1));
// check navigableKeySet()
NavigableSet <JobId> set = mapWithSize.NavigableKeySet();
NUnit.Framework.Assert.AreEqual("Incorrect navigableKeySet()", 1, set.Count);
NUnit.Framework.Assert.IsTrue("Incorrect navigableKeySet()", set.Contains(jobId));
// check values()
ICollection <HistoryFileManager.HistoryFileInfo> values = mapWithSize.Values();
NUnit.Framework.Assert.AreEqual("Incorrect values()", 1, values.Count);
NUnit.Framework.Assert.IsTrue("Incorrect values()", values.Contains(fileInfo1));
}
public virtual void TestRenameMapOutputForReduce()
{
JobConf conf = new JobConf();
MROutputFiles mrOutputFiles = new MROutputFiles();
mrOutputFiles.SetConf(conf);
// make sure both dirs are distinct
//
conf.Set(MRConfig.LocalDir, localDirs[0].ToString());
Path mapOut = mrOutputFiles.GetOutputFileForWrite(1);
conf.Set(MRConfig.LocalDir, localDirs[1].ToString());
Path mapOutIdx = mrOutputFiles.GetOutputIndexFileForWrite(1);
Assert.AssertNotEquals("Paths must be different!", mapOut.GetParent(), mapOutIdx.
GetParent());
// make both dirs part of LOCAL_DIR
conf.SetStrings(MRConfig.LocalDir, localDirs);
FileContext lfc = FileContext.GetLocalFSFileContext(conf);
lfc.Create(mapOut, EnumSet.Of(CreateFlag.Create)).Close();
lfc.Create(mapOutIdx, EnumSet.Of(CreateFlag.Create)).Close();
JobId jobId = MRBuilderUtils.NewJobId(12345L, 1, 2);
TaskId tid = MRBuilderUtils.NewTaskId(jobId, 0, TaskType.Map);
TaskAttemptId taid = MRBuilderUtils.NewTaskAttemptId(tid, 0);
LocalContainerLauncher.RenameMapOutputForReduce(conf, taid, mrOutputFiles);
}
public virtual void TestTimeout()
{
EventHandler mockHandler = Org.Mockito.Mockito.Mock <EventHandler>();
Clock clock = new SystemClock();
TaskHeartbeatHandler hb = new TaskHeartbeatHandler(mockHandler, clock, 1);
Configuration conf = new Configuration();
conf.SetInt(MRJobConfig.TaskTimeout, 10);
//10 ms
conf.SetInt(MRJobConfig.TaskTimeoutCheckIntervalMs, 10);
//10 ms
hb.Init(conf);
hb.Start();
try
{
ApplicationId appId = ApplicationId.NewInstance(0l, 5);
JobId jobId = MRBuilderUtils.NewJobId(appId, 4);
TaskId tid = MRBuilderUtils.NewTaskId(jobId, 3, TaskType.Map);
TaskAttemptId taid = MRBuilderUtils.NewTaskAttemptId(tid, 2);
hb.Register(taid);
Sharpen.Thread.Sleep(100);
//Events only happen when the task is canceled
Org.Mockito.Mockito.Verify(mockHandler, Org.Mockito.Mockito.Times(2)).Handle(Matchers.Any
<Org.Apache.Hadoop.Yarn.Event.Event>());
}
finally
{
hb.Stop();
}
}
public static TaskAttemptId MakeTaskAttemptId(long ts, int appId, int taskId, TaskType
taskType, int id)
{
ApplicationId aID = ApplicationId.NewInstance(ts, appId);
JobId jID = MRBuilderUtils.NewJobId(aID, id);
TaskId tID = MRBuilderUtils.NewTaskId(jID, taskId, taskType);
return(MRBuilderUtils.NewTaskAttemptId(tID, id));
}
public virtual void TestShuffleProviders()
{
ApplicationId appId = ApplicationId.NewInstance(1, 1);
JobId jobId = MRBuilderUtils.NewJobId(appId, 1);
TaskId taskId = MRBuilderUtils.NewTaskId(jobId, 1, TaskType.Map);
Path jobFile = Org.Mockito.Mockito.Mock <Path>();
EventHandler eventHandler = Org.Mockito.Mockito.Mock <EventHandler>();
TaskAttemptListener taListener = Org.Mockito.Mockito.Mock <TaskAttemptListener>();
Org.Mockito.Mockito.When(taListener.GetAddress()).ThenReturn(new IPEndPoint("localhost"
, 0));
JobConf jobConf = new JobConf();
jobConf.SetClass("fs.file.impl", typeof(TestShuffleProvider.StubbedFS), typeof(FileSystem
));
jobConf.SetBoolean("fs.file.impl.disable.cache", true);
jobConf.Set(JobConf.MapredMapTaskEnv, string.Empty);
jobConf.Set(YarnConfiguration.NmAuxServices, TestShuffleProvider.TestShuffleHandler1
.MapreduceTestShuffleServiceid + "," + TestShuffleProvider.TestShuffleHandler2.MapreduceTestShuffleServiceid
);
string serviceName = TestShuffleProvider.TestShuffleHandler1.MapreduceTestShuffleServiceid;
string serviceStr = string.Format(YarnConfiguration.NmAuxServiceFmt, serviceName);
jobConf.Set(serviceStr, typeof(TestShuffleProvider.TestShuffleHandler1).FullName);
serviceName = TestShuffleProvider.TestShuffleHandler2.MapreduceTestShuffleServiceid;
serviceStr = string.Format(YarnConfiguration.NmAuxServiceFmt, serviceName);
jobConf.Set(serviceStr, typeof(TestShuffleProvider.TestShuffleHandler2).FullName);
jobConf.Set(MRJobConfig.MapreduceJobShuffleProviderServices, TestShuffleProvider.TestShuffleHandler1
.MapreduceTestShuffleServiceid + "," + TestShuffleProvider.TestShuffleHandler2.MapreduceTestShuffleServiceid
);
Credentials credentials = new Credentials();
Org.Apache.Hadoop.Security.Token.Token <JobTokenIdentifier> jobToken = new Org.Apache.Hadoop.Security.Token.Token
<JobTokenIdentifier>(Sharpen.Runtime.GetBytesForString(("tokenid")), Sharpen.Runtime.GetBytesForString
(("tokenpw")), new Text("tokenkind"), new Text("tokenservice"));
TaskAttemptImpl taImpl = new MapTaskAttemptImpl(taskId, 1, eventHandler, jobFile,
1, Org.Mockito.Mockito.Mock <JobSplit.TaskSplitMetaInfo>(), jobConf, taListener,
jobToken, credentials, new SystemClock(), null);
jobConf.Set(MRJobConfig.ApplicationAttemptId, taImpl.GetID().ToString());
ContainerLaunchContext launchCtx = TaskAttemptImpl.CreateContainerLaunchContext(null
, jobConf, jobToken, taImpl.CreateRemoteTask(), TypeConverter.FromYarn(jobId), Org.Mockito.Mockito.Mock
<WrappedJvmID>(), taListener, credentials);
IDictionary <string, ByteBuffer> serviceDataMap = launchCtx.GetServiceData();
NUnit.Framework.Assert.IsNotNull("TestShuffleHandler1 is missing", serviceDataMap
[TestShuffleProvider.TestShuffleHandler1.MapreduceTestShuffleServiceid]);
NUnit.Framework.Assert.IsNotNull("TestShuffleHandler2 is missing", serviceDataMap
[TestShuffleProvider.TestShuffleHandler2.MapreduceTestShuffleServiceid]);
NUnit.Framework.Assert.IsTrue("mismatch number of services in map", serviceDataMap
.Count == 3);
}
public virtual void TestAddExisting()
{
HistoryFileManager.JobListCache cache = new HistoryFileManager.JobListCache(2, 1000
);
JobId jobId = MRBuilderUtils.NewJobId(1, 1, 1);
HistoryFileManager.HistoryFileInfo fileInfo = Org.Mockito.Mockito.Mock <HistoryFileManager.HistoryFileInfo
>();
Org.Mockito.Mockito.When(fileInfo.GetJobId()).ThenReturn(jobId);
cache.AddIfAbsent(fileInfo);
cache.AddIfAbsent(fileInfo);
NUnit.Framework.Assert.AreEqual("Incorrect number of cache entries", 1, cache.Values
().Count);
}
private static TaskAttemptCompletionEvent CreateTce(int eventId, bool isMap, TaskAttemptCompletionEventStatus
status)
{
JobId jid = MRBuilderUtils.NewJobId(12345, 1, 1);
TaskId tid = MRBuilderUtils.NewTaskId(jid, 0, isMap ? TaskType.Map : TaskType.Reduce
);
TaskAttemptId attemptId = MRBuilderUtils.NewTaskAttemptId(tid, 0);
RecordFactory recordFactory = RecordFactoryProvider.GetRecordFactory(null);
TaskAttemptCompletionEvent tce = recordFactory.NewRecordInstance <TaskAttemptCompletionEvent
>();
tce.SetEventId(eventId);
tce.SetAttemptId(attemptId);
tce.SetStatus(status);
return(tce);
}
/// <exception cref="System.Exception"/>
public virtual void TestPoolLimits()
{
ApplicationId appId = ApplicationId.NewInstance(12345, 67);
ApplicationAttemptId appAttemptId = ApplicationAttemptId.NewInstance(appId, 3);
JobId jobId = MRBuilderUtils.NewJobId(appId, 8);
TaskId taskId = MRBuilderUtils.NewTaskId(jobId, 9, TaskType.Map);
TaskAttemptId taskAttemptId = MRBuilderUtils.NewTaskAttemptId(taskId, 0);
ContainerId containerId = ContainerId.NewContainerId(appAttemptId, 10);
AppContext context = Org.Mockito.Mockito.Mock<AppContext>();
TestContainerLauncher.CustomContainerLauncher containerLauncher = new TestContainerLauncher.CustomContainerLauncher
(this, context);
Configuration conf = new Configuration();
conf.SetInt(MRJobConfig.MrAmContainerlauncherThreadCountLimit, 12);
containerLauncher.Init(conf);
containerLauncher.Start();
ThreadPoolExecutor threadPool = containerLauncher.GetThreadPool();
// 10 different hosts
containerLauncher.expectedCorePoolSize = containerLauncher.initialPoolSize;
for (int i = 0; i < 10; i++)
{
containerLauncher.Handle(new ContainerLauncherEvent(taskAttemptId, containerId, "host"
+ i + ":1234", null, ContainerLauncher.EventType.ContainerRemoteLaunch));
}
WaitForEvents(containerLauncher, 10);
NUnit.Framework.Assert.AreEqual(10, threadPool.GetPoolSize());
NUnit.Framework.Assert.IsNull(containerLauncher.foundErrors);
// 4 more different hosts, but thread pool size should be capped at 12
containerLauncher.expectedCorePoolSize = 12;
for (int i_1 = 1; i_1 <= 4; i_1++)
{
containerLauncher.Handle(new ContainerLauncherEvent(taskAttemptId, containerId, "host1"
+ i_1 + ":1234", null, ContainerLauncher.EventType.ContainerRemoteLaunch));
}
WaitForEvents(containerLauncher, 12);
NUnit.Framework.Assert.AreEqual(12, threadPool.GetPoolSize());
NUnit.Framework.Assert.IsNull(containerLauncher.foundErrors);
// Make some threads ideal so that remaining events are also done.
containerLauncher.finishEventHandling = true;
WaitForEvents(containerLauncher, 14);
NUnit.Framework.Assert.AreEqual(12, threadPool.GetPoolSize());
NUnit.Framework.Assert.IsNull(containerLauncher.foundErrors);
containerLauncher.Stop();
}
public virtual void TestAverageReduceTime()
{
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = Org.Mockito.Mockito.Mock <CompletedJob
>();
Task task1 = Org.Mockito.Mockito.Mock <Task>();
Task task2 = Org.Mockito.Mockito.Mock <Task>();
JobId jobId = MRBuilderUtils.NewJobId(1L, 1, 1);
TaskId taskId1 = MRBuilderUtils.NewTaskId(jobId, 1, TaskType.Reduce);
TaskId taskId2 = MRBuilderUtils.NewTaskId(jobId, 2, TaskType.Reduce);
TaskAttemptId taskAttemptId1 = MRBuilderUtils.NewTaskAttemptId(taskId1, 1);
TaskAttemptId taskAttemptId2 = MRBuilderUtils.NewTaskAttemptId(taskId2, 2);
TaskAttempt taskAttempt1 = Org.Mockito.Mockito.Mock <TaskAttempt>();
TaskAttempt taskAttempt2 = Org.Mockito.Mockito.Mock <TaskAttempt>();
JobReport jobReport = Org.Mockito.Mockito.Mock <JobReport>();
Org.Mockito.Mockito.When(taskAttempt1.GetState()).ThenReturn(TaskAttemptState.Succeeded
);
Org.Mockito.Mockito.When(taskAttempt1.GetLaunchTime()).ThenReturn(0L);
Org.Mockito.Mockito.When(taskAttempt1.GetShuffleFinishTime()).ThenReturn(4L);
Org.Mockito.Mockito.When(taskAttempt1.GetSortFinishTime()).ThenReturn(6L);
Org.Mockito.Mockito.When(taskAttempt1.GetFinishTime()).ThenReturn(8L);
Org.Mockito.Mockito.When(taskAttempt2.GetState()).ThenReturn(TaskAttemptState.Succeeded
);
Org.Mockito.Mockito.When(taskAttempt2.GetLaunchTime()).ThenReturn(5L);
Org.Mockito.Mockito.When(taskAttempt2.GetShuffleFinishTime()).ThenReturn(10L);
Org.Mockito.Mockito.When(taskAttempt2.GetSortFinishTime()).ThenReturn(22L);
Org.Mockito.Mockito.When(taskAttempt2.GetFinishTime()).ThenReturn(42L);
Org.Mockito.Mockito.When(task1.GetType()).ThenReturn(TaskType.Reduce);
Org.Mockito.Mockito.When(task2.GetType()).ThenReturn(TaskType.Reduce);
Org.Mockito.Mockito.When(task1.GetAttempts()).ThenReturn(new _Dictionary_120(taskAttemptId1
, taskAttempt1));
Org.Mockito.Mockito.When(task2.GetAttempts()).ThenReturn(new _Dictionary_123(taskAttemptId2
, taskAttempt2));
Org.Mockito.Mockito.When(job.GetTasks()).ThenReturn(new _Dictionary_127(taskId1,
task1, taskId2, task2));
Org.Mockito.Mockito.When(job.GetID()).ThenReturn(jobId);
Org.Mockito.Mockito.When(job.GetReport()).ThenReturn(jobReport);
Org.Mockito.Mockito.When(job.GetName()).ThenReturn("TestJobInfo");
Org.Mockito.Mockito.When(job.GetState()).ThenReturn(JobState.Succeeded);
JobInfo jobInfo = new JobInfo(job);
NUnit.Framework.Assert.AreEqual(11L, jobInfo.GetAvgReduceTime());
}
/// <exception cref="System.Exception"/>
public virtual void TestEviction()
{
int maxSize = 2;
HistoryFileManager.JobListCache cache = new HistoryFileManager.JobListCache(maxSize
, 1000);
JobId jobId1 = MRBuilderUtils.NewJobId(1, 1, 1);
HistoryFileManager.HistoryFileInfo fileInfo1 = Org.Mockito.Mockito.Mock <HistoryFileManager.HistoryFileInfo
>();
Org.Mockito.Mockito.When(fileInfo1.GetJobId()).ThenReturn(jobId1);
JobId jobId2 = MRBuilderUtils.NewJobId(2, 2, 2);
HistoryFileManager.HistoryFileInfo fileInfo2 = Org.Mockito.Mockito.Mock <HistoryFileManager.HistoryFileInfo
>();
Org.Mockito.Mockito.When(fileInfo2.GetJobId()).ThenReturn(jobId2);
JobId jobId3 = MRBuilderUtils.NewJobId(3, 3, 3);
HistoryFileManager.HistoryFileInfo fileInfo3 = Org.Mockito.Mockito.Mock <HistoryFileManager.HistoryFileInfo
>();
Org.Mockito.Mockito.When(fileInfo3.GetJobId()).ThenReturn(jobId3);
cache.AddIfAbsent(fileInfo1);
cache.AddIfAbsent(fileInfo2);
cache.AddIfAbsent(fileInfo3);
ICollection <HistoryFileManager.HistoryFileInfo> values;
for (int i = 0; i < 9; i++)
{
values = cache.Values();
if (values.Count > maxSize)
{
Sharpen.Thread.Sleep(100);
}
else
{
NUnit.Framework.Assert.IsFalse("fileInfo1 should have been evicted", values.Contains
(fileInfo1));
return;
}
}
NUnit.Framework.Assert.Fail("JobListCache didn't delete the extra entry");
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestAverageMergeTime()
{
string historyFileName = "job_1329348432655_0001-1329348443227-user-Sleep+job-1329348468601-10-1-SUCCEEDED-default.jhist";
string confFileName = "job_1329348432655_0001_conf.xml";
Configuration conf = new Configuration();
JobACLsManager jobAclsMgr = new JobACLsManager(conf);
Path fulleHistoryPath = new Path(typeof(TestJobHistoryEntities).GetClassLoader().
GetResource(historyFileName).GetFile());
Path fullConfPath = new Path(typeof(TestJobHistoryEntities).GetClassLoader().GetResource
(confFileName).GetFile());
HistoryFileManager.HistoryFileInfo info = Org.Mockito.Mockito.Mock <HistoryFileManager.HistoryFileInfo
>();
Org.Mockito.Mockito.When(info.GetConfFile()).ThenReturn(fullConfPath);
JobId jobId = MRBuilderUtils.NewJobId(1329348432655l, 1, 1);
CompletedJob completedJob = new CompletedJob(conf, jobId, fulleHistoryPath, true,
"user", info, jobAclsMgr);
JobInfo jobInfo = new JobInfo(completedJob);
// There are 2 tasks with merge time of 45 and 55 respectively. So average
// merge time should be 50.
NUnit.Framework.Assert.AreEqual(50L, jobInfo.GetAvgMergeTime());
}
/// <exception cref="System.Exception"/>
public virtual void TestKillJob()
{
JobConf conf = new JobConf();
AppContext context = Org.Mockito.Mockito.Mock <AppContext>();
// a simple event handler solely to detect the container cleaned event
CountDownLatch isDone = new CountDownLatch(1);
EventHandler handler = new _EventHandler_106(isDone);
Org.Mockito.Mockito.When(context.GetEventHandler()).ThenReturn(handler);
// create and start the launcher
LocalContainerLauncher launcher = new LocalContainerLauncher(context, Org.Mockito.Mockito.Mock
<TaskUmbilicalProtocol>());
launcher.Init(conf);
launcher.Start();
// create mocked job, task, and task attempt
// a single-mapper job
JobId jobId = MRBuilderUtils.NewJobId(Runtime.CurrentTimeMillis(), 1, 1);
TaskId taskId = MRBuilderUtils.NewTaskId(jobId, 1, TaskType.Map);
TaskAttemptId taId = MRBuilderUtils.NewTaskAttemptId(taskId, 0);
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job
>();
Org.Mockito.Mockito.When(job.GetTotalMaps()).ThenReturn(1);
Org.Mockito.Mockito.When(job.GetTotalReduces()).ThenReturn(0);
IDictionary <JobId, Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job> jobs = new Dictionary
<JobId, Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job>();
jobs[jobId] = job;
// app context returns the one and only job
Org.Mockito.Mockito.When(context.GetAllJobs()).ThenReturn(jobs);
Task ytask = Org.Mockito.Mockito.Mock <Task>();
Org.Mockito.Mockito.When(ytask.GetType()).ThenReturn(TaskType.Map);
Org.Mockito.Mockito.When(job.GetTask(taskId)).ThenReturn(ytask);
// create a sleeping mapper that runs beyond the test timeout
MapTask mapTask = Org.Mockito.Mockito.Mock <MapTask>();
Org.Mockito.Mockito.When(mapTask.IsMapOrReduce()).ThenReturn(true);
Org.Mockito.Mockito.When(mapTask.IsMapTask()).ThenReturn(true);
TaskAttemptID taskID = TypeConverter.FromYarn(taId);
Org.Mockito.Mockito.When(mapTask.GetTaskID()).ThenReturn(taskID);
Org.Mockito.Mockito.When(mapTask.GetJobID()).ThenReturn(((JobID)taskID.GetJobID()
));
Org.Mockito.Mockito.DoAnswer(new _Answer_152()).When(mapTask).Run(Matchers.IsA <JobConf
>(), Matchers.IsA <TaskUmbilicalProtocol>());
// sleep for a long time
// pump in a task attempt launch event
ContainerLauncherEvent launchEvent = new ContainerRemoteLaunchEvent(taId, null, CreateMockContainer
(), mapTask);
launcher.Handle(launchEvent);
Sharpen.Thread.Sleep(200);
// now pump in a container clean-up event
ContainerLauncherEvent cleanupEvent = new ContainerLauncherEvent(taId, null, null
, null, ContainerLauncher.EventType.ContainerRemoteCleanup);
launcher.Handle(cleanupEvent);
// wait for the event to fire: this should be received promptly
isDone.Await();
launcher.Close();
}
/// <exception cref="System.Exception"/>
public virtual void TestPoolSize()
{
ApplicationId appId = ApplicationId.NewInstance(12345, 67);
ApplicationAttemptId appAttemptId = ApplicationAttemptId.NewInstance(appId, 3);
JobId jobId = MRBuilderUtils.NewJobId(appId, 8);
TaskId taskId = MRBuilderUtils.NewTaskId(jobId, 9, TaskType.Map);
AppContext context = Org.Mockito.Mockito.Mock<AppContext>();
TestContainerLauncher.CustomContainerLauncher containerLauncher = new TestContainerLauncher.CustomContainerLauncher
(this, context);
containerLauncher.Init(new Configuration());
containerLauncher.Start();
ThreadPoolExecutor threadPool = containerLauncher.GetThreadPool();
// No events yet
NUnit.Framework.Assert.AreEqual(containerLauncher.initialPoolSize, MRJobConfig.DefaultMrAmContainerlauncherThreadpoolInitialSize
);
NUnit.Framework.Assert.AreEqual(0, threadPool.GetPoolSize());
NUnit.Framework.Assert.AreEqual(containerLauncher.initialPoolSize, threadPool.GetCorePoolSize
());
NUnit.Framework.Assert.IsNull(containerLauncher.foundErrors);
containerLauncher.expectedCorePoolSize = containerLauncher.initialPoolSize;
for (int i = 0; i < 10; i++)
{
ContainerId containerId = ContainerId.NewContainerId(appAttemptId, i);
TaskAttemptId taskAttemptId = MRBuilderUtils.NewTaskAttemptId(taskId, i);
containerLauncher.Handle(new ContainerLauncherEvent(taskAttemptId, containerId, "host"
+ i + ":1234", null, ContainerLauncher.EventType.ContainerRemoteLaunch));
}
WaitForEvents(containerLauncher, 10);
NUnit.Framework.Assert.AreEqual(10, threadPool.GetPoolSize());
NUnit.Framework.Assert.IsNull(containerLauncher.foundErrors);
// Same set of hosts, so no change
containerLauncher.finishEventHandling = true;
int timeOut = 0;
while (containerLauncher.numEventsProcessed.Get() < 10 && timeOut++ < 200)
{
Log.Info("Waiting for number of events processed to become " + 10 + ". It is now "
+ containerLauncher.numEventsProcessed.Get() + ". Timeout is " + timeOut);
Sharpen.Thread.Sleep(1000);
}
NUnit.Framework.Assert.AreEqual(10, containerLauncher.numEventsProcessed.Get());
containerLauncher.finishEventHandling = false;
for (int i_1 = 0; i_1 < 10; i_1++)
{
ContainerId containerId = ContainerId.NewContainerId(appAttemptId, i_1 + 10);
TaskAttemptId taskAttemptId = MRBuilderUtils.NewTaskAttemptId(taskId, i_1 + 10);
containerLauncher.Handle(new ContainerLauncherEvent(taskAttemptId, containerId, "host"
+ i_1 + ":1234", null, ContainerLauncher.EventType.ContainerRemoteLaunch));
}
WaitForEvents(containerLauncher, 20);
NUnit.Framework.Assert.AreEqual(10, threadPool.GetPoolSize());
NUnit.Framework.Assert.IsNull(containerLauncher.foundErrors);
// Different hosts, there should be an increase in core-thread-pool size to
// 21(11hosts+10buffer)
// Core pool size should be 21 but the live pool size should be only 11.
containerLauncher.expectedCorePoolSize = 11 + containerLauncher.initialPoolSize;
containerLauncher.finishEventHandling = false;
ContainerId containerId_1 = ContainerId.NewContainerId(appAttemptId, 21);
TaskAttemptId taskAttemptId_1 = MRBuilderUtils.NewTaskAttemptId(taskId, 21);
containerLauncher.Handle(new ContainerLauncherEvent(taskAttemptId_1, containerId_1
, "host11:1234", null, ContainerLauncher.EventType.ContainerRemoteLaunch));
WaitForEvents(containerLauncher, 21);
NUnit.Framework.Assert.AreEqual(11, threadPool.GetPoolSize());
NUnit.Framework.Assert.IsNull(containerLauncher.foundErrors);
containerLauncher.Stop();
// change configuration MR_AM_CONTAINERLAUNCHER_THREADPOOL_INITIAL_SIZE
// and verify initialPoolSize value.
Configuration conf = new Configuration();
conf.SetInt(MRJobConfig.MrAmContainerlauncherThreadpoolInitialSize, 20);
containerLauncher = new TestContainerLauncher.CustomContainerLauncher(this, context
);
containerLauncher.Init(conf);
NUnit.Framework.Assert.AreEqual(containerLauncher.initialPoolSize, 20);
}
private JobId CreateJobId(long clusterTimestamp, int idInt)
{
return(MRBuilderUtils.NewJobId(ApplicationId.NewInstance(clusterTimestamp, idInt)
, idInt));
}
public virtual void TestAttemptContainerRequest()
{
Text SecretKeyAlias = new Text("secretkeyalias");
byte[] SecretKey = Sharpen.Runtime.GetBytesForString(("secretkey"));
IDictionary <ApplicationAccessType, string> acls = new Dictionary <ApplicationAccessType
, string>(1);
acls[ApplicationAccessType.ViewApp] = "otheruser";
ApplicationId appId = ApplicationId.NewInstance(1, 1);
JobId jobId = MRBuilderUtils.NewJobId(appId, 1);
TaskId taskId = MRBuilderUtils.NewTaskId(jobId, 1, TaskType.Map);
Path jobFile = Org.Mockito.Mockito.Mock <Path>();
EventHandler eventHandler = Org.Mockito.Mockito.Mock <EventHandler>();
TaskAttemptListener taListener = Org.Mockito.Mockito.Mock <TaskAttemptListener>();
Org.Mockito.Mockito.When(taListener.GetAddress()).ThenReturn(new IPEndPoint("localhost"
, 0));
JobConf jobConf = new JobConf();
jobConf.SetClass("fs.file.impl", typeof(TestTaskAttemptContainerRequest.StubbedFS
), typeof(FileSystem));
jobConf.SetBoolean("fs.file.impl.disable.cache", true);
jobConf.Set(JobConf.MapredMapTaskEnv, string.Empty);
// setup UGI for security so tokens and keys are preserved
jobConf.Set(CommonConfigurationKeysPublic.HadoopSecurityAuthentication, "kerberos"
);
UserGroupInformation.SetConfiguration(jobConf);
Credentials credentials = new Credentials();
credentials.AddSecretKey(SecretKeyAlias, SecretKey);
Org.Apache.Hadoop.Security.Token.Token <JobTokenIdentifier> jobToken = new Org.Apache.Hadoop.Security.Token.Token
<JobTokenIdentifier>(Sharpen.Runtime.GetBytesForString(("tokenid")), Sharpen.Runtime.GetBytesForString
(("tokenpw")), new Text("tokenkind"), new Text("tokenservice"));
TaskAttemptImpl taImpl = new MapTaskAttemptImpl(taskId, 1, eventHandler, jobFile,
1, Org.Mockito.Mockito.Mock <JobSplit.TaskSplitMetaInfo>(), jobConf, taListener,
jobToken, credentials, new SystemClock(), null);
jobConf.Set(MRJobConfig.ApplicationAttemptId, taImpl.GetID().ToString());
ContainerLaunchContext launchCtx = TaskAttemptImpl.CreateContainerLaunchContext(acls
, jobConf, jobToken, taImpl.CreateRemoteTask(), TypeConverter.FromYarn(jobId), Org.Mockito.Mockito.Mock
<WrappedJvmID>(), taListener, credentials);
NUnit.Framework.Assert.AreEqual("ACLs mismatch", acls, launchCtx.GetApplicationACLs
());
Credentials launchCredentials = new Credentials();
DataInputByteBuffer dibb = new DataInputByteBuffer();
dibb.Reset(launchCtx.GetTokens());
launchCredentials.ReadTokenStorageStream(dibb);
// verify all tokens specified for the task attempt are in the launch context
foreach (Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> token in credentials
.GetAllTokens())
{
Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> launchToken = launchCredentials
.GetToken(token.GetService());
NUnit.Framework.Assert.IsNotNull("Token " + token.GetService() + " is missing", launchToken
);
NUnit.Framework.Assert.AreEqual("Token " + token.GetService() + " mismatch", token
, launchToken);
}
// verify the secret key is in the launch context
NUnit.Framework.Assert.IsNotNull("Secret key missing", launchCredentials.GetSecretKey
(SecretKeyAlias));
NUnit.Framework.Assert.IsTrue("Secret key mismatch", Arrays.Equals(SecretKey, launchCredentials
.GetSecretKey(SecretKeyAlias)));
}
public virtual void TestDelegationToken()
{
Logger rootLogger = LogManager.GetRootLogger();
rootLogger.SetLevel(Level.Debug);
YarnConfiguration conf = new YarnConfiguration(new JobConf());
// Just a random principle
conf.Set(JHAdminConfig.MrHistoryPrincipal, "RandomOrc/[email protected]");
conf.Set(CommonConfigurationKeysPublic.HadoopSecurityAuthentication, "kerberos");
UserGroupInformation.SetConfiguration(conf);
long initialInterval = 10000l;
long maxLifetime = 20000l;
long renewInterval = 10000l;
JobHistoryServer jobHistoryServer = null;
MRClientProtocol clientUsingDT = null;
long tokenFetchTime;
try
{
jobHistoryServer = new _JobHistoryServer_87(initialInterval, maxLifetime, renewInterval
);
// no keytab based login
// Don't need it, skip.;
// final JobHistoryServer jobHistoryServer = jhServer;
jobHistoryServer.Init(conf);
jobHistoryServer.Start();
MRClientProtocol hsService = jobHistoryServer.GetClientService().GetClientHandler
();
// Fake the authentication-method
UserGroupInformation loggedInUser = UserGroupInformation.CreateRemoteUser("*****@*****.**"
);
NUnit.Framework.Assert.AreEqual("testrenewer", loggedInUser.GetShortUserName());
// Default realm is APACHE.ORG
loggedInUser.SetAuthenticationMethod(UserGroupInformation.AuthenticationMethod.Kerberos
);
Token token = GetDelegationToken(loggedInUser, hsService, loggedInUser.GetShortUserName
());
tokenFetchTime = Runtime.CurrentTimeMillis();
Log.Info("Got delegation token at: " + tokenFetchTime);
// Now try talking to JHS using the delegation token
clientUsingDT = GetMRClientProtocol(token, jobHistoryServer.GetClientService().GetBindAddress
(), "TheDarkLord", conf);
GetJobReportRequest jobReportRequest = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord
<GetJobReportRequest>();
jobReportRequest.SetJobId(MRBuilderUtils.NewJobId(123456, 1, 1));
try
{
clientUsingDT.GetJobReport(jobReportRequest);
}
catch (IOException e)
{
NUnit.Framework.Assert.AreEqual("Unknown job job_123456_0001", e.Message);
}
// Renew after 50% of token age.
while (Runtime.CurrentTimeMillis() < tokenFetchTime + initialInterval / 2)
{
Sharpen.Thread.Sleep(500l);
}
long nextExpTime = RenewDelegationToken(loggedInUser, hsService, token);
long renewalTime = Runtime.CurrentTimeMillis();
Log.Info("Renewed token at: " + renewalTime + ", NextExpiryTime: " + nextExpTime);
// Wait for first expiry, but before renewed expiry.
while (Runtime.CurrentTimeMillis() > tokenFetchTime + initialInterval && Runtime.
CurrentTimeMillis() < nextExpTime)
{
Sharpen.Thread.Sleep(500l);
}
Sharpen.Thread.Sleep(50l);
// Valid token because of renewal.
try
{
clientUsingDT.GetJobReport(jobReportRequest);
}
catch (IOException e)
{
NUnit.Framework.Assert.AreEqual("Unknown job job_123456_0001", e.Message);
}
// Wait for expiry.
while (Runtime.CurrentTimeMillis() < renewalTime + renewInterval)
{
Sharpen.Thread.Sleep(500l);
}
Sharpen.Thread.Sleep(50l);
Log.Info("At time: " + Runtime.CurrentTimeMillis() + ", token should be invalid");
// Token should have expired.
try
{
clientUsingDT.GetJobReport(jobReportRequest);
NUnit.Framework.Assert.Fail("Should not have succeeded with an expired token");
}
catch (IOException e)
{
NUnit.Framework.Assert.IsTrue(e.InnerException.Message.Contains("is expired"));
}
// Test cancellation
// Stop the existing proxy, start another.
if (clientUsingDT != null)
{
// RPC.stopProxy(clientUsingDT);
clientUsingDT = null;
}
token = GetDelegationToken(loggedInUser, hsService, loggedInUser.GetShortUserName
());
tokenFetchTime = Runtime.CurrentTimeMillis();
Log.Info("Got delegation token at: " + tokenFetchTime);
// Now try talking to HSService using the delegation token
clientUsingDT = GetMRClientProtocol(token, jobHistoryServer.GetClientService().GetBindAddress
(), "loginuser2", conf);
try
{
clientUsingDT.GetJobReport(jobReportRequest);
}
catch (IOException e)
{
NUnit.Framework.Assert.Fail("Unexpected exception" + e);
}
CancelDelegationToken(loggedInUser, hsService, token);
// Testing the token with different renewer to cancel the token
Token tokenWithDifferentRenewer = GetDelegationToken(loggedInUser, hsService, "yarn"
);
CancelDelegationToken(loggedInUser, hsService, tokenWithDifferentRenewer);
if (clientUsingDT != null)
{
// RPC.stopProxy(clientUsingDT);
clientUsingDT = null;
}
// Creating a new connection.
clientUsingDT = GetMRClientProtocol(token, jobHistoryServer.GetClientService().GetBindAddress
(), "loginuser2", conf);
Log.Info("Cancelled delegation token at: " + Runtime.CurrentTimeMillis());
// Verify cancellation worked.
try
{
clientUsingDT.GetJobReport(jobReportRequest);
NUnit.Framework.Assert.Fail("Should not have succeeded with a cancelled delegation token"
);
}
catch (IOException)
{
}
}
finally
{
jobHistoryServer.Stop();
}
}