/// <exception cref="System.Exception"/>
public virtual void TestCompletedTask()
{
HistoryFileManager.HistoryFileInfo info = Org.Mockito.Mockito.Mock <HistoryFileManager.HistoryFileInfo
>();
Org.Mockito.Mockito.When(info.GetConfFile()).ThenReturn(fullConfPath);
completedJob = new CompletedJob(conf, jobId, fullHistoryPath, loadTasks, "user",
info, jobAclsManager);
TaskId mt1Id = MRBuilderUtils.NewTaskId(jobId, 0, TaskType.Map);
TaskId rt1Id = MRBuilderUtils.NewTaskId(jobId, 0, TaskType.Reduce);
IDictionary <TaskId, Task> mapTasks = completedJob.GetTasks(TaskType.Map);
IDictionary <TaskId, Task> reduceTasks = completedJob.GetTasks(TaskType.Reduce);
NUnit.Framework.Assert.AreEqual(10, mapTasks.Count);
NUnit.Framework.Assert.AreEqual(2, reduceTasks.Count);
Task mt1 = mapTasks[mt1Id];
NUnit.Framework.Assert.AreEqual(1, mt1.GetAttempts().Count);
NUnit.Framework.Assert.AreEqual(TaskState.Succeeded, mt1.GetState());
TaskReport mt1Report = mt1.GetReport();
NUnit.Framework.Assert.AreEqual(TaskState.Succeeded, mt1Report.GetTaskState());
NUnit.Framework.Assert.AreEqual(mt1Id, mt1Report.GetTaskId());
Task rt1 = reduceTasks[rt1Id];
NUnit.Framework.Assert.AreEqual(1, rt1.GetAttempts().Count);
NUnit.Framework.Assert.AreEqual(TaskState.Succeeded, rt1.GetState());
TaskReport rt1Report = rt1.GetReport();
NUnit.Framework.Assert.AreEqual(TaskState.Succeeded, rt1Report.GetTaskState());
NUnit.Framework.Assert.AreEqual(rt1Id, rt1Report.GetTaskId());
}
public virtual void TestTimeout()
{
EventHandler mockHandler = Org.Mockito.Mockito.Mock <EventHandler>();
Clock clock = new SystemClock();
TaskHeartbeatHandler hb = new TaskHeartbeatHandler(mockHandler, clock, 1);
Configuration conf = new Configuration();
conf.SetInt(MRJobConfig.TaskTimeout, 10);
//10 ms
conf.SetInt(MRJobConfig.TaskTimeoutCheckIntervalMs, 10);
//10 ms
hb.Init(conf);
hb.Start();
try
{
ApplicationId appId = ApplicationId.NewInstance(0l, 5);
JobId jobId = MRBuilderUtils.NewJobId(appId, 4);
TaskId tid = MRBuilderUtils.NewTaskId(jobId, 3, TaskType.Map);
TaskAttemptId taid = MRBuilderUtils.NewTaskAttemptId(tid, 2);
hb.Register(taid);
Sharpen.Thread.Sleep(100);
//Events only happen when the task is canceled
Org.Mockito.Mockito.Verify(mockHandler, Org.Mockito.Mockito.Times(2)).Handle(Matchers.Any
<Org.Apache.Hadoop.Yarn.Event.Event>());
}
finally
{
hb.Stop();
}
}
public virtual IList <AMInfo> GetAMInfos()
{
IList <AMInfo> amInfos = new List <AMInfo>();
foreach (JobHistoryParser.AMInfo jhAmInfo in jobInfo.GetAMInfos())
{
AMInfo amInfo = MRBuilderUtils.NewAMInfo(jhAmInfo.GetAppAttemptId(), jhAmInfo.GetStartTime
(), jhAmInfo.GetContainerId(), jhAmInfo.GetNodeManagerHost(), jhAmInfo.GetNodeManagerPort
(), jhAmInfo.GetNodeManagerHttpPort());
amInfos.AddItem(amInfo);
}
return(amInfos);
}
public virtual void TestAddExisting()
{
HistoryFileManager.JobListCache cache = new HistoryFileManager.JobListCache(2, 1000
);
JobId jobId = MRBuilderUtils.NewJobId(1, 1, 1);
HistoryFileManager.HistoryFileInfo fileInfo = Org.Mockito.Mockito.Mock <HistoryFileManager.HistoryFileInfo
>();
Org.Mockito.Mockito.When(fileInfo.GetJobId()).ThenReturn(jobId);
cache.AddIfAbsent(fileInfo);
cache.AddIfAbsent(fileInfo);
NUnit.Framework.Assert.AreEqual("Incorrect number of cache entries", 1, cache.Values
().Count);
}
public virtual void TestAverageReduceTime()
{
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = Org.Mockito.Mockito.Mock <CompletedJob
>();
Task task1 = Org.Mockito.Mockito.Mock <Task>();
Task task2 = Org.Mockito.Mockito.Mock <Task>();
JobId jobId = MRBuilderUtils.NewJobId(1L, 1, 1);
TaskId taskId1 = MRBuilderUtils.NewTaskId(jobId, 1, TaskType.Reduce);
TaskId taskId2 = MRBuilderUtils.NewTaskId(jobId, 2, TaskType.Reduce);
TaskAttemptId taskAttemptId1 = MRBuilderUtils.NewTaskAttemptId(taskId1, 1);
TaskAttemptId taskAttemptId2 = MRBuilderUtils.NewTaskAttemptId(taskId2, 2);
TaskAttempt taskAttempt1 = Org.Mockito.Mockito.Mock <TaskAttempt>();
TaskAttempt taskAttempt2 = Org.Mockito.Mockito.Mock <TaskAttempt>();
JobReport jobReport = Org.Mockito.Mockito.Mock <JobReport>();
Org.Mockito.Mockito.When(taskAttempt1.GetState()).ThenReturn(TaskAttemptState.Succeeded
);
Org.Mockito.Mockito.When(taskAttempt1.GetLaunchTime()).ThenReturn(0L);
Org.Mockito.Mockito.When(taskAttempt1.GetShuffleFinishTime()).ThenReturn(4L);
Org.Mockito.Mockito.When(taskAttempt1.GetSortFinishTime()).ThenReturn(6L);
Org.Mockito.Mockito.When(taskAttempt1.GetFinishTime()).ThenReturn(8L);
Org.Mockito.Mockito.When(taskAttempt2.GetState()).ThenReturn(TaskAttemptState.Succeeded
);
Org.Mockito.Mockito.When(taskAttempt2.GetLaunchTime()).ThenReturn(5L);
Org.Mockito.Mockito.When(taskAttempt2.GetShuffleFinishTime()).ThenReturn(10L);
Org.Mockito.Mockito.When(taskAttempt2.GetSortFinishTime()).ThenReturn(22L);
Org.Mockito.Mockito.When(taskAttempt2.GetFinishTime()).ThenReturn(42L);
Org.Mockito.Mockito.When(task1.GetType()).ThenReturn(TaskType.Reduce);
Org.Mockito.Mockito.When(task2.GetType()).ThenReturn(TaskType.Reduce);
Org.Mockito.Mockito.When(task1.GetAttempts()).ThenReturn(new _Dictionary_120(taskAttemptId1
, taskAttempt1));
Org.Mockito.Mockito.When(task2.GetAttempts()).ThenReturn(new _Dictionary_123(taskAttemptId2
, taskAttempt2));
Org.Mockito.Mockito.When(job.GetTasks()).ThenReturn(new _Dictionary_127(taskId1,
task1, taskId2, task2));
Org.Mockito.Mockito.When(job.GetID()).ThenReturn(jobId);
Org.Mockito.Mockito.When(job.GetReport()).ThenReturn(jobReport);
Org.Mockito.Mockito.When(job.GetName()).ThenReturn("TestJobInfo");
Org.Mockito.Mockito.When(job.GetState()).ThenReturn(JobState.Succeeded);
JobInfo jobInfo = new JobInfo(job);
NUnit.Framework.Assert.AreEqual(11L, jobInfo.GetAvgReduceTime());
}
/// <exception cref="System.Exception"/>
public virtual void TestEviction()
{
int maxSize = 2;
HistoryFileManager.JobListCache cache = new HistoryFileManager.JobListCache(maxSize
, 1000);
JobId jobId1 = MRBuilderUtils.NewJobId(1, 1, 1);
HistoryFileManager.HistoryFileInfo fileInfo1 = Org.Mockito.Mockito.Mock <HistoryFileManager.HistoryFileInfo
>();
Org.Mockito.Mockito.When(fileInfo1.GetJobId()).ThenReturn(jobId1);
JobId jobId2 = MRBuilderUtils.NewJobId(2, 2, 2);
HistoryFileManager.HistoryFileInfo fileInfo2 = Org.Mockito.Mockito.Mock <HistoryFileManager.HistoryFileInfo
>();
Org.Mockito.Mockito.When(fileInfo2.GetJobId()).ThenReturn(jobId2);
JobId jobId3 = MRBuilderUtils.NewJobId(3, 3, 3);
HistoryFileManager.HistoryFileInfo fileInfo3 = Org.Mockito.Mockito.Mock <HistoryFileManager.HistoryFileInfo
>();
Org.Mockito.Mockito.When(fileInfo3.GetJobId()).ThenReturn(jobId3);
cache.AddIfAbsent(fileInfo1);
cache.AddIfAbsent(fileInfo2);
cache.AddIfAbsent(fileInfo3);
ICollection <HistoryFileManager.HistoryFileInfo> values;
for (int i = 0; i < 9; i++)
{
values = cache.Values();
if (values.Count > maxSize)
{
Sharpen.Thread.Sleep(100);
}
else
{
NUnit.Framework.Assert.IsFalse("fileInfo1 should have been evicted", values.Contains
(fileInfo1));
return;
}
}
NUnit.Framework.Assert.Fail("JobListCache didn't delete the extra entry");
}
/// <exception cref="System.Exception"/>
public virtual void TestCompletedTaskAttempt()
{
HistoryFileManager.HistoryFileInfo info = Org.Mockito.Mockito.Mock <HistoryFileManager.HistoryFileInfo
>();
Org.Mockito.Mockito.When(info.GetConfFile()).ThenReturn(fullConfPath);
completedJob = new CompletedJob(conf, jobId, fullHistoryPath, loadTasks, "user",
info, jobAclsManager);
TaskId mt1Id = MRBuilderUtils.NewTaskId(jobId, 0, TaskType.Map);
TaskId rt1Id = MRBuilderUtils.NewTaskId(jobId, 0, TaskType.Reduce);
TaskAttemptId mta1Id = MRBuilderUtils.NewTaskAttemptId(mt1Id, 0);
TaskAttemptId rta1Id = MRBuilderUtils.NewTaskAttemptId(rt1Id, 0);
Task mt1 = completedJob.GetTask(mt1Id);
Task rt1 = completedJob.GetTask(rt1Id);
TaskAttempt mta1 = mt1.GetAttempt(mta1Id);
NUnit.Framework.Assert.AreEqual(TaskAttemptState.Succeeded, mta1.GetState());
NUnit.Framework.Assert.AreEqual("localhost:45454", mta1.GetAssignedContainerMgrAddress
());
NUnit.Framework.Assert.AreEqual("localhost:9999", mta1.GetNodeHttpAddress());
TaskAttemptReport mta1Report = mta1.GetReport();
NUnit.Framework.Assert.AreEqual(TaskAttemptState.Succeeded, mta1Report.GetTaskAttemptState
());
NUnit.Framework.Assert.AreEqual("localhost", mta1Report.GetNodeManagerHost());
NUnit.Framework.Assert.AreEqual(45454, mta1Report.GetNodeManagerPort());
NUnit.Framework.Assert.AreEqual(9999, mta1Report.GetNodeManagerHttpPort());
TaskAttempt rta1 = rt1.GetAttempt(rta1Id);
NUnit.Framework.Assert.AreEqual(TaskAttemptState.Succeeded, rta1.GetState());
NUnit.Framework.Assert.AreEqual("localhost:45454", rta1.GetAssignedContainerMgrAddress
());
NUnit.Framework.Assert.AreEqual("localhost:9999", rta1.GetNodeHttpAddress());
TaskAttemptReport rta1Report = rta1.GetReport();
NUnit.Framework.Assert.AreEqual(TaskAttemptState.Succeeded, rta1Report.GetTaskAttemptState
());
NUnit.Framework.Assert.AreEqual("localhost", rta1Report.GetNodeManagerHost());
NUnit.Framework.Assert.AreEqual(45454, rta1Report.GetNodeManagerPort());
NUnit.Framework.Assert.AreEqual(9999, rta1Report.GetNodeManagerHttpPort());
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestAverageMergeTime()
{
string historyFileName = "job_1329348432655_0001-1329348443227-user-Sleep+job-1329348468601-10-1-SUCCEEDED-default.jhist";
string confFileName = "job_1329348432655_0001_conf.xml";
Configuration conf = new Configuration();
JobACLsManager jobAclsMgr = new JobACLsManager(conf);
Path fulleHistoryPath = new Path(typeof(TestJobHistoryEntities).GetClassLoader().
GetResource(historyFileName).GetFile());
Path fullConfPath = new Path(typeof(TestJobHistoryEntities).GetClassLoader().GetResource
(confFileName).GetFile());
HistoryFileManager.HistoryFileInfo info = Org.Mockito.Mockito.Mock <HistoryFileManager.HistoryFileInfo
>();
Org.Mockito.Mockito.When(info.GetConfFile()).ThenReturn(fullConfPath);
JobId jobId = MRBuilderUtils.NewJobId(1329348432655l, 1, 1);
CompletedJob completedJob = new CompletedJob(conf, jobId, fulleHistoryPath, true,
"user", info, jobAclsMgr);
JobInfo jobInfo = new JobInfo(completedJob);
// There are 2 tasks with merge time of 45 and 55 respectively. So average
// merge time should be 50.
NUnit.Framework.Assert.AreEqual(50L, jobInfo.GetAvgMergeTime());
}
private TaskAttemptId CreateTaskAttemptId(long clusterTimestamp, int jobIdInt, int
taskIdInt, TaskType taskType, int taskAttemptIdInt)
{
return(MRBuilderUtils.NewTaskAttemptId(CreateTaskId(clusterTimestamp, jobIdInt, taskIdInt
, taskType), taskAttemptIdInt));
}
public virtual void TestAttemptContainerRequest()
{
Text SecretKeyAlias = new Text("secretkeyalias");
byte[] SecretKey = Sharpen.Runtime.GetBytesForString(("secretkey"));
IDictionary <ApplicationAccessType, string> acls = new Dictionary <ApplicationAccessType
, string>(1);
acls[ApplicationAccessType.ViewApp] = "otheruser";
ApplicationId appId = ApplicationId.NewInstance(1, 1);
JobId jobId = MRBuilderUtils.NewJobId(appId, 1);
TaskId taskId = MRBuilderUtils.NewTaskId(jobId, 1, TaskType.Map);
Path jobFile = Org.Mockito.Mockito.Mock <Path>();
EventHandler eventHandler = Org.Mockito.Mockito.Mock <EventHandler>();
TaskAttemptListener taListener = Org.Mockito.Mockito.Mock <TaskAttemptListener>();
Org.Mockito.Mockito.When(taListener.GetAddress()).ThenReturn(new IPEndPoint("localhost"
, 0));
JobConf jobConf = new JobConf();
jobConf.SetClass("fs.file.impl", typeof(TestTaskAttemptContainerRequest.StubbedFS
), typeof(FileSystem));
jobConf.SetBoolean("fs.file.impl.disable.cache", true);
jobConf.Set(JobConf.MapredMapTaskEnv, string.Empty);
// setup UGI for security so tokens and keys are preserved
jobConf.Set(CommonConfigurationKeysPublic.HadoopSecurityAuthentication, "kerberos"
);
UserGroupInformation.SetConfiguration(jobConf);
Credentials credentials = new Credentials();
credentials.AddSecretKey(SecretKeyAlias, SecretKey);
Org.Apache.Hadoop.Security.Token.Token <JobTokenIdentifier> jobToken = new Org.Apache.Hadoop.Security.Token.Token
<JobTokenIdentifier>(Sharpen.Runtime.GetBytesForString(("tokenid")), Sharpen.Runtime.GetBytesForString
(("tokenpw")), new Text("tokenkind"), new Text("tokenservice"));
TaskAttemptImpl taImpl = new MapTaskAttemptImpl(taskId, 1, eventHandler, jobFile,
1, Org.Mockito.Mockito.Mock <JobSplit.TaskSplitMetaInfo>(), jobConf, taListener,
jobToken, credentials, new SystemClock(), null);
jobConf.Set(MRJobConfig.ApplicationAttemptId, taImpl.GetID().ToString());
ContainerLaunchContext launchCtx = TaskAttemptImpl.CreateContainerLaunchContext(acls
, jobConf, jobToken, taImpl.CreateRemoteTask(), TypeConverter.FromYarn(jobId), Org.Mockito.Mockito.Mock
<WrappedJvmID>(), taListener, credentials);
NUnit.Framework.Assert.AreEqual("ACLs mismatch", acls, launchCtx.GetApplicationACLs
());
Credentials launchCredentials = new Credentials();
DataInputByteBuffer dibb = new DataInputByteBuffer();
dibb.Reset(launchCtx.GetTokens());
launchCredentials.ReadTokenStorageStream(dibb);
// verify all tokens specified for the task attempt are in the launch context
foreach (Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> token in credentials
.GetAllTokens())
{
Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> launchToken = launchCredentials
.GetToken(token.GetService());
NUnit.Framework.Assert.IsNotNull("Token " + token.GetService() + " is missing", launchToken
);
NUnit.Framework.Assert.AreEqual("Token " + token.GetService() + " mismatch", token
, launchToken);
}
// verify the secret key is in the launch context
NUnit.Framework.Assert.IsNotNull("Secret key missing", launchCredentials.GetSecretKey
(SecretKeyAlias));
NUnit.Framework.Assert.IsTrue("Secret key mismatch", Arrays.Equals(SecretKey, launchCredentials
.GetSecretKey(SecretKeyAlias)));
}
public virtual void TestDelegationToken()
{
Logger rootLogger = LogManager.GetRootLogger();
rootLogger.SetLevel(Level.Debug);
YarnConfiguration conf = new YarnConfiguration(new JobConf());
// Just a random principle
conf.Set(JHAdminConfig.MrHistoryPrincipal, "RandomOrc/[email protected]");
conf.Set(CommonConfigurationKeysPublic.HadoopSecurityAuthentication, "kerberos");
UserGroupInformation.SetConfiguration(conf);
long initialInterval = 10000l;
long maxLifetime = 20000l;
long renewInterval = 10000l;
JobHistoryServer jobHistoryServer = null;
MRClientProtocol clientUsingDT = null;
long tokenFetchTime;
try
{
jobHistoryServer = new _JobHistoryServer_87(initialInterval, maxLifetime, renewInterval
);
// no keytab based login
// Don't need it, skip.;
// final JobHistoryServer jobHistoryServer = jhServer;
jobHistoryServer.Init(conf);
jobHistoryServer.Start();
MRClientProtocol hsService = jobHistoryServer.GetClientService().GetClientHandler
();
// Fake the authentication-method
UserGroupInformation loggedInUser = UserGroupInformation.CreateRemoteUser("*****@*****.**"
);
NUnit.Framework.Assert.AreEqual("testrenewer", loggedInUser.GetShortUserName());
// Default realm is APACHE.ORG
loggedInUser.SetAuthenticationMethod(UserGroupInformation.AuthenticationMethod.Kerberos
);
Token token = GetDelegationToken(loggedInUser, hsService, loggedInUser.GetShortUserName
());
tokenFetchTime = Runtime.CurrentTimeMillis();
Log.Info("Got delegation token at: " + tokenFetchTime);
// Now try talking to JHS using the delegation token
clientUsingDT = GetMRClientProtocol(token, jobHistoryServer.GetClientService().GetBindAddress
(), "TheDarkLord", conf);
GetJobReportRequest jobReportRequest = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord
<GetJobReportRequest>();
jobReportRequest.SetJobId(MRBuilderUtils.NewJobId(123456, 1, 1));
try
{
clientUsingDT.GetJobReport(jobReportRequest);
}
catch (IOException e)
{
NUnit.Framework.Assert.AreEqual("Unknown job job_123456_0001", e.Message);
}
// Renew after 50% of token age.
while (Runtime.CurrentTimeMillis() < tokenFetchTime + initialInterval / 2)
{
Sharpen.Thread.Sleep(500l);
}
long nextExpTime = RenewDelegationToken(loggedInUser, hsService, token);
long renewalTime = Runtime.CurrentTimeMillis();
Log.Info("Renewed token at: " + renewalTime + ", NextExpiryTime: " + nextExpTime);
// Wait for first expiry, but before renewed expiry.
while (Runtime.CurrentTimeMillis() > tokenFetchTime + initialInterval && Runtime.
CurrentTimeMillis() < nextExpTime)
{
Sharpen.Thread.Sleep(500l);
}
Sharpen.Thread.Sleep(50l);
// Valid token because of renewal.
try
{
clientUsingDT.GetJobReport(jobReportRequest);
}
catch (IOException e)
{
NUnit.Framework.Assert.AreEqual("Unknown job job_123456_0001", e.Message);
}
// Wait for expiry.
while (Runtime.CurrentTimeMillis() < renewalTime + renewInterval)
{
Sharpen.Thread.Sleep(500l);
}
Sharpen.Thread.Sleep(50l);
Log.Info("At time: " + Runtime.CurrentTimeMillis() + ", token should be invalid");
// Token should have expired.
try
{
clientUsingDT.GetJobReport(jobReportRequest);
NUnit.Framework.Assert.Fail("Should not have succeeded with an expired token");
}
catch (IOException e)
{
NUnit.Framework.Assert.IsTrue(e.InnerException.Message.Contains("is expired"));
}
// Test cancellation
// Stop the existing proxy, start another.
if (clientUsingDT != null)
{
// RPC.stopProxy(clientUsingDT);
clientUsingDT = null;
}
token = GetDelegationToken(loggedInUser, hsService, loggedInUser.GetShortUserName
());
tokenFetchTime = Runtime.CurrentTimeMillis();
Log.Info("Got delegation token at: " + tokenFetchTime);
// Now try talking to HSService using the delegation token
clientUsingDT = GetMRClientProtocol(token, jobHistoryServer.GetClientService().GetBindAddress
(), "loginuser2", conf);
try
{
clientUsingDT.GetJobReport(jobReportRequest);
}
catch (IOException e)
{
NUnit.Framework.Assert.Fail("Unexpected exception" + e);
}
CancelDelegationToken(loggedInUser, hsService, token);
// Testing the token with different renewer to cancel the token
Token tokenWithDifferentRenewer = GetDelegationToken(loggedInUser, hsService, "yarn"
);
CancelDelegationToken(loggedInUser, hsService, tokenWithDifferentRenewer);
if (clientUsingDT != null)
{
// RPC.stopProxy(clientUsingDT);
clientUsingDT = null;
}
// Creating a new connection.
clientUsingDT = GetMRClientProtocol(token, jobHistoryServer.GetClientService().GetBindAddress
(), "loginuser2", conf);
Log.Info("Cancelled delegation token at: " + Runtime.CurrentTimeMillis());
// Verify cancellation worked.
try
{
clientUsingDT.GetJobReport(jobReportRequest);
NUnit.Framework.Assert.Fail("Should not have succeeded with a cancelled delegation token"
);
}
catch (IOException)
{
}
}
finally
{
jobHistoryServer.Stop();
}
}
