public virtual void TestRetriesOnAMConnectionFailures()
{
if (!isAMReachableFromClient)
{
return;
}
ResourceMgrDelegate rm = Org.Mockito.Mockito.Mock <ResourceMgrDelegate>();
Org.Mockito.Mockito.When(rm.GetApplicationReport(TypeConverter.ToYarn(oldJobId).GetAppId
())).ThenReturn(GetRunningApplicationReport("am1", 78));
// throw exception in 1st, 2nd, 3rd and 4th call of getJobReport, and
// succeed in the 5th call.
MRClientProtocol amProxy = Org.Mockito.Mockito.Mock <MRClientProtocol>();
Org.Mockito.Mockito.When(amProxy.GetJobReport(Matchers.Any <Org.Apache.Hadoop.Mapreduce.V2.Api.Protocolrecords.GetJobReportRequest
>())).ThenThrow(new RuntimeException("11")).ThenThrow(new RuntimeException("22")
).ThenThrow(new RuntimeException("33")).ThenThrow(new RuntimeException("44")).ThenReturn
(GetJobReportResponse());
Configuration conf = new YarnConfiguration();
conf.Set(MRConfig.FrameworkName, MRConfig.YarnFrameworkName);
conf.SetBoolean(MRJobConfig.JobAmAccessDisabled, !isAMReachableFromClient);
ClientServiceDelegate clientServiceDelegate = new _ClientServiceDelegate_167(amProxy
, conf, rm, oldJobId, null);
JobStatus jobStatus = clientServiceDelegate.GetJobStatus(oldJobId);
NUnit.Framework.Assert.IsNotNull(jobStatus);
// assert maxClientRetry is not decremented.
NUnit.Framework.Assert.AreEqual(conf.GetInt(MRJobConfig.MrClientMaxRetries, MRJobConfig
.DefaultMrClientMaxRetries), clientServiceDelegate.GetMaxClientRetry());
Org.Mockito.Mockito.Verify(amProxy, Org.Mockito.Mockito.Times(5)).GetJobReport(Matchers.Any
<Org.Apache.Hadoop.Mapreduce.V2.Api.Protocolrecords.GetJobReportRequest>());
}
public virtual void TestRMDownRestoreForJobStatusBeforeGetAMReport()
{
Configuration conf = new YarnConfiguration();
conf.SetInt(MRJobConfig.MrClientMaxRetries, 3);
conf.Set(MRConfig.FrameworkName, MRConfig.YarnFrameworkName);
conf.SetBoolean(MRJobConfig.JobAmAccessDisabled, !isAMReachableFromClient);
MRClientProtocol historyServerProxy = Org.Mockito.Mockito.Mock <MRClientProtocol>(
);
Org.Mockito.Mockito.When(historyServerProxy.GetJobReport(Matchers.Any <Org.Apache.Hadoop.Mapreduce.V2.Api.Protocolrecords.GetJobReportRequest
>())).ThenReturn(GetJobReportResponse());
ResourceMgrDelegate rmDelegate = Org.Mockito.Mockito.Mock <ResourceMgrDelegate>();
try
{
Org.Mockito.Mockito.When(rmDelegate.GetApplicationReport(jobId.GetAppId())).ThenThrow
(new UndeclaredThrowableException(new IOException("Connection refuced1"))).ThenThrow
(new UndeclaredThrowableException(new IOException("Connection refuced2"))).ThenReturn
(GetFinishedApplicationReport());
ClientServiceDelegate clientServiceDelegate = new ClientServiceDelegate(conf, rmDelegate
, oldJobId, historyServerProxy);
JobStatus jobStatus = clientServiceDelegate.GetJobStatus(oldJobId);
Org.Mockito.Mockito.Verify(rmDelegate, Org.Mockito.Mockito.Times(3)).GetApplicationReport
(Matchers.Any <ApplicationId>());
NUnit.Framework.Assert.IsNotNull(jobStatus);
}
catch (YarnException e)
{
throw new IOException(e);
}
}
public virtual void TestAMAccessDisabled()
{
//test only applicable when AM not reachable
if (isAMReachableFromClient)
{
return;
}
MRClientProtocol historyServerProxy = Org.Mockito.Mockito.Mock <MRClientProtocol>(
);
Org.Mockito.Mockito.When(historyServerProxy.GetJobReport(GetJobReportRequest())).
ThenReturn(GetJobReportResponseFromHistoryServer());
ResourceMgrDelegate rmDelegate = Org.Mockito.Mockito.Mock <ResourceMgrDelegate>();
try
{
Org.Mockito.Mockito.When(rmDelegate.GetApplicationReport(jobId.GetAppId())).ThenReturn
(GetRunningApplicationReport("am1", 78)).ThenReturn(GetRunningApplicationReport(
"am1", 78)).ThenReturn(GetRunningApplicationReport("am1", 78)).ThenReturn(GetFinishedApplicationReport
());
}
catch (YarnException e)
{
throw new IOException(e);
}
ClientServiceDelegate clientServiceDelegate = Org.Mockito.Mockito.Spy(GetClientServiceDelegate
(historyServerProxy, rmDelegate));
JobStatus jobStatus = clientServiceDelegate.GetJobStatus(oldJobId);
NUnit.Framework.Assert.IsNotNull(jobStatus);
NUnit.Framework.Assert.AreEqual("N/A", jobStatus.GetJobName());
Org.Mockito.Mockito.Verify(clientServiceDelegate, Org.Mockito.Mockito.Times(0)).InstantiateAMProxy
(Matchers.Any <IPEndPoint>());
// Should not reach AM even for second and third times too.
jobStatus = clientServiceDelegate.GetJobStatus(oldJobId);
NUnit.Framework.Assert.IsNotNull(jobStatus);
NUnit.Framework.Assert.AreEqual("N/A", jobStatus.GetJobName());
Org.Mockito.Mockito.Verify(clientServiceDelegate, Org.Mockito.Mockito.Times(0)).InstantiateAMProxy
(Matchers.Any <IPEndPoint>());
jobStatus = clientServiceDelegate.GetJobStatus(oldJobId);
NUnit.Framework.Assert.IsNotNull(jobStatus);
NUnit.Framework.Assert.AreEqual("N/A", jobStatus.GetJobName());
Org.Mockito.Mockito.Verify(clientServiceDelegate, Org.Mockito.Mockito.Times(0)).InstantiateAMProxy
(Matchers.Any <IPEndPoint>());
// The third time around, app is completed, so should go to JHS
JobStatus jobStatus1 = clientServiceDelegate.GetJobStatus(oldJobId);
NUnit.Framework.Assert.IsNotNull(jobStatus1);
NUnit.Framework.Assert.AreEqual("TestJobFilePath", jobStatus1.GetJobFile());
NUnit.Framework.Assert.AreEqual("http://TestTrackingUrl", jobStatus1.GetTrackingUrl
());
NUnit.Framework.Assert.AreEqual(1.0f, jobStatus1.GetMapProgress(), 0.0f);
NUnit.Framework.Assert.AreEqual(1.0f, jobStatus1.GetReduceProgress(), 0.0f);
Org.Mockito.Mockito.Verify(clientServiceDelegate, Org.Mockito.Mockito.Times(0)).InstantiateAMProxy
(Matchers.Any <IPEndPoint>());
}
public virtual void TestUnknownAppInRM()
{
MRClientProtocol historyServerProxy = Org.Mockito.Mockito.Mock <MRClientProtocol>(
);
Org.Mockito.Mockito.When(historyServerProxy.GetJobReport(GetJobReportRequest())).
ThenReturn(GetJobReportResponse());
ClientServiceDelegate clientServiceDelegate = GetClientServiceDelegate(historyServerProxy
, GetRMDelegate());
JobStatus jobStatus = clientServiceDelegate.GetJobStatus(oldJobId);
NUnit.Framework.Assert.IsNotNull(jobStatus);
}
/// <exception cref="Com.Google.Protobuf.ServiceException"/>
public virtual MRServiceProtos.GetJobReportResponseProto GetJobReport(RpcController
controller, MRServiceProtos.GetJobReportRequestProto proto)
{
GetJobReportRequestPBImpl request = new GetJobReportRequestPBImpl(proto);
try
{
GetJobReportResponse response = real.GetJobReport(request);
return(((GetJobReportResponsePBImpl)response).GetProto());
}
catch (IOException e)
{
throw new ServiceException(e);
}
}
public virtual void TestRetriesOnConnectionFailure()
{
MRClientProtocol historyServerProxy = Org.Mockito.Mockito.Mock <MRClientProtocol>(
);
Org.Mockito.Mockito.When(historyServerProxy.GetJobReport(GetJobReportRequest())).
ThenThrow(new RuntimeException("1")).ThenThrow(new RuntimeException("2")).ThenReturn
(GetJobReportResponse());
ResourceMgrDelegate rm = Org.Mockito.Mockito.Mock <ResourceMgrDelegate>();
Org.Mockito.Mockito.When(rm.GetApplicationReport(TypeConverter.ToYarn(oldJobId).GetAppId
())).ThenReturn(null);
ClientServiceDelegate clientServiceDelegate = GetClientServiceDelegate(historyServerProxy
, rm);
JobStatus jobStatus = clientServiceDelegate.GetJobStatus(oldJobId);
NUnit.Framework.Assert.IsNotNull(jobStatus);
Org.Mockito.Mockito.Verify(historyServerProxy, Org.Mockito.Mockito.Times(3)).GetJobReport
(Matchers.Any <Org.Apache.Hadoop.Mapreduce.V2.Api.Protocolrecords.GetJobReportRequest
>());
}
public virtual void TestJobReportFromHistoryServer()
{
MRClientProtocol historyServerProxy = Org.Mockito.Mockito.Mock <MRClientProtocol>(
);
Org.Mockito.Mockito.When(historyServerProxy.GetJobReport(GetJobReportRequest())).
ThenReturn(GetJobReportResponseFromHistoryServer());
ResourceMgrDelegate rm = Org.Mockito.Mockito.Mock <ResourceMgrDelegate>();
Org.Mockito.Mockito.When(rm.GetApplicationReport(TypeConverter.ToYarn(oldJobId).GetAppId
())).ThenReturn(null);
ClientServiceDelegate clientServiceDelegate = GetClientServiceDelegate(historyServerProxy
, rm);
JobStatus jobStatus = clientServiceDelegate.GetJobStatus(oldJobId);
NUnit.Framework.Assert.IsNotNull(jobStatus);
NUnit.Framework.Assert.AreEqual("TestJobFilePath", jobStatus.GetJobFile());
NUnit.Framework.Assert.AreEqual("http://TestTrackingUrl", jobStatus.GetTrackingUrl
());
NUnit.Framework.Assert.AreEqual(1.0f, jobStatus.GetMapProgress(), 0.0f);
NUnit.Framework.Assert.AreEqual(1.0f, jobStatus.GetReduceProgress(), 0.0f);
}
public virtual void TestNoRetryOnAMAuthorizationException()
{
if (!isAMReachableFromClient)
{
return;
}
ResourceMgrDelegate rm = Org.Mockito.Mockito.Mock <ResourceMgrDelegate>();
Org.Mockito.Mockito.When(rm.GetApplicationReport(TypeConverter.ToYarn(oldJobId).GetAppId
())).ThenReturn(GetRunningApplicationReport("am1", 78));
// throw authorization exception on first invocation
MRClientProtocol amProxy = Org.Mockito.Mockito.Mock <MRClientProtocol>();
Org.Mockito.Mockito.When(amProxy.GetJobReport(Matchers.Any <Org.Apache.Hadoop.Mapreduce.V2.Api.Protocolrecords.GetJobReportRequest
>())).ThenThrow(new AuthorizationException("Denied"));
Configuration conf = new YarnConfiguration();
conf.Set(MRConfig.FrameworkName, MRConfig.YarnFrameworkName);
conf.SetBoolean(MRJobConfig.JobAmAccessDisabled, !isAMReachableFromClient);
ClientServiceDelegate clientServiceDelegate = new _ClientServiceDelegate_205(amProxy
, conf, rm, oldJobId, null);
try
{
clientServiceDelegate.GetJobStatus(oldJobId);
NUnit.Framework.Assert.Fail("Exception should be thrown upon AuthorizationException"
);
}
catch (IOException e)
{
NUnit.Framework.Assert.AreEqual(typeof(AuthorizationException).FullName + ": Denied"
, e.Message);
}
// assert maxClientRetry is not decremented.
NUnit.Framework.Assert.AreEqual(conf.GetInt(MRJobConfig.MrClientMaxRetries, MRJobConfig
.DefaultMrClientMaxRetries), clientServiceDelegate.GetMaxClientRetry());
Org.Mockito.Mockito.Verify(amProxy, Org.Mockito.Mockito.Times(1)).GetJobReport(Matchers.Any
<Org.Apache.Hadoop.Mapreduce.V2.Api.Protocolrecords.GetJobReportRequest>());
}
public virtual void TestRemoteExceptionFromHistoryServer()
{
MRClientProtocol historyServerProxy = Org.Mockito.Mockito.Mock <MRClientProtocol>(
);
Org.Mockito.Mockito.When(historyServerProxy.GetJobReport(GetJobReportRequest())).
ThenThrow(new IOException("Job ID doesnot Exist"));
ResourceMgrDelegate rm = Org.Mockito.Mockito.Mock <ResourceMgrDelegate>();
Org.Mockito.Mockito.When(rm.GetApplicationReport(TypeConverter.ToYarn(oldJobId).GetAppId
())).ThenReturn(null);
ClientServiceDelegate clientServiceDelegate = GetClientServiceDelegate(historyServerProxy
, rm);
try
{
clientServiceDelegate.GetJobStatus(oldJobId);
NUnit.Framework.Assert.Fail("Invoke should throw exception after retries.");
}
catch (IOException e)
{
NUnit.Framework.Assert.IsTrue(e.Message.Contains("Job ID doesnot Exist"));
}
}
public virtual void Test()
{
TestMRClientService.MRAppWithClientService app = new TestMRClientService.MRAppWithClientService
(this, 1, 0, false);
Configuration conf = new Configuration();
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(conf);
app.WaitForState(job, JobState.Running);
NUnit.Framework.Assert.AreEqual("Num tasks not correct", 1, job.GetTasks().Count);
IEnumerator <Task> it = job.GetTasks().Values.GetEnumerator();
Task task = it.Next();
app.WaitForState(task, TaskState.Running);
TaskAttempt attempt = task.GetAttempts().Values.GetEnumerator().Next();
app.WaitForState(attempt, TaskAttemptState.Running);
// send the diagnostic
string diagnostic1 = "Diagnostic1";
string diagnostic2 = "Diagnostic2";
app.GetContext().GetEventHandler().Handle(new TaskAttemptDiagnosticsUpdateEvent(attempt
.GetID(), diagnostic1));
// send the status update
TaskAttemptStatusUpdateEvent.TaskAttemptStatus taskAttemptStatus = new TaskAttemptStatusUpdateEvent.TaskAttemptStatus
();
taskAttemptStatus.id = attempt.GetID();
taskAttemptStatus.progress = 0.5f;
taskAttemptStatus.stateString = "RUNNING";
taskAttemptStatus.taskState = TaskAttemptState.Running;
taskAttemptStatus.phase = Phase.Map;
// send the status update
app.GetContext().GetEventHandler().Handle(new TaskAttemptStatusUpdateEvent(attempt
.GetID(), taskAttemptStatus));
//verify that all object are fully populated by invoking RPCs.
YarnRPC rpc = YarnRPC.Create(conf);
MRClientProtocol proxy = (MRClientProtocol)rpc.GetProxy(typeof(MRClientProtocol),
app.clientService.GetBindAddress(), conf);
GetCountersRequest gcRequest = recordFactory.NewRecordInstance <GetCountersRequest
>();
gcRequest.SetJobId(job.GetID());
NUnit.Framework.Assert.IsNotNull("Counters is null", proxy.GetCounters(gcRequest)
.GetCounters());
GetJobReportRequest gjrRequest = recordFactory.NewRecordInstance <GetJobReportRequest
>();
gjrRequest.SetJobId(job.GetID());
JobReport jr = proxy.GetJobReport(gjrRequest).GetJobReport();
VerifyJobReport(jr);
GetTaskAttemptCompletionEventsRequest gtaceRequest = recordFactory.NewRecordInstance
<GetTaskAttemptCompletionEventsRequest>();
gtaceRequest.SetJobId(job.GetID());
gtaceRequest.SetFromEventId(0);
gtaceRequest.SetMaxEvents(10);
NUnit.Framework.Assert.IsNotNull("TaskCompletionEvents is null", proxy.GetTaskAttemptCompletionEvents
(gtaceRequest).GetCompletionEventList());
GetDiagnosticsRequest gdRequest = recordFactory.NewRecordInstance <GetDiagnosticsRequest
>();
gdRequest.SetTaskAttemptId(attempt.GetID());
NUnit.Framework.Assert.IsNotNull("Diagnostics is null", proxy.GetDiagnostics(gdRequest
).GetDiagnosticsList());
GetTaskAttemptReportRequest gtarRequest = recordFactory.NewRecordInstance <GetTaskAttemptReportRequest
>();
gtarRequest.SetTaskAttemptId(attempt.GetID());
TaskAttemptReport tar = proxy.GetTaskAttemptReport(gtarRequest).GetTaskAttemptReport
();
VerifyTaskAttemptReport(tar);
GetTaskReportRequest gtrRequest = recordFactory.NewRecordInstance <GetTaskReportRequest
>();
gtrRequest.SetTaskId(task.GetID());
NUnit.Framework.Assert.IsNotNull("TaskReport is null", proxy.GetTaskReport(gtrRequest
).GetTaskReport());
GetTaskReportsRequest gtreportsRequest = recordFactory.NewRecordInstance <GetTaskReportsRequest
>();
gtreportsRequest.SetJobId(job.GetID());
gtreportsRequest.SetTaskType(TaskType.Map);
NUnit.Framework.Assert.IsNotNull("TaskReports for map is null", proxy.GetTaskReports
(gtreportsRequest).GetTaskReportList());
gtreportsRequest = recordFactory.NewRecordInstance <GetTaskReportsRequest>();
gtreportsRequest.SetJobId(job.GetID());
gtreportsRequest.SetTaskType(TaskType.Reduce);
NUnit.Framework.Assert.IsNotNull("TaskReports for reduce is null", proxy.GetTaskReports
(gtreportsRequest).GetTaskReportList());
IList <string> diag = proxy.GetDiagnostics(gdRequest).GetDiagnosticsList();
NUnit.Framework.Assert.AreEqual("Num diagnostics not correct", 1, diag.Count);
NUnit.Framework.Assert.AreEqual("Diag 1 not correct", diagnostic1, diag[0].ToString
());
TaskReport taskReport = proxy.GetTaskReport(gtrRequest).GetTaskReport();
NUnit.Framework.Assert.AreEqual("Num diagnostics not correct", 1, taskReport.GetDiagnosticsCount
());
//send the done signal to the task
app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(task.GetAttempts()
.Values.GetEnumerator().Next().GetID(), TaskAttemptEventType.TaDone));
app.WaitForState(job, JobState.Succeeded);
// For invalid jobid, throw IOException
gtreportsRequest = recordFactory.NewRecordInstance <GetTaskReportsRequest>();
gtreportsRequest.SetJobId(TypeConverter.ToYarn(JobID.ForName("job_1415730144495_0001"
)));
gtreportsRequest.SetTaskType(TaskType.Reduce);
try
{
proxy.GetTaskReports(gtreportsRequest);
NUnit.Framework.Assert.Fail("IOException not thrown for invalid job id");
}
catch (IOException)
{
}
}
public virtual void TestReconnectOnAMRestart()
{
//test not applicable when AM not reachable
//as instantiateAMProxy is not called at all
if (!isAMReachableFromClient)
{
return;
}
MRClientProtocol historyServerProxy = Org.Mockito.Mockito.Mock <MRClientProtocol>(
);
// RM returns AM1 url, null, null and AM2 url on invocations.
// Nulls simulate the time when AM2 is in the process of restarting.
ResourceMgrDelegate rmDelegate = Org.Mockito.Mockito.Mock <ResourceMgrDelegate>();
try
{
Org.Mockito.Mockito.When(rmDelegate.GetApplicationReport(jobId.GetAppId())).ThenReturn
(GetRunningApplicationReport("am1", 78)).ThenReturn(GetRunningApplicationReport(
null, 0)).ThenReturn(GetRunningApplicationReport(null, 0)).ThenReturn(GetRunningApplicationReport
("am2", 90));
}
catch (YarnException e)
{
throw new IOException(e);
}
Org.Apache.Hadoop.Mapreduce.V2.Api.Protocolrecords.GetJobReportResponse jobReportResponse1
= Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Mapreduce.V2.Api.Protocolrecords.GetJobReportResponse
>();
Org.Mockito.Mockito.When(jobReportResponse1.GetJobReport()).ThenReturn(MRBuilderUtils
.NewJobReport(jobId, "jobName-firstGen", "user", JobState.Running, 0, 0, 0, 0, 0
, 0, 0, "anything", null, false, string.Empty));
// First AM returns a report with jobName firstGen and simulates AM shutdown
// on second invocation.
MRClientProtocol firstGenAMProxy = Org.Mockito.Mockito.Mock <MRClientProtocol>();
Org.Mockito.Mockito.When(firstGenAMProxy.GetJobReport(Matchers.Any <Org.Apache.Hadoop.Mapreduce.V2.Api.Protocolrecords.GetJobReportRequest
>())).ThenReturn(jobReportResponse1).ThenThrow(new RuntimeException("AM is down!"
));
Org.Apache.Hadoop.Mapreduce.V2.Api.Protocolrecords.GetJobReportResponse jobReportResponse2
= Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Mapreduce.V2.Api.Protocolrecords.GetJobReportResponse
>();
Org.Mockito.Mockito.When(jobReportResponse2.GetJobReport()).ThenReturn(MRBuilderUtils
.NewJobReport(jobId, "jobName-secondGen", "user", JobState.Running, 0, 0, 0, 0,
0, 0, 0, "anything", null, false, string.Empty));
// Second AM generation returns a report with jobName secondGen
MRClientProtocol secondGenAMProxy = Org.Mockito.Mockito.Mock <MRClientProtocol>();
Org.Mockito.Mockito.When(secondGenAMProxy.GetJobReport(Matchers.Any <Org.Apache.Hadoop.Mapreduce.V2.Api.Protocolrecords.GetJobReportRequest
>())).ThenReturn(jobReportResponse2);
ClientServiceDelegate clientServiceDelegate = Org.Mockito.Mockito.Spy(GetClientServiceDelegate
(historyServerProxy, rmDelegate));
// First time, connection should be to AM1, then to AM2. Further requests
// should use the same proxy to AM2 and so instantiateProxy shouldn't be
// called.
Org.Mockito.Mockito.DoReturn(firstGenAMProxy).DoReturn(secondGenAMProxy).When(clientServiceDelegate
).InstantiateAMProxy(Matchers.Any <IPEndPoint>());
JobStatus jobStatus = clientServiceDelegate.GetJobStatus(oldJobId);
NUnit.Framework.Assert.IsNotNull(jobStatus);
NUnit.Framework.Assert.AreEqual("jobName-firstGen", jobStatus.GetJobName());
jobStatus = clientServiceDelegate.GetJobStatus(oldJobId);
NUnit.Framework.Assert.IsNotNull(jobStatus);
NUnit.Framework.Assert.AreEqual("jobName-secondGen", jobStatus.GetJobName());
jobStatus = clientServiceDelegate.GetJobStatus(oldJobId);
NUnit.Framework.Assert.IsNotNull(jobStatus);
NUnit.Framework.Assert.AreEqual("jobName-secondGen", jobStatus.GetJobName());
Org.Mockito.Mockito.Verify(clientServiceDelegate, Org.Mockito.Mockito.Times(2)).InstantiateAMProxy
(Matchers.Any <IPEndPoint>());
}
public virtual void TestDelegationToken()
{
Logger rootLogger = LogManager.GetRootLogger();
rootLogger.SetLevel(Level.Debug);
YarnConfiguration conf = new YarnConfiguration(new JobConf());
// Just a random principle
conf.Set(JHAdminConfig.MrHistoryPrincipal, "RandomOrc/[email protected]");
conf.Set(CommonConfigurationKeysPublic.HadoopSecurityAuthentication, "kerberos");
UserGroupInformation.SetConfiguration(conf);
long initialInterval = 10000l;
long maxLifetime = 20000l;
long renewInterval = 10000l;
JobHistoryServer jobHistoryServer = null;
MRClientProtocol clientUsingDT = null;
long tokenFetchTime;
try
{
jobHistoryServer = new _JobHistoryServer_87(initialInterval, maxLifetime, renewInterval
);
// no keytab based login
// Don't need it, skip.;
// final JobHistoryServer jobHistoryServer = jhServer;
jobHistoryServer.Init(conf);
jobHistoryServer.Start();
MRClientProtocol hsService = jobHistoryServer.GetClientService().GetClientHandler
();
// Fake the authentication-method
UserGroupInformation loggedInUser = UserGroupInformation.CreateRemoteUser("*****@*****.**"
);
NUnit.Framework.Assert.AreEqual("testrenewer", loggedInUser.GetShortUserName());
// Default realm is APACHE.ORG
loggedInUser.SetAuthenticationMethod(UserGroupInformation.AuthenticationMethod.Kerberos
);
Token token = GetDelegationToken(loggedInUser, hsService, loggedInUser.GetShortUserName
());
tokenFetchTime = Runtime.CurrentTimeMillis();
Log.Info("Got delegation token at: " + tokenFetchTime);
// Now try talking to JHS using the delegation token
clientUsingDT = GetMRClientProtocol(token, jobHistoryServer.GetClientService().GetBindAddress
(), "TheDarkLord", conf);
GetJobReportRequest jobReportRequest = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord
<GetJobReportRequest>();
jobReportRequest.SetJobId(MRBuilderUtils.NewJobId(123456, 1, 1));
try
{
clientUsingDT.GetJobReport(jobReportRequest);
}
catch (IOException e)
{
NUnit.Framework.Assert.AreEqual("Unknown job job_123456_0001", e.Message);
}
// Renew after 50% of token age.
while (Runtime.CurrentTimeMillis() < tokenFetchTime + initialInterval / 2)
{
Sharpen.Thread.Sleep(500l);
}
long nextExpTime = RenewDelegationToken(loggedInUser, hsService, token);
long renewalTime = Runtime.CurrentTimeMillis();
Log.Info("Renewed token at: " + renewalTime + ", NextExpiryTime: " + nextExpTime);
// Wait for first expiry, but before renewed expiry.
while (Runtime.CurrentTimeMillis() > tokenFetchTime + initialInterval && Runtime.
CurrentTimeMillis() < nextExpTime)
{
Sharpen.Thread.Sleep(500l);
}
Sharpen.Thread.Sleep(50l);
// Valid token because of renewal.
try
{
clientUsingDT.GetJobReport(jobReportRequest);
}
catch (IOException e)
{
NUnit.Framework.Assert.AreEqual("Unknown job job_123456_0001", e.Message);
}
// Wait for expiry.
while (Runtime.CurrentTimeMillis() < renewalTime + renewInterval)
{
Sharpen.Thread.Sleep(500l);
}
Sharpen.Thread.Sleep(50l);
Log.Info("At time: " + Runtime.CurrentTimeMillis() + ", token should be invalid");
// Token should have expired.
try
{
clientUsingDT.GetJobReport(jobReportRequest);
NUnit.Framework.Assert.Fail("Should not have succeeded with an expired token");
}
catch (IOException e)
{
NUnit.Framework.Assert.IsTrue(e.InnerException.Message.Contains("is expired"));
}
// Test cancellation
// Stop the existing proxy, start another.
if (clientUsingDT != null)
{
// RPC.stopProxy(clientUsingDT);
clientUsingDT = null;
}
token = GetDelegationToken(loggedInUser, hsService, loggedInUser.GetShortUserName
());
tokenFetchTime = Runtime.CurrentTimeMillis();
Log.Info("Got delegation token at: " + tokenFetchTime);
// Now try talking to HSService using the delegation token
clientUsingDT = GetMRClientProtocol(token, jobHistoryServer.GetClientService().GetBindAddress
(), "loginuser2", conf);
try
{
clientUsingDT.GetJobReport(jobReportRequest);
}
catch (IOException e)
{
NUnit.Framework.Assert.Fail("Unexpected exception" + e);
}
CancelDelegationToken(loggedInUser, hsService, token);
// Testing the token with different renewer to cancel the token
Token tokenWithDifferentRenewer = GetDelegationToken(loggedInUser, hsService, "yarn"
);
CancelDelegationToken(loggedInUser, hsService, tokenWithDifferentRenewer);
if (clientUsingDT != null)
{
// RPC.stopProxy(clientUsingDT);
clientUsingDT = null;
}
// Creating a new connection.
clientUsingDT = GetMRClientProtocol(token, jobHistoryServer.GetClientService().GetBindAddress
(), "loginuser2", conf);
Log.Info("Cancelled delegation token at: " + Runtime.CurrentTimeMillis());
// Verify cancellation worked.
try
{
clientUsingDT.GetJobReport(jobReportRequest);
NUnit.Framework.Assert.Fail("Should not have succeeded with a cancelled delegation token"
);
}
catch (IOException)
{
}
}
finally
{
jobHistoryServer.Stop();
}
}
