public virtual void TestRetriesOnAMConnectionFailures() { if (!isAMReachableFromClient) { return; } ResourceMgrDelegate rm = Org.Mockito.Mockito.Mock <ResourceMgrDelegate>(); Org.Mockito.Mockito.When(rm.GetApplicationReport(TypeConverter.ToYarn(oldJobId).GetAppId ())).ThenReturn(GetRunningApplicationReport("am1", 78)); // throw exception in 1st, 2nd, 3rd and 4th call of getJobReport, and // succeed in the 5th call. MRClientProtocol amProxy = Org.Mockito.Mockito.Mock <MRClientProtocol>(); Org.Mockito.Mockito.When(amProxy.GetJobReport(Matchers.Any <Org.Apache.Hadoop.Mapreduce.V2.Api.Protocolrecords.GetJobReportRequest >())).ThenThrow(new RuntimeException("11")).ThenThrow(new RuntimeException("22") ).ThenThrow(new RuntimeException("33")).ThenThrow(new RuntimeException("44")).ThenReturn (GetJobReportResponse()); Configuration conf = new YarnConfiguration(); conf.Set(MRConfig.FrameworkName, MRConfig.YarnFrameworkName); conf.SetBoolean(MRJobConfig.JobAmAccessDisabled, !isAMReachableFromClient); ClientServiceDelegate clientServiceDelegate = new _ClientServiceDelegate_167(amProxy , conf, rm, oldJobId, null); JobStatus jobStatus = clientServiceDelegate.GetJobStatus(oldJobId); NUnit.Framework.Assert.IsNotNull(jobStatus); // assert maxClientRetry is not decremented. NUnit.Framework.Assert.AreEqual(conf.GetInt(MRJobConfig.MrClientMaxRetries, MRJobConfig .DefaultMrClientMaxRetries), clientServiceDelegate.GetMaxClientRetry()); Org.Mockito.Mockito.Verify(amProxy, Org.Mockito.Mockito.Times(5)).GetJobReport(Matchers.Any <Org.Apache.Hadoop.Mapreduce.V2.Api.Protocolrecords.GetJobReportRequest>()); }
public virtual void TestRMDownRestoreForJobStatusBeforeGetAMReport() { Configuration conf = new YarnConfiguration(); conf.SetInt(MRJobConfig.MrClientMaxRetries, 3); conf.Set(MRConfig.FrameworkName, MRConfig.YarnFrameworkName); conf.SetBoolean(MRJobConfig.JobAmAccessDisabled, !isAMReachableFromClient); MRClientProtocol historyServerProxy = Org.Mockito.Mockito.Mock <MRClientProtocol>( ); Org.Mockito.Mockito.When(historyServerProxy.GetJobReport(Matchers.Any <Org.Apache.Hadoop.Mapreduce.V2.Api.Protocolrecords.GetJobReportRequest >())).ThenReturn(GetJobReportResponse()); ResourceMgrDelegate rmDelegate = Org.Mockito.Mockito.Mock <ResourceMgrDelegate>(); try { Org.Mockito.Mockito.When(rmDelegate.GetApplicationReport(jobId.GetAppId())).ThenThrow (new UndeclaredThrowableException(new IOException("Connection refuced1"))).ThenThrow (new UndeclaredThrowableException(new IOException("Connection refuced2"))).ThenReturn (GetFinishedApplicationReport()); ClientServiceDelegate clientServiceDelegate = new ClientServiceDelegate(conf, rmDelegate , oldJobId, historyServerProxy); JobStatus jobStatus = clientServiceDelegate.GetJobStatus(oldJobId); Org.Mockito.Mockito.Verify(rmDelegate, Org.Mockito.Mockito.Times(3)).GetApplicationReport (Matchers.Any <ApplicationId>()); NUnit.Framework.Assert.IsNotNull(jobStatus); } catch (YarnException e) { throw new IOException(e); } }
public virtual void TestAMAccessDisabled() { //test only applicable when AM not reachable if (isAMReachableFromClient) { return; } MRClientProtocol historyServerProxy = Org.Mockito.Mockito.Mock <MRClientProtocol>( ); Org.Mockito.Mockito.When(historyServerProxy.GetJobReport(GetJobReportRequest())). ThenReturn(GetJobReportResponseFromHistoryServer()); ResourceMgrDelegate rmDelegate = Org.Mockito.Mockito.Mock <ResourceMgrDelegate>(); try { Org.Mockito.Mockito.When(rmDelegate.GetApplicationReport(jobId.GetAppId())).ThenReturn (GetRunningApplicationReport("am1", 78)).ThenReturn(GetRunningApplicationReport( "am1", 78)).ThenReturn(GetRunningApplicationReport("am1", 78)).ThenReturn(GetFinishedApplicationReport ()); } catch (YarnException e) { throw new IOException(e); } ClientServiceDelegate clientServiceDelegate = Org.Mockito.Mockito.Spy(GetClientServiceDelegate (historyServerProxy, rmDelegate)); JobStatus jobStatus = clientServiceDelegate.GetJobStatus(oldJobId); NUnit.Framework.Assert.IsNotNull(jobStatus); NUnit.Framework.Assert.AreEqual("N/A", jobStatus.GetJobName()); Org.Mockito.Mockito.Verify(clientServiceDelegate, Org.Mockito.Mockito.Times(0)).InstantiateAMProxy (Matchers.Any <IPEndPoint>()); // Should not reach AM even for second and third times too. jobStatus = clientServiceDelegate.GetJobStatus(oldJobId); NUnit.Framework.Assert.IsNotNull(jobStatus); NUnit.Framework.Assert.AreEqual("N/A", jobStatus.GetJobName()); Org.Mockito.Mockito.Verify(clientServiceDelegate, Org.Mockito.Mockito.Times(0)).InstantiateAMProxy (Matchers.Any <IPEndPoint>()); jobStatus = clientServiceDelegate.GetJobStatus(oldJobId); NUnit.Framework.Assert.IsNotNull(jobStatus); NUnit.Framework.Assert.AreEqual("N/A", jobStatus.GetJobName()); Org.Mockito.Mockito.Verify(clientServiceDelegate, Org.Mockito.Mockito.Times(0)).InstantiateAMProxy (Matchers.Any <IPEndPoint>()); // The third time around, app is completed, so should go to JHS JobStatus jobStatus1 = clientServiceDelegate.GetJobStatus(oldJobId); NUnit.Framework.Assert.IsNotNull(jobStatus1); NUnit.Framework.Assert.AreEqual("TestJobFilePath", jobStatus1.GetJobFile()); NUnit.Framework.Assert.AreEqual("http://TestTrackingUrl", jobStatus1.GetTrackingUrl ()); NUnit.Framework.Assert.AreEqual(1.0f, jobStatus1.GetMapProgress(), 0.0f); NUnit.Framework.Assert.AreEqual(1.0f, jobStatus1.GetReduceProgress(), 0.0f); Org.Mockito.Mockito.Verify(clientServiceDelegate, Org.Mockito.Mockito.Times(0)).InstantiateAMProxy (Matchers.Any <IPEndPoint>()); }
public virtual void TestUnknownAppInRM() { MRClientProtocol historyServerProxy = Org.Mockito.Mockito.Mock <MRClientProtocol>( ); Org.Mockito.Mockito.When(historyServerProxy.GetJobReport(GetJobReportRequest())). ThenReturn(GetJobReportResponse()); ClientServiceDelegate clientServiceDelegate = GetClientServiceDelegate(historyServerProxy , GetRMDelegate()); JobStatus jobStatus = clientServiceDelegate.GetJobStatus(oldJobId); NUnit.Framework.Assert.IsNotNull(jobStatus); }
/// <exception cref="Com.Google.Protobuf.ServiceException"/> public virtual MRServiceProtos.GetJobReportResponseProto GetJobReport(RpcController controller, MRServiceProtos.GetJobReportRequestProto proto) { GetJobReportRequestPBImpl request = new GetJobReportRequestPBImpl(proto); try { GetJobReportResponse response = real.GetJobReport(request); return(((GetJobReportResponsePBImpl)response).GetProto()); } catch (IOException e) { throw new ServiceException(e); } }
public virtual void TestRetriesOnConnectionFailure() { MRClientProtocol historyServerProxy = Org.Mockito.Mockito.Mock <MRClientProtocol>( ); Org.Mockito.Mockito.When(historyServerProxy.GetJobReport(GetJobReportRequest())). ThenThrow(new RuntimeException("1")).ThenThrow(new RuntimeException("2")).ThenReturn (GetJobReportResponse()); ResourceMgrDelegate rm = Org.Mockito.Mockito.Mock <ResourceMgrDelegate>(); Org.Mockito.Mockito.When(rm.GetApplicationReport(TypeConverter.ToYarn(oldJobId).GetAppId ())).ThenReturn(null); ClientServiceDelegate clientServiceDelegate = GetClientServiceDelegate(historyServerProxy , rm); JobStatus jobStatus = clientServiceDelegate.GetJobStatus(oldJobId); NUnit.Framework.Assert.IsNotNull(jobStatus); Org.Mockito.Mockito.Verify(historyServerProxy, Org.Mockito.Mockito.Times(3)).GetJobReport (Matchers.Any <Org.Apache.Hadoop.Mapreduce.V2.Api.Protocolrecords.GetJobReportRequest >()); }
public virtual void TestJobReportFromHistoryServer() { MRClientProtocol historyServerProxy = Org.Mockito.Mockito.Mock <MRClientProtocol>( ); Org.Mockito.Mockito.When(historyServerProxy.GetJobReport(GetJobReportRequest())). ThenReturn(GetJobReportResponseFromHistoryServer()); ResourceMgrDelegate rm = Org.Mockito.Mockito.Mock <ResourceMgrDelegate>(); Org.Mockito.Mockito.When(rm.GetApplicationReport(TypeConverter.ToYarn(oldJobId).GetAppId ())).ThenReturn(null); ClientServiceDelegate clientServiceDelegate = GetClientServiceDelegate(historyServerProxy , rm); JobStatus jobStatus = clientServiceDelegate.GetJobStatus(oldJobId); NUnit.Framework.Assert.IsNotNull(jobStatus); NUnit.Framework.Assert.AreEqual("TestJobFilePath", jobStatus.GetJobFile()); NUnit.Framework.Assert.AreEqual("http://TestTrackingUrl", jobStatus.GetTrackingUrl ()); NUnit.Framework.Assert.AreEqual(1.0f, jobStatus.GetMapProgress(), 0.0f); NUnit.Framework.Assert.AreEqual(1.0f, jobStatus.GetReduceProgress(), 0.0f); }
public virtual void TestNoRetryOnAMAuthorizationException() { if (!isAMReachableFromClient) { return; } ResourceMgrDelegate rm = Org.Mockito.Mockito.Mock <ResourceMgrDelegate>(); Org.Mockito.Mockito.When(rm.GetApplicationReport(TypeConverter.ToYarn(oldJobId).GetAppId ())).ThenReturn(GetRunningApplicationReport("am1", 78)); // throw authorization exception on first invocation MRClientProtocol amProxy = Org.Mockito.Mockito.Mock <MRClientProtocol>(); Org.Mockito.Mockito.When(amProxy.GetJobReport(Matchers.Any <Org.Apache.Hadoop.Mapreduce.V2.Api.Protocolrecords.GetJobReportRequest >())).ThenThrow(new AuthorizationException("Denied")); Configuration conf = new YarnConfiguration(); conf.Set(MRConfig.FrameworkName, MRConfig.YarnFrameworkName); conf.SetBoolean(MRJobConfig.JobAmAccessDisabled, !isAMReachableFromClient); ClientServiceDelegate clientServiceDelegate = new _ClientServiceDelegate_205(amProxy , conf, rm, oldJobId, null); try { clientServiceDelegate.GetJobStatus(oldJobId); NUnit.Framework.Assert.Fail("Exception should be thrown upon AuthorizationException" ); } catch (IOException e) { NUnit.Framework.Assert.AreEqual(typeof(AuthorizationException).FullName + ": Denied" , e.Message); } // assert maxClientRetry is not decremented. NUnit.Framework.Assert.AreEqual(conf.GetInt(MRJobConfig.MrClientMaxRetries, MRJobConfig .DefaultMrClientMaxRetries), clientServiceDelegate.GetMaxClientRetry()); Org.Mockito.Mockito.Verify(amProxy, Org.Mockito.Mockito.Times(1)).GetJobReport(Matchers.Any <Org.Apache.Hadoop.Mapreduce.V2.Api.Protocolrecords.GetJobReportRequest>()); }
public virtual void TestRemoteExceptionFromHistoryServer() { MRClientProtocol historyServerProxy = Org.Mockito.Mockito.Mock <MRClientProtocol>( ); Org.Mockito.Mockito.When(historyServerProxy.GetJobReport(GetJobReportRequest())). ThenThrow(new IOException("Job ID doesnot Exist")); ResourceMgrDelegate rm = Org.Mockito.Mockito.Mock <ResourceMgrDelegate>(); Org.Mockito.Mockito.When(rm.GetApplicationReport(TypeConverter.ToYarn(oldJobId).GetAppId ())).ThenReturn(null); ClientServiceDelegate clientServiceDelegate = GetClientServiceDelegate(historyServerProxy , rm); try { clientServiceDelegate.GetJobStatus(oldJobId); NUnit.Framework.Assert.Fail("Invoke should throw exception after retries."); } catch (IOException e) { NUnit.Framework.Assert.IsTrue(e.Message.Contains("Job ID doesnot Exist")); } }
public virtual void Test() { TestMRClientService.MRAppWithClientService app = new TestMRClientService.MRAppWithClientService (this, 1, 0, false); Configuration conf = new Configuration(); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(conf); app.WaitForState(job, JobState.Running); NUnit.Framework.Assert.AreEqual("Num tasks not correct", 1, job.GetTasks().Count); IEnumerator <Task> it = job.GetTasks().Values.GetEnumerator(); Task task = it.Next(); app.WaitForState(task, TaskState.Running); TaskAttempt attempt = task.GetAttempts().Values.GetEnumerator().Next(); app.WaitForState(attempt, TaskAttemptState.Running); // send the diagnostic string diagnostic1 = "Diagnostic1"; string diagnostic2 = "Diagnostic2"; app.GetContext().GetEventHandler().Handle(new TaskAttemptDiagnosticsUpdateEvent(attempt .GetID(), diagnostic1)); // send the status update TaskAttemptStatusUpdateEvent.TaskAttemptStatus taskAttemptStatus = new TaskAttemptStatusUpdateEvent.TaskAttemptStatus (); taskAttemptStatus.id = attempt.GetID(); taskAttemptStatus.progress = 0.5f; taskAttemptStatus.stateString = "RUNNING"; taskAttemptStatus.taskState = TaskAttemptState.Running; taskAttemptStatus.phase = Phase.Map; // send the status update app.GetContext().GetEventHandler().Handle(new TaskAttemptStatusUpdateEvent(attempt .GetID(), taskAttemptStatus)); //verify that all object are fully populated by invoking RPCs. YarnRPC rpc = YarnRPC.Create(conf); MRClientProtocol proxy = (MRClientProtocol)rpc.GetProxy(typeof(MRClientProtocol), app.clientService.GetBindAddress(), conf); GetCountersRequest gcRequest = recordFactory.NewRecordInstance <GetCountersRequest >(); gcRequest.SetJobId(job.GetID()); NUnit.Framework.Assert.IsNotNull("Counters is null", proxy.GetCounters(gcRequest) .GetCounters()); GetJobReportRequest gjrRequest = recordFactory.NewRecordInstance <GetJobReportRequest >(); gjrRequest.SetJobId(job.GetID()); JobReport jr = proxy.GetJobReport(gjrRequest).GetJobReport(); VerifyJobReport(jr); GetTaskAttemptCompletionEventsRequest gtaceRequest = recordFactory.NewRecordInstance <GetTaskAttemptCompletionEventsRequest>(); gtaceRequest.SetJobId(job.GetID()); gtaceRequest.SetFromEventId(0); gtaceRequest.SetMaxEvents(10); NUnit.Framework.Assert.IsNotNull("TaskCompletionEvents is null", proxy.GetTaskAttemptCompletionEvents (gtaceRequest).GetCompletionEventList()); GetDiagnosticsRequest gdRequest = recordFactory.NewRecordInstance <GetDiagnosticsRequest >(); gdRequest.SetTaskAttemptId(attempt.GetID()); NUnit.Framework.Assert.IsNotNull("Diagnostics is null", proxy.GetDiagnostics(gdRequest ).GetDiagnosticsList()); GetTaskAttemptReportRequest gtarRequest = recordFactory.NewRecordInstance <GetTaskAttemptReportRequest >(); gtarRequest.SetTaskAttemptId(attempt.GetID()); TaskAttemptReport tar = proxy.GetTaskAttemptReport(gtarRequest).GetTaskAttemptReport (); VerifyTaskAttemptReport(tar); GetTaskReportRequest gtrRequest = recordFactory.NewRecordInstance <GetTaskReportRequest >(); gtrRequest.SetTaskId(task.GetID()); NUnit.Framework.Assert.IsNotNull("TaskReport is null", proxy.GetTaskReport(gtrRequest ).GetTaskReport()); GetTaskReportsRequest gtreportsRequest = recordFactory.NewRecordInstance <GetTaskReportsRequest >(); gtreportsRequest.SetJobId(job.GetID()); gtreportsRequest.SetTaskType(TaskType.Map); NUnit.Framework.Assert.IsNotNull("TaskReports for map is null", proxy.GetTaskReports (gtreportsRequest).GetTaskReportList()); gtreportsRequest = recordFactory.NewRecordInstance <GetTaskReportsRequest>(); gtreportsRequest.SetJobId(job.GetID()); gtreportsRequest.SetTaskType(TaskType.Reduce); NUnit.Framework.Assert.IsNotNull("TaskReports for reduce is null", proxy.GetTaskReports (gtreportsRequest).GetTaskReportList()); IList <string> diag = proxy.GetDiagnostics(gdRequest).GetDiagnosticsList(); NUnit.Framework.Assert.AreEqual("Num diagnostics not correct", 1, diag.Count); NUnit.Framework.Assert.AreEqual("Diag 1 not correct", diagnostic1, diag[0].ToString ()); TaskReport taskReport = proxy.GetTaskReport(gtrRequest).GetTaskReport(); NUnit.Framework.Assert.AreEqual("Num diagnostics not correct", 1, taskReport.GetDiagnosticsCount ()); //send the done signal to the task app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(task.GetAttempts() .Values.GetEnumerator().Next().GetID(), TaskAttemptEventType.TaDone)); app.WaitForState(job, JobState.Succeeded); // For invalid jobid, throw IOException gtreportsRequest = recordFactory.NewRecordInstance <GetTaskReportsRequest>(); gtreportsRequest.SetJobId(TypeConverter.ToYarn(JobID.ForName("job_1415730144495_0001" ))); gtreportsRequest.SetTaskType(TaskType.Reduce); try { proxy.GetTaskReports(gtreportsRequest); NUnit.Framework.Assert.Fail("IOException not thrown for invalid job id"); } catch (IOException) { } }
public virtual void TestReconnectOnAMRestart() { //test not applicable when AM not reachable //as instantiateAMProxy is not called at all if (!isAMReachableFromClient) { return; } MRClientProtocol historyServerProxy = Org.Mockito.Mockito.Mock <MRClientProtocol>( ); // RM returns AM1 url, null, null and AM2 url on invocations. // Nulls simulate the time when AM2 is in the process of restarting. ResourceMgrDelegate rmDelegate = Org.Mockito.Mockito.Mock <ResourceMgrDelegate>(); try { Org.Mockito.Mockito.When(rmDelegate.GetApplicationReport(jobId.GetAppId())).ThenReturn (GetRunningApplicationReport("am1", 78)).ThenReturn(GetRunningApplicationReport( null, 0)).ThenReturn(GetRunningApplicationReport(null, 0)).ThenReturn(GetRunningApplicationReport ("am2", 90)); } catch (YarnException e) { throw new IOException(e); } Org.Apache.Hadoop.Mapreduce.V2.Api.Protocolrecords.GetJobReportResponse jobReportResponse1 = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Mapreduce.V2.Api.Protocolrecords.GetJobReportResponse >(); Org.Mockito.Mockito.When(jobReportResponse1.GetJobReport()).ThenReturn(MRBuilderUtils .NewJobReport(jobId, "jobName-firstGen", "user", JobState.Running, 0, 0, 0, 0, 0 , 0, 0, "anything", null, false, string.Empty)); // First AM returns a report with jobName firstGen and simulates AM shutdown // on second invocation. MRClientProtocol firstGenAMProxy = Org.Mockito.Mockito.Mock <MRClientProtocol>(); Org.Mockito.Mockito.When(firstGenAMProxy.GetJobReport(Matchers.Any <Org.Apache.Hadoop.Mapreduce.V2.Api.Protocolrecords.GetJobReportRequest >())).ThenReturn(jobReportResponse1).ThenThrow(new RuntimeException("AM is down!" )); Org.Apache.Hadoop.Mapreduce.V2.Api.Protocolrecords.GetJobReportResponse jobReportResponse2 = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Mapreduce.V2.Api.Protocolrecords.GetJobReportResponse >(); Org.Mockito.Mockito.When(jobReportResponse2.GetJobReport()).ThenReturn(MRBuilderUtils .NewJobReport(jobId, "jobName-secondGen", "user", JobState.Running, 0, 0, 0, 0, 0, 0, 0, "anything", null, false, string.Empty)); // Second AM generation returns a report with jobName secondGen MRClientProtocol secondGenAMProxy = Org.Mockito.Mockito.Mock <MRClientProtocol>(); Org.Mockito.Mockito.When(secondGenAMProxy.GetJobReport(Matchers.Any <Org.Apache.Hadoop.Mapreduce.V2.Api.Protocolrecords.GetJobReportRequest >())).ThenReturn(jobReportResponse2); ClientServiceDelegate clientServiceDelegate = Org.Mockito.Mockito.Spy(GetClientServiceDelegate (historyServerProxy, rmDelegate)); // First time, connection should be to AM1, then to AM2. Further requests // should use the same proxy to AM2 and so instantiateProxy shouldn't be // called. Org.Mockito.Mockito.DoReturn(firstGenAMProxy).DoReturn(secondGenAMProxy).When(clientServiceDelegate ).InstantiateAMProxy(Matchers.Any <IPEndPoint>()); JobStatus jobStatus = clientServiceDelegate.GetJobStatus(oldJobId); NUnit.Framework.Assert.IsNotNull(jobStatus); NUnit.Framework.Assert.AreEqual("jobName-firstGen", jobStatus.GetJobName()); jobStatus = clientServiceDelegate.GetJobStatus(oldJobId); NUnit.Framework.Assert.IsNotNull(jobStatus); NUnit.Framework.Assert.AreEqual("jobName-secondGen", jobStatus.GetJobName()); jobStatus = clientServiceDelegate.GetJobStatus(oldJobId); NUnit.Framework.Assert.IsNotNull(jobStatus); NUnit.Framework.Assert.AreEqual("jobName-secondGen", jobStatus.GetJobName()); Org.Mockito.Mockito.Verify(clientServiceDelegate, Org.Mockito.Mockito.Times(2)).InstantiateAMProxy (Matchers.Any <IPEndPoint>()); }
public virtual void TestDelegationToken() { Logger rootLogger = LogManager.GetRootLogger(); rootLogger.SetLevel(Level.Debug); YarnConfiguration conf = new YarnConfiguration(new JobConf()); // Just a random principle conf.Set(JHAdminConfig.MrHistoryPrincipal, "RandomOrc/[email protected]"); conf.Set(CommonConfigurationKeysPublic.HadoopSecurityAuthentication, "kerberos"); UserGroupInformation.SetConfiguration(conf); long initialInterval = 10000l; long maxLifetime = 20000l; long renewInterval = 10000l; JobHistoryServer jobHistoryServer = null; MRClientProtocol clientUsingDT = null; long tokenFetchTime; try { jobHistoryServer = new _JobHistoryServer_87(initialInterval, maxLifetime, renewInterval ); // no keytab based login // Don't need it, skip.; // final JobHistoryServer jobHistoryServer = jhServer; jobHistoryServer.Init(conf); jobHistoryServer.Start(); MRClientProtocol hsService = jobHistoryServer.GetClientService().GetClientHandler (); // Fake the authentication-method UserGroupInformation loggedInUser = UserGroupInformation.CreateRemoteUser("*****@*****.**" ); NUnit.Framework.Assert.AreEqual("testrenewer", loggedInUser.GetShortUserName()); // Default realm is APACHE.ORG loggedInUser.SetAuthenticationMethod(UserGroupInformation.AuthenticationMethod.Kerberos ); Token token = GetDelegationToken(loggedInUser, hsService, loggedInUser.GetShortUserName ()); tokenFetchTime = Runtime.CurrentTimeMillis(); Log.Info("Got delegation token at: " + tokenFetchTime); // Now try talking to JHS using the delegation token clientUsingDT = GetMRClientProtocol(token, jobHistoryServer.GetClientService().GetBindAddress (), "TheDarkLord", conf); GetJobReportRequest jobReportRequest = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <GetJobReportRequest>(); jobReportRequest.SetJobId(MRBuilderUtils.NewJobId(123456, 1, 1)); try { clientUsingDT.GetJobReport(jobReportRequest); } catch (IOException e) { NUnit.Framework.Assert.AreEqual("Unknown job job_123456_0001", e.Message); } // Renew after 50% of token age. while (Runtime.CurrentTimeMillis() < tokenFetchTime + initialInterval / 2) { Sharpen.Thread.Sleep(500l); } long nextExpTime = RenewDelegationToken(loggedInUser, hsService, token); long renewalTime = Runtime.CurrentTimeMillis(); Log.Info("Renewed token at: " + renewalTime + ", NextExpiryTime: " + nextExpTime); // Wait for first expiry, but before renewed expiry. while (Runtime.CurrentTimeMillis() > tokenFetchTime + initialInterval && Runtime. CurrentTimeMillis() < nextExpTime) { Sharpen.Thread.Sleep(500l); } Sharpen.Thread.Sleep(50l); // Valid token because of renewal. try { clientUsingDT.GetJobReport(jobReportRequest); } catch (IOException e) { NUnit.Framework.Assert.AreEqual("Unknown job job_123456_0001", e.Message); } // Wait for expiry. while (Runtime.CurrentTimeMillis() < renewalTime + renewInterval) { Sharpen.Thread.Sleep(500l); } Sharpen.Thread.Sleep(50l); Log.Info("At time: " + Runtime.CurrentTimeMillis() + ", token should be invalid"); // Token should have expired. try { clientUsingDT.GetJobReport(jobReportRequest); NUnit.Framework.Assert.Fail("Should not have succeeded with an expired token"); } catch (IOException e) { NUnit.Framework.Assert.IsTrue(e.InnerException.Message.Contains("is expired")); } // Test cancellation // Stop the existing proxy, start another. if (clientUsingDT != null) { // RPC.stopProxy(clientUsingDT); clientUsingDT = null; } token = GetDelegationToken(loggedInUser, hsService, loggedInUser.GetShortUserName ()); tokenFetchTime = Runtime.CurrentTimeMillis(); Log.Info("Got delegation token at: " + tokenFetchTime); // Now try talking to HSService using the delegation token clientUsingDT = GetMRClientProtocol(token, jobHistoryServer.GetClientService().GetBindAddress (), "loginuser2", conf); try { clientUsingDT.GetJobReport(jobReportRequest); } catch (IOException e) { NUnit.Framework.Assert.Fail("Unexpected exception" + e); } CancelDelegationToken(loggedInUser, hsService, token); // Testing the token with different renewer to cancel the token Token tokenWithDifferentRenewer = GetDelegationToken(loggedInUser, hsService, "yarn" ); CancelDelegationToken(loggedInUser, hsService, tokenWithDifferentRenewer); if (clientUsingDT != null) { // RPC.stopProxy(clientUsingDT); clientUsingDT = null; } // Creating a new connection. clientUsingDT = GetMRClientProtocol(token, jobHistoryServer.GetClientService().GetBindAddress (), "loginuser2", conf); Log.Info("Cancelled delegation token at: " + Runtime.CurrentTimeMillis()); // Verify cancellation worked. try { clientUsingDT.GetJobReport(jobReportRequest); NUnit.Framework.Assert.Fail("Should not have succeeded with a cancelled delegation token" ); } catch (IOException) { } } finally { jobHistoryServer.Stop(); } }